novox/mesh-delivery
19
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Indexer: the catalog's values are the seat's types and the catalog client's, the gate's, checker's and builder's states are read by their own names, the go.mod leaves the old seat paths; no go/delivery name is left (D-11: its holders and buckets are its own constants checked against its manifest; ADR 0634 §1, §2, §5; issue 1966)

Browse Source 
Claude-Session: https://claude.ai/code/session_01NcbgfFSnn24M3GcqttoU7i
This commit is contained in:
jschoubben
2026-11-05 16:25:35 +01:00
parent 5125365189
commit 9869e34b8b
88 changed files with 2081 additions and 1293 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/catalog.go
+8 -7
View File
@@ -3,8 +3,9 @@ package main
import (
"fmt"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/views"
"git.novox.be/novox/mesh-delivery/sdk/graph"
@@ -16,12 +17,12 @@ import (
func catalogRead(modules, manifests, nodes stdio.KeptState, seatDefinitions, marks views.Source) index.CatalogRead {
return index.CatalogRead{
SeatDefinitions: seatDefinitions, Marks: marks,
Modules: func() (map[string]delivery.Module, error) {
Modules: func() (map[string]meshcatalog.ModulesByModuleValue, error) {
keys, err := modules.Keys()
if err != nil {
return nil, err
}
out := map[string]delivery.Module{}
out := map[string]meshcatalog.ModulesByModuleValue{}
for _, k := range keys {
e, err := modules.Get(k)
if err != nil {
@@ -30,14 +31,14 @@ func catalogRead(modules, manifests, nodes stdio.KeptState, seatDefinitions, mar
if e == nil {
continue
}
if m, err := delivery.DecodeModule(e.Value); err == nil {
if m, err := meshcatalogclient.DecodeModule(e.Value); err == nil {
out[k] = m
}
}
return out, nil
},
Manifest: func(digest string) (string, error) {
key, err := delivery.ManifestsKey(digest)
key, err := meshcatalogclient.ManifestsKey(digest)
if err != nil {
return "", err
}
@@ -48,8 +49,8 @@ func catalogRead(modules, manifests, nodes stdio.KeptState, seatDefinitions, mar
if e == nil {
return "", fmt.Errorf("the manifests state holds no %s", digest)
}
kept, err := delivery.DecodeManifest(e.Value)
return kept.JSON, err
kept, err := meshcatalogclient.DecodeManifest(e.Value)
return kept.Manifest, err
},
Graph: func() (graph.Graph, error) {
keys, err := nodes.Keys()
modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/holds.go
+2 -2
View File
@@ -10,7 +10,7 @@ import (
"git.novox.be/novox/mesh-delivery/sdk/enginereads"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
"mesh-delivery-indexer/internal/index"
)
@@ -45,7 +45,7 @@ func holdsTool(read stdio.KeptState, ask asker, now func() time.Time, acts meshh
}
if release {
what := "moved the mesh-wide hold of " + module + " by hand"
if mk, err := delivery.ModulesKey(module); err == nil {
if mk, err := meshcatalogclient.ModulesKey(module); err == nil {
if e, err := read.Get(mk); err == nil && e != nil {
if h, err := index.DecodeHold(e.Value); err == nil {
what += ", " + index.Strands(h)
modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/views.go
+13 -12
View File
@@ -6,8 +6,9 @@ import (
"time"
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/health"
"git.novox.be/novox/mesh-sdk/go/views"
@@ -22,14 +23,14 @@ func viewTools(getenv func(string) string, state func(string) views.Source, now
marks := state(views.MarksState)
var out []stdio.Tool
for _, t := range index.FirstViewTools {
out = append(out, views.FirstViewTool(delivery.IndexerModule, t.Tool, t.State, t.Words,
out = append(out, views.FirstViewTool(index.Module, t.Tool, t.State, t.Words,
views.ParseCallers(getenv(views.CallersEnv(t.Tool))), state(t.State), marks, now))
}
for _, t := range index.ReadTools {
out = append(out, views.ReadTool(delivery.IndexerModule, t.Tool, t.State, t.Words, state(t.State), marks, now))
out = append(out, views.ReadTool(index.Module, t.Tool, t.State, t.Words, state(t.State), marks, now))
}
// Its own first views, of the assigner's placement (DLV-F3r): read, and asked again by hand.
return append(out, views.StatusTools(delivery.IndexerModule, "indexer_first_views", "indexer_first_view_again", stdio.State(views.StatusState), meshhandactsclient.Recorder{Ask: stdio.Ask}, now)...)
return append(out, views.StatusTools(index.Module, "indexer_first_views", "indexer_first_view_again", stdio.State(views.StatusState), meshhandactsclient.Recorder{Ask: stdio.Ask}, now)...)
}
// keptSource is one of the indexer's states as its runtime reads it.
@@ -51,30 +52,30 @@ func configReadersTool(watched *stdio.WatchedStates) stdio.Tool {
if module == "" {
return nil, fmt.Errorf("%s names the module", index.ConfigReadersTool)
}
trunk := map[string]delivery.Module{}
trunk := map[string]meshcatalog.ModulesByModuleValue{}
mods, err := watched.Values("modules")
if err != nil {
return nil, fmt.Errorf("the modules state cannot be read: %w", err)
}
for k, v := range mods {
if m, err := delivery.DecodeModule(v); err == nil {
if m, err := meshcatalogclient.DecodeModule(v); err == nil {
trunk[k] = m
}
}
kept := map[string]delivery.Manifest{}
kept := map[string]meshcatalog.ManifestsByDigestValue{}
raws, err := watched.Values("manifests")
if err != nil {
return nil, fmt.Errorf("the manifests state cannot be read: %w", err)
}
for k, v := range raws {
if m, err := delivery.DecodeManifest(v); err == nil {
if m, err := meshcatalogclient.DecodeManifest(v); err == nil {
kept[k] = m
}
}
var pulls []delivery.ConfigCandidate
if cands, err := watched.Values(delivery.IndexerConfigCandidatesState); err == nil {
var pulls []meshcatalog.ConfigCandidatesByPullValue
if cands, err := watched.Values(index.ConfigCandidatesState); err == nil {
for _, v := range cands {
var c delivery.ConfigCandidate
var c meshcatalog.ConfigCandidatesByPullValue
if json.Unmarshal(v, &c) == nil {
pulls = append(pulls, c)
}
@@ -140,7 +141,7 @@ func packageUsersTool(state func(string) views.Source) stdio.Tool {
"package": map[string]any{"type": "string", "description": "one package by its name in its own ecosystem (a Go module path, an npm package name); all when absent"}}},
Run: func(args map[string]any) (any, error) {
only, _ := args["package"].(string)
got, err := index.PackageUsersOf(state("modules"), state("manifests"), state(delivery.PackagesState), only)
got, err := index.PackageUsersOf(state("modules"), state("manifests"), state(index.PackagesState), only)
if err != nil {
return nil, err
}
modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/bridgemessage.go
+2 -2
View File
@@ -5,13 +5,13 @@ import (
"encoding/json"
"fmt"
"git.novox.be/novox/mesh-sdk/go/delivery"
"mesh-delivery-indexer/internal/index"
)
// mergedTool is the forge bridge's tool that answers a merge's merged message again (issue 1585): arguments repository
// and merge, answer {message}, the message whole as the bridge writes it on FORGE. The indexer's manifest lists it in
// invokes, which is the grant to call it.
const mergedTool = delivery.ForgeBridgeModule + ".bridge_merged_message"
const mergedTool = index.ForgeBridgeModule + ".bridge_merged_message"
// messageVia asks the forge bridge for a merge's merged message, for a held merge the indexer does not hold in memory (a
// restart, a new build). An error is the bridge's own words.
modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go
+128 -115
View File
@@ -17,14 +17,25 @@ import (
"testing"
"time"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/nodebuildgo"
sdkstep "git.novox.be/novox/mesh-delivery/sdk/buildstep"
"github.com/nats-io/nats-server/v2/server"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-contracts/placement/go/clients/meshmoduleassignerclient"
"git.novox.be/novox/mesh-contracts/placement/go/seats/meshmoduleassigner"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdeliverychecker"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein"
"git.novox.be/novox/mesh-delivery/sdk/trunk"
"git.novox.be/novox/mesh-sdk/go/buscompose"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/process"
"git.novox.be/novox/mesh-delivery/sdk/graph"
@@ -74,20 +85,20 @@ func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10; 1585/T14, T15
builds.end(t, "novox/mesh-catalog", "c", "modules/c")
}, func(ctx context.Context, js jetstream.JetStream) {
// The states and the stream the controller makes from the indexer's manifest (old sections, ADR 0595 §11).
for bucket, history := range map[string]uint8{index.PendingBucket: 1, delivery.VerdictsBucket: delivery.VerdictsHistory,
graph.Bucket: 1, delivery.TakenBucket: 1, delivery.PackagesBucket: 1,
for bucket, history := range map[string]uint8{index.PendingBucket: 1, index.VerdictsBucket: index.VerdictsHistory,
graph.Bucket: 1, index.TakenBucket: 1, index.PackagesBucket: 1,
index.SeatDefinitionsBucket: 1, index.RequirementsBucket: 1, findingsBucket: 1, index.HoldsBucket: 1} {
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: bucket, History: history}); err != nil {
t.Fatal(err)
}
}
// The take-in events are the indexer's ordinary events (ADR 0633 §5), on the path manifest-accepted takes.
tc, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: delivery.ToolchainsBucket("mesh-build-go"), History: 1})
tc, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: index.ToolchainsBucket("mesh-build-go"), History: 1})
if err != nil {
t.Fatal(err)
}
key, _ := delivery.ToolchainKey("n1", "go")
value, err := delivery.EncodeToolchain(delivery.Toolchain{Language: "go", Version: "go1.27.1", Recipe: "1", At: time.Now()})
key, _ := index.ToolchainKey("n1", "go")
value, err := index.EncodeToolchain(nodebuildgo.Toolchain{Language: "go", Version: "go1.27.1", Recipe: "1", At: ptr(time.Now())})
if err != nil {
t.Fatal(err)
}
@@ -137,18 +148,18 @@ func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10; 1585/T14, T15
}
}
hash, _ := delivery.HeadsHash([]delivery.Head{{Repository: "novox/mesh-delivery", Number: 12, Head: h1}, {Repository: "novox/mesh-catalog", Number: 300, Head: h2}})
var c delivery.Candidate
hash, _ := meshdeliveryclient.HeadsHash([]meshdeliveryclient.Head{{Repository: "novox/mesh-delivery", Number: 12, Head: h1}, {Repository: "novox/mesh-catalog", Number: 300, Head: h2}})
var c meshcatalog.CandidatesByGroupValue
for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
e, err := candidates.Get(ctx, "feat/x")
if err != nil {
continue
}
if c, err = delivery.DecodeCandidate(e.Value()); err == nil && len(c.Pulls) == 2 {
if c, err = index.DecodeCandidate(e.Value()); err == nil && len(c.Pulls) == 2 {
break
}
}
if c.Group != "feat/x" || c.Base != 0 || c.HeadsHash != hash || len(c.Pulls) != 2 {
if c.Group != "feat/x" || c.Base != 0 || c.Heads != hash || len(c.Pulls) != 2 {
t.Fatalf("candidate = %+v within 20 s", c)
}
for _, p := range c.Pulls {
@@ -167,9 +178,9 @@ func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10; 1585/T14, T15
t.Errorf("views event = %+v, bucket at %d (%v)", views0, cur.Revision(), err)
}
marks, _ := js.KeyValue(ctx, index.MarksBucket)
f := &views.Follower{Owner: delivery.IndexerModule, Tool: "indexer_read_candidates", State: "candidates",
f := &views.Follower{Owner: index.Module, Tool: "indexer_read_candidates", State: "candidates",
Ask: func(context.Context, string) (views.Answer, error) {
return views.FirstView(delivery.IndexerModule, "candidates", jsSource{ctx, candidates}, jsSource{ctx, marks}, "", time.Now())
return views.FirstView(index.Module, "candidates", jsSource{ctx, candidates}, jsSource{ctx, marks}, "", time.Now())
}}
if err := f.Start(ctx); err != nil {
t.Fatal(err)
@@ -191,7 +202,7 @@ func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10; 1585/T14, T15
if err != nil {
t.Fatal(err)
}
const ix = delivery.IndexerModule
const ix = index.Module
m1 := forgemsg.Forge{Kind: forgemsg.ForgeMerged, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: h1,
Merge: "1111111111111111111111111111111111111111", Root: "r0",
Trees: map[string]string{"modules/" + ix: "tree-indexer-2", "modules/a": "tree-a-2"},
@@ -262,35 +273,35 @@ func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10; 1585/T14, T15
return e.Value(), true
}
// Only what building and moving the named indexer needs is written.
raw, ok := get(delivery.ModulesBucket, ix)
own, err := delivery.DecodeModule(raw)
if !ok || err != nil || own.FP2 != named || own.Commit != m1.Merge {
raw, ok := get(index.ModulesBucket, ix)
own, err := meshcatalogclient.DecodeModule(raw)
if !ok || err != nil || own.Fp2 != named || own.Commit != m1.Merge {
t.Fatalf("modules.%s = %+v (%v), want the named build %s at the merge", ix, own, err, named)
}
if _, ok := get(delivery.ManifestsBucket, own.Manifest); !ok {
if _, ok := get(index.ManifestsBucket, own.Manifest); !ok {
t.Errorf("manifests.%s is not written", own.Manifest)
}
if _, ok := get(delivery.VerdictsBucket, own.Manifest); !ok {
if _, ok := get(index.VerdictsBucket, own.Manifest); !ok {
t.Errorf("the verdict on the indexer's manifest %s is not written", own.Manifest)
}
if gk, _ := graph.Key(ix); !func() bool { _, ok := get(graph.Bucket, gk); return ok }() {
t.Errorf("the indexer's key %s of the module graph is not written", gk)
}
repoKey, _ := delivery.RepositoryKey(m1.Repository)
if _, ok := get(delivery.ModulesBucket, "a"); ok {
repoKey, _ := trunk.RepositoryKey(m1.Repository)
if _, ok := get(index.ModulesBucket, "a"); ok {
t.Errorf("modules.a is written while the revision is pending")
}
if _, ok := get(delivery.TakenBucket, repoKey); ok {
if _, ok := get(index.TakenBucket, repoKey); ok {
t.Errorf("taken.%s is written while the revision is pending", repoKey)
}
if e, err := candidates.Get(ctx, "feat/x"); err != nil {
t.Errorf("the candidate feat/x is gone while the merge of #12 is pending: %v", err)
} else if c, err := delivery.DecodeCandidate(e.Value()); err != nil || len(c.Pulls) != 2 {
} else if c, err := index.DecodeCandidate(e.Value()); err != nil || len(c.Pulls) != 2 {
t.Errorf("the candidate feat/x = %+v while the merge of #12 is pending, want its two pull requests", c)
}
// A group's first member arrives meanwhile; its end is still to come (the knowledge base's remark: it stays across the restart).
members := []delivery.Head{{Repository: "novox/mesh-catalog", Number: 40, Head: "4040404040404040404040404040404040404040"},
members := []meshdeliveryclient.Head{{Repository: "novox/mesh-catalog", Number: 40, Head: "4040404040404040404040404040404040404040"},
{Repository: "novox/mesh-controller", Number: 41, Head: "4141414141414141414141414141414141414141"}}
g1 := forgemsg.Forge{Kind: forgemsg.ForgeMerged, Repository: "novox/mesh-catalog", Branch: "feat/g", Number: 40, Head: members[0].Head,
Merge: "4000000000000000000000000000000000000000", Root: "r0", Trees: map[string]string{"modules/c": "tree-c-2"},
@@ -337,16 +348,16 @@ func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10; 1585/T14, T15
break
}
}
if raw, ok := get(delivery.ModulesBucket, "a"); !ok || !strings.Contains(string(raw), m1.Merge) {
if raw, ok := get(index.ModulesBucket, "a"); !ok || !strings.Contains(string(raw), m1.Merge) {
t.Errorf("modules.a = %s after the take-in at the start, want it at the merge %s", raw, m1.Merge)
}
if _, ok := get(delivery.TakenBucket, repoKey); !ok {
if _, ok := get(index.TakenBucket, repoKey); !ok {
t.Errorf("taken.%s is not written by the take-in at the start", repoKey)
}
if len(w) != 1 || w[0].Key != groupKey || len(w[0].Keys) != 1 || w[0].Keys[0] != g1Key {
t.Fatalf("indexer_pending after the start = %+v, want only the group gathered, its first member %s", w, g1Key)
}
if _, ok := get(delivery.ModulesBucket, "c"); ok {
if _, ok := get(index.ModulesBucket, "c"); ok {
t.Errorf("modules.c is written while its group is gathered")
}
@@ -365,8 +376,8 @@ func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10; 1585/T14, T15
}
var hasC, hasK bool
for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline) && !(hasC && hasK && len(w) == 0); time.Sleep(50 * time.Millisecond) {
_, hasC = get(delivery.ModulesBucket, "c")
_, hasK = get(delivery.ModulesBucket, "k")
_, hasC = get(index.ModulesBucket, "c")
_, hasK = get(index.ModulesBucket, "k")
w = waits()
}
if !hasC || !hasK || len(w) != 0 {
@@ -511,21 +522,21 @@ func startIndexerRun(t *testing.T, put func(context.Context, buildLog), before .
t.Fatal(err)
}
}
candidates, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: delivery.CandidatesBucket, History: delivery.CandidatesHistory})
candidates, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: index.CandidatesBucket, History: index.CandidatesHistory})
if err != nil {
t.Fatal(err)
}
modules, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: delivery.ModulesBucket, History: delivery.ModulesHistory})
modules, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: index.ModulesBucket, History: index.ModulesHistory})
if err != nil {
t.Fatal(err)
}
// The bucket of the manifests, made by the controller from the manifest's state entry manifests (history 1, issue 1449).
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: delivery.ManifestsBucket, History: delivery.ManifestsHistory}); err != nil {
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: index.ManifestsBucket, History: index.ManifestsHistory}); err != nil {
t.Fatal(err)
}
// The builder's module log, as the runtime raises it (ADR 0297), filled by put before the indexer starts.
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: buscompose.LogStreamName(delivery.BuilderModule, delivery.BuilderLog),
Subjects: []string{buscompose.LogSubject(delivery.BuilderModule, delivery.BuilderLog) + ".>"}, AllowDirect: true}); err != nil {
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: buscompose.LogStreamName(builds.BuilderModule, builds.BuilderLog),
Subjects: []string{buscompose.LogSubject(builds.BuilderModule, builds.BuilderLog) + ".>"}, AllowDirect: true}); err != nil {
t.Fatal(err)
}
// The indexer's marks (ADR 0615 §5), and a waiter on manifest-accepted, which the views family never wakes (its events
@@ -586,13 +597,13 @@ type buildLog struct {
// end appends the end of a done trunk build of module from repo's directory path.
func (l buildLog) end(t *testing.T, repo, module, path string) {
t.Helper()
raw, err := delivery.EncodeBuildLogEntry(delivery.BuildLogEntry{Event: delivery.BuildEndedEvent, Module: module, FP: "src2:" + module,
raw, err := builds.EncodeBuildLogEntry(builds.BuildLogEntry{Event: builds.BuildEndedEvent, Module: module, FP: "src2:" + module,
Repository: repo, Directory: path, Commit: "7a74cee83f88e6585a5511c3cc53bdf0f5303d20", Node: "n1",
At: time.Date(2026, 10, 19, 9, 38, 0, 0, time.UTC), Outcome: delivery.BuildDone})
At: time.Date(2026, 10, 19, 9, 38, 0, 0, time.UTC), Outcome: builds.BuildDone})
if err != nil {
t.Fatal(err)
}
if _, err := l.js.Publish(l.ctx, buscompose.LogSubject(delivery.BuilderModule, delivery.BuilderLog)+"."+module, raw); err != nil {
if _, err := l.js.Publish(l.ctx, buscompose.LogSubject(builds.BuilderModule, builds.BuilderLog)+"."+module, raw); err != nil {
t.Fatal(err)
}
}
@@ -614,15 +625,15 @@ func TestIndexerWritesModulesAtAMerge(t *testing.T) { // 1239/AC9; I1, I3: the m
if _, err := js.Publish(ctx, "mesh.stream.mesh-delivery-forge-bridge.event.forge.novox.mesh-delivery.merged", data); err != nil {
t.Fatal(err)
}
var m delivery.Module
var m meshcatalog.ModulesByModuleValue
for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
if e, err := modules.Get(ctx, "a"); err == nil {
if m, err = delivery.DecodeModule(e.Value()); err == nil {
if m, err = meshcatalogclient.DecodeModule(e.Value()); err == nil {
break
}
}
}
if m.Repository != "novox/mesh-delivery" || m.Commit != merge || m.Path != "modules/a" || m.Manifest == "" || m.FP != "" || m.None != "" { // 1606/AC2
if m.Repository != "novox/mesh-delivery" || m.Commit != merge || m.Path != "modules/a" || m.Manifest == "" || m.Fp != "" || m.None != "" { // 1606/AC2
t.Errorf("module = %+v within 20 s, want path modules/a, its manifest, and no value of today's form", m)
}
}
@@ -652,7 +663,7 @@ func TestIndexerKeepsManifestsAtAMerge(t *testing.T) { // 1444/AC4, 1449/AC4, 15
builds.end(t, "novox/mesh-catalog", "avahi", "modules/avahi")
}, func(ctx context.Context, js jetstream.JetStream) {
// The verdicts state and the stream of the indexer's events, as the controller makes them from its manifest.
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: delivery.VerdictsBucket, History: delivery.VerdictsHistory}); err != nil {
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: index.VerdictsBucket, History: index.VerdictsHistory}); err != nil {
t.Fatal(err)
}
})
@@ -671,20 +682,20 @@ func TestIndexerKeepsManifestsAtAMerge(t *testing.T) { // 1444/AC4, 1449/AC4, 15
if _, err := js.Publish(ctx, "mesh.stream.mesh-delivery-forge-bridge.event.forge.novox.mesh-catalog.merged", data); err != nil {
t.Fatal(err)
}
var m delivery.Module
var m meshcatalog.ModulesByModuleValue
for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
if e, err := modules.Get(ctx, "avahi"); err == nil {
if m, err = delivery.DecodeModule(e.Value()); err == nil {
if m, err = meshcatalogclient.DecodeModule(e.Value()); err == nil {
break
}
}
}
digest := delivery.ManifestDigest(manifest)
if m.Manifest != digest || m.None2 != "toolchain not yet reported" || m.FP2 != "" || m.Inputs2 == nil || m.Inputs2.Tree != "newtree" ||
digest := meshcatalogclient.ManifestDigest(manifest)
if m.Manifest != digest || m.None2 != "toolchain not yet reported" || m.Fp2 != "" || m.Inputs2 == nil || m.Inputs2.Tree != "newtree" ||
m.Inputs2.Locks["tools"] != goSum || m.Path != "modules/avahi" { // 1606/AC2
t.Errorf("module = %+v within 20 s, want its path modules/avahi, its manifest digest %s, its inputs and none2 'toolchain not yet reported'", m, digest)
}
kv, err := js.KeyValue(ctx, delivery.ManifestsBucket)
kv, err := js.KeyValue(ctx, index.ManifestsBucket)
if err != nil {
t.Fatal(err)
}
@@ -692,20 +703,20 @@ func TestIndexerKeepsManifestsAtAMerge(t *testing.T) { // 1444/AC4, 1449/AC4, 15
if err != nil {
t.Fatalf("manifests.%s: %v", digest, err)
}
if got, err := delivery.DecodeManifest(e.Value()); err != nil || got.Module != "avahi" || got.JSON != string(manifest) {
if got, err := meshcatalogclient.DecodeManifest(e.Value()); err != nil || got.Module != "avahi" || got.Manifest != string(manifest) {
t.Errorf("manifest = %+v err %v, want avahi and the file's bytes", got, err)
}
verdicts, err := js.KeyValue(ctx, delivery.VerdictsBucket)
verdicts, err := js.KeyValue(ctx, index.VerdictsBucket)
if err != nil {
t.Fatal(err)
}
want := map[string]string{"good": delivery.VerdictAccepted, "bad": delivery.VerdictRefused, "later": delivery.VerdictManifestWaits}
got := map[string]delivery.ManifestVerdict{}
want := map[string]string{"good": meshcatalogclient.VerdictAccepted, "bad": meshcatalogclient.VerdictRefused, "later": meshcatalogclient.VerdictManifestWaits}
got := map[string]meshcatalog.VerdictsByDigestValue{}
for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline) && len(got) < len(want); time.Sleep(50 * time.Millisecond) {
for name := range want {
if e, err := verdicts.Get(ctx, delivery.ManifestDigest([]byte(judged[name]))); err == nil {
if v, err := delivery.DecodeManifestVerdict(e.Value()); err == nil {
if e, err := verdicts.Get(ctx, meshcatalogclient.ManifestDigest([]byte(judged[name]))); err == nil {
if v, err := meshcatalogclient.DecodeVerdict(e.Value()); err == nil {
got[name] = v
}
}
@@ -732,8 +743,8 @@ func TestIndexerKeepsManifestsAtAMerge(t *testing.T) { // 1444/AC4, 1449/AC4, 15
if err != nil {
break
}
var r delivery.ManifestRefused
if msg.Subject == "mesh.seat.mesh-catalog.event."+delivery.ManifestRefusedEvent && json.Unmarshal(msg.Data, &r) == nil {
var r meshcatalog.ManifestRefusedEvent
if msg.Subject == "mesh.seat.mesh-catalog.event."+meshcatalog.EventManifestRefused && json.Unmarshal(msg.Data, &r) == nil {
said[r.Module] = true
}
}
@@ -745,9 +756,9 @@ func TestIndexerKeepsManifestsAtAMerge(t *testing.T) { // 1444/AC4, 1449/AC4, 15
}
// TestIndexerWritesAGoModuleAgainWhenTheBuildModuleReportsItsToolchain: the bucket mesh-build-go_toolchains exists (the state
// toolchains of the build module, delivery.ToolchainsBucket) and holds no key; a merged message of mesh-catalog carrying
// toolchains of the build module, index.ToolchainsBucket) and holds no key; a merged message of mesh-catalog carrying
// avahi's real manifest makes modules.avahi with none2 'toolchain not yet reported'; the bucket made after the indexer started, a put of the key n1.go and the build module's event toolchain.reported (the build
// module's own layout, delivery.ToolchainKey and EncodeToolchain) makes, within 20 s and with no merge, its fingerprint:
// module's own layout, index.ToolchainKey and EncodeToolchain) makes, within 20 s and with no merge, its fingerprint:
// no builder input is waited for (ADR 0530 §1; issue 1887 I1).
func TestIndexerWritesAGoModuleAgainWhenTheBuildModuleReportsItsToolchain(t *testing.T) { // 1444/AC10
manifest, err := os.ReadFile("../../internal/index/testdata/catalogue-a67f0196-avahi.module.json")
@@ -773,7 +784,7 @@ func TestIndexerWritesAGoModuleAgainWhenTheBuildModuleReportsItsToolchain(t *tes
var got string
for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
if e, err := modules.Get(ctx, "avahi"); err == nil {
if m, err := delivery.DecodeModule(e.Value()); err == nil {
if m, err := meshcatalogclient.DecodeModule(e.Value()); err == nil {
if got = m.None2; got == want {
break
}
@@ -782,20 +793,20 @@ func TestIndexerWritesAGoModuleAgainWhenTheBuildModuleReportsItsToolchain(t *tes
}
return got
}
if got := none2(delivery.ToolchainNoReport); got != delivery.ToolchainNoReport {
t.Fatalf("none2 = %q within 20 s, want %q", got, delivery.ToolchainNoReport)
if got := none2(sdkstep.ToolchainNoReport); got != sdkstep.ToolchainNoReport {
t.Fatalf("none2 = %q within 20 s, want %q", got, sdkstep.ToolchainNoReport)
}
// The bucket is made only now, as when the build module is assigned after the indexer started (review finding 9): the
// indexer finds it on its own.
kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: delivery.ToolchainsBucket("mesh-build-go"), History: 1})
kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: index.ToolchainsBucket("mesh-build-go"), History: 1})
if err != nil {
t.Fatal(err)
}
key, err := delivery.ToolchainKey("n1", "go")
key, err := index.ToolchainKey("n1", "go")
if err != nil {
t.Fatal(err)
}
value, err := delivery.EncodeToolchain(delivery.Toolchain{Language: "go", Version: "go1.27.1", Recipe: "1", At: time.Now()})
value, err := index.EncodeToolchain(nodebuildgo.Toolchain{Language: "go", Version: "go1.27.1", Recipe: "1", At: ptr(time.Now())})
if err != nil {
t.Fatal(err)
}
@@ -833,15 +844,15 @@ func TestIndexerGivesACandidateItsFingerprintWhenTheCheckerCreatesThePreviewsBuc
if _, err := js.CreateOrUpdateConsumer(ctx, PreviewsStream, jetstream.ConsumerConfig{Durable: "n1_mesh-delivery-indexer", AckPolicy: jetstream.AckExplicitPolicy}); err != nil {
t.Fatal(err)
}
source, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: delivery.SourceBucket, History: 4})
source, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: index.SourceBucket, History: 4})
if err != nil {
t.Fatal(err)
}
data, err := delivery.EncodeSource(delivery.Source{Group: "g", Heads: map[string]string{"novox/mesh-catalog": trunk}})
data, err := meshdeliveryclient.EncodeSource(meshdelivery.SourceByMainValue{Group: "g", Heads: map[string]string{"novox/mesh-catalog": trunk}})
if err != nil {
t.Fatal(err)
}
if _, err := source.Put(ctx, delivery.SourceMainKey, data); err != nil {
if _, err := source.Put(ctx, meshdeliveryclient.SourceMainKey, data); err != nil {
t.Fatal(err)
}
})
@@ -857,7 +868,7 @@ func TestIndexerGivesACandidateItsFingerprintWhenTheCheckerCreatesThePreviewsBuc
var got string
for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
if e, err := candidates.Get(ctx, "feat/x"); err == nil {
if c, err := delivery.DecodeCandidate(e.Value()); err == nil && len(c.Pulls) == 1 {
if c, err := index.DecodeCandidate(e.Value()); err == nil && len(c.Pulls) == 1 {
got = c.Pulls[0].Fps["avahi"].None2
if got == want {
break
@@ -871,14 +882,14 @@ func TestIndexerGivesACandidateItsFingerprintWhenTheCheckerCreatesThePreviewsBuc
t.Fatalf("none2 = %q within 20 s of the pull request, want 'no merge preview is computed'", got)
}
// The checker's bucket is made only now, as when the checker is assigned after the indexer started.
kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: delivery.PreviewsBucket, History: delivery.PreviewsHistory})
kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: index.PreviewsBucket, History: index.PreviewsHistory})
if err != nil {
t.Fatal(err)
}
pv := delivery.Preview{Repository: "novox/mesh-catalog", Head: head, Trunk: trunk, Merge: "6666666666666666666666666666666666666666",
Modules: map[string]delivery.PreviewModule{"modules/avahi": {Tree: "newtree", Manifest: delivery.ManifestDigest(manifest), Locks: map[string]string{"go.sum": goSum}}}}
key, _ := delivery.PreviewKey(pv.Repository, pv.Head, pv.Trunk)
data, err := delivery.EncodePreview(pv)
pv := meshdeliverychecker.PreviewsByRepositoryHeadTrunkValue{Repository: "novox/mesh-catalog", Head: head, Trunk: trunk, Merge: "6666666666666666666666666666666666666666",
Modules: map[string]meshdeliverychecker.PreviewsByRepositoryHeadTrunkValueModulesValue{"modules/avahi": {Tree: "newtree", Manifest: meshcatalogclient.ManifestDigest(manifest), Locks: map[string]string{"go.sum": goSum}}}}
key, _ := index.PreviewKey(pv.Repository, pv.Head, pv.Trunk)
data, err := index.EncodePreview(pv)
if err != nil {
t.Fatal(err)
}
@@ -888,8 +899,8 @@ func TestIndexerGivesACandidateItsFingerprintWhenTheCheckerCreatesThePreviewsBuc
if _, err := js.Publish(ctx, previewWrittenSubject, []byte(`{"key":"`+key+`"}`)); err != nil {
t.Fatal(err)
}
if got := none2(delivery.ToolchainNoReport); got != delivery.ToolchainNoReport {
t.Errorf("none2 = %q within 20 s of the preview, want %q (computed from the preview)", got, delivery.ToolchainNoReport)
if got := none2(sdkstep.ToolchainNoReport); got != sdkstep.ToolchainNoReport {
t.Errorf("none2 = %q within 20 s of the preview, want %q (computed from the preview)", got, sdkstep.ToolchainNoReport)
}
}
@@ -903,45 +914,45 @@ func TestIndexerJudgesAnUnreadableVerdictAgainAtStart(t *testing.T) { // 1923/AC
t.Fatal(err)
}
const goSum = "3b5028f0c1e9bde04dd766abd63eb5931bc516bae695cf46f00abcd8510b0381"
digest := delivery.ManifestDigest(manifest)
digest := meshcatalogclient.ManifestDigest(manifest)
_, ctx, _, modules := startIndexer(t, func(context.Context, buildLog) {}, func(ctx context.Context, js jetstream.JetStream) {
mkey, err := delivery.ManifestsKey(digest)
mkey, err := meshcatalogclient.ManifestsKey(digest)
if err != nil {
t.Fatal(err)
}
mdata, err := delivery.EncodeManifest(delivery.Manifest{Module: "avahi", JSON: string(manifest)})
mdata, err := meshcatalogclient.EncodeManifest(meshcatalog.ManifestsByDigestValue{Module: "avahi", Manifest: string(manifest)})
if err != nil {
t.Fatal(err)
}
manifests, err := js.KeyValue(ctx, delivery.ManifestsBucket)
manifests, err := js.KeyValue(ctx, index.ManifestsBucket)
if err != nil {
t.Fatal(err)
}
if _, err := manifests.Put(ctx, mkey, mdata); err != nil {
t.Fatal(err)
}
stale, err := delivery.EncodeModule(delivery.Module{Repository: "novox/mesh-catalog", Commit: "89abcdef0123456789abcdef0123456789abcdef", Path: "modules/avahi",
stale, err := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Repository: "novox/mesh-catalog", Commit: "89abcdef0123456789abcdef0123456789abcdef", Path: "modules/avahi",
Manifest: digest, None2: "the manifest cannot be read: claims node-build-go, which is a seat in the mesh's own namespace that it does not define",
Inputs2: &delivery.KeptInputs2{Tree: "tree-avahi", Locks: map[string]string{"go.sum": goSum}}})
Inputs2: &meshcatalog.ModulesByModuleValueInputs2{Tree: "tree-avahi", Locks: map[string]string{"go.sum": goSum}}})
if err != nil {
t.Fatal(err)
}
held, err := js.KeyValue(ctx, delivery.ModulesBucket)
held, err := js.KeyValue(ctx, index.ModulesBucket)
if err != nil {
t.Fatal(err)
}
if _, err := held.Put(ctx, "avahi", stale); err != nil {
t.Fatal(err)
}
kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: delivery.ToolchainsBucket("mesh-build-go"), History: 1})
kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: index.ToolchainsBucket("mesh-build-go"), History: 1})
if err != nil {
t.Fatal(err)
}
key, err := delivery.ToolchainKey("n1", "go")
key, err := index.ToolchainKey("n1", "go")
if err != nil {
t.Fatal(err)
}
value, err := delivery.EncodeToolchain(delivery.Toolchain{Language: "go", Version: "go1.27.1", Recipe: "1", At: time.Now()})
value, err := index.EncodeToolchain(nodebuildgo.Toolchain{Language: "go", Version: "go1.27.1", Recipe: "1", At: ptr(time.Now())})
if err != nil {
t.Fatal(err)
}
@@ -949,18 +960,18 @@ func TestIndexerJudgesAnUnreadableVerdictAgainAtStart(t *testing.T) { // 1923/AC
t.Fatal(err)
}
})
var got delivery.Module
var got meshcatalog.ModulesByModuleValue
for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
if e, err := modules.Get(ctx, "avahi"); err == nil {
if m, err := delivery.DecodeModule(e.Value()); err == nil {
if got = m; got.FP2 != "" {
if m, err := meshcatalogclient.DecodeModule(e.Value()); err == nil {
if got = m; got.Fp2 != "" {
break
}
}
}
}
if got.FP2 == "" || got.None2 != "" {
t.Errorf("avahi = fp2 %q none2 %q within 20 s of the start, want a fingerprint and no reason", got.FP2, got.None2)
if got.Fp2 == "" || got.None2 != "" {
t.Errorf("avahi = fp2 %q none2 %q within 20 s of the start, want a fingerprint and no reason", got.Fp2, got.None2)
}
}
@@ -978,16 +989,16 @@ func TestIndexerCountsOnlyThePlacedBuildNodesToolchains(t *testing.T) {
const goSum = "3b5028f0c1e9bde04dd766abd63eb5931bc516bae695cf46f00abcd8510b0381"
var placement, toolchains jetstream.KeyValue
place := func(ctx context.Context, t *testing.T, node string, modules ...string) {
p := delivery.Placement{Node: node}
p := meshmoduleassigner.Placement{Node: node}
for _, m := range modules {
p.Modules = append(p.Modules, delivery.PlacedModule{Module: m})
p.Modules = append(p.Modules, meshmoduleassigner.PlacedModule{Module: m})
}
// Its node key, the commit that names the modules placed (research 072 row 37).
values, err := delivery.PlacementValues(p)
values, err := meshmoduleassignerclient.PlacementValues(p)
if err != nil {
t.Fatal(err)
}
key, _ := delivery.PlacementNodeKey(node)
key := meshmoduleassigner.PlacementByNodeNodeKey(node)
if _, err := placement.Put(ctx, key, values[key]); err != nil {
t.Fatal(err)
}
@@ -998,14 +1009,14 @@ func TestIndexerCountsOnlyThePlacedBuildNodesToolchains(t *testing.T) {
assignerLate = up
t.Cleanup(func() { assignerLate = nil })
js, ctx, _, modules := startIndexer(t, func(context.Context, buildLog) {}, func(ctx context.Context, js jetstream.JetStream) {
kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: delivery.ToolchainsBucket("mesh-build-go"), History: 1})
kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: index.ToolchainsBucket("mesh-build-go"), History: 1})
if err != nil {
t.Fatal(err)
}
toolchains = kv
for node, version := range map[string]string{"n1": "go1.27.3", "n2": "go1.27.2"} {
key, _ := delivery.ToolchainKey(node, "go")
value, err := delivery.EncodeToolchain(delivery.Toolchain{Language: "go", Version: version, Recipe: "1", At: time.Now()})
key, _ := index.ToolchainKey(node, "go")
value, err := index.EncodeToolchain(nodebuildgo.Toolchain{Language: "go", Version: version, Recipe: "1", At: ptr(time.Now())})
if err != nil {
t.Fatal(err)
}
@@ -1014,7 +1025,7 @@ func TestIndexerCountsOnlyThePlacedBuildNodesToolchains(t *testing.T) {
}
}
// The seat mesh-placement's placement, as the assigner writes it (ADR 0567): n1 places mesh-build-go, n2 nothing.
if placement, err = js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: delivery.PlacementBucket, History: delivery.PlacementHistory}); err != nil {
if placement, err = js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: index.PlacementBucket, History: index.PlacementHistory}); err != nil {
t.Fatal(err)
}
place(ctx, t, "n1", "mesh-build-go")
@@ -1032,11 +1043,11 @@ func TestIndexerCountsOnlyThePlacedBuildNodesToolchains(t *testing.T) {
if _, err := js.Publish(ctx, "mesh.stream.mesh-delivery-forge-bridge.event.forge.novox.mesh-catalog.merged", data); err != nil {
t.Fatal(err)
}
read := func(ok func(delivery.Module) bool) delivery.Module {
var got delivery.Module
read := func(ok func(meshcatalog.ModulesByModuleValue) bool) meshcatalog.ModulesByModuleValue {
var got meshcatalog.ModulesByModuleValue
for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
if e, err := modules.Get(ctx, "avahi"); err == nil {
if m, err := delivery.DecodeModule(e.Value()); err == nil {
if m, err := meshcatalogclient.DecodeModule(e.Value()); err == nil {
if got = m; ok(m) {
break
}
@@ -1045,14 +1056,16 @@ func TestIndexerCountsOnlyThePlacedBuildNodesToolchains(t *testing.T) {
}
return got
}
agreed := func(m delivery.Module) bool { return m.FP2 != "" && m.Toolchains["go"].Version == "go1.27.3" }
agreed := func(m meshcatalog.ModulesByModuleValue) bool {
return m.Fp2 != "" && m.Toolchains["go"].Version == "go1.27.3"
}
held := read(agreed)
if !agreed(held) {
t.Fatalf("within 20 s avahi has fp2 %q, none2 %q, toolchains %v: want a fingerprint of n1's go1.27.3 alone", held.FP2, held.None2, held.Toolchains)
t.Fatalf("within 20 s avahi has fp2 %q, none2 %q, toolchains %v: want a fingerprint of n1's go1.27.3 alone", held.Fp2, held.None2, held.Toolchains)
}
place(ctx, t, "n2", "mesh-build-go")
key, _ := delivery.ToolchainKey("n2", "go")
value, err := delivery.EncodeToolchain(delivery.Toolchain{Language: "go", Version: "go1.27.2", Recipe: "1", At: time.Now().Add(time.Second)})
key, _ := index.ToolchainKey("n2", "go")
value, err := index.EncodeToolchain(nodebuildgo.Toolchain{Language: "go", Version: "go1.27.2", Recipe: "1", At: ptr(time.Now().Add(time.Second))})
if err != nil {
t.Fatal(err)
}
@@ -1064,8 +1077,8 @@ func TestIndexerCountsOnlyThePlacedBuildNodesToolchains(t *testing.T) {
time.Sleep(500 * time.Millisecond)
if e, err := modules.Get(ctx, "avahi"); err != nil {
t.Fatal(err)
} else if m, err := delivery.DecodeModule(e.Value()); err != nil || m.FP2 != held.FP2 {
t.Fatalf("after n2's placement avahi has fp2 %q, none2 %q: want its fingerprint %q kept", m.FP2, m.None2, held.FP2)
} else if m, err := meshcatalogclient.DecodeModule(e.Value()); err != nil || m.Fp2 != held.Fp2 {
t.Fatalf("after n2's placement avahi has fp2 %q, none2 %q: want its fingerprint %q kept", m.Fp2, m.None2, held.Fp2)
}
moved := strings.Replace(string(data), `"newtree"`, `"movedtree"`, 1)
if moved == string(data) {
@@ -1074,15 +1087,15 @@ func TestIndexerCountsOnlyThePlacedBuildNodesToolchains(t *testing.T) {
if _, err := js.Publish(ctx, "mesh.stream.mesh-delivery-forge-bridge.event.forge.novox.mesh-catalog.merged", []byte(moved)); err != nil {
t.Fatal(err)
}
disagree := func(m delivery.Module) bool {
return m.FP2 == "" && strings.HasPrefix(m.None2, "builders disagree: go1.27.2 recipe 1 on n2")
disagree := func(m meshcatalog.ModulesByModuleValue) bool {
return m.Fp2 == "" && strings.HasPrefix(m.None2, "builders disagree: go1.27.2 recipe 1 on n2")
}
if m := read(disagree); !disagree(m) {
t.Fatalf("within 20 s of a merge after n2's placement avahi has fp2 %q, none2 %q: want the builders to disagree", m.FP2, m.None2)
t.Fatalf("within 20 s of a merge after n2's placement avahi has fp2 %q, none2 %q: want the builders to disagree", m.Fp2, m.None2)
}
place(ctx, t, "n2")
if m := read(agreed); !agreed(m) {
t.Fatalf("within 20 s of n2's placement taken away avahi has fp2 %q, none2 %q", m.FP2, m.None2)
t.Fatalf("within 20 s of n2's placement taken away avahi has fp2 %q, none2 %q", m.Fp2, m.None2)
}
}
@@ -1095,23 +1108,23 @@ var assignerLate chan struct{}
// and its views family said by the views emitter onto ASSIGNER_VIEWS, which the indexer's durable consumer reads.
func serveAssignerPlacement(t *testing.T, ctx context.Context, js jetstream.JetStream, nc *nats.Conn) {
t.Helper()
placement, err := js.KeyValue(ctx, delivery.PlacementBucket)
placement, err := js.KeyValue(ctx, index.PlacementBucket)
if err != nil {
if placement, err = js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: delivery.PlacementBucket, History: delivery.PlacementHistory}); err != nil {
if placement, err = js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: index.PlacementBucket, History: index.PlacementHistory}); err != nil {
t.Fatal(err)
}
}
marks, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: delivery.AssignerModule + "_" + views.MarksState, History: 1})
marks, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: index.AssignerModule + "_" + views.MarksState, History: 1})
if err != nil {
t.Fatal(err)
}
if _, err := js.CreateOrUpdateConsumer(ctx, "EVENTS", jetstream.ConsumerConfig{Durable: "n1_mesh-delivery-indexer", AckPolicy: jetstream.AckExplicitPolicy,
FilterSubjects: []string{"mesh.mod." + delivery.AssignerModule + ".event.views.>"}}); err != nil {
FilterSubjects: []string{"mesh.mod." + index.AssignerModule + ".event.views.>"}}); err != nil {
t.Fatal(err)
}
em := views.Emitter{Owner: delivery.AssignerModule, Subject: process.Account{Node: "n1", Module: delivery.AssignerModule}.EventSubject,
em := views.Emitter{Owner: index.AssignerModule, Subject: process.Account{Node: "n1", Module: index.AssignerModule}.EventSubject,
Publish: views.JetStreamPublish(js), Marks: marks}
if err := em.Start(ctx, delivery.PlacementState, placement, nil); err != nil {
if err := em.Start(ctx, index.PlacementState, placement, nil); err != nil {
t.Fatal(err)
}
late := assignerLate
@@ -1123,7 +1136,7 @@ func serveAssignerPlacement(t *testing.T, ctx context.Context, js jetstream.JetS
return
}
}
a, err := views.FirstView(delivery.AssignerModule, delivery.PlacementState, jsSource{ctx, placement}, jsSource{ctx, marks}, "", time.Now())
a, err := views.FirstView(index.AssignerModule, index.PlacementState, jsSource{ctx, placement}, jsSource{ctx, marks}, "", time.Now())
out := map[string]any{"result": a, "node": "n0"}
if err != nil {
out = map[string]any{"error": err.Error()}
modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go
+16 -16
View File
@@ -26,9 +26,9 @@ import (
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-delivery/sdk/enginereads"
"git.novox.be/novox/mesh-delivery/sdk/graph"
@@ -190,7 +190,7 @@ type delivered interface {
// groupEndSubject is the whole subject of a delivery group's end on the forge bridge's stream: compared exactly, so a
// repository named forge/group, whose merged message ends the same way, is never taken for one.
var groupEndSubject = process.ModuleStreamSubject(delivery.ForgeBridgeModule, forgemsg.ForgeGroupEndEvent)
var groupEndSubject = process.ModuleStreamSubject(index.ForgeBridgeModule, forgemsg.ForgeGroupEndEvent)
// handleGroupEnd takes a group's end in (index.EndGroup): a body that does not read is terminated and said; an end
// whose members are not all held yet, or while the build this program runs is not known, is left for redelivery.
@@ -359,21 +359,21 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
return err
}
defer conn.Close()
candidates, err := process.EssentialBucket(ctx, js, "indexer", delivery.CandidatesBucket, "write a candidate's fingerprints")
candidates, err := process.EssentialBucket(ctx, js, "indexer", index.CandidatesBucket, "write a candidate's fingerprints")
if err != nil {
return err
}
modules, err := process.EssentialBucket(ctx, js, "indexer", delivery.ModulesBucket, "write a module's fingerprint")
modules, err := process.EssentialBucket(ctx, js, "indexer", index.ModulesBucket, "write a module's fingerprint")
if err != nil {
return err
}
manifests, err := process.EssentialBucket(ctx, js, "indexer", delivery.ManifestsBucket, "keep a module's manifest")
manifests, err := process.EssentialBucket(ctx, js, "indexer", index.ManifestsBucket, "keep a module's manifest")
if err != nil {
return err
}
// Its taken state (ADR 0556 clause 1) is bound when it appears, as every state a delivery adds (issue 1790): until then
// a merge's taken key is said and not written, and the rest of the indexer runs.
taken := process.AwaitBucket(ctx, js, conn, delivery.TakenBucket, "no repository's taken key is written", logger.Printf)
taken := process.AwaitBucket(ctx, js, conn, index.TakenBucket, "no repository's taken key is written", logger.Printf)
// The control records (ADR 0565 §2), new with this delivery and bound when they appear: until then nothing is written to them.
predefined := process.AwaitBucket(ctx, js, conn, index.PredefinedBucket, "no predefined list is written", logger.Printf)
configCandidates := process.AwaitBucket(ctx, js, conn, index.ConfigCandidatesBucket, "no pull request's config declarations are written", logger.Printf)
@@ -394,7 +394,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
verdicts := process.AwaitBucket(ctx, js, conn, index.VerdictsBucket, "no verdict on a manifest is written", logger.Printf)
// The published packages (ADR 0593 §2): a state this delivery brings, bound when it appears; until then every
// fingerprint takes the version line.
packages := process.AwaitBucket(ctx, js, conn, delivery.PackagesBucket, "no package is published, so every fingerprint takes the version line", logger.Printf)
packages := process.AwaitBucket(ctx, js, conn, index.PackagesBucket, "no package is published, so every fingerprint takes the version line", logger.Printf)
// How many modules use each old section (issue 1966 D4): a state this delivery brings, bound when it appears; until then
// the judge refuses no use of an old section.
oldSections := process.AwaitBucket(ctx, js, conn, index.OldSectionsBucket, "the count of the old sections is not kept, so the judge refuses no use of one", logger.Printf)
@@ -425,9 +425,9 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
// new one indexes the module it names when that module is registered and the bucket holds none (registered.go).
var started atomic.Bool
var startEnds []stdio.LogEntry // the log's entries read at the start, whose imports are kept once the packages are published
if err := calls.ReadLog(delivery.BuilderModule, delivery.BuilderLog).Watch("", stdio.LogFromAll, func(le stdio.LogEntry) error {
if err := calls.ReadLog(builds.BuilderModule, builds.BuilderLog).Watch("", stdio.LogFromAll, func(le stdio.LogEntry) error {
if err := known.Put(le.Value); err != nil {
logger.Printf("%s entry %d is not read: %v", delivery.BuildLogRead, le.Sequence, err)
logger.Printf("%s entry %d is not read: %v", builds.BuildLogRead, le.Sequence, err)
return nil
}
if started.Load() {
@@ -438,7 +438,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
ix.ApplyOutOfStepBuild(ctx, le.Value)
// A done build's imports are kept on its module (ADR 0593 §2); at the start they are kept once the packages are.
if err := ix.ApplyBuildEnd(ctx, le.Value); err != nil {
logger.Printf("%s entry %d: its imports are not kept: %v", delivery.BuildLogRead, le.Sequence, err)
logger.Printf("%s entry %d: its imports are not kept: %v", builds.BuildLogRead, le.Sequence, err)
}
} else {
startEnds = append(startEnds, le)
@@ -446,21 +446,21 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
return nil
}); err != nil {
logger.Printf("%s is not read (%v): a module whose key names no directory is not given one, and no registered module is "+
"indexed from it, until the indexer starts again", delivery.BuildLogRead, err)
"indexed from it, until the indexer starts again", builds.BuildLogRead, err)
}
started.Store(true)
// The gate's source bucket (mesh-delivery-gate_source) comes with the gate (release 3e, novox/hq ADR 0450 §1): its key
// main is the mesh's source revision, which stamps the modules (issue 1344). Where it does not exist yet the indexer does
// not wait for it (that needs a timer, ADR 0437 §8): it says so, and is started again with the gate.
sourceErrs := make(chan error, 1)
if source := process.OptionalBucket(ctx, js, delivery.SourceBucket, "no source revision is read until the indexer starts again", logger.Printf); source != nil {
if source := process.OptionalBucket(ctx, js, index.SourceBucket, "no source revision is read until the indexer starts again", logger.Printf); source != nil {
each := func(e jetstream.KeyValueEntry) {
if opOf(e) {
applySource(ctx, ix, e.Key(), e.Revision(), e.Value(), logger)
}
}
if sw, err := settle(ctx, source, each); err != nil {
process.NotWatched(logger.Printf, delivery.SourceBucket, err, "no source revision is read until the indexer starts again")
process.NotWatched(logger.Printf, index.SourceBucket, err, "no source revision is read until the indexer starts again")
} else {
go func() { sourceErrs <- follow(ctx, sw, each) }()
}
@@ -542,7 +542,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
for _, le := range startEnds {
ix.ApplyOutOfStepBuild(ctx, le.Value)
if err := ix.ApplyBuildEnd(ctx, le.Value); err != nil {
logger.Printf("%s entry %d: its imports are not kept: %v", delivery.BuildLogRead, le.Sequence, err)
logger.Printf("%s entry %d: its imports are not kept: %v", builds.BuildLogRead, le.Sequence, err)
}
}
startEnds = nil
@@ -673,8 +673,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
startViews(ctx, js, cred, marks,
findings,
map[string]readyOf{"manifests": boundNow(manifests), "modules": boundNow(modules), "candidates": boundNow(candidates),
index.VerdictsState: awaited(verdicts), delivery.IndexerPredefinedState: awaited(predefined),
delivery.IndexerConfigCandidatesState: awaited(configCandidates), delivery.PackagesState: awaited(packages),
index.VerdictsState: awaited(verdicts), index.PredefinedState: awaited(predefined),
index.ConfigCandidatesState: awaited(configCandidates), index.PackagesState: awaited(packages),
graphState: awaited(graph), index.HeardState: awaited(heard.b), "taken": awaited(taken), "findings": awaited(findings),
index.SeatDefinitionsState: awaited(seatDefs)}, errs, logger.Printf)
cc, err := process.HearEvents(cons, func(m jetstream.Msg) { handle(ctx, m, ix, heard, logger); ix.KickFill(ctx) }, logger.Printf)
modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main_test.go
+8 -5
View File
@@ -11,8 +11,9 @@ import (
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/process"
"mesh-delivery-indexer/internal/index"
@@ -110,7 +111,7 @@ func TestAMessageIsIndexedAndAcked(t *testing.T) { // 1238/AC2 (unit part)
m := &fakeMsg{subject: "mesh.stream.mesh-delivery-forge-bridge.event.forge.novox.mesh-delivery.pull", data: data}
heard := &memStore{vals: map[string][]byte{}}
handle(context.Background(), m, ix, heard, log.New(&bytes.Buffer{}, "", 0))
c, err := delivery.DecodeCandidate(st.vals["feat/x"])
c, err := index.DecodeCandidate(st.vals["feat/x"])
if err != nil || len(c.Pulls) != 1 || c.Pulls[0].Modules[0] != "a" || m.acked != 1 || m.doubled != 1 || m.termd != 0 {
t.Errorf("candidate %+v err %v acked %d terminated %d", c, err, m.acked, m.termd)
}
@@ -157,18 +158,20 @@ func TestTheSourceMainKeyIsTakenAtItsRevision(t *testing.T) { // 1344/AC1 (the w
logger := log.New(&logs, "", 0)
mods := &memStore{vals: map[string][]byte{}}
ix := &index.Indexer{Store: &memStore{vals: map[string][]byte{}}, Modules: mods, Builds: index.NewBuilds()}
value, err := delivery.EncodeSource(delivery.Source{Group: "g", Heads: map[string]string{"novox/mesh-delivery": head}})
value, err := meshdeliveryclient.EncodeSource(meshdelivery.SourceByMainValue{Group: "g", Heads: map[string]string{"novox/mesh-delivery": head}})
if err != nil {
t.Fatal(err)
}
applySource(context.Background(), ix, "other", 3, value, logger) // not the key of the mesh's source
applySource(context.Background(), ix, delivery.SourceMainKey, 4, []byte(`{"v":1`), logger)
applySource(context.Background(), ix, meshdeliveryclient.SourceMainKey, 4, []byte(`{"v":1`), logger)
if !strings.Contains(logs.String(), "source") {
t.Errorf("a source that cannot be read was not said: %q", logs.String())
}
logs.Reset()
applySource(context.Background(), ix, delivery.SourceMainKey, 5, value, logger)
applySource(context.Background(), ix, meshdeliveryclient.SourceMainKey, 5, value, logger)
if logs.Len() != 0 {
t.Errorf("a good source said %q", logs.String())
}
}
func ptr[T any](v T) *T { return &v }
modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named.go
+9 -8
View File
@@ -9,8 +9,9 @@ import (
"path"
"strings"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-delivery/sdk/trunk"
"mesh-delivery-indexer/internal/fingerprint"
"mesh-delivery-indexer/internal/index"
@@ -29,7 +30,7 @@ type toolCaller func(key string, body any) (json.RawMessage, error)
// asked is an error naming it.
func namedVia(call toolCaller) func(ctx context.Context, repository, sha string, paths map[string][]string) (map[string]map[string]string, error) {
return func(_ context.Context, repository, sha string, paths map[string][]string) (map[string]map[string]string, error) {
owner, repo, ok := strings.Cut(delivery.RepositoryName(repository), "/")
owner, repo, ok := strings.Cut(trunk.RepositoryName(repository), "/")
if !ok {
return nil, fmt.Errorf("%q is not <owner>/<name>", repository)
}
@@ -58,7 +59,7 @@ func namedVia(call toolCaller) func(ctx context.Context, repository, sha string,
// answers without paths, one per directory that holds a module.json (the root as ""; the tool lists no other).
func trunkVia(call toolCaller) func(ctx context.Context, repository, sha string) ([]string, error) {
return func(_ context.Context, repository, sha string) ([]string, error) {
owner, repo, ok := strings.Cut(delivery.RepositoryName(repository), "/")
owner, repo, ok := strings.Cut(trunk.RepositoryName(repository), "/")
if !ok {
return nil, fmt.Errorf("%q is not <owner>/<name>", repository)
}
@@ -85,7 +86,7 @@ func trunkVia(call toolCaller) func(ctx context.Context, repository, sha string)
// directory asked is an error naming it; a cut listing fails the call in the tool.
func filesVia(call toolCaller) func(ctx context.Context, repository, sha string, dirs []string) (map[string]map[string]string, error) {
return func(_ context.Context, repository, sha string, dirs []string) (map[string]map[string]string, error) {
owner, repo, ok := strings.Cut(delivery.RepositoryName(repository), "/")
owner, repo, ok := strings.Cut(trunk.RepositoryName(repository), "/")
if !ok {
return nil, fmt.Errorf("%q is not <owner>/<name>", repository)
}
@@ -124,7 +125,7 @@ func goModVia(call toolCaller) func(ctx context.Context, repository, sha, dir st
// not hold is none.
func goFileVia(call toolCaller, name string) func(ctx context.Context, repository, sha, dir string) ([]byte, error) {
return func(_ context.Context, repository, sha, dir string) ([]byte, error) {
owner, repo, ok := strings.Cut(delivery.RepositoryName(repository), "/")
owner, repo, ok := strings.Cut(trunk.RepositoryName(repository), "/")
if !ok {
return nil, fmt.Errorf("%q is not <owner>/<name>", repository)
}
@@ -132,7 +133,7 @@ func goFileVia(call toolCaller, name string) func(ctx context.Context, repositor
switch {
case name == "": // readFileVia: dir is the file's whole path
p = dir
case dir != "" && dir != delivery.RootPath:
case dir != "" && dir != meshcatalogclient.RootPath:
p = dir + "/" + name
}
raw, err := call(getFileTool, map[string]string{"owner": owner, "repo": repo, "path": p, "ref": sha})
@@ -157,7 +158,7 @@ func goFileVia(call toolCaller, name string) func(ctx context.Context, repositor
// containsVia asks the forge whether a commit is in another (gitea.gitea_contains: contains is true for the same commit or an ancestor).
func containsVia(call toolCaller) func(ctx context.Context, repository, commit, in string) (bool, error) {
return func(_ context.Context, repository, commit, in string) (bool, error) {
owner, repo, ok := strings.Cut(delivery.RepositoryName(repository), "/")
owner, repo, ok := strings.Cut(trunk.RepositoryName(repository), "/")
if !ok {
return false, fmt.Errorf("%q is not <owner>/<name>", repository)
}
@@ -187,7 +188,7 @@ var lockFiles = []string{"go.sum", "package-lock.json"}
// forge does not have is an error; a directory with no module.json in the trees is answered with no manifest; a lock file it does not have is left out.
func readVia(call toolCaller) func(ctx context.Context, repository, sha, dir string) (index.RepoRead, error) {
return func(_ context.Context, repository, sha, dir string) (index.RepoRead, error) {
owner, repo, ok := strings.Cut(delivery.RepositoryName(repository), "/")
owner, repo, ok := strings.Cut(trunk.RepositoryName(repository), "/")
if !ok {
return index.RepoRead{}, fmt.Errorf("%q is not <owner>/<name>", repository)
}
modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/ownbuild.go
+2 -2
View File
@@ -6,7 +6,7 @@ import (
"errors"
"fmt"
"git.novox.be/novox/mesh-sdk/go/delivery"
"mesh-delivery-indexer/internal/index"
)
// compositionVerb is the node-engine seat's read verb that says what runs of a module on a machine (its Runs.FP), asked of
@@ -17,7 +17,7 @@ const compositionVerb = "seat:node-engine.composition"
// ownBuildVia asks the engine of node which fingerprint of the indexer runs there.
func ownBuildVia(call toolCaller, node string) func(ctx context.Context) (string, error) {
return func(context.Context) (string, error) {
raw, err := call(compositionVerb+"@"+node, map[string]any{"module": delivery.IndexerModule})
raw, err := call(compositionVerb+"@"+node, map[string]any{"module": index.Module})
if err != nil {
return "", fmt.Errorf("%s: %w", compositionVerb, err)
}
modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/packages_integration_test.go
+41 -34
View File
@@ -11,17 +11,22 @@ import (
"testing"
"time"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/nodebuildgo"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
"git.novox.be/novox/mesh-sdk/go/buscompose"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/seatdef"
"git.novox.be/novox/mesh-contracts/go/seats/meshdeliverychecker"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdeliverychecker"
"mesh-delivery-indexer/internal/fingerprint"
"mesh-delivery-indexer/internal/index"
)
@@ -59,20 +64,20 @@ func TestAMeshSDKMergeMovesOnlyTheModulesImportingAChangedPackage(t *testing.T)
var packages, seatDefs, findings, takeIns jetstream.KeyValue
js, ctx, _, modules := startIndexer(t, func(context.Context, buildLog) {}, func(ctx context.Context, js jetstream.JetStream) {
var err error
if packages, err = js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: delivery.PackagesBucket, History: delivery.PackagesHistory}); err != nil {
if packages, err = js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: index.PackagesBucket, History: index.PackagesHistory}); err != nil {
t.Fatal(err)
}
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: delivery.TakenBucket, History: 1}); err != nil {
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: index.TakenBucket, History: 1}); err != nil {
t.Fatal(err)
}
// Both languages' toolchains, so mesh-sdk (a TypeScript package) gets a fingerprint that moves with its source.
for module, language := range map[string]string{"mesh-build-go": "go", "mesh-build-node": "typescript"} {
tc, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: delivery.ToolchainsBucket(module), History: 1})
tc, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: index.ToolchainsBucket(module), History: 1})
if err != nil {
t.Fatal(err)
}
key, _ := delivery.ToolchainKey("n1", language)
value, _ := delivery.EncodeToolchain(delivery.Toolchain{Language: language, Version: "v1", Recipe: "1", At: time.Now()})
key, _ := index.ToolchainKey("n1", language)
value, _ := index.EncodeToolchain(nodebuildgo.Toolchain{Language: language, Version: "v1", Recipe: "1", At: ptr(time.Now())})
if _, err := tc.Put(ctx, key, value); err != nil {
t.Fatal(err)
}
@@ -90,7 +95,7 @@ func TestAMeshSDKMergeMovesOnlyTheModulesImportingAChangedPackage(t *testing.T)
if takeIns, err = js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: index.TakeInsBucket, History: 1}); err != nil {
t.Fatal(err)
}
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: checkerAsksStream, Subjects: []string{meshdeliverychecker.HolderSubjectVerdict(delivery.CheckerModule)}}); err != nil {
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: checkerAsksStream, Subjects: []string{meshdeliverychecker.HolderSubjectVerdict(index.CheckerModule)}}); err != nil {
t.Fatal(err)
}
if _, err := js.CreateOrUpdateConsumer(ctx, checkerAsksStream, jetstream.ConsumerConfig{Durable: "n1_mesh-delivery-indexer", AckPolicy: jetstream.AckExplicitPolicy}); err != nil {
@@ -108,12 +113,12 @@ func TestAMeshSDKMergeMovesOnlyTheModulesImportingAChangedPackage(t *testing.T)
t.Fatal(err)
}
}
module := func(name string, ok func(delivery.Module) bool) delivery.Module {
module := func(name string, ok func(meshcatalog.ModulesByModuleValue) bool) meshcatalog.ModulesByModuleValue {
t.Helper()
var m delivery.Module
var m meshcatalog.ModulesByModuleValue
for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
if e, err := modules.Get(ctx, name); err == nil {
if m, err = delivery.DecodeModule(e.Value()); err == nil && ok(m) {
if m, err = meshcatalogclient.DecodeModule(e.Value()); err == nil && ok(m) {
return m
}
}
@@ -129,7 +134,7 @@ func TestAMeshSDKMergeMovesOnlyTheModulesImportingAChangedPackage(t *testing.T)
// mesh-sdk at m1, then the catalogue's two modules: their go.sum names mesh-sdk, read by the indexer itself.
merged("novox/mesh-sdk", 70, m1, forgemsg.Forge{Root: sdkRoot1, Trees: map[string]string{"": sdkRoot1},
Manifests: map[string]string{"": string(sdkManifest)}, Paths: sdkObjects("6666666666666666666666666666666666666666")})
module("mesh-sdk", func(m delivery.Module) bool { return m.Commit == m1 })
module("mesh-sdk", func(m meshcatalog.ModulesByModuleValue) bool { return m.Commit == m1 })
merged("novox/mesh-catalog", 301, catalogMerge, forgemsg.Forge{Root: "r0",
Trees: map[string]string{"modules/importer": "t1", "modules/plain": "t2"},
Manifests: map[string]string{"modules/importer": named("importer"), "modules/plain": named("plain")},
@@ -140,26 +145,28 @@ func TestAMeshSDKMergeMovesOnlyTheModulesImportingAChangedPackage(t *testing.T)
t.Fatal(err)
}
}
hasFP := func(m delivery.Module) bool { return m.FP2 != "" && m.Packages[sdkGo] == m1 }
hasFP := func(m meshcatalog.ModulesByModuleValue) bool { return m.Fp2 != "" && m.Packages[sdkGo] == m1 }
importer, plain := module("importer", hasFP), module("plain", hasFP)
// Each build ends reporting its imports: kept, no fingerprint moving.
for _, b := range []struct {
m delivery.Module
m meshcatalog.ModulesByModuleValue
name string
imports []string
}{{importer, "importer", []string{sdkGo, sdkGo + "/buildstep"}}, {plain, "plain", []string{sdkGo, sdkGo + "/announce"}}} {
raw, err := delivery.EncodeBuildLogEntry(delivery.BuildLogEntry{Event: delivery.BuildEndedEvent, Module: b.name, FP: b.m.FP2,
raw, err := builds.EncodeBuildLogEntry(builds.BuildLogEntry{Event: builds.BuildEndedEvent, Module: b.name, FP: b.m.Fp2,
Repository: "novox/mesh-catalog", Directory: "modules/" + b.name, Commit: catalogMerge, Ref: "main", Node: "n1",
At: time.Now().UTC(), Outcome: delivery.BuildDone, Imports: b.imports})
At: time.Now().UTC(), Outcome: builds.BuildDone, Imports: b.imports})
if err != nil {
t.Fatal(err)
}
if _, err := js.Publish(ctx, buscompose.LogSubject(delivery.BuilderModule, delivery.BuilderLog)+"."+b.name, raw); err != nil {
if _, err := js.Publish(ctx, buscompose.LogSubject(builds.BuilderModule, builds.BuilderLog)+"."+b.name, raw); err != nil {
t.Fatal(err)
}
fp := b.m.FP2
module(b.name, func(m delivery.Module) bool { return m.Inputs2 != nil && m.Inputs2.ImportForm != "" && m.FP2 == fp })
fp := b.m.Fp2
module(b.name, func(m meshcatalog.ModulesByModuleValue) bool {
return m.Inputs2 != nil && m.Inputs2.ImportForm != "" && m.Fp2 == fp
})
}
// mesh-sdk merges a change of go/ alone that leaves its own named paths' objects: its value, a TypeScript package's
@@ -172,8 +179,8 @@ func TestAMeshSDKMergeMovesOnlyTheModulesImportingAChangedPackage(t *testing.T)
// mesh-sdk merges a change of buildstep alone (its version moves: src changed too).
merged("novox/mesh-sdk", 71, m2, forgemsg.Forge{Root: sdk2, Trees: map[string]string{"": sdk2},
Manifests: map[string]string{"": string(sdkManifest)}, Paths: sdkObjects("7777777777777777777777777777777777777777")})
sdkAt2 := module("mesh-sdk", func(m delivery.Module) bool { return m.Commit == m2 })
if sdkAt2.Inputs2 == nil || !strings.HasPrefix(sdkAt2.Inputs2.Tree, delivery.NamedTreePrefix) {
sdkAt2 := module("mesh-sdk", func(m meshcatalog.ModulesByModuleValue) bool { return m.Commit == m2 })
if sdkAt2.Inputs2 == nil || !strings.HasPrefix(sdkAt2.Inputs2.Tree, fingerprint.NamedTreePrefix) {
t.Errorf("mesh-sdk's kept inputs %+v, want a named tree", sdkAt2.Inputs2)
}
history, err := modules.History(ctx, "mesh-sdk")
@@ -181,15 +188,15 @@ func TestAMeshSDKMergeMovesOnlyTheModulesImportingAChangedPackage(t *testing.T)
t.Fatal(err)
}
for _, e := range history {
if m, err := delivery.DecodeModule(e.Value()); err == nil && m.Commit == goOnly {
if m, err := meshcatalogclient.DecodeModule(e.Value()); err == nil && m.Commit == goOnly {
t.Errorf("the go/-only merge wrote mesh-sdk's value: %+v", m)
}
}
key, _ := delivery.PackagesKey(sdkGo)
var pub delivery.PublishedPackages
key, _ := index.PackagesKey(sdkGo)
var pub meshcatalog.PackagesByModuleValue
for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
if e, err := packages.Get(ctx, key); err == nil {
if pub, err = delivery.DecodePublishedPackages(e.Value()); err == nil && pub.Commit == m2 {
if pub, err = index.DecodePublishedPackages(e.Value()); err == nil && pub.Commit == m2 {
break
}
}
@@ -197,13 +204,13 @@ func TestAMeshSDKMergeMovesOnlyTheModulesImportingAChangedPackage(t *testing.T)
if pub.Commit != m2 || len(pub.Packages) != 3 {
t.Fatalf("packages.%s = %+v within 20 s, want the merge %s with a digest of go, announce and buildstep", key, pub, m2)
}
moved := module("importer", func(m delivery.Module) bool { return m.FP2 != importer.FP2 })
moved := module("importer", func(m meshcatalog.ModulesByModuleValue) bool { return m.Fp2 != importer.Fp2 })
if moved.Packages[sdkGo] != m2 {
t.Errorf("importer moved to %+v, want it built against %s", moved.Packages, m2)
}
// plain imports nothing that changed: its value keeps its fingerprint (one recompute has run: importer moved in it).
if now := module("plain", func(delivery.Module) bool { return true }); now.FP2 != plain.FP2 {
t.Errorf("plain moved from %s to %s, though it imports nothing the merge changed", plain.FP2, now.FP2)
if now := module("plain", func(meshcatalog.ModulesByModuleValue) bool { return true }); now.Fp2 != plain.Fp2 {
t.Errorf("plain moved from %s to %s, though it imports nothing the merge changed", plain.Fp2, now.Fp2)
}
// DLV-I17, I18 (research 081 test 21, design 26): mesh-contracts merges a valid definition, said once; a pull
@@ -281,13 +288,13 @@ func TestAMeshSDKMergeMovesOnlyTheModulesImportingAChangedPackage(t *testing.T)
// naming them raises generated-types-out-of-step, and only a clean build end of the same module clears it. DLV-I19 [i]:
// indexer_out_of_step answers both, from the states the indexer wrote.
verdict, _ := json.Marshal(meshdeliverychecker.VerdictHolderEvent{Repository: "novox/mesh-contracts", Pull: 82, Head: badHead, Verdict: "fail", OutOfStep: []string{"go/seats/git/types.go"}})
if _, err := js.Publish(ctx, meshdeliverychecker.HolderSubjectVerdict(delivery.CheckerModule), verdict); err != nil {
if _, err := js.Publish(ctx, meshdeliverychecker.HolderSubjectVerdict(index.CheckerModule), verdict); err != nil {
t.Fatal(err)
}
logEnd := func(outcome string, files ...string) {
t.Helper()
raw, err := delivery.EncodeBuildLogEntry(delivery.BuildLogEntry{Event: delivery.BuildEndedEvent, Module: "mesh-contracts", FP: "src2:mesh-contracts",
Repository: "novox/mesh-contracts", Directory: delivery.RootPath, Commit: contractsMerge, Node: "n1", At: time.Now().UTC(), Outcome: outcome})
raw, err := builds.EncodeBuildLogEntry(builds.BuildLogEntry{Event: builds.BuildEndedEvent, Module: "mesh-contracts", FP: "src2:mesh-contracts",
Repository: "novox/mesh-contracts", Directory: meshcatalogclient.RootPath, Commit: contractsMerge, Node: "n1", At: time.Now().UTC(), Outcome: outcome})
if err != nil {
t.Fatal(err)
}
@@ -297,11 +304,11 @@ func TestAMeshSDKMergeMovesOnlyTheModulesImportingAChangedPackage(t *testing.T)
e["out-of-step"] = files // BuildLogEntry.OutOfStep, the builder's field (J3), written by its name here
}
raw, _ = json.Marshal(e)
if _, err := js.Publish(ctx, buscompose.LogSubject(delivery.BuilderModule, delivery.BuilderLog)+".mesh-contracts", raw); err != nil {
if _, err := js.Publish(ctx, buscompose.LogSubject(builds.BuilderModule, builds.BuilderLog)+".mesh-contracts", raw); err != nil {
t.Fatal(err)
}
}
logEnd(delivery.BuildFailed, "go/seats/git/types.go")
logEnd(builds.BuildFailed, "go/seats/git/types.go")
outOfStep := func(want int) []index.OutOfStepRow {
t.Helper()
var rows []index.OutOfStepRow
@@ -316,7 +323,7 @@ func TestAMeshSDKMergeMovesOnlyTheModulesImportingAChangedPackage(t *testing.T)
if rows := outOfStep(2); rows[0].Module != "mesh-contracts" || !strings.Contains(rows[0].Summary, "types.go") || rows[1].Pull != "novox/mesh-contracts#82" {
t.Errorf("indexer_out_of_step = %+v, want the trunk's finding and the pull request's files", rows)
}
logEnd(delivery.BuildDone)
logEnd(builds.BuildDone)
if rows := outOfStep(1); rows[0].Source != index.SourcePull {
t.Errorf("after a clean build end indexer_out_of_step = %+v, want the pull request's files alone", rows)
}
modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/previews.go
+6 -8
View File
@@ -11,8 +11,6 @@ import (
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/delivery"
"mesh-delivery-indexer/internal/index"
)
@@ -21,7 +19,7 @@ import (
const PreviewsStream = "CHECKS"
// previewWrittenSubject is the subject of the checker's preview.written event.
const previewWrittenSubject = "mesh.stream." + delivery.CheckerModule + ".event.preview.written"
const previewWrittenSubject = "mesh.stream." + index.CheckerModule + ".event.preview.written"
// previewWatch reads the merge previews the checker writes (mesh-delivery-checker_previews) and follows them: every change
// writes the candidates' fingerprints again (index.Indexer.ApplyPreview). The checker's bucket is made when it is assigned,
@@ -50,26 +48,26 @@ func (w *previewWatch) ensure() error {
if w.watched {
return nil
}
kv, err := w.js.KeyValue(w.ctx, delivery.PreviewsBucket)
kv, err := w.js.KeyValue(w.ctx, index.PreviewsBucket)
switch {
case errors.Is(err, jetstream.ErrBucketNotFound):
w.logger.Printf("the bucket %s does not exist yet: it is watched when the checker says it wrote a preview", delivery.PreviewsBucket)
w.logger.Printf("the bucket %s does not exist yet: it is watched when the checker says it wrote a preview", index.PreviewsBucket)
return nil
case err != nil:
// Refused or not answered (its grant not on the bus yet, issue 1945): said, and opened again at the checker's next preview.
w.logger.Printf("the bucket %s cannot be read (%v): it is watched when the checker says it wrote a preview", delivery.PreviewsBucket, err)
w.logger.Printf("the bucket %s cannot be read (%v): it is watched when the checker says it wrote a preview", index.PreviewsBucket, err)
return nil
}
watcher, err := settle(w.ctx, kv, func(e jetstream.KeyValueEntry) { w.ix.PutPreview(e.Key(), e.Value(), !opOf(e)) })
if err != nil {
process.NotWatched(w.logger.Printf, delivery.PreviewsBucket, err, "it is watched when the checker says it wrote a preview")
process.NotWatched(w.logger.Printf, index.PreviewsBucket, err, "it is watched when the checker says it wrote a preview")
return nil
}
w.watched = true
go func() {
w.errs <- follow(w.ctx, watcher, func(e jetstream.KeyValueEntry) {
if err := w.ix.ApplyPreview(w.ctx, e.Key(), e.Value(), !opOf(e)); err != nil {
w.logger.Printf("%s.%s could not be taken into the candidates: %v", delivery.PreviewsBucket, e.Key(), err)
w.logger.Printf("%s.%s could not be taken into the candidates: %v", index.PreviewsBucket, e.Key(), err)
}
})
}()
modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/seatdefs.go
+2 -3
View File
@@ -8,8 +8,7 @@ import (
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-contracts/go/seats/meshdeliverychecker"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdeliverychecker"
"git.novox.be/novox/mesh-sdk/go/process"
"mesh-delivery-indexer/internal/index"
@@ -65,7 +64,7 @@ func saySeatDefinition(js jetstream.JetStream, cred process.Account) func(contex
// heardVerdict takes the checker's verdict event: a pull request's out-of-step files are kept (a clean verdict removes
// them). Another event of the stream is acknowledged and passed over; a write that failed is delivered again.
func heardVerdict(ctx context.Context, ix *index.Indexer, m jetstream.Msg, logf func(string, ...any)) {
if m.Subject() != meshdeliverychecker.HolderSubjectVerdict(delivery.CheckerModule) {
if m.Subject() != meshdeliverychecker.HolderSubjectVerdict(index.CheckerModule) {
_ = m.Ack()
return
}
modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/series.go
+4 -4
View File
@@ -6,9 +6,9 @@ import (
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-contracts/go/seats/gopackageregistry"
"git.novox.be/novox/mesh-contracts/go/seats/npmpackageregistry"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/gopackageregistry"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/npmpackageregistry"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
"mesh-delivery-indexer/internal/index"
)
@@ -21,7 +21,7 @@ var (
gopackageregistry.SubjectPublished: gopackageregistry.Seat,
npmpackageregistry.SubjectPublished: npmpackageregistry.Seat,
}
numberedSubject = "mesh.mod." + delivery.BuilderModule + ".event.version.numbered"
numberedSubject = "mesh.mod." + builds.BuilderModule + ".event.version.numbered"
)
// numbered is the builder's version.numbered body as the indexer reads it.
modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/source.go
+3 -3
View File
@@ -4,7 +4,7 @@ import (
"context"
"log"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient"
"mesh-delivery-indexer/internal/index"
)
@@ -13,10 +13,10 @@ import (
// source revision (mesh-sdk go/delivery SourceMainKey, Source). Any other key is not the mesh's source. A value that is no
// source is said; so is a write that failed, and it is tried again only at the next revision (novox/hq ADR 0437 §8).
func applySource(ctx context.Context, ix *index.Indexer, key string, rev uint64, value []byte, logger *log.Logger) {
if key != delivery.SourceMainKey {
if key != meshdeliveryclient.SourceMainKey {
return
}
s, err := delivery.DecodeSource(value)
s, err := meshdeliveryclient.DecodeSource(value)
if err != nil {
logger.Printf("source.%s at revision %d is not a source, and is left: %v", key, rev, err)
return
modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/toolchains.go
+2 -4
View File
@@ -14,8 +14,6 @@ import (
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/delivery"
"mesh-delivery-indexer/internal/index"
)
@@ -33,7 +31,7 @@ type bucketOpener interface {
}
// toolchainWatch reads the toolchains each build module reports (mesh-build-go_toolchains, mesh-build-node_toolchains: the
// module's state toolchains, delivery.ToolchainsBucket; novox/hq ADR 0465 clause 2, issue 1444) and follows them: every
// module's state toolchains, index.ToolchainsBucket; novox/hq ADR 0465 clause 2, issue 1444) and follows them: every
// change of a language's agreed toolchain writes the modules of that language again (index.Indexer.Recompute). A bucket
// is made when its build module is assigned, after the indexer started, and the indexer holds no timer (ADR 0437 §8):
// it watches the buckets that exist at start, and a build module says each time it wrote its toolchains key
@@ -67,7 +65,7 @@ func (w *toolchainWatch) start() error {
func (w *toolchainWatch) ensure(module string) error {
w.mu.Lock()
defer w.mu.Unlock()
name := delivery.ToolchainsBucket(module)
name := index.ToolchainsBucket(module)
if !w.watched[module] {
kv, err := w.js.KeyValue(w.ctx, name)
switch {
modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/toolchains_test.go
+6 -4
View File
@@ -10,9 +10,11 @@ import (
"testing"
"time"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/nodebuildgo"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/delivery"
"mesh-delivery-indexer/internal/index"
)
// A build module's bucket is made when the module is assigned, after the indexer started (issue 1444, review finding 9).
@@ -81,7 +83,7 @@ func (e fakeEntry) Operation() jetstream.KeyValueOp { return jetstream.KeyValueP
func TestABuildModulesBucketThatAppearsLaterIsWatchedWhenItsEventArrives(t *testing.T) { // 1444 (review finding 9)
ctx, stop := context.WithCancel(context.Background())
defer stop()
value, err := delivery.EncodeToolchain(delivery.Toolchain{Language: "go", Version: "go1.27.1", Recipe: "1", At: time.Now()})
value, err := index.EncodeToolchain(nodebuildgo.Toolchain{Language: "go", Version: "go1.27.1", Recipe: "1", At: ptr(time.Now())})
if err != nil {
t.Fatal(err)
}
@@ -120,7 +122,7 @@ func TestABuildModulesBucketThatAppearsLaterIsWatchedWhenItsEventArrives(t *test
}
// The bucket of mesh-build-go is made and already holds a key, and the module says it reported.
bucket := &fakeBucket{held: []jetstream.KeyValueEntry{fakeEntry{key: "novox.go", value: value}}}
buckets.make(delivery.ToolchainsBucket("mesh-build-go"), bucket)
buckets.make(index.ToolchainsBucket("mesh-build-go"), bucket)
msg := &fakeMsg{subject: "mesh.stream.mesh-build-go.event.toolchain.reported"}
w.Reported(msg)
if msg.acked != 1 || len(puts) != 1 || puts[0] != "mesh-build-go novox.go" || recomputes != 1 {
@@ -140,7 +142,7 @@ func TestABuildModulesBucketThatAppearsLaterIsWatchedWhenItsEventArrives(t *test
t.Errorf("applies = %v, want the key put later", applies)
}
w.Reported(&fakeMsg{subject: "mesh.stream.mesh-build-go.event.toolchain.reported"})
if len(puts) != 1 || recomputes != 2 || buckets.asked[delivery.ToolchainsBucket("mesh-build-go")] != 2 {
if len(puts) != 1 || recomputes != 2 || buckets.asked[index.ToolchainsBucket("mesh-build-go")] != 2 {
t.Errorf("puts %v recomputes %d asked %v, want the bucket opened again only at the first report", puts, recomputes, buckets.asked)
}
// An event of no build module is terminated.
modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/verdicts.go
+10 -10
View File
@@ -6,8 +6,8 @@ import (
"fmt"
"time"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/process"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
@@ -15,14 +15,14 @@ import (
// sayRefused says the indexer's event manifest-refused (novox/hq ADR 0578 §1): mesh.seat.mesh-catalog.event.manifest-refused,
// with the SDK's envelope headers and the digest and judge as the bus's dedup id, so one refusal is said once.
func sayRefused(js jetstream.JetStream, cred process.Account, logf func(string, ...any)) func(context.Context, delivery.ManifestRefused) {
return func(ctx context.Context, r delivery.ManifestRefused) {
func sayRefused(js jetstream.JetStream, cred process.Account, logf func(string, ...any)) func(context.Context, meshcatalog.ManifestRefusedEvent) {
return func(ctx context.Context, r meshcatalog.ManifestRefusedEvent) {
data, err := json.Marshal(r)
if err != nil {
logf("judge: %s for %s is not said: %v", delivery.ManifestRefusedEvent, r.Module, err)
logf("judge: %s for %s is not said: %v", meshcatalog.EventManifestRefused, r.Module, err)
return
}
msg := nats.NewMsg(seatEventSubject(delivery.ManifestRefusedEvent))
msg := nats.NewMsg(seatEventSubject(meshcatalog.EventManifestRefused))
id := "manifest-refused-" + r.Digest + "-" + r.Judge
msg.Header.Set("x-event-id", id)
msg.Header.Set("x-source", cred.Module)
@@ -33,21 +33,21 @@ func sayRefused(js jetstream.JetStream, cred process.Account, logf func(string,
pub, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
defer cancel()
if _, err := js.PublishMsg(pub, msg, jetstream.WithMsgID(id)); err != nil {
logf("judge: %s", fmt.Errorf("saying %s for %s: %w", delivery.ManifestRefusedEvent, r.Module, err))
logf("judge: %s", fmt.Errorf("saying %s for %s: %w", meshcatalog.EventManifestRefused, r.Module, err))
}
}
}
// sayAccepted says the indexer's event manifest-accepted (issue 1966 D3, novox/hq ADR 0595 §4):
// mesh.seat.mesh-catalog.event.manifest-accepted, carrying the module's bus-state entries whole, with the digest and judge as
// the bus's dedup id (delivery.ManifestAccepted.EventID), so one acceptance is said once.
func sayAccepted(js jetstream.JetStream, cred process.Account) func(context.Context, delivery.ManifestAccepted, string) error {
return func(ctx context.Context, a delivery.ManifestAccepted, id string) error {
// the bus's dedup id (meshcatalog.ManifestAcceptedEvent.EventID), so one acceptance is said once.
func sayAccepted(js jetstream.JetStream, cred process.Account) func(context.Context, meshcatalog.ManifestAcceptedEvent, string) error {
return func(ctx context.Context, a meshcatalog.ManifestAcceptedEvent, id string) error {
data, err := json.Marshal(a)
if err != nil {
return err
}
msg := nats.NewMsg(seatEventSubject(delivery.ManifestAcceptedEvent))
msg := nats.NewMsg(seatEventSubject(meshcatalog.EventManifestAccepted))
msg.Header.Set("x-event-id", id)
msg.Header.Set("x-source", cred.Module)
msg.Header.Set("x-node", cred.Node)
modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/views.go
+11 -10
View File
@@ -9,7 +9,6 @@ import (
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/health"
"git.novox.be/novox/mesh-sdk/go/process"
@@ -17,11 +16,13 @@ import (
"git.novox.be/novox/mesh-delivery/sdk/graph"
"git.novox.be/novox/mesh-delivery/sdk/heard"
"git.novox.be/novox/mesh-sdk/go/views"
"mesh-delivery-indexer/internal/index"
)
// findingsBucket is the indexer's own findings, which the condition keeper raises and clears (ADR 0564 §2): a value too
// large for its views event, and its first view of the assigner's placement unanswered past its bound.
var findingsBucket = health.FindingsBucket(delivery.IndexerModule)
var findingsBucket = health.FindingsBucket(index.Module)
// readyOf binds a state's bucket: now for one bound at the start, once it appears for one awaited.
type readyOf func(func(jetstream.KeyValue))
@@ -38,7 +39,7 @@ func startViews(ctx context.Context, js jetstream.JetStream, cred process.Accoun
if kv == nil {
return
}
if err := sdkfindings.Writer(ctx, findings.KV).Put(views.TooLargeFinding(delivery.IndexerModule, state, key, size, time.Now())); err != nil {
if err := sdkfindings.Writer(ctx, findings.KV).Put(views.TooLargeFinding(index.Module, state, key, size, time.Now())); err != nil {
logf("views: the finding of %s.%s too large is not written: %v", state, key, err)
}
}
@@ -58,7 +59,7 @@ func startViews(ctx context.Context, js jetstream.JetStream, cred process.Accoun
// view-unsaid, in its own findings once bound (review round 2).
func viewsEmitter(ctx context.Context, js jetstream.JetStream, cred process.Account, mk jetstream.KeyValue, findings *process.AwaitedBucket,
tooLarge func(string, string, int), logf func(string, ...any)) views.Emitter {
return views.Emitter{Owner: delivery.IndexerModule, Subject: cred.EventSubject, Publish: views.JetStreamPublish(js),
return views.Emitter{Owner: index.Module, Subject: cred.EventSubject, Publish: views.JetStreamPublish(js),
Marks: mk, TooLarge: tooLarge, Findings: sdkfindings.Writer(ctx, findings.KV), Logf: logf}
}
@@ -79,7 +80,7 @@ func sortedStates(states map[string]readyOf) []string {
// notHeardWriter writes the indexer's finding of a family it does not hear into its findings state, once that is bound.
func notHeardWriter(ctx context.Context, findings *process.AwaitedBucket, logf func(string, ...any)) heard.Writer {
return heard.Writer{Observer: delivery.IndexerModule, Logf: logf,
return heard.Writer{Observer: index.Module, Logf: logf,
Put: func(key string, value []byte) error {
findings.Ready(func(kv jetstream.KeyValue) {
if _, err := kv.Put(ctx, key, value); err != nil {
@@ -106,10 +107,10 @@ const graphState = graph.State
// firstViewsBucket is the indexer's own state of its first views (views.StatusState): what indexer_first_views reads and
// where indexer_first_view_again asks.
var firstViewsBucket = delivery.IndexerModule + "_" + views.StatusState
var firstViewsBucket = index.Module + "_" + views.StatusState
// placementTool is the assigner's first-view tool of its placement, as the indexer's invokes names it (ADR 0621 §2).
const placementTool = delivery.AssignerModule + "." + delivery.PlacementState
const placementTool = index.AssignerModule + "." + index.PlacementState
// followPlacement reads the seat mesh-placement's placement through the assigner's first view and then its views
// family, never its bucket (ADR 0615 §1, DLV-F1r), into gate: each key as it changes, settled once answered, with
@@ -120,7 +121,7 @@ func followPlacement(ctx context.Context, hub *process.EventHub, cred process.Ac
gate *buildrun.ToolchainGate, changed func(context.Context), findings, status *process.AwaitedBucket, logf func(string, ...any)) *views.Follower {
finding := func(s views.Status, raised bool) {
w := sdkfindings.Writer(ctx, findings.KV)
f := views.UnansweredFinding(delivery.IndexerModule, s, time.Now())
f := views.UnansweredFinding(index.Module, s, time.Now())
var err error
if raised {
err = w.Put(f)
@@ -132,7 +133,7 @@ func followPlacement(ctx context.Context, hub *process.EventHub, cred process.Ac
}
}
var settled atomic.Bool // the first view answered: the gate's nodes are the counted ones from now on
f := &views.Follower{Owner: delivery.AssignerModule, Tool: placementTool, State: delivery.PlacementState,
f := &views.Follower{Owner: index.AssignerModule, Tool: placementTool, State: index.PlacementState,
Ask: views.AskVia(call, placementTool), Logf: logf,
Raise: func(s views.Status) { finding(s, true) }, Clear: func(s views.Status) { finding(s, false) },
Changed: func(key string, value json.RawMessage, deleted bool, at time.Time) {
@@ -166,7 +167,7 @@ func followPlacement(ctx context.Context, hub *process.EventHub, cred process.Ac
// The assigner's views family, on the module's one hub (the host starts it once every handler is registered); a family the
// hub cannot register is the finding family-not-heard until the indexer starts again (ADR 0615 §5).
_ = notHeardWriter(ctx, findings, logf).Bind(process.EventsStream, "the placement is the first view's alone, asked again by hand, until the indexer starts again", func() error {
return hub.Handle("mesh.mod."+delivery.AssignerModule+".event."+views.EventPrefix+"*", views.Hear(ctx, map[string]*views.Follower{delivery.PlacementState: f}, logf))
return hub.Handle("mesh.mod."+index.AssignerModule+".event."+views.EventPrefix+"*", views.Hear(ctx, map[string]*views.Follower{index.PlacementState: f}, logf))
})
wait, cancel := context.WithTimeout(ctx, views.Bound) // a context's bound, no timer (ADR 0437 §8)
defer cancel()
modules/mesh-delivery-indexer/go.mod
+14 -4
View File
@@ -3,9 +3,20 @@ module mesh-delivery-indexer
go 1.26.0
require (
git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-contracts/placement/go/clients/meshmoduleassignerclient v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-contracts/placement/go/seats/meshmoduleassigner v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliverycheckerclient v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-delivery/contracts/go/seats/gopackageregistry v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdeliverychecker v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-delivery/contracts/go/seats/nodebuildgo v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-delivery/contracts/go/seats/npmpackageregistry v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104174359-05b3e2e1b94f
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
@@ -35,9 +46,6 @@ require (
require (
git.novox.be/novox/mesh-contracts/go/seats/meshbroker v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-contracts/go/seats/gopackageregistry v0.0.0-20261104174344-ae5266e77639
git.novox.be/novox/mesh-contracts/go/seats/meshdeliverychecker v0.0.0-20261104174344-ae5266e77639
git.novox.be/novox/mesh-contracts/go/seats/npmpackageregistry v0.0.0-20261104174344-ae5266e77639
)
require git.novox.be/novox/mesh-contracts/go v0.0.0-20261104174344-ae5266e77639 // indirect
@@ -47,3 +55,5 @@ replace git.novox.be/novox/mesh-delivery/sdk => ../../sdk
replace git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein => ./takein
replace git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg => ../mesh-delivery-forge-bridge/forgemsg
replace git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds => ../mesh-delivery-builder/builds
modules/mesh-delivery-indexer/internal/fingerprint/fingerprint.go
+52 -31
View File
@@ -1,7 +1,7 @@
// Package fingerprint is the one computation of a module's source fingerprint, of a named tree, of a mesh package's
// content digest and of a package's published version (novox/hq ADR 0593 §2): the indexer's own, and nobody else's. It
// left mesh-sdk's delivery package, which keeps only the fingerprint's form (delivery.SourcePrefix2, the key part) and
// the shapes the indexer writes (delivery.Module, delivery.PublishedPackages). No other part computes any of it: the
// left mesh-sdk's delivery package, which keeps only the fingerprint's form (meshcatalogclient.SourcePrefix2, the key part) and
// the shapes the indexer writes (meshcatalog.ModulesByModuleValue, meshcatalog.PackagesByModuleValue). No other part computes any of it: the
// builder, the checker and the build modules read what the indexer writes.
package fingerprint
@@ -15,7 +15,10 @@ import (
"sort"
"strings"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
sdkstep "git.novox.be/novox/mesh-delivery/sdk/buildstep"
"git.novox.be/novox/mesh-delivery/sdk/trunk"
)
// NoPackageVersion opens the reason a module has no fingerprint while a mesh package it uses has no published version
@@ -25,7 +28,7 @@ const NoPackageVersion = "no published version of the package "
// Inputs are what a module's source fingerprint is of (novox/hq ADR 0429 clause 5 as ADR 0437 clause 5 changed it; ADR
// 0453 clause 6; ADR 0593 §2). A base enters by its module's fingerprint, an artifact that resolves packages when it is
// built by the hash of its lock file (an entry with an empty hash is such an artifact with no lock, which has no
// fingerprint), an artifact's toolchain as delivery.Toolchain2.
// fingerprint), an artifact's toolchain as meshcatalogclient.Toolchain2.
//
// The mesh packages enter in one of two forms. **The import form** (ADR 0593 §2): Imports, the content digest of each
// mesh package the module's programs import (by import path), and Requires, the digest of each mesh Go module's go.mod
@@ -38,7 +41,7 @@ type Inputs struct {
Builder string
Contexts map[string]string
Locks map[string]string
Toolchains map[string]delivery.Toolchain2
Toolchains map[string]meshcatalogclient.Toolchain2
Packages map[string]string
Imports map[string]string
Requires map[string]string
@@ -102,16 +105,16 @@ func Of(in Inputs) (fp, none string) {
}
sort.Strings(lines[3:])
sum := sha256.Sum256([]byte(strings.Join(lines, "\n")))
return delivery.SourcePrefix2 + hex.EncodeToString(sum[:]), ""
return meshcatalogclient.SourcePrefix2 + hex.EncodeToString(sum[:]), ""
}
// NamedTree is the tree input of a module whose every artifact names the paths it is built from (novox/hq ADR 0499
// clause 2): `named:` and the hex sha256 of the lines "manifest <digest>" and, sorted, "path <path> <object>", joined by
// a newline. manifest is the module's delivery.ManifestDigest; paths are the named paths, each clean and relative to
// a newline. manifest is the module's meshcatalogclient.ManifestDigest; paths are the named paths, each clean and relative to
// the module's directory, with the git object id each has at the commit. It refuses no path, a named path with no
// object, a path that is not clean, and a manifest that is not a digest, naming them: the module then has none.
func NamedTree(manifest string, paths map[string]string) (string, error) {
if _, err := delivery.ManifestsKey(manifest); err != nil {
if _, err := meshcatalogclient.ManifestsKey(manifest); err != nil {
return "", err
}
if len(paths) == 0 {
@@ -129,20 +132,20 @@ func NamedTree(manifest string, paths map[string]string) (string, error) {
}
sort.Strings(lines)
sum := sha256.Sum256([]byte("manifest " + manifest + "\n" + strings.Join(lines, "\n")))
return delivery.NamedTreePrefix + hex.EncodeToString(sum[:]), nil
return NamedTreePrefix + hex.EncodeToString(sum[:]), nil
}
// MeshPackagesOf are the mesh Go modules a go.sum names (the module path of each line under
// delivery.ForgeModulePrefix), sorted and each once; nil for none. They are what a module with no reported import set
// trunk.ForgeModulePrefix), sorted and each once; nil for none. They are what a module with no reported import set
// takes the version line of (ADR 0593 §2).
func MeshPackagesOf(goSum []byte) []string {
seen := map[string]bool{}
for _, line := range strings.Split(string(goSum), "\n") {
f := strings.Fields(line)
if len(f) < 2 || !strings.HasPrefix(f[0], delivery.ForgeModulePrefix) {
if len(f) < 2 || !strings.HasPrefix(f[0], trunk.ForgeModulePrefix) {
continue
}
if _, _, ok := delivery.PackageRepository(f[0]); ok {
if _, _, ok := trunk.PackageRepository(f[0]); ok {
seen[f[0]] = true
}
}
@@ -156,7 +159,7 @@ func MeshPackagesOf(goSum []byte) []string {
func OtherPackages(packages []string, repository string) []string {
var out []string
for _, p := range packages {
if repo, _, ok := delivery.PackageRepository(p); ok && !delivery.SameRepository(repo, repository) {
if repo, _, ok := trunk.PackageRepository(p); ok && !trunk.SameRepository(repo, repository) {
out = append(out, p)
}
}
@@ -195,7 +198,7 @@ func ReplacedSiblings(goMod []byte, repository string) []string {
if len(from) == 0 || len(to) != 1 || !(strings.HasPrefix(to[0], "./") || strings.HasPrefix(to[0], "../")) {
continue
}
if repo, _, ok := delivery.PackageRepository(from[0]); ok && delivery.SameRepository(repo, repository) {
if repo, _, ok := trunk.PackageRepository(from[0]); ok && trunk.SameRepository(repo, repository) {
seen[from[0]] = true
}
}
@@ -210,14 +213,14 @@ func ReplacedSiblings(goMod []byte, repository string) []string {
// (ReplacedSiblings), by the content its directory publishes, `content:` and the hex sha256 of its requirements' digest
// and each package's digest (empty while it is not published, which gives no fingerprint until it is, as a version
// none publishes does).
func VersionLine(repository string, packages []string, versions delivery.PackageVersions, published map[string]delivery.PublishedPackages) map[string]string {
func VersionLine(repository string, packages []string, versions sdkstep.PackageVersions, published map[string]meshcatalog.PackagesByModuleValue) map[string]string {
if len(packages) == 0 {
return nil
}
out := map[string]string{}
for _, p := range packages {
repo, _, ok := delivery.PackageRepository(p)
if !ok || !delivery.SameRepository(repo, repository) {
repo, _, ok := trunk.PackageRepository(p)
if !ok || !trunk.SameRepository(repo, repository) {
out[p] = versions[p]
continue
}
@@ -243,7 +246,7 @@ func VersionLine(repository string, packages []string, versions delivery.Package
type Resolved struct {
Digests map[string]string
Requires map[string]string
Versions delivery.PackageVersions
Versions sdkstep.PackageVersions
Missing []string
}
@@ -251,17 +254,17 @@ type Resolved struct {
// import is held by the published module with the longest path that publishes it. A module of repository itself is
// built from its directory, so it is given no version (a same-repository replace, ADR 0593 §1); it still enters by
// its digests.
func Resolve(repository string, imports []string, published map[string]delivery.PublishedPackages) Resolved {
func Resolve(repository string, imports []string, published map[string]meshcatalog.PackagesByModuleValue) Resolved {
r := Resolved{Digests: map[string]string{}, Requires: map[string]string{}}
for _, imp := range imports {
var holder *delivery.PublishedPackages
var holder *meshcatalog.PackagesByModuleValue
digest := ""
for _, m := range slices.Sorted(maps.Keys(published)) {
p := published[m]
if imp != m && !strings.HasPrefix(imp, m+"/") {
continue
}
d, ok := p.DigestOf(imp)
d, ok := digestOf(p, imp)
if !ok {
continue
}
@@ -275,9 +278,9 @@ func Resolve(repository string, imports []string, published map[string]delivery.
}
r.Digests[imp] = digest
r.Requires[holder.Module] = holder.Requires
if !delivery.SameRepository(holder.Repository, repository) {
if !trunk.SameRepository(holder.Repository, repository) {
if r.Versions == nil {
r.Versions = delivery.PackageVersions{}
r.Versions = sdkstep.PackageVersions{}
}
r.Versions[holder.Module] = holder.Commit
}
@@ -296,7 +299,7 @@ func Resolve(repository string, imports []string, published map[string]delivery.
// leaving out `*_test.go` and everything under a sub-directory that is itself a package or holds a go.mod: a file
// //go:embed can reach counts, a test or a sub-package does not. The requirements' digest is the hex sha256 of the
// lines "go.mod <object>" and "go.sum <object>" (the second absent with no go.sum).
func Packages(module string, files map[string]string) (requires string, packages []delivery.PackageDigest, err error) {
func Packages(module string, files map[string]string) (requires string, packages []meshcatalog.PackagesByModuleValuePackagesItem, err error) {
goMod, ok := files["go.mod"]
if !ok {
return "", nil, fmt.Errorf("the module %s has no go.mod at its directory", module)
@@ -351,9 +354,11 @@ func Packages(module string, files map[string]string) (requires string, packages
if dir != "" {
imp = module + "/" + dir
}
packages = append(packages, delivery.PackageDigest{Package: imp, Digest: hex.EncodeToString(s[:])})
packages = append(packages, meshcatalog.PackagesByModuleValuePackagesItem{Package: imp, Digest: hex.EncodeToString(s[:])})
}
slices.SortFunc(packages, func(a, b delivery.PackageDigest) int { return strings.Compare(a.Package, b.Package) })
slices.SortFunc(packages, func(a, b meshcatalog.PackagesByModuleValuePackagesItem) int {
return strings.Compare(a.Package, b.Package)
})
return requires, packages, nil
}
@@ -398,21 +403,21 @@ func sortedKeys[V any](m map[string]V) []string { return slices.Sorted(maps.Keys
// commit of the held module of the package's repository whose directory holds the package's directory (the deepest, a
// module at the repository's root holding every directory). A package no held module holds, or whose module names no
// commit or no directory, is in missing, sorted; versions is nil when packages is empty.
func HeldVersionsOf(packages []string, modules map[string]delivery.Module) (versions delivery.PackageVersions, missing []string) {
func HeldVersionsOf(packages []string, modules map[string]meshcatalog.ModulesByModuleValue) (versions sdkstep.PackageVersions, missing []string) {
for _, pkg := range packages {
repository, dir, ok := delivery.PackageRepository(pkg)
repository, dir, ok := trunk.PackageRepository(pkg)
commit, depth := "", -1
if ok {
for _, key := range slices.Sorted(maps.Keys(modules)) {
m := modules[key]
if !delivery.SameRepository(m.Repository, repository) || m.Commit == "" {
if !trunk.SameRepository(m.Repository, repository) || m.Commit == "" {
continue
}
at := strings.Trim(m.Path, "/")
switch {
case at == "":
continue // a value that names no directory says nothing of where the package lies
case at == delivery.RootPath:
case at == meshcatalogclient.RootPath:
at = ""
case dir != at && !strings.HasPrefix(dir, at+"/"):
continue
@@ -427,10 +432,26 @@ func HeldVersionsOf(packages []string, modules map[string]delivery.Module) (vers
continue
}
if versions == nil {
versions = delivery.PackageVersions{}
versions = sdkstep.PackageVersions{}
}
versions[pkg] = commit
}
slices.Sort(missing)
return versions, missing
}
// digestOf is the content digest of the package at import path pkg in a published module, and whether it holds it
// (its packages are sorted by import path).
func digestOf(p meshcatalog.PackagesByModuleValue, pkg string) (string, bool) {
i, found := slices.BinarySearchFunc(p.Packages, pkg, func(d meshcatalog.PackagesByModuleValuePackagesItem, s string) int {
return strings.Compare(d.Package, s)
})
if !found {
return "", false
}
return p.Packages[i].Digest, true
}
// NamedTreePrefix opens a module's tree when it is computed over the paths its artifacts name, not the directory's own
// git tree (novox/hq ADR 0499 clause 2): the indexer alone computes it (NamedTree).
const NamedTreePrefix = "named:"
modules/mesh-delivery-indexer/internal/fingerprint/fingerprint_test.go
+14 -12
View File
@@ -4,7 +4,9 @@ import (
"strings"
"testing"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
sdkstep "git.novox.be/novox/mesh-delivery/sdk/buildstep"
)
const sdk = "git.novox.be/novox/mesh-sdk/go"
@@ -29,11 +31,11 @@ func versionLineTakesASiblingDirectoryByItsContent(t *testing.T) {
if got := ReplacedSiblings(goMod, repo); len(got) != 1 || got[0] != buildrun {
t.Fatalf("siblings %v, want buildrun alone (mesh-sdk is another repository, replaced by a version)", got)
}
published := func(digest string) map[string]delivery.PublishedPackages {
return map[string]delivery.PublishedPackages{buildrun: {Module: buildrun, Repository: repo, Requires: "r1",
Packages: []delivery.PackageDigest{{Package: buildrun, Digest: digest}}}}
published := func(digest string) map[string]meshcatalog.PackagesByModuleValue {
return map[string]meshcatalog.PackagesByModuleValue{buildrun: {Module: buildrun, Repository: repo, Requires: "r1",
Packages: []meshcatalog.PackagesByModuleValuePackagesItem{{Package: buildrun, Digest: digest}}}}
}
versions := delivery.PackageVersions{sdk: "c1"}
versions := sdkstep.PackageVersions{sdk: "c1"}
line := func(digest string) string {
in := base()
in.Packages = VersionLine(repo, []string{sdk, buildrun}, versions, published(digest))
@@ -53,7 +55,7 @@ func versionLineTakesASiblingDirectoryByItsContent(t *testing.T) {
func base() Inputs {
return Inputs{Module: "netcheck", Tree: "t1", Bases: []string{"src2:b"}, Contexts: map[string]string{"c": "x"},
Locks: map[string]string{"bin": "l1"}, Toolchains: map[string]delivery.Toolchain2{"bin": {Language: "go", Version: "1.26.0", Recipe: "7"}}}
Locks: map[string]string{"bin": "l1"}, Toolchains: map[string]meshcatalogclient.Toolchain2{"bin": {Language: "go", Version: "1.26.0", Recipe: "7"}}}
}
// The form moved out of mesh-sdk computes what mesh-sdk's SourceFingerprint2 and NamedTree computed for the same
@@ -77,10 +79,10 @@ func movedComputationGivesTheValuesItGaveInMeshSDK(t *testing.T) {
}
}
func published(module, repo, dir, commit, requires string, digests map[string]string) delivery.PublishedPackages {
p := delivery.PublishedPackages{Module: module, Repository: repo, Dir: dir, Commit: commit, Requires: requires}
func published(module, repo, dir, commit, requires string, digests map[string]string) meshcatalog.PackagesByModuleValue {
p := meshcatalog.PackagesByModuleValue{Module: module, Repository: repo, Dir: dir, Commit: commit, Requires: requires}
for _, pkg := range sortedKeys(digests) {
p.Packages = append(p.Packages, delivery.PackageDigest{Package: pkg, Digest: digests[pkg]})
p.Packages = append(p.Packages, meshcatalog.PackagesByModuleValuePackagesItem{Package: pkg, Digest: digests[pkg]})
}
return p
}
@@ -90,13 +92,13 @@ func published(module, repo, dir, commit, requires string, digests map[string]st
func importFormMovesOnlyWithWhatTheModuleImports(t *testing.T) {
const sibling = "git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep"
imports := []string{sdk, sdk + "/announce", sibling}
pubs := func(announce, buildstep, sib, requires string) map[string]delivery.PublishedPackages {
return map[string]delivery.PublishedPackages{
pubs := func(announce, buildstep, sib, requires string) map[string]meshcatalog.PackagesByModuleValue {
return map[string]meshcatalog.PackagesByModuleValue{
sdk: published(sdk, "novox/mesh-sdk", "go", "c1", requires, map[string]string{sdk: "r", sdk + "/announce": announce, sdk + "/buildstep": buildstep}),
sibling: published(sibling, "novox/mesh-catalog", "modules/mesh-build-go/buildstep", "c9", "q", map[string]string{sibling: sib}),
}
}
fpOf := func(p map[string]delivery.PublishedPackages) string {
fpOf := func(p map[string]meshcatalog.PackagesByModuleValue) string {
t.Helper()
r := Resolve("novox/mesh-catalog", imports, p)
if len(r.Missing) > 0 {
modules/mesh-delivery-indexer/internal/index/accepted.go
+27 -29
View File
@@ -8,9 +8,10 @@ import (
"strings"
"time"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-sdk/go/health"
"git.novox.be/novox/mesh-sdk/go/keys"
"mesh-delivery-indexer/judge"
)
@@ -29,33 +30,30 @@ const (
// acceptedOf is the event body for the verdict v (its revision in the verdicts state) on the manifest raw of digest, its
// entries an empty list when it asks for none.
func acceptedOf(digest string, v delivery.ManifestVerdict, revision uint64, raw []byte) delivery.ManifestAccepted {
func acceptedOf(digest string, v meshcatalog.VerdictsByDigestValue, revision uint64, raw []byte) meshcatalog.ManifestAcceptedEvent {
m, _ := judge.Manifest(raw, nil)
a := delivery.AcceptedOf(digest, v, m)
if a.BusState == nil {
a.BusState = []catalogue.BusStateEntry{}
}
a.Revision = revision
a := acceptedEvent(digest, v, m)
a.Revision = int64(revision)
return a
}
// leftOf is the empty manifest-accepted for module, which left the trunk, judged by judgeID; revision is the modules
// state's revision of the value that left (0 when none is held), so its event id (delivery.ManifestAccepted.EventID, the
// state's revision of the value that left (0 when none is held), so its event id (acceptedEventID, the
// module and the revision, since its body names no digest) differs for each leave.
func leftOf(module string, revision uint64, judgeID string) (delivery.ManifestAccepted, string) {
a := delivery.ManifestAccepted{V: delivery.Version, Module: module, Judge: judgeID, BusState: []catalogue.BusStateEntry{}, Revision: revision}
return a, a.EventID()
func leftOf(module string, revision uint64, judgeID string) (meshcatalog.ManifestAcceptedEvent, string) {
a := meshcatalog.ManifestAcceptedEvent{V: keys.Version, Module: module, Judge: judgeID, BusState: busStateOf(nil), Revision: int64(revision)}
return a, acceptedEventID(a)
}
// sayAccepted says a with the event id id, once more when the first publish fails; the module's finding is written when
// both fail and deleted when it is said. Nil SayAccepted says nothing.
func (ix *Indexer) sayAccepted(ctx context.Context, a delivery.ManifestAccepted, id string) error {
func (ix *Indexer) sayAccepted(ctx context.Context, a meshcatalog.ManifestAcceptedEvent, id string) error {
if ix.SayAccepted == nil {
return nil
}
err := ix.SayAccepted(ctx, a, id)
if err != nil {
ix.logf("judge: %s for %s failed, published again: %v", delivery.ManifestAcceptedEvent, a.Module, err)
ix.logf("judge: %s for %s failed, published again: %v", meshcatalog.EventManifestAccepted, a.Module, err)
err = ix.SayAccepted(ctx, a, id)
}
ix.unsaid(ctx, a.Module, err)
@@ -64,7 +62,7 @@ func (ix *Indexer) sayAccepted(ctx context.Context, a delivery.ManifestAccepted,
// unsaidKey is the key of module's unsaid finding.
func unsaidKey(module string) string {
return health.FindingKey(health.ScopeModule, delivery.IndexerModule+".accepted."+module, UnsaidKind)
return health.FindingKey(health.ScopeModule, Module+".accepted."+module, UnsaidKind)
}
// unsaid writes module's finding for err, or deletes it with none.
@@ -78,19 +76,19 @@ func (ix *Indexer) unsaid(ctx context.Context, module string, err error) {
return nil, found, nil
}
at := ix.now().UTC()
summary := fmt.Sprintf("%s of %s was not published: %v", delivery.ManifestAcceptedEvent, module, err)
summary := fmt.Sprintf("%s of %s was not published: %v", meshcatalog.EventManifestAccepted, module, err)
data, ferr := health.EncodeFinding(health.Finding{Key: key, Kind: UnsaidKind,
Subject: health.Subject{Scope: health.ScopeModule, ID: delivery.IndexerModule + ".accepted." + module}, Severity: health.Warning,
Subject: health.Subject{Scope: health.ScopeModule, ID: Module + ".accepted." + module}, Severity: health.Warning,
Summary: summary,
Headline: "The indexer could not tell the bus module that " + module + "'s manifest was accepted",
Explanation: "The bus module creates and removes a module's bus state from this event. The indexer says it again at its " +
"next judging pass and clears this once it is said; indexer_say_accepted says it now.",
Resolved: "manifest-accepted of " + module + " was said.",
Evidence: []health.Evidence{{At: at, Said: summary}}, Source: UnsaidKind, Observer: delivery.IndexerModule, At: at})
Evidence: []health.Evidence{{At: at, Said: summary}}, Source: UnsaidKind, Observer: Module, At: at})
return data, false, ferr
})
if werr != nil {
ix.logf("judge: the finding of %s for %s is not written: %v", delivery.ManifestAcceptedEvent, module, werr)
ix.logf("judge: the finding of %s for %s is not written: %v", meshcatalog.EventManifestAccepted, module, werr)
}
}
@@ -104,7 +102,7 @@ func (ix *Indexer) sayFor(ctx context.Context, module string, allowLeft, again b
return "", fmt.Errorf("the modules state cannot be read: %w", err)
}
m, ok := held[module]
var a delivery.ManifestAccepted
var a meshcatalog.ManifestAcceptedEvent
var id string
switch {
case !ok && !allowLeft:
@@ -114,7 +112,7 @@ func (ix *Indexer) sayFor(ctx context.Context, module string, allowLeft, again b
case m.Manifest == "" || ix.Verdicts == nil || ix.Manifests == nil:
return "", fmt.Errorf("%s has no accepted verdict the indexer holds", module)
default:
key, err := delivery.VerdictsKey(m.Manifest)
key, err := meshcatalogclient.VerdictsKey(m.Manifest)
if err != nil {
return "", err
}
@@ -122,11 +120,11 @@ func (ix *Indexer) sayFor(ctx context.Context, module string, allowLeft, again b
if err != nil {
return "", fmt.Errorf("the verdict on %s cannot be read: %w", module, err)
}
var v delivery.ManifestVerdict
var v meshcatalog.VerdictsByDigestValue
if found {
v, err = delivery.DecodeManifestVerdict(value)
v, err = meshcatalogclient.DecodeVerdict(value)
}
if !found || err != nil || v.Refused() || v.Waits() {
if !found || err != nil || meshcatalogclient.VerdictHeld(v) {
if !allowLeft {
return "", fmt.Errorf("%s has no accepted verdict on its manifest %s", module, m.Manifest)
}
@@ -134,18 +132,18 @@ func (ix *Indexer) sayFor(ctx context.Context, module string, allowLeft, again b
return "none: its manifest is not accepted", nil
}
raw, _, found, err := ix.Manifests.Get(ctx, m.Manifest)
kept, derr := delivery.DecodeManifest(raw)
kept, derr := meshcatalogclient.DecodeManifest(raw)
if err != nil || !found || derr != nil {
return "", fmt.Errorf("the manifest %s of %s cannot be read", m.Manifest, module)
}
a = acceptedOf(m.Manifest, v, revision, []byte(kept.JSON))
id = a.EventID()
a = acceptedOf(m.Manifest, v, revision, []byte(kept.Manifest))
id = acceptedEventID(a)
}
if again {
id += "-again-" + ix.now().UTC().Format(time.RFC3339Nano)
}
if err := ix.sayAccepted(ctx, a, id); err != nil {
return "", fmt.Errorf("%s of %s was not published: %w", delivery.ManifestAcceptedEvent, module, err)
return "", fmt.Errorf("%s of %s was not published: %w", meshcatalog.EventManifestAccepted, module, err)
}
if a.Digest == "" {
return "said empty: the trunk holds no module " + module, nil
@@ -174,7 +172,7 @@ func (ix *Indexer) SayUnsaid(ctx context.Context) {
if err != nil || f.Kind != UnsaidKind {
continue
}
module := strings.TrimPrefix(f.Subject.ID, delivery.IndexerModule+".accepted.")
module := strings.TrimPrefix(f.Subject.ID, Module+".accepted.")
if _, err := ix.sayFor(ctx, module, true, false); err != nil {
ix.logf("judge: %v; its finding stands", err)
}
modules/mesh-delivery-indexer/internal/index/builds.go
+21 -20
View File
@@ -9,7 +9,8 @@ import (
"time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-delivery/sdk/buildrecord"
"git.novox.be/novox/mesh-delivery/sdk/trunk"
)
// Builds holds the builder's log (mesh-delivery-builder.build-log, novox/hq ADR 0571 clause 1) for what the indexer reads
@@ -22,11 +23,11 @@ import (
// (ADR 0501 clause 6, issue 1851). Safe for use from a watch and from the message handler at once.
type Builds struct {
mu sync.Mutex
records map[string]delivery.BuildRecord // by the entry's event, module, fingerprint and time
records map[string]buildrecord.BuildRecord // by the entry's event, module, fingerprint and time
}
// NewBuilds is an empty Builds.
func NewBuilds() *Builds { return &Builds{records: map[string]delivery.BuildRecord{}} }
func NewBuilds() *Builds { return &Builds{records: map[string]buildrecord.BuildRecord{}} }
// Put takes one entry of the builder's log. A build's end is an outcome: done registers, failed or queued does not; its
// start is a request, which registers nothing either. An entry of another kind (the old reference form's close, design 51
@@ -36,21 +37,21 @@ func (b *Builds) Put(value []byte) error {
if !builds.IsBuildEntry(value) {
return nil
}
e, err := delivery.DecodeBuildLogEntry(value)
e, err := builds.DecodeBuildLogEntry(value)
if err != nil {
return err
}
if e.Module == "" || e.Repository == "" {
return fmt.Errorf("a build log entry naming no module or repository: %s", value)
}
r := delivery.BuildRecord{ID: e.At.UTC().Format(time.RFC3339Nano) + "." + e.FP, Repository: e.Repository, Ref: e.Ref, Module: e.Module,
r := buildrecord.BuildRecord{ID: e.At.UTC().Format(time.RFC3339Nano) + "." + e.FP, Repository: e.Repository, Ref: e.Ref, Module: e.Module,
Path: e.Directory, Commit: e.Commit, At: e.At}
switch {
case e.Event != delivery.BuildEndedEvent:
case e.Event != builds.BuildEndedEvent:
r.State = e.Event
case e.Outcome == delivery.BuildFailed:
case e.Outcome == builds.BuildFailed:
r.Failed = "the build failed"
case e.Outcome != delivery.BuildDone:
case e.Outcome != builds.BuildDone:
r.State = e.Outcome
}
b.mu.Lock()
@@ -59,7 +60,7 @@ func (b *Builds) Put(value []byte) error {
return nil
}
func newer(a, b delivery.BuildRecord) bool {
func newer(a, b buildrecord.BuildRecord) bool {
return a.At.After(b.At) || (a.At.Equal(b.At) && a.ID > b.ID)
}
@@ -67,18 +68,18 @@ func newer(a, b delivery.BuildRecord) bool {
// newest successful trunk build's first ("." for the repository's root), and false when no record of the repository
// names the module (issue 1614).
func (b *Builds) PathOf(module, repository string) (string, bool) {
var best *delivery.BuildRecord
trunk := false
var best *buildrecord.BuildRecord
onTrunk := false
b.mu.Lock()
defer b.mu.Unlock()
for _, r := range b.records {
if r.Module != module || !delivery.SameRepository(r.Repository, repository) {
if r.Module != module || !trunk.SameRepository(r.Repository, repository) {
continue
}
t := registered(r) && trunkLike[r.Ref]
if best == nil || (t && !trunk) || (t == trunk && newer(r, *best)) {
if best == nil || (t && !onTrunk) || (t == onTrunk && newer(r, *best)) {
c := r
best, trunk = &c, t
best, onTrunk = &c, t
}
}
if best == nil {
@@ -97,7 +98,7 @@ type Registration struct {
// registered is whether the record is an outcome that did not fail: what registers a module. It asks for no particular branch: a
// repository's trunk is named main, master, or what its owner chose (the registered photos, de-spiegel and link2pay were
// built from nox-mesh, and every record of them names that ref).
func registered(r delivery.BuildRecord) bool {
func registered(r buildrecord.BuildRecord) bool {
return r.State == "" && r.Failed == ""
}
@@ -107,11 +108,11 @@ func registered(r delivery.BuildRecord) bool {
// first that holds the module's manifest is the one indexed.
func (b *Builds) RegistrationsOf(module string) []Registration {
type place struct{ repository, path string }
byPlace := map[place][]delivery.BuildRecord{}
byPlace := map[place][]buildrecord.BuildRecord{}
b.mu.Lock()
for _, r := range b.records {
if r.Module == module && registered(r) {
k := place{delivery.RepositoryName(r.Repository), pathOf(strings.Trim(r.Path, "/"))}
k := place{trunk.RepositoryName(r.Repository), pathOf(strings.Trim(r.Path, "/"))}
byPlace[k] = append(byPlace[k], r)
}
}
@@ -119,9 +120,9 @@ func (b *Builds) RegistrationsOf(module string) []Registration {
// Within one place the record is that of the ref a registration comes from, never a newer pull request's or try's: a
// trunk's ref (main, master, trunk, none), else the ref most of the place's records name (the registered photos, de-spiegel
// and link2pay were built from nox-mesh), the newer of two equal; then its newest record.
best := map[place]delivery.BuildRecord{}
best := map[place]buildrecord.BuildRecord{}
for k, recs := range byPlace {
count, latest := map[string]int{}, map[string]delivery.BuildRecord{}
count, latest := map[string]int{}, map[string]buildrecord.BuildRecord{}
for _, r := range recs {
count[r.Ref]++
if cur, ok := latest[r.Ref]; !ok || newer(r, cur) {
@@ -149,7 +150,7 @@ func (b *Builds) RegistrationsOf(module string) []Registration {
best[k] = latest[pick]
}
recs := slices.Collect(maps.Values(best))
slices.SortFunc(recs, func(x, y delivery.BuildRecord) int {
slices.SortFunc(recs, func(x, y buildrecord.BuildRecord) int {
switch {
case trunkLike[x.Ref] != trunkLike[y.Ref]:
if trunkLike[x.Ref] {
modules/mesh-delivery-indexer/internal/index/bus_state.go
+7 -7
View File
@@ -4,8 +4,8 @@ import (
"fmt"
"slices"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/views"
)
@@ -58,11 +58,11 @@ func BusStateOf(modules, verdicts, manifests, marks views.Source) (BusStateAnswe
if !ok {
continue
}
mod, err := delivery.DecodeModule(e.Value)
mod, err := meshcatalogclient.DecodeModule(e.Value)
if err != nil || mod.Manifest == "" {
continue // no manifest held for it, so no verdict
}
key, err := delivery.VerdictsKey(mod.Manifest)
key, err := meshcatalogclient.VerdictsKey(mod.Manifest)
if err != nil {
continue
}
@@ -73,8 +73,8 @@ func BusStateOf(modules, verdicts, manifests, marks views.Source) (BusStateAnswe
if !ok {
continue
}
v, err := delivery.DecodeManifestVerdict(ve.Value)
if err != nil || v.State != delivery.VerdictAccepted {
v, err := meshcatalogclient.DecodeVerdict(ve.Value)
if err != nil || v.State != meshcatalogclient.VerdictAccepted {
continue
}
me, ok, err := manifests.Get(mod.Manifest)
@@ -84,11 +84,11 @@ func BusStateOf(modules, verdicts, manifests, marks views.Source) (BusStateAnswe
if !ok {
continue
}
kept, err := delivery.DecodeManifest(me.Value)
kept, err := meshcatalogclient.DecodeManifest(me.Value)
if err != nil {
return BusStateAnswer{}, fmt.Errorf("the manifest of %s: %w", name, err)
}
m, err := catalogue.Decode([]byte(kept.JSON))
m, err := catalogue.Decode([]byte(kept.Manifest))
if err != nil {
return BusStateAnswer{}, fmt.Errorf("the accepted manifest of %s does not read: %w", name, err)
}
modules/mesh-delivery-indexer/internal/index/candidate_manifests_test.go
+17 -15
View File
@@ -7,12 +7,14 @@ import (
"strings"
"testing"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/keys"
)
func manifestPull(t *testing.T, number int, head, manifest string) forgemsg.Forge {
f := forgemsg.Forge{V: delivery.Version, Kind: forgemsg.ForgePull, Repository: catalogueRepo, Branch: "feat/x", Number: number, Head: head,
f := forgemsg.Forge{V: keys.Version, Kind: forgemsg.ForgePull, Repository: catalogueRepo, Branch: "feat/x", Number: number, Head: head,
Action: forgemsg.PullUpdated, Dirs: []string{"modules/avahi"}, DirsKnown: true}
if manifest != "" {
f.Manifests = map[string]string{"modules/avahi": manifest}
@@ -20,9 +22,9 @@ func manifestPull(t *testing.T, number int, head, manifest string) forgemsg.Forg
return f
}
func candidateOf(t *testing.T, st *memStore) delivery.Candidate {
func candidateOf(t *testing.T, st *memStore) meshcatalog.CandidatesByGroupValue {
t.Helper()
key, err := delivery.CandidatesKey("feat/x")
key, err := CandidatesKey("feat/x")
if err != nil {
t.Fatal(err)
}
@@ -30,7 +32,7 @@ func candidateOf(t *testing.T, st *memStore) delivery.Candidate {
if !ok {
t.Fatalf("candidates holds %d keys, none %s", len(st.vals), key)
}
c, err := delivery.DecodeCandidate(raw)
c, err := DecodeCandidate(raw)
if err != nil {
t.Fatal(err)
}
@@ -44,12 +46,12 @@ func TestACandidateGivesEachTouchedModuleItsManifestAndTheReasonItHasNoPreview(t
t.Fatal(err)
}
c := candidateOf(t, ix.Store.(*memStore)) // DecodeCandidate has run Candidate.check
want := delivery.CandidateModule{Manifest: delivery.ManifestDigest([]byte(avahi)), None2: "no merge preview is computed"}
want := meshcatalog.CandidatesByGroupValuePullsItemFpsValue{Manifest: meshcatalogclient.ManifestDigest([]byte(avahi)), None2: "no merge preview is computed"}
if len(c.Pulls) != 1 || len(c.Pulls[0].Fps) != 1 || !reflect.DeepEqual(c.Pulls[0].Fps["avahi"], want) {
t.Fatalf("pulls = %+v, want Fps[avahi] = %+v", c.Pulls, want)
}
if c.FPSet == "" || c.FPSet != delivery.FingerprintSet(c.Pulls) {
t.Errorf("fpset = %q, want %q", c.FPSet, delivery.FingerprintSet(c.Pulls))
if c.Fpset == "" || c.Fpset != meshcatalogclient.FingerprintSet(c.Pulls) {
t.Errorf("fpset = %q, want %q", c.Fpset, meshcatalogclient.FingerprintSet(c.Pulls))
}
// The same pull request heard again without its directories (the head unchanged) keeps what it had.
again := manifestPull(t, 301, h1, "")
@@ -57,8 +59,8 @@ func TestACandidateGivesEachTouchedModuleItsManifestAndTheReasonItHasNoPreview(t
if err := ix.Apply(context.Background(), again); err != nil {
t.Fatal(err)
}
if c2 := candidateOf(t, ix.Store.(*memStore)); !reflect.DeepEqual(c2.Pulls[0].Fps["avahi"], want) || c2.FPSet != c.FPSet {
t.Errorf("after a message without directories: %+v fpset %q", c2.Pulls[0].Fps, c2.FPSet)
if c2 := candidateOf(t, ix.Store.(*memStore)); !reflect.DeepEqual(c2.Pulls[0].Fps["avahi"], want) || c2.Fpset != c.Fpset {
t.Errorf("after a message without directories: %+v fpset %q", c2.Pulls[0].Fps, c2.Fpset)
}
}
@@ -68,7 +70,7 @@ func TestAPullWithoutManifestsWritesTheCandidateAsToday(t *testing.T) { // 1451/
t.Fatal(err)
}
c := candidateOf(t, ix.Store.(*memStore))
if len(c.Pulls) != 1 || c.Pulls[0].Fps != nil || c.FPSet != "" || len(c.Pulls[0].Modules) != 1 || c.Pulls[0].Modules[0] != "avahi" {
if len(c.Pulls) != 1 || c.Pulls[0].Fps != nil || c.Fpset != "" || len(c.Pulls[0].Modules) != 1 || c.Pulls[0].Modules[0] != "avahi" {
t.Errorf("candidate = %+v, want the module avahi and no Fps and no fpset", c)
}
}
@@ -93,15 +95,15 @@ func TestTwoPullsGivingOneModuleDifferentManifestsAreRefusedNamingBoth(t *testin
}
}
// Candidate.check refuses the group too, which is what the refusal stands for.
if _, encErr := delivery.EncodeCandidate(delivery.Candidate{V: delivery.Version, Group: "feat.x", HeadsHash: before.HeadsHash, Pulls: []delivery.CandidatePull{before.Pulls[0], {
Head: delivery.Head{Repository: catalogueRepo, Number: 302, Head: h2}, Modules: []string{"avahi"},
Fps: map[string]delivery.CandidateModule{"avahi": {Manifest: delivery.ManifestDigest([]byte(other)), None2: NoneNoPreview}}}}}); encErr == nil {
if _, encErr := EncodeCandidate(meshcatalog.CandidatesByGroupValue{V: keys.Version, Group: "feat.x", Heads: before.Heads, Pulls: []meshcatalog.CandidatesByGroupValuePullsItem{before.Pulls[0], {
Repository: catalogueRepo, Number: 302, Head: h2, Modules: []string{"avahi"},
Fps: map[string]meshcatalog.CandidatesByGroupValuePullsItemFpsValue{"avahi": {Manifest: meshcatalogclient.ManifestDigest([]byte(other)), None2: NoneNoPreview}}}}}); encErr == nil {
t.Error("the candidate check accepts two manifests for one module")
}
if ix.Store.(*memStore).writes != writes {
t.Error("a partial write was made")
}
if after := candidateOf(t, ix.Store.(*memStore)); len(after.Pulls) != 1 || after.FPSet != before.FPSet {
if after := candidateOf(t, ix.Store.(*memStore)); len(after.Pulls) != 1 || after.Fpset != before.Fpset {
t.Errorf("the candidate changed: %+v", after)
}
}
modules/mesh-delivery-indexer/internal/index/candidate_values.go
+132
View File
@@ -0,0 +1,132 @@
package index
import (
"encoding/json"
"fmt"
"path"
"reflect"
"strings"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-sdk/go/keys"
sdkstep "git.novox.be/novox/mesh-delivery/sdk/buildstep"
)
// The values of the state candidates (novox/hq ADR 0437 §5, ADR 0453 clauses 6 and 16): the contract's types, written by
// the indexer alone, so their key and writing rules are the indexer's (ADR 0634 §1).
// CandidateNoneNoModule is the None2 of a candidate's module whose directory the merge preview names no manifest for: the
// change removes the module. The indexer writes it and the assigner reads it as a removal (issue 1297).
const CandidateNoneNoModule = "the merge preview names no manifest for the module's directory"
// CandidatesKey is the key, in the candidates state, of a group: `<group>`, the branch name the group's pull requests share
// (a slash is allowed), escaped so a branch such as `fix/1.2` keys; a lone pull request is a group of one under its branch.
func CandidatesKey(group string) (string, error) {
if group == "" {
return "", fmt.Errorf("index: the group name is empty")
}
return meshcatalog.CandidatesByGroupKey(keys.EscapeName(group)), nil
}
func checkCandidateModule(m meshcatalog.CandidatesByGroupValuePullsItemFpsValue) error {
switch {
case m.Fp2 != "" && m.None2 != "":
return fmt.Errorf("index: a module has both fp2 and none2")
case m.Fp2 == "" && m.None2 == "":
return fmt.Errorf("index: a module has neither fp2 nor none2")
}
for p, object := range m.Paths {
if p == "" || p != path.Clean(p) || strings.HasPrefix(p, "/") || p == ".." || strings.HasPrefix(p, "../") {
return fmt.Errorf("index: the named path %q is not a clean path inside the module's directory", p)
}
if object == "" {
return fmt.Errorf("index: the named path %q names no git object", p)
}
}
if err := sdkstep.PackageVersions(m.Packages).Check(); err != nil {
return err
}
if m.Fp2 != "" {
if _, err := meshcatalogclient.KeyPart(m.Fp2); err != nil {
return err
}
}
if m.Manifest != "" {
if _, err := meshcatalogclient.ManifestsKey(m.Manifest); err != nil {
return err
}
}
return nil
}
func checkCandidate(c meshcatalog.CandidatesByGroupValue) error {
if len(c.Pulls) == 0 {
return fmt.Errorf("index: a candidate names no pull request")
}
heads := make([]meshdeliveryclient.Head, len(c.Pulls))
for i, p := range c.Pulls {
heads[i] = meshdeliveryclient.Head{Repository: p.Repository, Number: int(p.Number), Head: p.Head}
}
want, _ := meshdeliveryclient.HeadsHash(heads)
if c.Heads != want {
return fmt.Errorf("index: the candidate's heads hash %q is not the hash %q of its pull requests' heads", c.Heads, want)
}
seen := map[string]meshcatalog.CandidatesByGroupValuePullsItemFpsValue{}
for _, p := range c.Pulls {
touched := map[string]bool{}
for _, m := range p.Modules {
touched[m] = true
}
for module, m := range p.Fps {
if !touched[module] {
return fmt.Errorf("index: the candidate gives a fingerprint of the module %q, which pull request %s#%d does not touch", module, p.Repository, p.Number)
}
if err := checkCandidateModule(m); err != nil {
return fmt.Errorf("index: the candidate's module %q: %w", module, err)
}
if before, ok := seen[module]; ok && !reflect.DeepEqual(before, m) {
return fmt.Errorf("index: two pull requests of the candidate give the module %q different fingerprints", module)
}
seen[module] = m
}
}
if wantSet := meshcatalogclient.FingerprintSet(c.Pulls); c.Fpset != wantSet {
return fmt.Errorf("index: the candidate's fpset %q is not the hash %q of its fingerprints", c.Fpset, wantSet)
}
return nil
}
// EncodeCandidate is c as JSON at the kept values' version; it refuses what a reader could not take.
func EncodeCandidate(c meshcatalog.CandidatesByGroupValue) ([]byte, error) {
if err := checkCandidate(c); err != nil {
return nil, err
}
c.V = keys.Version
return json.Marshal(c)
}
// DecodeCandidate reads a candidate; it refuses an unknown version (keys.ErrUnknownVersion) and what EncodeCandidate refuses.
func DecodeCandidate(data []byte) (meshcatalog.CandidatesByGroupValue, error) {
if err := keys.CheckVersion(data); err != nil {
return meshcatalog.CandidatesByGroupValue{}, err
}
var c meshcatalog.CandidatesByGroupValue
if err := json.Unmarshal(data, &c); err != nil {
return meshcatalog.CandidatesByGroupValue{}, fmt.Errorf("index: a candidate does not fit its shape: %w", err)
}
if err := checkCandidate(c); err != nil {
return meshcatalog.CandidatesByGroupValue{}, err
}
return c, nil
}
// SameCandidate says whether two candidates are the same value.
func SameCandidate(a, b meshcatalog.CandidatesByGroupValue) bool { return reflect.DeepEqual(a, b) }
// headOf is a candidate's pull request as the head the heads hash is over.
func headOf(p meshcatalog.CandidatesByGroupValuePullsItem) meshdeliveryclient.Head {
return meshdeliveryclient.Head{Repository: p.Repository, Number: int(p.Number), Head: p.Head}
}
modules/mesh-delivery-indexer/internal/index/candidates.go
+58 -49
View File
@@ -1,6 +1,7 @@
package index
import (
"cmp"
"context"
"errors"
"fmt"
@@ -11,9 +12,15 @@ import (
"sync/atomic"
"time"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/nodebuildgo"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdeliverychecker"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/keys"
)
// ErrConflict is what a Store says when a write found the key changed or created since it was read.
@@ -67,15 +74,15 @@ type Indexer struct {
// JudgeID names the judge a verdict says made it; empty is judge.Identity, the running program's.
JudgeID string
// SayRefused says the event manifest-refused for a refusal newly written; nil says nothing.
SayRefused func(ctx context.Context, r delivery.ManifestRefused)
SayRefused func(ctx context.Context, r meshcatalog.ManifestRefusedEvent)
// SayAccepted publishes the event manifest-accepted a under the bus's duplicate id id (accepted.go; issue 1966 D3,
// DLV-I9): for every accepted verdict newly written, carrying the entries whole, and an empty one for a module leaving
// the trunk; nil says nothing.
SayAccepted func(ctx context.Context, a delivery.ManifestAccepted, id string) error
SayAccepted func(ctx context.Context, a meshcatalog.ManifestAcceptedEvent, id string) error
// Findings is the indexer's findings state, where a manifest-accepted whose publish failed twice is kept until said
// (accepted.go, DLV-I9b); nil writes none.
Findings Store
// Packages is the packages state (delivery.PackagesBucket, packages.go; novox/hq ADR 0593 §2); nil publishes none,
// Packages is the packages state (PackagesBucket, packages.go; novox/hq ADR 0593 §2); nil publishes none,
// and every fingerprint takes the version line.
Packages Store
// SeatDefinitions is the state seat-definitions (seatdefs.go; design 26, DLV-I17); nil keeps none.
@@ -133,19 +140,19 @@ type Indexer struct {
mu sync.Mutex // guards source, toolchains and previews, which the watches and the forge messages all reach
source sourceNow
previews map[string]delivery.Preview // by previews key (previews.go)
toolchains map[string]map[string]delivery.Toolchain // language, then node (toolchains.go)
tried map[string]bool // repository, commit and directory a registered module's manifest was tried at (registered.go)
unread map[string]bool // registered modules whose last read the forge could not answer, read again by RetryUnread
moving map[string]forgemsg.Forge // by modules key: a merge naming a module another repository holds, waiting for that trunk to drop it (moves.go)
retrying atomic.Bool // a RetryUnread runs in the background (kickRetry)
retryAgain atomic.Bool // an event asked for one more RetryUnread after the running one
retryWG sync.WaitGroup // the background RetryUnread, for tests to wait on
recompute sync.Mutex // one Recompute at a time
untaken map[string]string // repository to the latest merge applied while the taken state was missing (taken.go)
filling atomic.Bool // a FillTaken runs in the background (KickFill)
fillAgain atomic.Bool // a kick arrived while it ran
atHeads map[string]map[string]delivery.PublishedPackages // a member head's published Go modules, by repository@head (candidatePublished)
previews map[string]meshdeliverychecker.PreviewsByRepositoryHeadTrunkValue // by previews key (previews.go)
toolchains map[string]map[string]nodebuildgo.Toolchain // language, then node (toolchains.go)
tried map[string]bool // repository, commit and directory a registered module's manifest was tried at (registered.go)
unread map[string]bool // registered modules whose last read the forge could not answer, read again by RetryUnread
moving map[string]forgemsg.Forge // by modules key: a merge naming a module another repository holds, waiting for that trunk to drop it (moves.go)
retrying atomic.Bool // a RetryUnread runs in the background (kickRetry)
retryAgain atomic.Bool // an event asked for one more RetryUnread after the running one
retryWG sync.WaitGroup // the background RetryUnread, for tests to wait on
recompute sync.Mutex // one Recompute at a time
untaken map[string]string // repository to the latest merge applied while the taken state was missing (taken.go)
filling atomic.Bool // a FillTaken runs in the background (KickFill)
fillAgain atomic.Bool // a kick arrived while it ran
atHeads map[string]map[string]meshcatalog.PackagesByModuleValue // a member head's published Go modules, by repository@head (candidatePublished)
// OwnBuild answers the fingerprint of the build this program runs, from the node-engine's seat; nil knows none.
OwnBuild func(ctx context.Context) (string, error)
judgeMu sync.Mutex // one judge pass at a time (verdicts.go)
@@ -232,7 +239,7 @@ func (ix *Indexer) ApplyMessage(ctx context.Context, f forgemsg.Forge) (Applied,
}
}
var digests map[string]string
var held map[string]delivery.Module
var held map[string]meshcatalog.ModulesByModuleValue
if f.Kind == forgemsg.ForgePull {
var err error
if digests, err = ix.keepManifests(ctx, f); err != nil {
@@ -249,7 +256,7 @@ func (ix *Indexer) ApplyMessage(ctx context.Context, f forgemsg.Forge) (Applied,
break
}
if value, _, found, err := ix.Manifests.Get(ctx, digest); err == nil && found {
if kept, err := delivery.DecodeManifest(value); err == nil {
if kept, err := meshcatalogclient.DecodeManifest(value); err == nil {
over[kept.Module] = digest
}
}
@@ -272,19 +279,19 @@ func (ix *Indexer) removeCandidate(ctx context.Context, f forgemsg.Forge) error
}
// writeCandidate writes the group's candidate with the pull request f names (taken out for a merged or closed one).
func (ix *Indexer) writeCandidate(ctx context.Context, f forgemsg.Forge, digests map[string]string, held map[string]delivery.Module) error {
func (ix *Indexer) writeCandidate(ctx context.Context, f forgemsg.Forge, digests map[string]string, held map[string]meshcatalog.ModulesByModuleValue) error {
if trunkLike[f.Branch] {
ix.logf("%s#%d: its branch %q is no delivery group, so no candidate", f.Repository, f.Number, f.Branch)
return nil
}
key, err := delivery.CandidatesKey(f.Branch)
key, err := CandidatesKey(f.Branch)
if err != nil {
return fmt.Errorf("%w: %s#%d: %v", ErrUnreadable, f.Repository, f.Number, err)
}
return write(ctx, ix.Store, key, func(value []byte, found bool) ([]byte, bool, error) {
var cur *delivery.Candidate
var cur *meshcatalog.CandidatesByGroupValue
if found {
c, err := delivery.DecodeCandidate(value)
c, err := DecodeCandidate(value)
if err != nil {
return nil, false, fmt.Errorf("candidates.%s holds what is not a candidate: %w", key, err)
}
@@ -298,12 +305,12 @@ func (ix *Indexer) writeCandidate(ctx context.Context, f forgemsg.Forge, digests
}
}
switch {
case next == nil && cur == nil, next != nil && cur != nil && delivery.SameCandidate(*cur, *next):
case next == nil && cur == nil, next != nil && cur != nil && SameCandidate(*cur, *next):
return nil, false, nil
case next == nil:
return nil, true, nil
}
data, err := delivery.EncodeCandidate(*next)
data, err := EncodeCandidate(*next)
return data, false, err
})
}
@@ -325,12 +332,12 @@ func (ix *Indexer) applyOpen(ctx context.Context, f forgemsg.Forge) error {
if !found {
return nil, false, nil
}
c, err := delivery.DecodeCandidate(value)
c, err := DecodeCandidate(value)
if err != nil {
return nil, false, fmt.Errorf("candidates.%s holds what is not a candidate: %w", key, err)
}
kept := slices.DeleteFunc(slices.Clone(c.Pulls), func(p delivery.CandidatePull) bool {
return !named[forgemsg.OpenPull{Repository: p.Repository, Number: p.Number}]
kept := slices.DeleteFunc(slices.Clone(c.Pulls), func(p meshcatalog.CandidatesByGroupValuePullsItem) bool {
return !named[forgemsg.OpenPull{Repository: p.Repository, Number: int(p.Number)}]
})
switch {
case len(kept) == len(c.Pulls):
@@ -339,16 +346,16 @@ func (ix *Indexer) applyOpen(ctx context.Context, f forgemsg.Forge) error {
ix.logf("%s: no pull request of the group is open, so no candidate", key)
return nil, true, nil
}
heads := make([]delivery.Head, len(kept))
heads := make([]meshdeliveryclient.Head, len(kept))
for i, p := range kept {
heads[i] = p.Head
heads[i] = meshdeliveryclient.Head{Repository: p.Repository, Number: int(p.Number), Head: p.Head}
}
hash, err := delivery.HeadsHash(heads)
hash, err := meshdeliveryclient.HeadsHash(heads)
if err != nil {
return nil, false, fmt.Errorf("%w: candidates.%s: %v", ErrUnreadable, key, err)
}
c.Pulls, c.HeadsHash = kept, hash
data, err := delivery.EncodeCandidate(c)
c.Pulls, c.Heads = kept, hash
data, err := EncodeCandidate(c)
return data, false, err
})
if err != nil {
@@ -401,20 +408,22 @@ func write(ctx context.Context, st Store, key string, compute func(value []byte,
// its manifest and why it has no fingerprint of the new form (NoneNoPreview), and the candidate its FPSet (issue 1451); a
// pull request whose head is held again without its directories keeps the Fps it had. digests are the manifests' digests by
// directory; held are the modules the modules bucket holds, which name the module of a changed directory (moduleOfDir).
func (ix *Indexer) next(cur *delivery.Candidate, f forgemsg.Forge, digests map[string]string, held map[string]delivery.Module) *delivery.Candidate {
var pulls []delivery.CandidatePull
var base uint64
func (ix *Indexer) next(cur *meshcatalog.CandidatesByGroupValue, f forgemsg.Forge, digests map[string]string, held map[string]meshcatalog.ModulesByModuleValue) *meshcatalog.CandidatesByGroupValue {
var pulls []meshcatalog.CandidatesByGroupValuePullsItem
var base int64
if cur != nil {
pulls, base = slices.Clone(cur.Pulls), cur.Base
}
at := slices.IndexFunc(pulls, func(p delivery.CandidatePull) bool { return p.Repository == f.Repository && p.Number == f.Number })
at := slices.IndexFunc(pulls, func(p meshcatalog.CandidatesByGroupValuePullsItem) bool {
return p.Repository == f.Repository && p.Number == int64(f.Number)
})
if f.Kind == forgemsg.ForgeMerged || f.Action == forgemsg.PullClosed {
if at < 0 {
return cur
}
pulls = slices.Delete(pulls, at, at+1)
} else {
entry := delivery.CandidatePull{Head: delivery.Head{Repository: f.Repository, Number: f.Number, Head: f.Head}, Modules: []string{}}
entry := meshcatalog.CandidatesByGroupValuePullsItem{Repository: f.Repository, Number: int64(f.Number), Head: f.Head, Modules: []string{}}
switch {
case f.DirsKnown:
for _, dir := range f.Dirs {
@@ -424,12 +433,12 @@ func (ix *Indexer) next(cur *delivery.Candidate, f forgemsg.Forge, digests map[s
}
if digest, ok := digests[dir]; ok {
if entry.Fps == nil {
entry.Fps = map[string]delivery.CandidateModule{}
entry.Fps = map[string]meshcatalog.CandidatesByGroupValuePullsItemFpsValue{}
}
entry.Fps[m] = delivery.CandidateModule{Manifest: digest, None2: NoneNoPreview}
entry.Fps[m] = meshcatalog.CandidatesByGroupValuePullsItemFpsValue{Manifest: digest, None2: NoneNoPreview}
}
}
case at >= 0 && pulls[at].Head.Head == f.Head:
case at >= 0 && pulls[at].Head == f.Head:
// Its directories are not known now, for the head the group already holds: what it knew stands.
entry.Modules, entry.Fps = pulls[at].Modules, pulls[at].Fps
}
@@ -442,18 +451,18 @@ func (ix *Indexer) next(cur *delivery.Candidate, f forgemsg.Forge, digests map[s
if len(pulls) == 0 {
return nil
}
slices.SortFunc(pulls, func(a, b delivery.CandidatePull) int {
slices.SortFunc(pulls, func(a, b meshcatalog.CandidatesByGroupValuePullsItem) int {
if c := strings.Compare(a.Repository, b.Repository); c != 0 {
return c
}
return a.Number - b.Number
return cmp.Compare(a.Number, b.Number)
})
heads := make([]delivery.Head, len(pulls))
heads := make([]meshdeliveryclient.Head, len(pulls))
for i, p := range pulls {
heads[i] = p.Head
heads[i] = meshdeliveryclient.Head{Repository: p.Repository, Number: int(p.Number), Head: p.Head}
}
hash, _ := delivery.HeadsHash(heads)
return &delivery.Candidate{V: delivery.Version, Group: f.Branch, Base: base, HeadsHash: hash, Pulls: pulls, FPSet: delivery.FingerprintSet(pulls)}
hash, _ := meshdeliveryclient.HeadsHash(heads)
return &meshcatalog.CandidatesByGroupValue{V: keys.Version, Group: f.Branch, Base: base, Heads: hash, Pulls: pulls, Fpset: meshcatalogclient.FingerprintSet(pulls)}
}
// NoneNoPreview is why a candidate's touched module has no fingerprint of the new form: the merge preview is not
@@ -464,8 +473,8 @@ const NoneNoPreview = "no merge preview is computed"
// candidate's own check does (mesh-sdk Candidate.check), naming both pull requests so the log says which; nothing is
// written, and the message is dropped as unreadable and said in the log (main.go handle): the next message of either pull
// request is judged again.
func (ix *Indexer) refuseDisagreement(c delivery.Candidate) error {
first := map[string]delivery.CandidatePull{}
func (ix *Indexer) refuseDisagreement(c meshcatalog.CandidatesByGroupValue) error {
first := map[string]meshcatalog.CandidatesByGroupValuePullsItem{}
for _, p := range c.Pulls {
for module, m := range p.Fps {
before, ok := first[module]
modules/mesh-delivery-indexer/internal/index/catalog.go
+5 -6
View File
@@ -9,7 +9,6 @@ import (
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/views"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
@@ -25,7 +24,7 @@ import (
// CatalogRead reads the states the seat's verbs answer from.
type CatalogRead struct {
// Modules is every value of the modules state, by module.
Modules func() (map[string]delivery.Module, error)
Modules func() (map[string]meshcatalog.ModulesByModuleValue, error)
// Manifest is the manifest kept under a digest in the manifests state, as its JSON.
Manifest func(digest string) (string, error)
// Graph is the module graph, by module.
@@ -35,8 +34,8 @@ type CatalogRead struct {
SeatDefinitions, Marks views.Source
}
func entryOf(name string, m delivery.Module) catalogue.CatalogEntry {
return catalogue.CatalogEntry{Module: name, Repository: m.Repository, Path: m.Path, Commit: m.Commit, FP: m.FP2, None: m.None2}
func entryOf(name string, m meshcatalog.ModulesByModuleValue) catalogue.CatalogEntry {
return catalogue.CatalogEntry{Module: name, Repository: m.Repository, Path: m.Path, Commit: m.Commit, FP: m.Fp2, None: m.None2}
}
// CatalogModulesAnswer is the verb modules: every module the modules state holds, by name.
@@ -104,7 +103,7 @@ func (c CatalogRead) CatalogProvidesAnswer(provision string) (catalogue.CatalogP
}
// scopeOf is the scope the module's manifest offers provision at, "" when its manifest cannot be read.
func (c CatalogRead) scopeOf(m delivery.Module, provision string) string {
func (c CatalogRead) scopeOf(m meshcatalog.ModulesByModuleValue, provision string) string {
if m.Manifest == "" {
return ""
}
@@ -142,7 +141,7 @@ func (c CatalogRead) CatalogDependentsAnswer(module string) (catalogue.CatalogDe
return catalogue.CatalogDependents{Module: module, Dependents: out}, nil
}
func (c CatalogRead) graphAndModules() (graph.Graph, map[string]delivery.Module, error) {
func (c CatalogRead) graphAndModules() (graph.Graph, map[string]meshcatalog.ModulesByModuleValue, error) {
g, err := c.Graph()
if err != nil {
return nil, nil, fmt.Errorf("the module graph cannot be read: %w", err)
modules/mesh-delivery-indexer/internal/index/catalog_test.go
+3 -3
View File
@@ -10,7 +10,7 @@ import (
"strings"
"testing"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/sdk/graph"
)
@@ -57,8 +57,8 @@ func catalogVerbsAnswerFromTheStates(t *testing.T) {
mod{"ca-trust", catalogueManifest(t, "ca-trust")}, mod{"acme", withAcme})
reportToolchain(t, ix, "mesh-build-go", "novox", "go", "go1.27.1")
c := CatalogRead{
Modules: func() (map[string]delivery.Module, error) {
out := map[string]delivery.Module{}
Modules: func() (map[string]meshcatalog.ModulesByModuleValue, error) {
out := map[string]meshcatalog.ModulesByModuleValue{}
for k := range mods.vals {
m, _ := moduleOf(t, mods, k)
out[k] = m
modules/mesh-delivery-indexer/internal/index/config_readers.go
+3 -3
View File
@@ -4,7 +4,7 @@ import (
"encoding/json"
"slices"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
)
// ConfigReadersTool is the indexer's tool that says who reads a module's config keys (design 32 *A value no longer
@@ -51,7 +51,7 @@ func configKeys(raw string) (values, secrets []string) {
// ConfigReaders are the rows of module's config keys: every key any manifest of the module the indexer keeps ever
// declared (so a key no manifest declares any more is a row with no reader), each with who reads it now. trunk is the
// modules state, manifests the manifests state by digest, candidates the open pull requests' config declarations.
func ConfigReaders(module string, trunk map[string]delivery.Module, manifests map[string]delivery.Manifest, candidates []delivery.ConfigCandidate) []ConfigReaderRow {
func ConfigReaders(module string, trunk map[string]meshcatalog.ModulesByModuleValue, manifests map[string]meshcatalog.ManifestsByDigestValue, candidates []meshcatalog.ConfigCandidatesByPullValue) []ConfigReaderRow {
rows := map[string]*ConfigReaderRow{}
add := func(kind, key, reader string) {
id := kind + "|" + key
@@ -76,7 +76,7 @@ func ConfigReaders(module string, trunk map[string]delivery.Module, manifests ma
if digest == current {
reader = module
}
values, secrets := configKeys(kept.JSON)
values, secrets := configKeys(kept.Manifest)
for _, k := range values {
add(ConfigKindValue, k, reader)
}
modules/mesh-delivery-indexer/internal/index/control_records.go
+48 -43
View File
@@ -10,8 +10,10 @@ import (
"strings"
"time"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/keys"
"git.novox.be/novox/mesh-sdk/go/process"
)
@@ -22,8 +24,8 @@ import (
// The two states' buckets.
var (
PredefinedBucket = delivery.IndexerModule + "_" + delivery.IndexerPredefinedState
ConfigCandidatesBucket = delivery.IndexerModule + "_" + delivery.IndexerConfigCandidatesState
PredefinedBucket = Module + "_" + PredefinedState
ConfigCandidatesBucket = Module + "_" + ConfigCandidatesState
)
// writePredefined keeps the predefined list a merged message carries, unless the key names a later merge: a merge older
@@ -32,9 +34,12 @@ func (ix *Indexer) writePredefined(ctx context.Context, f forgemsg.Forge) error
if ix.Predefined == nil || len(f.Predefined) == 0 {
return nil
}
return ix.notYet(PredefinedBucket, write(ctx, ix.Predefined, delivery.PredefinedListKey, func(value []byte, found bool) ([]byte, bool, error) {
return ix.notYet(PredefinedBucket, write(ctx, ix.Predefined, PredefinedListKey, func(value []byte, found bool) ([]byte, bool, error) {
if found {
var cur delivery.PredefinedKept
var cur struct { // the list is compared as it was kept, byte for byte
Commit string `json:"commit"`
List json.RawMessage `json:"list"`
}
if err := json.Unmarshal(value, &cur); err == nil {
if cur.Commit == f.Merge && string(cur.List) == string(f.Predefined) {
return nil, false, nil
@@ -49,7 +54,7 @@ func (ix *Indexer) writePredefined(ctx context.Context, f forgemsg.Forge) error
}
}
}
next, err := json.Marshal(delivery.PredefinedKept{V: delivery.Version, Commit: f.Merge, At: ix.now(), List: f.Predefined})
next, err := json.Marshal(meshcatalog.PredefinedByListValue{V: keys.Version, Commit: f.Merge, At: ptr(ix.now()), List: f.Predefined})
return next, false, err
}))
}
@@ -76,10 +81,10 @@ func PullName(repository string, number int) string { return repository + "#" +
// DeclarationsOf is a manifest's config declarations by key, in the layout of the controller's declared-config mirror: the
// config form's `config`, else the settings form's `settings`, each default and enum encoded the one way.
func DeclarationsOf(raw string) (map[string]delivery.ConfigDeclaration, error) {
func DeclarationsOf(raw string) (map[string]meshcatalog.ConfigCandidatesByPullValueDeclaresValueValue, error) {
var m struct {
Config map[string]delivery.ConfigDeclaration `json:"config"`
Settings map[string]delivery.ConfigDeclaration `json:"settings"`
Config map[string]meshcatalog.ConfigCandidatesByPullValueDeclaresValueValue `json:"config"`
Settings map[string]meshcatalog.ConfigCandidatesByPullValueDeclaresValueValue `json:"settings"`
}
if err := json.Unmarshal([]byte(raw), &m); err != nil {
return nil, err
@@ -88,30 +93,17 @@ func DeclarationsOf(raw string) (map[string]delivery.ConfigDeclaration, error) {
if declared == nil {
declared = m.Settings
}
out := map[string]delivery.ConfigDeclaration{}
// A default and an enum decoded into the contract's `any` are written again in the one canonical way (keys sorted).
out := make(map[string]meshcatalog.ConfigCandidatesByPullValueDeclaresValueValue, len(declared))
for key, d := range declared {
for _, field := range []*json.RawMessage{&d.Default, &d.Enum} {
if len(*field) == 0 {
continue
}
var v any
if err := json.Unmarshal(*field, &v); err != nil {
return nil, fmt.Errorf("%s: %w", key, err)
}
canonical, err := json.Marshal(v)
if err != nil {
return nil, err
}
*field = canonical
}
out[key] = d
}
return out, nil
}
// changedDeclarations are the declarations head adds or changes against trunk.
func changedDeclarations(head, trunk map[string]delivery.ConfigDeclaration) map[string]delivery.ConfigDeclaration {
out := map[string]delivery.ConfigDeclaration{}
func changedDeclarations(head, trunk map[string]meshcatalog.ConfigCandidatesByPullValueDeclaresValueValue) map[string]meshcatalog.ConfigCandidatesByPullValueDeclaresValueValue {
out := map[string]meshcatalog.ConfigCandidatesByPullValueDeclaresValueValue{}
for key, d := range head {
if before, there := trunk[key]; !there || !reflect.DeepEqual(before, d) {
out[key] = d
@@ -121,7 +113,7 @@ func changedDeclarations(head, trunk map[string]delivery.ConfigDeclaration) map[
}
// trunkManifest is the manifest the indexer holds for a module on the trunk, "" when it holds none.
func (ix *Indexer) trunkManifest(ctx context.Context, held map[string]delivery.Module, module string) (string, error) {
func (ix *Indexer) trunkManifest(ctx context.Context, held map[string]meshcatalog.ModulesByModuleValue, module string) (string, error) {
m, ok := held[module]
if !ok || m.Manifest == "" || ix.Manifests == nil {
return "", nil
@@ -130,17 +122,17 @@ func (ix *Indexer) trunkManifest(ctx context.Context, held map[string]delivery.M
if err != nil || !found {
return "", err
}
kept, err := delivery.DecodeManifest(raw)
kept, err := meshcatalogclient.DecodeManifest(raw)
if err != nil {
return "", err
}
return kept.JSON, nil
return kept.Manifest, nil
}
// candidateDeclares are, per module of a pull request's head, the declarations its manifest adds or changes against the
// trunk's (an empty set for a module that adds none): heads are the head's manifests by module.
func (ix *Indexer) candidateDeclares(ctx context.Context, held map[string]delivery.Module, heads map[string]string) (map[string]map[string]delivery.ConfigDeclaration, error) {
out := map[string]map[string]delivery.ConfigDeclaration{}
func (ix *Indexer) candidateDeclares(ctx context.Context, held map[string]meshcatalog.ModulesByModuleValue, heads map[string]string) (map[string]map[string]meshcatalog.ConfigCandidatesByPullValueDeclaresValueValue, error) {
out := map[string]map[string]meshcatalog.ConfigCandidatesByPullValueDeclaresValueValue{}
for module, raw := range heads {
head, err := DeclarationsOf(raw)
if err != nil {
@@ -150,7 +142,7 @@ func (ix *Indexer) candidateDeclares(ctx context.Context, held map[string]delive
if err != nil {
return nil, err
}
trunk := map[string]delivery.ConfigDeclaration{}
trunk := map[string]meshcatalog.ConfigCandidatesByPullValueDeclaresValueValue{}
if trunkRaw != "" {
if trunk, err = DeclarationsOf(trunkRaw); err != nil {
return nil, fmt.Errorf("the trunk's manifest of %s: %w", module, err)
@@ -165,12 +157,12 @@ func (ix *Indexer) candidateDeclares(ctx context.Context, held map[string]delive
// writeConfigCandidate writes a pull request's key of config-candidates from its head's manifests by module: written when
// what it says differs (its time apart), never for the same head and declarations twice.
func (ix *Indexer) writeConfigCandidate(ctx context.Context, repository string, number int, head string, held map[string]delivery.Module, heads map[string]string) error {
func (ix *Indexer) writeConfigCandidate(ctx context.Context, repository string, number int, head string, held map[string]meshcatalog.ModulesByModuleValue, heads map[string]string) error {
if ix.ConfigCandidates == nil {
return nil
}
pull := PullName(repository, number)
key, err := delivery.ConfigCandidateKey(pull)
key, err := ConfigCandidateKey(pull)
if err != nil {
return fmt.Errorf("%w: %v", ErrUnreadable, err)
}
@@ -180,12 +172,12 @@ func (ix *Indexer) writeConfigCandidate(ctx context.Context, repository string,
}
return ix.notYet(ConfigCandidatesBucket, write(ctx, ix.ConfigCandidates, key, func(value []byte, found bool) ([]byte, bool, error) {
if found {
var cur delivery.ConfigCandidate
var cur meshcatalog.ConfigCandidatesByPullValue
if json.Unmarshal(value, &cur) == nil && cur.Head == head && reflect.DeepEqual(cur.Declares, declares) {
return nil, false, nil
}
}
next, err := json.Marshal(delivery.ConfigCandidate{V: delivery.Version, Pull: pull, Head: head, Declares: declares, At: ix.now()})
next, err := json.Marshal(meshcatalog.ConfigCandidatesByPullValue{V: keys.Version, Pull: pull, Head: head, Declares: declares, At: ptr(ix.now())})
return next, false, err
}))
}
@@ -195,7 +187,7 @@ func (ix *Indexer) dropConfigCandidate(ctx context.Context, repository string, n
if ix.ConfigCandidates == nil {
return nil
}
key, err := delivery.ConfigCandidateKey(PullName(repository, number))
key, err := ConfigCandidateKey(PullName(repository, number))
if err != nil {
return nil
}
@@ -204,7 +196,7 @@ func (ix *Indexer) dropConfigCandidate(ctx context.Context, repository string, n
// configCandidateOfPull is a pull message's part in config-candidates: a head whose manifests the message carries is written,
// a closed one deleted. A head held again without its directories leaves the key as it is.
func (ix *Indexer) configCandidateOfPull(ctx context.Context, f forgemsg.Forge, held map[string]delivery.Module) error {
func (ix *Indexer) configCandidateOfPull(ctx context.Context, f forgemsg.Forge, held map[string]meshcatalog.ModulesByModuleValue) error {
if f.Action == forgemsg.PullClosed {
return ix.dropConfigCandidate(ctx, f.Repository, f.Number)
}
@@ -248,7 +240,7 @@ func (ix *Indexer) configCandidatesAfterMerge(ctx context.Context, f forgemsg.Fo
if err != nil || !found {
continue
}
c, err := delivery.DecodeCandidate(raw)
c, err := DecodeCandidate(raw)
if err != nil {
continue
}
@@ -262,11 +254,11 @@ func (ix *Indexer) configCandidatesAfterMerge(ctx context.Context, f forgemsg.Fo
if err != nil || !found {
continue
}
if kept, err := delivery.DecodeManifest(mraw); err == nil {
heads[module] = kept.JSON
if kept, err := meshcatalogclient.DecodeManifest(mraw); err == nil {
heads[module] = kept.Manifest
}
}
if err := ix.writeConfigCandidate(ctx, p.Repository, p.Number, p.Head.Head, held, heads); err != nil {
if err := ix.writeConfigCandidate(ctx, p.Repository, int(p.Number), p.Head, held, heads); err != nil {
errs = append(errs, err)
}
}
@@ -274,7 +266,7 @@ func (ix *Indexer) configCandidatesAfterMerge(ctx context.Context, f forgemsg.Fo
return errors.Join(errs...)
}
func touchesAny(p delivery.CandidatePull, modules []string) bool {
func touchesAny(p meshcatalog.CandidatesByGroupValuePullsItem, modules []string) bool {
for _, m := range modules {
if _, ok := p.Fps[m]; ok {
return true
@@ -291,3 +283,16 @@ func writtenModules(written []string) []string {
}
return out
}
func ptr[T any](v T) *T { return &v }
// ConfigCandidateKey is a pull request's key in config-candidates, `owner/repository#number` with `/` and `#` as dots, the form
// mesh-config keeps a pull request's values under.
func ConfigCandidateKey(pull string) (string, error) {
owner, rest, ok := strings.Cut(strings.TrimSpace(pull), "/")
repo, number, ok2 := strings.Cut(rest, "#")
if !ok || !ok2 || owner == "" || repo == "" || number == "" || strings.ContainsAny(owner+repo+number, ".*> ") {
return "", fmt.Errorf("index: %q is not a pull request, owner/repository#number", pull)
}
return meshcatalog.ConfigCandidatesByPullKey(owner + "." + repo + "." + number), nil
}
modules/mesh-delivery-indexer/internal/index/control_records_test.go
+41 -37
View File
@@ -9,9 +9,13 @@ import (
"testing"
"time"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
"git.novox.be/novox/mesh-sdk/go/delivery"
sdkstep "git.novox.be/novox/mesh-delivery/sdk/buildstep"
"git.novox.be/novox/mesh-sdk/go/health"
"git.novox.be/novox/mesh-sdk/go/keys"
"git.novox.be/novox/mesh-sdk/go/process"
"git.novox.be/novox/mesh-sdk/go/seatdef"
"git.novox.be/novox/mesh-sdk/go/views"
@@ -26,13 +30,13 @@ func TestTheControlRecords(t *testing.T) {
// manifest declares now has no reader.
t.Run("indexer_config_readers", func(t *testing.T) {
d1, d2 := strings.Repeat("1", 64), strings.Repeat("2", 64)
manifests := map[string]delivery.Manifest{
d1: {Module: "web", JSON: `{"module":"web","config":{"old":{"kind":"value","why":"w"},"port":{"kind":"value","why":"w"}},"own-secrets":{"key":"x"}}`},
d2: {Module: "web", JSON: `{"module":"web","config":{"port":{"kind":"value","why":"w"}}}`},
strings.Repeat("3", 64): {Module: "db", JSON: `{"module":"db","config":{"old":{"kind":"value","why":"w"}}}`},
manifests := map[string]meshcatalog.ManifestsByDigestValue{
d1: {Module: "web", Manifest: `{"module":"web","config":{"old":{"kind":"value","why":"w"},"port":{"kind":"value","why":"w"}},"own-secrets":{"key":"x"}}`},
d2: {Module: "web", Manifest: `{"module":"web","config":{"port":{"kind":"value","why":"w"}}}`},
strings.Repeat("3", 64): {Module: "db", Manifest: `{"module":"db","config":{"old":{"kind":"value","why":"w"}}}`},
}
trunk := map[string]delivery.Module{"web": {Manifest: d2}}
pulls := []delivery.ConfigCandidate{{Pull: "novox/web#7", Declares: map[string]map[string]delivery.ConfigDeclaration{"web": {"new": {}}}}}
trunk := map[string]meshcatalog.ModulesByModuleValue{"web": {Manifest: d2}}
pulls := []meshcatalog.ConfigCandidatesByPullValue{{Pull: "novox/web#7", Declares: map[string]map[string]meshcatalog.ConfigCandidatesByPullValueDeclaresValueValue{"web": {"new": {}}}}}
got := ConfigReaders("web", trunk, manifests, pulls)
want := []ConfigReaderRow{{Module: "web", Key: "key", Kind: ConfigKindSecret, Readers: []string{}},
{Module: "web", Key: "new", Kind: ConfigKindValue, Readers: []string{"novox/web#7"}},
@@ -56,15 +60,15 @@ func TestTheControlRecords(t *testing.T) {
s[key] = &views.Entry{Value: raw, Revision: rev}
}
modules, verdicts, manifests, marks := memSource{}, memSource{}, memSource{}, src
putVerdict := func(digest string, rev uint64, v delivery.ManifestVerdict) {
raw, err := delivery.EncodeManifestVerdict(v)
putVerdict := func(digest string, rev uint64, v meshcatalog.VerdictsByDigestValue) {
raw, err := meshcatalogclient.EncodeVerdict(v)
if err != nil {
t.Fatal(err)
}
verdicts[digest] = &views.Entry{Value: raw, Revision: rev}
}
for name, digest := range map[string]string{"cache": dAsk, "plain": dNone, "bad": dRefused, "bare": ""} {
raw, err := delivery.EncodeModule(delivery.Module{FP2: "src2:" + strings.Repeat("a", 64), Manifest: digest,
raw, err := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: "src2:" + strings.Repeat("a", 64), Manifest: digest,
Repository: "novox/mesh-catalog", Commit: "c"})
if err != nil {
t.Fatal(err)
@@ -73,11 +77,11 @@ func TestTheControlRecords(t *testing.T) {
}
asks := `{"module":"cache","version":"1","requires":[{"name":"c","storage":"key-value","template":"state"}]}`
for digest, raw := range map[string]string{dAsk: asks, dNone: `{"module":"plain","version":"1"}`, dRefused: asks} {
put(manifests, digest, 1, delivery.Manifest{V: delivery.Version, Module: "m", JSON: raw})
put(manifests, digest, 1, meshcatalog.ManifestsByDigestValue{V: keys.Version, Module: "m", Manifest: raw})
}
putVerdict(dAsk, 7, delivery.ManifestVerdict{Module: "cache", State: delivery.VerdictAccepted, Judge: "j"})
putVerdict(dNone, 8, delivery.ManifestVerdict{Module: "plain", State: delivery.VerdictAccepted, Judge: "j"})
putVerdict(dRefused, 9, delivery.ManifestVerdict{Module: "bad", State: delivery.VerdictRefused, Judge: "j", Problems: []string{"p"}})
putVerdict(dAsk, 7, meshcatalog.VerdictsByDigestValue{Module: "cache", State: meshcatalogclient.VerdictAccepted, Judge: "j"})
putVerdict(dNone, 8, meshcatalog.VerdictsByDigestValue{Module: "plain", State: meshcatalogclient.VerdictAccepted, Judge: "j"})
putVerdict(dRefused, 9, meshcatalog.VerdictsByDigestValue{Module: "bad", State: meshcatalogclient.VerdictRefused, Judge: "j", Problems: []string{"p"}})
put(marks, VerdictsState, 1, views.Mark{V: views.Version, State: VerdictsState, Revision: 9})
got, err := BusStateOf(modules, verdicts, manifests, marks)
if err != nil || got.Revision != 9 || len(got.Modules) != 1 || got.Modules[0].Module != "cache" ||
@@ -157,7 +161,7 @@ func TestTheControlRecords(t *testing.T) {
merged := func(merge string) {
t.Helper()
if err := ix.ApplySeatDefinitions(ctx, forgemsg.Forge{Kind: forgemsg.ForgeMerged, Repository: repo, Number: 1, Merge: merge,
Manifests: map[string]string{delivery.RootPath: manifest, "modules/other": `{"module":"other","version":"1"}`}}); err != nil {
Manifests: map[string]string{meshcatalogclient.RootPath: manifest, "modules/other": `{"module":"other","version":"1"}`}}); err != nil {
t.Fatal(err)
}
}
@@ -213,7 +217,7 @@ func TestTheControlRecords(t *testing.T) {
mergedOther := func(merge string) {
t.Helper()
if err := ix.ApplySeatDefinitions(ctx, forgemsg.Forge{Kind: forgemsg.ForgeMerged, Repository: other, Number: 2, Merge: merge,
Manifests: map[string]string{delivery.RootPath: otherManifest}}); err != nil {
Manifests: map[string]string{meshcatalogclient.RootPath: otherManifest}}); err != nil {
t.Fatal(err)
}
}
@@ -235,7 +239,7 @@ func TestTheControlRecords(t *testing.T) {
}
// A pull request's head: its own entry, the trunk's answered unless it is named.
if err := ix.ApplySeatDefinitions(ctx, forgemsg.Forge{Kind: forgemsg.ForgePull, Action: forgemsg.PullUpdated, Repository: repo, Number: 9, Head: head,
Dirs: []string{delivery.RootPath}, Manifests: map[string]string{delivery.RootPath: manifest}}); err != nil {
Dirs: []string{meshcatalogclient.RootPath}, Manifests: map[string]string{meshcatalogclient.RootPath: manifest}}); err != nil {
t.Fatal(err)
}
pk := SeatDefinitionKey("git", SourcePull, repo, 9)
@@ -257,7 +261,7 @@ func TestTheControlRecords(t *testing.T) {
t.Fatal(err)
}
end := func(outcome string, files ...string) []byte {
raw, _ := json.Marshal(map[string]any{"v": 1, "event": delivery.BuildEndedEvent, "module": "mesh-contracts", "ref": "main",
raw, _ := json.Marshal(map[string]any{"v": 1, "event": builds.BuildEndedEvent, "module": "mesh-contracts", "ref": "main",
"commit": m3, "outcome": outcome, "out-of-step": files})
return raw
}
@@ -272,7 +276,7 @@ func TestTheControlRecords(t *testing.T) {
rows[1].Pull != repo+"#9" || !reflect.DeepEqual(rows[1].Files, []string{"go/a.go", "go/b.go"}) {
t.Errorf("out of step %+v, %v; want the trunk's module and the pull request's sorted files", rows, err)
}
ix.ApplyOutOfStepBuild(ctx, end(delivery.BuildDone))
ix.ApplyOutOfStepBuild(ctx, end(builds.BuildDone))
if len(findings.vals) != 0 {
t.Errorf("findings %v stand after a clean build end", findings.vals)
}
@@ -280,10 +284,10 @@ func TestTheControlRecords(t *testing.T) {
// generated-from: a trunk that names none, a head that names it, is read as the contracts module the head is.
ix.Modules, ix.Manifests = newMem(), newMem()
plain := `{"module":"mesh-contracts","version":"1"}`
kept, _ := delivery.EncodeManifest(delivery.Manifest{Module: "mesh-contracts", JSON: plain})
mkey, _ := delivery.ManifestsKey(delivery.ManifestDigest([]byte(plain)))
kept, _ := meshcatalogclient.EncodeManifest(meshcatalog.ManifestsByDigestValue{Module: "mesh-contracts", Manifest: plain})
mkey, _ := meshcatalogclient.ManifestsKey(meshcatalogclient.ManifestDigest([]byte(plain)))
ix.Manifests.(*memStore).vals[mkey] = kept
mod, _ := delivery.EncodeModule(delivery.Module{FP2: "src2:" + strings.Repeat("a", 64), Manifest: delivery.ManifestDigest([]byte(plain)), Repository: repo, Commit: m3, Path: delivery.RootPath})
mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: "src2:" + strings.Repeat("a", 64), Manifest: meshcatalogclient.ManifestDigest([]byte(plain)), Repository: repo, Commit: m3, Path: meshcatalogclient.RootPath})
ix.Modules.(*memStore).vals["mesh-contracts"] = mod
files["h2"] = map[string]string{"module.json": manifest, "seats/git/seat.json": strings.Replace(git, "ADR 0111", "ADR 0400", 1)}
if r := ix.SeatDefinitionsAgain(ctx, "mesh-contracts", 9, "h2"); !r.Stored {
@@ -335,7 +339,7 @@ func TestTheControlRecords(t *testing.T) {
files["p1"] = map[string]string{"seats/wants/seat.json": seat("wants", "Placement")}
files["i1"] = map[string]string{"seats/vault/seat.json": seat("vault", "Identity and access")}
domainManifest := func(module string) map[string]string {
return map[string]string{delivery.RootPath: strings.Replace(manifest, `"mesh-contracts"`, `"`+module+`"`, 1)}
return map[string]string{meshcatalogclient.RootPath: strings.Replace(manifest, `"mesh-contracts"`, `"`+module+`"`, 1)}
}
mergeAt := func(repository, merge string, manifests map[string]string) {
t.Helper()
@@ -348,7 +352,7 @@ func TestTheControlRecords(t *testing.T) {
before = len(said)
// mesh-contracts defines wants and vault, seats of other domains: refused with a finding naming both modules, the
// held entries stand, and lone (Core) is taken.
mergeAt(repo, "c1", map[string]string{delivery.RootPath: manifest})
mergeAt(repo, "c1", map[string]string{meshcatalogclient.RootPath: manifest})
if v, ok := findings.vals[outside]; !ok || !strings.Contains(string(v), "mesh-placement-contracts") || !strings.Contains(string(v), "mesh-identity-and-access-contracts") ||
!strings.Contains(string(v), "mesh-contracts defines it") {
t.Errorf("finding %s = %s, want it naming both modules", outside, v)
@@ -365,7 +369,7 @@ func TestTheControlRecords(t *testing.T) {
if w, _ := entry("wants.trunk"); w.Module != "mesh-placement-contracts" || w.Commit != "p1" {
t.Errorf("the domain's module did not take wants over: %+v", w)
}
mergeAt(repo, "c2", map[string]string{delivery.RootPath: manifest})
mergeAt(repo, "c2", map[string]string{meshcatalogclient.RootPath: manifest})
if w, _ := entry("wants.trunk"); w.Module != "mesh-placement-contracts" {
t.Errorf("mesh-contracts dropping wants removed or replaced the domain module's entry: %+v", w)
}
@@ -390,7 +394,7 @@ func TestTheControlRecords(t *testing.T) {
files["h2"] = map[string]string{"seats/wants/seat.json": seat("wants", "Placement")}
ix.ReadFile = func(_ context.Context, _, sha, p string) ([]byte, error) { return []byte(files[sha][p]), nil }
if err := ix.ApplySeatDefinitions(ctx, forgemsg.Forge{Kind: forgemsg.ForgePull, Action: forgemsg.PullUpdated, Repository: repo, Number: 11, Head: "h2",
Dirs: []string{delivery.RootPath}, Manifests: map[string]string{delivery.RootPath: manifest}}); err != nil {
Dirs: []string{meshcatalogclient.RootPath}, Manifests: map[string]string{meshcatalogclient.RootPath: manifest}}); err != nil {
t.Fatal(err)
}
if e, _ := entry(SeatDefinitionKey("wants", SourcePull, repo, 11)); !strings.Contains(e.Unparsed, "mesh-placement-contracts") || e.Definition != nil {
@@ -423,7 +427,7 @@ func TestTheControlRecords(t *testing.T) {
sdk := `{"module":"mesh-sdk","version":"1","build":{"artifacts":[{"name":"go","kind":"package","language":"go","paths":["go"]},` +
`{"name":"ts","kind":"package","language":"typescript","paths":["src","package.json"],"series":"0.3"}]}}`
if err := ix.KeepPackageNames(ctx, forgemsg.Forge{Kind: forgemsg.ForgeMerged, Repository: "novox/mesh-sdk", Merge: "m1",
Manifests: map[string]string{delivery.RootPath: sdk}}); err != nil {
Manifests: map[string]string{meshcatalogclient.RootPath: sdk}}); err != nil {
t.Fatal(err)
}
named := ix.SeriesAll(ctx)
@@ -487,13 +491,13 @@ func TestTheControlRecords(t *testing.T) {
"none": {"novox/mesh-catalog", `{"module":"none","version":"1"}`, nil, nil, ""},
} {
digest := fmt.Sprintf("%064x", len(name)*16+int(name[0]))
putJSON(manifests, digest, delivery.Manifest{V: delivery.Version, Module: name, JSON: u.manifest})
mod := delivery.Module{FP2: "src2:" + strings.Repeat("a", 64), Manifest: digest, Repository: u.repo, Commit: "c",
Inputs2: &delivery.KeptInputs2{Packages: u.pkgs, Imports: u.imports}}
putJSON(manifests, digest, meshcatalog.ManifestsByDigestValue{V: keys.Version, Module: name, Manifest: u.manifest})
mod := meshcatalog.ModulesByModuleValue{Fp2: "src2:" + strings.Repeat("a", 64), Manifest: digest, Repository: u.repo, Commit: "c",
Inputs2: &meshcatalog.ModulesByModuleValueInputs2{Packages: u.pkgs, Imports: u.imports}}
if u.pinned != "" {
mod.Packages = delivery.PackageVersions{sdkGo: u.pinned}
mod.Packages = sdkstep.PackageVersions{sdkGo: u.pinned}
}
raw, err := delivery.EncodeModule(mod)
raw, err := meshcatalogclient.EncodeModule(mod)
if err != nil {
t.Fatal(err)
}
@@ -528,8 +532,8 @@ func TestTheControlRecords(t *testing.T) {
if err := ix.Apply(ctx, f); err != nil {
t.Fatal(err)
}
var kept delivery.PredefinedKept
if err := json.Unmarshal(pre.vals[delivery.PredefinedListKey], &kept); err != nil || kept.Commit != merge || string(kept.List) != string(f.Predefined) {
var kept meshcatalog.PredefinedByListValue
if err := json.Unmarshal(pre.vals[PredefinedListKey], &kept); err != nil || kept.Commit != merge || string(rawList(pre.vals[PredefinedListKey])) != string(f.Predefined) {
t.Fatalf("kept = %+v (%v)", kept, err)
}
writes := pre.writes
@@ -561,9 +565,9 @@ func TestTheControlRecords(t *testing.T) {
if err := ix.Apply(ctx, p); err != nil {
t.Fatal(err)
}
var c delivery.ConfigCandidate
var c meshcatalog.ConfigCandidatesByPullValue
if err := json.Unmarshal(cands.vals[key], &c); err != nil || c.Pull != "novox/mesh-catalog#7" || c.Head != h1 || len(c.Declares["avahi"]) != 1 ||
string(c.Declares["avahi"]["name"].Default) != `"x"` {
c.Declares["avahi"]["name"].Default != "x" {
t.Fatalf("candidate = %+v (%v)", c, err)
}
writes := cands.writes
@@ -574,7 +578,7 @@ func TestTheControlRecords(t *testing.T) {
if err := ix.Apply(ctx, same); err != nil {
t.Fatal(err)
}
c = delivery.ConfigCandidate{}
c = meshcatalog.ConfigCandidatesByPullValue{}
if err := json.Unmarshal(cands.vals[key], &c); err != nil || len(c.Declares) != 0 || c.Declares == nil {
t.Errorf("a head declaring nothing new = %+v, want an empty set", c)
}
modules/mesh-delivery-indexer/internal/index/events.go
+50
View File
@@ -0,0 +1,50 @@
package index
import (
"encoding/json"
"strconv"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/keys"
)
// The bodies of the seat's events manifest-accepted and manifest-refused are the contract's types; the indexer, who says
// them, makes them from a verdict (novox/hq ADR 0634 §1: the logic of one holder stays with it).
// refusedOf is the body of manifest-refused for the verdict v on the manifest of digest.
func refusedOf(digest string, v meshcatalog.VerdictsByDigestValue) meshcatalog.ManifestRefusedEvent {
return meshcatalog.ManifestRefusedEvent{V: keys.Version, Module: v.Module, Digest: digest, Judge: v.Judge, Problems: v.Problems}
}
// acceptedEvent is the body of manifest-accepted for the verdict v accepting the manifest m of digest, its bus-state
// entries whole (novox/hq ADR 0595 §4); an empty list when m asks for none.
func acceptedEvent(digest string, v meshcatalog.VerdictsByDigestValue, m catalogue.Manifest) meshcatalog.ManifestAcceptedEvent {
return meshcatalog.ManifestAcceptedEvent{V: keys.Version, Module: v.Module, Digest: digest, Judge: v.Judge, BusState: busStateOf(m.BusState)}
}
// busStateOf is the manifest's bus-state entries in the event's form (the same JSON).
func busStateOf(entries []catalogue.BusStateEntry) []meshcatalog.ManifestAcceptedEventBusStateItem {
out := []meshcatalog.ManifestAcceptedEventBusStateItem{}
if len(entries) == 0 {
return out
}
raw, err := json.Marshal(entries)
if err == nil {
err = json.Unmarshal(raw, &out)
}
if err != nil {
return []meshcatalog.ManifestAcceptedEventBusStateItem{}
}
return out
}
// acceptedEventID is the event's id, one per manifest digest and judge, so the bus takes it once:
// `manifest-accepted-<digest>-<judge>`. The empty one of a module leaving the trunk carries no digest, so it is named by
// its module and the indexer's revision: `manifest-accepted-left-<module>-<revision>`, so two leaves are both kept.
func acceptedEventID(a meshcatalog.ManifestAcceptedEvent) string {
if a.Digest == "" {
return meshcatalog.EventManifestAccepted + "-left-" + a.Module + "-" + strconv.FormatInt(a.Revision, 10)
}
return meshcatalog.EventManifestAccepted + "-" + a.Digest + "-" + a.Judge
}
modules/mesh-delivery-indexer/internal/index/form.go
+9 -7
View File
@@ -4,7 +4,9 @@ import (
"slices"
"strings"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
sdkstep "git.novox.be/novox/mesh-delivery/sdk/buildstep"
"mesh-delivery-indexer/internal/fingerprint"
)
@@ -13,7 +15,7 @@ import (
// its build is pinned to.
type form struct {
fp, none string
versions delivery.PackageVersions
versions sdkstep.PackageVersions
}
// formOf is the module's fingerprint from its kept inputs (bases already its bases' fingerprints) and toolchains, in the
@@ -35,7 +37,7 @@ type form struct {
// digests of that version's commit.
//
// It sets kept.ImportForm when it writes the import form.
func formOf(module, repository string, kept *delivery.KeptInputs2, toolchains map[string]delivery.Toolchain2, prev delivery.Module, held map[string]delivery.Module, published map[string]delivery.PublishedPackages, pins delivery.PackageVersions, logf func(string, ...any)) form {
func formOf(module, repository string, kept *meshcatalog.ModulesByModuleValueInputs2, toolchains map[string]meshcatalogclient.Toolchain2, prev meshcatalog.ModulesByModuleValue, held map[string]meshcatalog.ModulesByModuleValue, published map[string]meshcatalog.PackagesByModuleValue, pins sdkstep.PackageVersions, logf func(string, ...any)) form {
in := inputsFrom(module, *kept, toolchains)
if len(kept.Imports) > 0 {
r := fingerprint.Resolve(repository, kept.Imports, published)
@@ -45,8 +47,8 @@ func formOf(module, repository string, kept *delivery.KeptInputs2, toolchains ma
switch {
case fp == "":
return form{none: none}
case kept.ImportForm == fp && prev.FP2 != "":
return form{fp: prev.FP2, versions: prev.Packages} // the build that reported the imports stands
case kept.ImportForm == fp && prev.Fp2 != "":
return form{fp: prev.Fp2, versions: prev.Packages} // the build that reported the imports stands
}
kept.ImportForm = fp
return form{fp: fp, versions: r.Versions}
@@ -60,7 +62,7 @@ func formOf(module, repository string, kept *delivery.KeptInputs2, toolchains ma
for p, v := range pins { // an accepted series' newest published version (acceptedSeries, DLV-I6)
if slices.Contains(others, p) {
if versions == nil {
versions = delivery.PackageVersions{}
versions = sdkstep.PackageVersions{}
}
versions[p] = v
unpublished = slices.DeleteFunc(unpublished, func(u string) bool { return u == p })
@@ -79,7 +81,7 @@ func formOf(module, repository string, kept *delivery.KeptInputs2, toolchains ma
// inputsFrom is the fingerprint's inputs of module m from its kept inputs and the given toolchains, no package of
// either form yet.
func inputsFrom(module string, k delivery.KeptInputs2, toolchains map[string]delivery.Toolchain2) fingerprint.Inputs {
func inputsFrom(module string, k meshcatalog.ModulesByModuleValueInputs2, toolchains map[string]meshcatalogclient.Toolchain2) fingerprint.Inputs {
return fingerprint.Inputs{Module: module, Tree: k.Tree, Bases: k.Bases, Builder: k.Builder, Contexts: k.Contexts,
Locks: k.Locks, Toolchains: toolchains}
}
modules/mesh-delivery-indexer/internal/index/graph.go
+5 -4
View File
@@ -7,8 +7,9 @@ import (
"maps"
"slices"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
sdkstep "git.novox.be/novox/mesh-delivery/sdk/buildstep"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/process"
"git.novox.be/novox/mesh-delivery/sdk/graph"
@@ -28,11 +29,11 @@ const GraphMost = 2000
// graphEdges is what a recompute knows of one module for its graph node.
type graphEdges struct {
module string
cur delivery.Module
cur meshcatalog.ModulesByModuleValue
manifest catalogue.Manifest
fp, none string
tcs map[string]delivery.ToolchainUsed
pkgs delivery.PackageVersions
tcs map[string]meshcatalog.ToolchainUsed
pkgs sdkstep.PackageVersions
}
// graphNode is the graph node of e: its edges, the awaited provisions judged against providers (catalogue.Awaited, as
modules/mesh-delivery-indexer/internal/index/heard.go
+1 -3
View File
@@ -4,8 +4,6 @@ import (
"context"
"encoding/json"
"time"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// What the indexer heard of the forge bridge's stream FORGE (issue 1126 note 76): no tool read FORGE's messages, so a merge
@@ -15,7 +13,7 @@ import (
// the last 64 values (its history); mesh-delivery-view's indexer_heard reads them, newest first, with their times.
const (
HeardState = "heard"
HeardBucket = delivery.IndexerModule + "_" + HeardState
HeardBucket = Module + "_" + HeardState
HeardKey = "forge"
// What the indexer did with a message.
HeardApplied = "applied"
modules/mesh-delivery-indexer/internal/index/helpers_test.go
+12
View File
@@ -0,0 +1,12 @@
package index
import "encoding/json"
// rawList is the list a kept predefined value holds, as it was written.
func rawList(value []byte) json.RawMessage {
var kept struct {
List json.RawMessage `json:"list"`
}
_ = json.Unmarshal(value, &kept)
return kept.List
}
modules/mesh-delivery-indexer/internal/index/holds.go
+17 -15
View File
@@ -12,8 +12,10 @@ import (
"strings"
"time"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/sdk/trunk"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/process"
)
@@ -33,12 +35,12 @@ import (
const (
// HoldsState is the indexer's state of held entries: the bucket mesh-delivery-indexer_holds, keyed as the modules state.
HoldsState = "holds"
HoldsBucket = delivery.IndexerModule + "_" + HoldsState
HoldsBucket = Module + "_" + HoldsState
// HoldsTool reads the holds, asks one again now, or moves one by a person's word.
HoldsTool = "indexer_holds"
// EngineReadsState is the indexer's state of what each counted node's engine states it reads (sdk/enginereads).
EngineReadsState = "engine-reads"
EngineReadsBucket = delivery.IndexerModule + "_" + EngineReadsState
EngineReadsBucket = Module + "_" + EngineReadsState
// EngineReadsTool lists, asks again and drops those statements.
EngineReadsTool = "indexer_engine_reads"
)
@@ -58,7 +60,7 @@ const (
HoldStandingKind = "manifest-held"
HoldBound = 30 * time.Minute
HoldBoundsState = "hold-bounds"
HoldBoundsBucket = delivery.IndexerModule + "_" + HoldBoundsState
HoldBoundsBucket = Module + "_" + HoldBoundsState
)
// Bounds is the state hold-bounds: Arm writes key to expire at ends, a standing key keeping its end; Disarm deletes it.
@@ -161,7 +163,7 @@ func lackingWords(lacking map[string][]string, unknown string) string {
// a changed manifest of a module the modules state holds an entry of: a new module has no previous entry to keep, and is
// held on each node by the assigner (ADR 0614 §4). An error is a hold that could not be kept: the merge is left, and
// delivered again, rather than writing what a counted engine cannot read.
func (ix *Indexer) holdBack(ctx context.Context, key string, next, cur delivery.Module, has bool) (bool, error) {
func (ix *Indexer) holdBack(ctx context.Context, key string, next, cur meshcatalog.ModulesByModuleValue, has bool) (bool, error) {
if ix.Holds == nil || ix.Counted == nil {
return false, nil
}
@@ -176,7 +178,7 @@ func (ix *Indexer) holdBack(ctx context.Context, key string, next, cur delivery.
if len(lacking) == 0 && unknown == "" {
return false, ix.dropHold(ctx, key, "its merge "+next.Commit+" brought a manifest every counted node-engine reads")
}
raw, err := delivery.EncodeModule(next)
raw, err := meshcatalogclient.EncodeModule(next)
if err != nil {
return false, err
}
@@ -297,7 +299,7 @@ func (ix *Indexer) moveHold(ctx context.Context, key, by string) (bool, string,
ix.logf("holds.%s holds what is not a hold, left: %v", key, err)
return moved, "", fmt.Errorf("holds.%s: %w", key, err)
}
next, err := delivery.DecodeModule(h.Next)
next, err := meshcatalogclient.DecodeModule(h.Next)
if err != nil {
return moved, "", fmt.Errorf("holds.%s: its entry: %w", key, err)
}
@@ -334,8 +336,8 @@ func (ix *Indexer) moveHold(ctx context.Context, key, by string) (bool, string,
ix.recompute.Unlock()
return moved, "", fmt.Errorf("modules.%s: %w", key, err)
}
cur, derr := delivery.DecodeModule(curRaw)
if !has || derr != nil || !delivery.SameRepository(cur.Repository, next.Repository) {
cur, derr := meshcatalogclient.DecodeModule(curRaw)
if !has || derr != nil || !trunk.SameRepository(cur.Repository, next.Repository) {
err := ix.Holds.Delete(ctx, key, rev)
ix.recompute.Unlock()
if errors.Is(err, ErrConflict) {
@@ -347,9 +349,9 @@ func (ix *Indexer) moveHold(ctx context.Context, key, by string) (bool, string,
ix.logf("holds.%s dropped: the modules state no longer holds the module of %s", key, next.Repository)
return moved, "dropped: the modules state no longer holds the module of " + next.Repository, err
}
next.Src = ix.srcOf(next.Repository, next.Commit)
next.Src = int64(ix.srcOf(next.Repository, next.Commit))
w := &mergeWrite{repository: next.Repository, merge: next.Commit}
err = ix.recomputeLocked(ctx, map[string]delivery.Module{key: next}, w)
err = ix.recomputeLocked(ctx, map[string]meshcatalog.ModulesByModuleValue{key: next}, w)
if err == nil {
err = ix.Holds.Delete(ctx, key, rev)
}
@@ -382,8 +384,8 @@ func (ix *Indexer) moveHold(ctx context.Context, key, by string) (bool, string,
}
// sameHeld says a held entry is next but for where it came from: its first merge stands.
func sameHeld(held json.RawMessage, next delivery.Module) bool {
prev, err := delivery.DecodeModule(held)
func sameHeld(held json.RawMessage, next meshcatalog.ModulesByModuleValue) bool {
prev, err := meshcatalogclient.DecodeModule(held)
return err == nil && sameButWhere(prev, next)
}
@@ -453,7 +455,7 @@ func cmpOr(a, b string) string {
// lack what it needs once more and weighs it now; by moves it by that person's word, whatever the engines state.
func (ix *Indexer) HoldAgain(ctx context.Context, module string, release bool, by string) TakenIn {
r := TakenIn{Module: module}
key, err := delivery.ModulesKey(module)
key, err := meshcatalogclient.ModulesKey(module)
if err != nil {
r.Answer = err.Error()
return r
@@ -504,7 +506,7 @@ func (ix *Indexer) HoldAgain(ctx context.Context, module string, release bool, b
// ListHolds is every hold the state holds, by module, or one module's; a value that does not read is named with why.
func ListHolds(keys []string, get func(key string) ([]byte, error), module string) (map[string]any, error) {
if module != "" {
key, err := delivery.ModulesKey(module)
key, err := meshcatalogclient.ModulesKey(module)
if err != nil {
return nil, err
}
modules/mesh-delivery-indexer/internal/index/holds_test.go
+6 -6
View File
@@ -7,8 +7,8 @@ import (
"testing"
"time"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/health"
)
@@ -58,7 +58,7 @@ func storeHoldersNewManifestWaitsForEveryCountedEngine(t *testing.T) {
}
kept, _ := moduleOf(t, mods, "distribution")
keptDocker, _ := moduleOf(t, mods, "docker")
if kept.Manifest != delivery.ManifestDigest([]byte(old)) || len(holds.vals) != 0 {
if kept.Manifest != meshcatalogclient.ManifestDigest([]byte(old)) || len(holds.vals) != 0 {
t.Fatalf("after the first merge modules.distribution names %s with %d holds, want the old manifest and none", kept.Manifest, len(holds.vals))
}
second := withManifests(mergedMsg(catalogueRepo, map[string]string{dir: "tree2", dockerDir: "d2"}), map[string]string{dir: next, dockerDir: docker}, nil)
@@ -70,12 +70,12 @@ func storeHoldersNewManifestWaitsForEveryCountedEngine(t *testing.T) {
stays := func(when string) {
t.Helper()
m, _ := moduleOf(t, mods, "distribution")
if m.Manifest != kept.Manifest || m.FP2 != kept.FP2 || m.Commit != kept.Commit || m.Inputs2 == nil || m.Inputs2.Tree != "tree1" {
if m.Manifest != kept.Manifest || m.Fp2 != kept.Fp2 || m.Commit != kept.Commit || m.Inputs2 == nil || m.Inputs2.Tree != "tree1" {
t.Fatalf("%s modules.distribution = manifest %s fp2 %q commit %s, want the old entry kept whole (%s, %q, %s)",
when, m.Manifest, m.FP2, m.Commit, kept.Manifest, kept.FP2, kept.Commit)
when, m.Manifest, m.Fp2, m.Commit, kept.Manifest, kept.Fp2, kept.Commit)
}
h, err := DecodeHold(holds.vals["distribution"])
if err != nil || h.Manifest != delivery.ManifestDigest([]byte(next)) || h.Kept != kept.Manifest || !slices.Equal(h.Lacking["n2"], []string{"form:seat-reach"}) || len(h.Lacking) != 1 {
if err != nil || h.Manifest != meshcatalogclient.ManifestDigest([]byte(next)) || h.Kept != kept.Manifest || !slices.Equal(h.Lacking["n2"], []string{"form:seat-reach"}) || len(h.Lacking) != 1 {
t.Fatalf("%s holds.distribution = %+v (%v), want the new manifest held, n2 alone lacking form:seat-reach", when, h, err)
}
}
@@ -108,7 +108,7 @@ func storeHoldersNewManifestWaitsForEveryCountedEngine(t *testing.T) {
t.Fatal(err)
}
m, _ := moduleOf(t, mods, "distribution")
if m.Manifest != delivery.ManifestDigest([]byte(next)) || m.Commit != second.Merge || m.Inputs2 == nil || m.Inputs2.Tree != "tree2" || len(holds.vals) != 0 || findings.vals[heldKey] != nil || len(bounds) != 0 {
if m.Manifest != meshcatalogclient.ManifestDigest([]byte(next)) || m.Commit != second.Merge || m.Inputs2 == nil || m.Inputs2.Tree != "tree2" || len(holds.vals) != 0 || findings.vals[heldKey] != nil || len(bounds) != 0 {
t.Fatalf("once every counted engine states form:seat-reach modules.distribution = manifest %s commit %s, %d holds, finding %s, bounds %v; want the new entry and none",
m.Manifest, m.Commit, len(holds.vals), findings.vals[heldKey], bounds)
}
modules/mesh-delivery-indexer/internal/index/imports.go
+20 -18
View File
@@ -7,13 +7,15 @@ import (
"slices"
"strings"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-delivery/sdk/trunk"
"mesh-delivery-indexer/internal/fingerprint"
)
// ApplyBuildEnd keeps the imports a done build reports at its end (delivery.BuildLogEntry.Imports; novox/hq ADR 0593
// ApplyBuildEnd keeps the imports a done build reports at its end (builds.BuildLogEntry.Imports; novox/hq ADR 0593
// §2) on the module it built, when the build's fingerprint is the one the module holds: the import set of the module's
// last trunk build. With them it keeps ImportForm, the import-form fingerprint of exactly what the build was built from:
// the module's kept inputs and the toolchains its fingerprint was computed with, and each imported package's digest at
@@ -28,14 +30,14 @@ func (ix *Indexer) ApplyBuildEnd(ctx context.Context, value []byte) error {
if !builds.IsBuildEntry(value) {
return nil
}
e, err := delivery.DecodeBuildLogEntry(value)
e, err := builds.DecodeBuildLogEntry(value)
if err != nil {
return err
}
if e.Event != delivery.BuildEndedEvent || e.Outcome != delivery.BuildDone || len(e.Imports) == 0 || ix.Modules == nil || e.FP == "" {
if e.Event != builds.BuildEndedEvent || e.Outcome != builds.BuildDone || len(e.Imports) == 0 || ix.Modules == nil || e.FP == "" {
return nil
}
key, err := delivery.ModulesKey(e.Module)
key, err := meshcatalogclient.ModulesKey(e.Module)
if err != nil {
return nil
}
@@ -45,8 +47,8 @@ func (ix *Indexer) ApplyBuildEnd(ctx context.Context, value []byte) error {
if err != nil || !found {
return err
}
cur, err := delivery.DecodeModule(raw)
if err != nil || cur.FP2 != e.FP || cur.Inputs2 == nil || cur.Manifest == "" {
cur, err := meshcatalogclient.DecodeModule(raw)
if err != nil || cur.Fp2 != e.FP || cur.Inputs2 == nil || cur.Manifest == "" {
return nil // a build of a fingerprint the module no longer holds says nothing of its inputs now
}
if slices.Equal(cur.Inputs2.Imports, e.Imports) && cur.Inputs2.ImportForm != "" {
@@ -56,14 +58,14 @@ func (ix *Indexer) ApplyBuildEnd(ctx context.Context, value []byte) error {
if err != nil {
return fmt.Errorf("modules.%s: its manifest: %w", key, err)
}
toolchains := map[string]delivery.Toolchain2{}
toolchains := map[string]meshcatalogclient.Toolchain2{}
for artifact, language := range languagesOf(m) {
used, ok := cur.Toolchains[language]
if !ok {
ix.logf("%s: its fingerprint names no %s toolchain, so the imports of its build are not kept", e.Module, language)
return nil
}
toolchains[artifact] = delivery.Toolchain2{Language: language, Version: used.Version, Recipe: used.Recipe}
toolchains[artifact] = meshcatalogclient.Toolchain2{Language: language, Version: used.Version, Recipe: used.Recipe}
}
kept := *cur.Inputs2
kept.Imports, kept.ImportForm = slices.Clone(e.Imports), ""
@@ -82,14 +84,14 @@ func (ix *Indexer) ApplyBuildEnd(ctx context.Context, value []byte) error {
if !found {
return nil, false, nil
}
now, err := delivery.DecodeModule(value)
if err != nil || now.FP2 != e.FP || now.Manifest != cur.Manifest || now.Inputs2 == nil {
now, err := meshcatalogclient.DecodeModule(value)
if err != nil || now.Fp2 != e.FP || now.Manifest != cur.Manifest || now.Inputs2 == nil {
return nil, false, nil
}
in := *now.Inputs2
in.Imports, in.ImportForm = kept.Imports, kept.ImportForm
now.Inputs2 = &in
data, err := delivery.EncodeModule(now)
data, err := meshcatalogclient.EncodeModule(now)
return data, false, err
})
}
@@ -98,7 +100,7 @@ func (ix *Indexer) ApplyBuildEnd(ctx context.Context, value []byte) error {
// it: a module of the built module's own repository at the commit built, one of another repository at the version the
// module's fingerprint pins (cur.Packages), else its latest published version. A module whose published commit is the
// one built is taken from the packages state; any other is read from the forge's listing at that commit.
func (ix *Indexer) publishedAsBuilt(ctx context.Context, cur delivery.Module, e delivery.BuildLogEntry) (map[string]delivery.PublishedPackages, error) {
func (ix *Indexer) publishedAsBuilt(ctx context.Context, cur meshcatalog.ModulesByModuleValue, e builds.BuildLogEntry) (map[string]meshcatalog.PackagesByModuleValue, error) {
published := ix.publishedAll(ctx)
if r := fingerprint.Resolve(cur.Repository, e.Imports, published); len(r.Missing) > 0 {
if err := ix.FillPackages(ctx); err != nil {
@@ -107,27 +109,27 @@ func (ix *Indexer) publishedAsBuilt(ctx context.Context, cur delivery.Module, e
published = ix.publishedAll(ctx)
}
// A pin that is an accepted series' version (DLV-I6) is taken by the contents its series keeps for it.
byVersion := map[string]delivery.PublishedPackages{}
byVersion := map[string]meshcatalog.PackagesByModuleValue{}
for _, s := range ix.SeriesAll(ctx) {
if s.GoModule != "" && s.Newest != nil && s.Newest.Contents != nil {
byVersion[s.GoModule+"@v"+s.Newest.Version] = *s.Newest.Contents
byVersion[s.GoModule+"@v"+s.Newest.Version] = publishedOfContents(*s.Newest.Contents)
}
}
type at struct{ repository, commit string }
reread := map[at][]string{}
var asVersion map[string]delivery.PublishedPackages
var asVersion map[string]meshcatalog.PackagesByModuleValue
for _, imp := range e.Imports {
r := fingerprint.Resolve(cur.Repository, []string{imp}, published)
for mod := range r.Requires {
p := published[mod]
commit := p.Commit
switch pin := cur.Packages[mod]; {
case delivery.SameRepository(p.Repository, cur.Repository) && e.Commit != "":
case trunk.SameRepository(p.Repository, cur.Repository) && e.Commit != "":
commit = e.Commit
case strings.HasPrefix(pin, "v"):
if c, ok := byVersion[mod+"@"+pin]; ok {
if asVersion == nil {
asVersion = map[string]delivery.PublishedPackages{}
asVersion = map[string]meshcatalog.PackagesByModuleValue{}
}
c.Commit, asVersion[mod] = pin, c
} else {
modules/mesh-delivery-indexer/internal/index/imports_test.go
+30 -28
View File
@@ -6,7 +6,9 @@ import (
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
)
const meshSDK = "git.novox.be/novox/mesh-sdk/go"
@@ -57,7 +59,7 @@ func TestImportsReportedMoveTheFingerprintOnlyWithAnImportedPackage(t *testing.T
ix.Packages, ix.Files = newMem(), listings.files
// avahi's go.sum names mesh-sdk (the version line), as every Go module's does; mesh-sdk's module is held at c1.
setPackages(t, mods, "avahi", []string{meshSDK})
sdkModule, err := delivery.EncodeModule(delivery.Module{Repository: "novox/mesh-sdk", Path: ".", Commit: c1, None2: "a TypeScript package"})
sdkModule, err := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Repository: "novox/mesh-sdk", Path: ".", Commit: c1, None2: "a TypeScript package"})
if err != nil {
t.Fatal(err)
}
@@ -74,12 +76,12 @@ func TestImportsReportedMoveTheFingerprintOnlyWithAnImportedPackage(t *testing.T
t.Fatal(err)
}
before, _ := moduleOf(t, mods, "avahi")
if before.FP2 == "" {
if before.Fp2 == "" {
t.Fatalf("no fingerprint to start from: %q", before.None2)
}
// The build of that fingerprint ends, reporting its imports: kept, and the fingerprint does not move.
end := buildEnd(t, "avahi", before.FP2, meshSDK, meshSDK+"/announce")
end := buildEnd(t, "avahi", before.Fp2, meshSDK, meshSDK+"/announce")
if err := ix.ApplyBuildEnd(ctx, end); err != nil {
t.Fatal(err)
}
@@ -87,9 +89,9 @@ func TestImportsReportedMoveTheFingerprintOnlyWithAnImportedPackage(t *testing.T
t.Fatal(err)
}
after, _ := moduleOf(t, mods, "avahi")
if after.FP2 != before.FP2 || strings.Join(after.Inputs2.Imports, ",") != meshSDK+","+meshSDK+"/announce" || after.Inputs2.ImportForm == "" {
if after.Fp2 != before.Fp2 || strings.Join(after.Inputs2.Imports, ",") != meshSDK+","+meshSDK+"/announce" || after.Inputs2.ImportForm == "" {
t.Fatalf("after the build's imports: fp2 %q (was %q), imports %v, import form %q; want the same fingerprint and the imports kept",
after.FP2, before.FP2, after.Inputs2.Imports, after.Inputs2.ImportForm)
after.Fp2, before.Fp2, after.Inputs2.Imports, after.Inputs2.ImportForm)
}
// mesh-sdk merges a change of buildstep only: avahi does not import it.
@@ -100,8 +102,8 @@ func TestImportsReportedMoveTheFingerprintOnlyWithAnImportedPackage(t *testing.T
if err := ix.Recompute(ctx); err != nil {
t.Fatal(err)
}
if m, _ := moduleOf(t, mods, "avahi"); m.FP2 != before.FP2 {
t.Fatalf("a mesh-sdk merge changing only buildstep moved avahi's fingerprint to %q", m.FP2)
if m, _ := moduleOf(t, mods, "avahi"); m.Fp2 != before.Fp2 {
t.Fatalf("a mesh-sdk merge changing only buildstep moved avahi's fingerprint to %q", m.Fp2)
}
// mesh-sdk merges a change of announce: avahi moves, to the import form, pinned to the new version.
@@ -113,9 +115,9 @@ func TestImportsReportedMoveTheFingerprintOnlyWithAnImportedPackage(t *testing.T
t.Fatal(err)
}
moved, _ := moduleOf(t, mods, "avahi")
if moved.FP2 == before.FP2 || moved.FP2 != moved.Inputs2.ImportForm || moved.Packages[meshSDK] != c3 {
if moved.Fp2 == before.Fp2 || moved.Fp2 != moved.Inputs2.ImportForm || moved.Packages[meshSDK] != c3 {
t.Fatalf("a change of an imported package: fp2 %q (was %q), import form %q, versions %v; want the import form at c3",
moved.FP2, before.FP2, moved.Inputs2.ImportForm, moved.Packages)
moved.Fp2, before.Fp2, moved.Inputs2.ImportForm, moved.Packages)
}
}
@@ -158,7 +160,7 @@ func aSiblingDirectoryAloneMovesItsImporter(t *testing.T) {
return out, nil
}
setPackages(t, mods, "avahi", []string{sibling})
fpAt := func(commit string) delivery.Module {
fpAt := func(commit string) meshcatalog.ModulesByModuleValue {
t.Helper()
held, _ := ix.heldModules(ctx)
if err := ix.publish(ctx, catalogueRepo, commit, held, false); err != nil {
@@ -168,18 +170,18 @@ func aSiblingDirectoryAloneMovesItsImporter(t *testing.T) {
t.Fatal(err)
}
m, _ := moduleOf(t, mods, "avahi")
if m.FP2 == "" {
if m.Fp2 == "" {
t.Fatalf("no fingerprint at %s: %q", commit, m.None2)
}
return m
}
first := fpAt(cat1)
moved := fpAt(strings.Repeat("b", 40))
if moved.FP2 == first.FP2 {
t.Fatalf("a change of the sibling directory alone left avahi at %q", first.FP2)
if moved.Fp2 == first.Fp2 {
t.Fatalf("a change of the sibling directory alone left avahi at %q", first.Fp2)
}
if again := fpAt(strings.Repeat("c", 40)); again.FP2 != moved.FP2 {
t.Errorf("a merge changing nothing avahi reaches moved it from %q to %q", moved.FP2, again.FP2)
if again := fpAt(strings.Repeat("c", 40)); again.Fp2 != moved.Fp2 {
t.Errorf("a merge changing nothing avahi reaches moved it from %q to %q", moved.Fp2, again.Fp2)
}
if _, pinned := moved.Packages[sibling]; pinned {
t.Errorf("the build is pinned to a version of its sibling directory: %v", moved.Packages)
@@ -188,9 +190,9 @@ func aSiblingDirectoryAloneMovesItsImporter(t *testing.T) {
func buildEnd(t *testing.T, module, fp string, imports ...string) []byte {
t.Helper()
raw, err := delivery.EncodeBuildLogEntry(delivery.BuildLogEntry{Event: delivery.BuildEndedEvent, Module: module, FP: fp,
raw, err := builds.EncodeBuildLogEntry(builds.BuildLogEntry{Event: builds.BuildEndedEvent, Module: module, FP: fp,
Repository: catalogueRepo, Directory: "modules/" + module, Commit: cat1, Ref: "main", Node: "n1", At: time.Now(),
Outcome: delivery.BuildDone, Imports: imports})
Outcome: builds.BuildDone, Imports: imports})
if err != nil {
t.Fatal(err)
}
@@ -204,7 +206,7 @@ func setPackages(t *testing.T, mods *memStore, module string, packages []string)
in := *m.Inputs2
in.Packages = packages
m.Inputs2 = &in
raw, err := delivery.EncodeModule(m)
raw, err := meshcatalogclient.EncodeModule(m)
if err != nil {
t.Fatal(err)
}
@@ -221,12 +223,12 @@ func anAcceptedSeriesPinsItsNewest(t *testing.T) {
reportToolchain(t, ix, "mesh-build-go", "novox", "go", "go1.27.1")
ix.Packages, ix.Files = newMem(), forge{c1: sdkFiles("a1", "b1"), c2: sdkFiles("a1", "b2"), c3: sdkFiles("a2", "b2")}.files
setPackages(t, mods, "avahi", []string{meshSDK})
sdkModule, err := delivery.EncodeModule(delivery.Module{Repository: "novox/mesh-sdk", Path: ".", Commit: c1, None2: "a TypeScript package"})
sdkModule, err := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Repository: "novox/mesh-sdk", Path: ".", Commit: c1, None2: "a TypeScript package"})
if err != nil {
t.Fatal(err)
}
mods.put("mesh-sdk", sdkModule)
step := func(what string, act func() error) delivery.Module {
step := func(what string, act func() error) meshcatalog.ModulesByModuleValue {
t.Helper()
if err := act(); err != nil {
t.Fatalf("%s: %v", what, err)
@@ -255,16 +257,16 @@ func anAcceptedSeriesPinsItsNewest(t *testing.T) {
t.Fatalf("with no series published, versions %v, want the commit form c1", before.Packages)
}
step("the build reports its imports", func() error {
return ix.ApplyBuildEnd(ctx, buildEnd(t, "avahi", before.FP2, meshSDK, meshSDK+"/announce"))
return ix.ApplyBuildEnd(ctx, buildEnd(t, "avahi", before.Fp2, meshSDK, meshSDK+"/announce"))
})
pinned := step("0.1.15 published from c3", hear("0.1.15", c3))
if pinned.FP2 == before.FP2 || pinned.Packages[meshSDK] != "v0.1.15" {
t.Fatalf("after 0.1.15: fp2 %q (was %q), versions %v; want moved by c3's announce, pinned to v0.1.15", pinned.FP2, before.FP2, pinned.Packages)
if pinned.Fp2 == before.Fp2 || pinned.Packages[meshSDK] != "v0.1.15" {
t.Fatalf("after 0.1.15: fp2 %q (was %q), versions %v; want moved by c3's announce, pinned to v0.1.15", pinned.Fp2, before.Fp2, pinned.Packages)
}
if m := step("0.2.0 published from c1", hear("0.2.0", c1)); m.FP2 != pinned.FP2 || m.Packages[meshSDK] != "v0.1.15" {
t.Errorf("a publish of a series avahi does not accept moved it: %q %v", m.FP2, m.Packages)
if m := step("0.2.0 published from c1", hear("0.2.0", c1)); m.Fp2 != pinned.Fp2 || m.Packages[meshSDK] != "v0.1.15" {
t.Errorf("a publish of a series avahi does not accept moved it: %q %v", m.Fp2, m.Packages)
}
if m := step("trunk at c2", trunk(c2)); m.FP2 != pinned.FP2 || m.Packages[meshSDK] != "v0.1.15" {
t.Errorf("a trunk merge moved avahi off its accepted series' newest: %q %v", m.FP2, m.Packages)
if m := step("trunk at c2", trunk(c2)); m.Fp2 != pinned.Fp2 || m.Packages[meshSDK] != "v0.1.15" {
t.Errorf("a trunk merge moved avahi off its accepted series' newest: %q %v", m.Fp2, m.Packages)
}
}
modules/mesh-delivery-indexer/internal/index/index_test.go
+26 -24
View File
@@ -6,8 +6,10 @@ import (
"strings"
"testing"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/keys"
)
const (
@@ -120,7 +122,7 @@ func buildRecord(t *testing.T, repo, module, path, at string) []byte {
}
func pull(repo string, number int, branch, head string, dirs ...string) forgemsg.Forge {
return forgemsg.Forge{V: delivery.Version, Kind: forgemsg.ForgePull, Repository: repo, Branch: branch, Number: number, Head: head,
return forgemsg.Forge{V: keys.Version, Kind: forgemsg.ForgePull, Repository: repo, Branch: branch, Number: number, Head: head,
Action: forgemsg.PullUpdated, Dirs: dirs, DirsKnown: dirs != nil}
}
@@ -137,13 +139,13 @@ func newIndexer(t *testing.T) (*Indexer, *memStore, *strings.Builder) {
}}, st, logs
}
func read(t *testing.T, st *memStore, key string) (delivery.Candidate, bool) {
func read(t *testing.T, st *memStore, key string) (meshcatalog.CandidatesByGroupValue, bool) {
t.Helper()
v, ok := st.vals[key]
if !ok {
return delivery.Candidate{}, false
return meshcatalog.CandidatesByGroupValue{}, false
}
c, err := delivery.DecodeCandidate(v)
c, err := DecodeCandidate(v)
if err != nil {
t.Fatal(err)
}
@@ -163,8 +165,8 @@ func TestTwoRepositoriesFormOneGroupByBranch(t *testing.T) { // 1238/AC2 (the gr
if !ok {
t.Fatal("no candidate under feat/x")
}
hash, _ := delivery.HeadsHash([]delivery.Head{{Repository: "novox/mesh-delivery", Number: 12, Head: h1}, {Repository: "novox/mesh-catalog", Number: 300, Head: h2}})
if c.Group != "feat/x" || c.Base != 0 || c.HeadsHash != hash || len(c.Pulls) != 2 {
hash, _ := meshdeliveryclient.HeadsHash([]meshdeliveryclient.Head{{Repository: "novox/mesh-delivery", Number: 12, Head: h1}, {Repository: "novox/mesh-catalog", Number: 300, Head: h2}})
if c.Group != "feat/x" || c.Base != 0 || c.Heads != hash || len(c.Pulls) != 2 {
t.Fatalf("candidate = %+v", c)
}
got := map[string][]string{}
@@ -194,7 +196,7 @@ func TestModuleOfADirectory(t *testing.T) { // 1238/AC3, as R4e has it: the modu
if strings.Join(c.Pulls[0].Modules, ",") != "odd-name,plain" {
t.Errorf("modules = %v, want the held module for modules/odd, the last element for modules/plain, no repeats", c.Pulls[0].Modules)
}
held := map[string]delivery.Module{"x": {Repository: "novox/other", Path: "modules/a"}}
held := map[string]meshcatalog.ModulesByModuleValue{"x": {Repository: "novox/other", Path: "modules/a"}}
if name, ok := moduleAt(held, "novox/mesh-delivery", "modules/a"); ok {
t.Errorf("modules/a of novox/mesh-delivery is %q, held only of novox/other", name)
}
@@ -223,7 +225,7 @@ func TestDirsNotKnownKeepTheModulesOfTheSameHead(t *testing.T) { // 1238/AC4
t.Fatal(err)
}
c, _ = read(t, st, "feat/x")
if c.Pulls[0].Head.Head != h3 || len(c.Pulls[0].Modules) != 0 {
if c.Pulls[0].Head != h3 || len(c.Pulls[0].Modules) != 0 {
t.Errorf("pull = %+v, want the new head with no modules", c.Pulls[0])
}
// Known and empty is no modules, not unknown.
@@ -246,7 +248,7 @@ func TestClosedAndMergedTakeAPullOutAndTheLastDeletesTheKey(t *testing.T) { // 1
f.Action = forgemsg.PullClosed
return f
}
return forgemsg.Forge{V: delivery.Version, Kind: forgemsg.ForgeMerged, Repository: repo, Branch: "feat/x", Number: n, Head: head, Merge: h3, Root: "r", Trees: map[string]string{"a": "t"}}
return forgemsg.Forge{V: keys.Version, Kind: forgemsg.ForgeMerged, Repository: repo, Branch: "feat/x", Number: n, Head: head, Merge: h3, Root: "r", Trees: map[string]string{"a": "t"}}
}
if err := ix.Apply(ctx, gone("novox/mesh-delivery", 12, h1)); err != nil {
t.Fatal(err)
@@ -255,9 +257,9 @@ func TestClosedAndMergedTakeAPullOutAndTheLastDeletesTheKey(t *testing.T) { // 1
if !ok || len(c.Pulls) != 1 || c.Pulls[0].Repository != "novox/mesh-catalog" {
t.Fatalf("%s: candidate = %+v (kept %v)", how, c, ok)
}
hash, _ := delivery.HeadsHash([]delivery.Head{c.Pulls[0].Head})
if c.HeadsHash != hash {
t.Errorf("%s: heads hash %q is not the remaining pull request's %q", how, c.HeadsHash, hash)
hash, _ := meshdeliveryclient.HeadsHash([]meshdeliveryclient.Head{headOf(c.Pulls[0])})
if c.Heads != hash {
t.Errorf("%s: heads hash %q is not the remaining pull request's %q", how, c.Heads, hash)
}
if err := ix.Apply(ctx, gone("novox/mesh-catalog", 300, h2)); err != nil {
t.Fatal(err)
@@ -321,11 +323,11 @@ func TestAConflictReadsAgain(t *testing.T) { // 1238 (compare-and-set: Update wi
_ = ix.Apply(ctx, pull("novox/mesh-delivery", 12, "feat/x", h1, "modules/a"))
// Between the indexer's read and its write another writer adds #300 to the group.
st.beforeWrite = func() {
other := delivery.Candidate{V: delivery.Version, Group: "feat/x", Pulls: []delivery.CandidatePull{
{Head: delivery.Head{Repository: "novox/mesh-delivery", Number: 12, Head: h1}, Modules: []string{"a"}},
{Head: delivery.Head{Repository: "novox/mesh-catalog", Number: 300, Head: h2}, Modules: []string{"c"}}}}
other.HeadsHash, _ = delivery.HeadsHash([]delivery.Head{other.Pulls[0].Head, other.Pulls[1].Head})
data, _ := delivery.EncodeCandidate(other)
other := meshcatalog.CandidatesByGroupValue{V: keys.Version, Group: "feat/x", Pulls: []meshcatalog.CandidatesByGroupValuePullsItem{
{Repository: "novox/mesh-delivery", Number: 12, Head: h1, Modules: []string{"a"}},
{Repository: "novox/mesh-catalog", Number: 300, Head: h2, Modules: []string{"c"}}}}
other.Heads, _ = meshdeliveryclient.HeadsHash([]meshdeliveryclient.Head{headOf(other.Pulls[0]), headOf(other.Pulls[1])})
data, _ := EncodeCandidate(other)
st.put("feat/x", data)
}
if err := ix.Apply(ctx, pull("novox/mesh-delivery", 12, "feat/x", h3, "modules/a")); err != nil {
@@ -336,8 +338,8 @@ func TestAConflictReadsAgain(t *testing.T) { // 1238 (compare-and-set: Update wi
t.Fatalf("pulls = %+v, want #12 moved and #300 kept (the other writer's)", c.Pulls)
}
for _, p := range c.Pulls {
if p.Number == 12 && p.Head.Head != h3 {
t.Errorf("#12 head = %s", p.Head.Head)
if p.Number == 12 && p.Head != h3 {
t.Errorf("#12 head = %s", p.Head)
}
}
}
@@ -349,7 +351,7 @@ func TestAnOddBranchNameKeysEscaped(t *testing.T) {
if err := ix.Apply(context.Background(), pull("novox/mesh-delivery", 12, "feat x?", h1, "modules/a")); err != nil {
t.Fatal(err)
}
key, err := delivery.CandidatesKey("feat x?")
key, err := CandidatesKey("feat x?")
if err != nil {
t.Fatal(err)
}
@@ -359,7 +361,7 @@ func TestAnOddBranchNameKeysEscaped(t *testing.T) {
}
func open(pulls ...forgemsg.OpenPull) forgemsg.Forge {
return forgemsg.Forge{V: delivery.Version, Kind: forgemsg.ForgeOpen, Open: pulls}
return forgemsg.Forge{V: keys.Version, Kind: forgemsg.ForgeOpen, Open: pulls}
}
// 1279/AC7 (ADR 0443 §5): on an open message, candidates lose every pull request it does not name, a group left empty is
@@ -383,8 +385,8 @@ func TestAnOpenMessageDropsEveryCandidateItDoesNotName(t *testing.T) { // 1279/A
t.Fatal(err)
}
c, ok := read(t, st, "feat/x")
hash, _ := delivery.HeadsHash([]delivery.Head{{Repository: "novox/mesh-delivery", Number: 12, Head: h1}})
if !ok || len(c.Pulls) != 1 || c.Pulls[0].Number != 12 || c.Pulls[0].Head.Head != h1 || strings.Join(c.Pulls[0].Modules, ",") != "a" || c.HeadsHash != hash || c.HeadsHash == before.HeadsHash {
hash, _ := meshdeliveryclient.HeadsHash([]meshdeliveryclient.Head{{Repository: "novox/mesh-delivery", Number: 12, Head: h1}})
if !ok || len(c.Pulls) != 1 || c.Pulls[0].Number != 12 || c.Pulls[0].Head != h1 || strings.Join(c.Pulls[0].Modules, ",") != "a" || c.Heads != hash || c.Heads == before.Heads {
t.Fatalf("feat/x after the open message = %+v, want only mesh-delivery#12 with its modules and the heads hash of it alone", c)
}
for _, gone := range []string{"feat/y", "feat/old"} {
modules/mesh-delivery-indexer/internal/index/inputs.go
+10 -7
View File
@@ -7,15 +7,18 @@ import (
"sort"
"strings"
sdkstep "git.novox.be/novox/mesh-delivery/sdk/buildstep"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
"mesh-delivery-indexer/internal/fingerprint"
"mesh-delivery-indexer/judge"
)
// The reasons a module has no fingerprint of the new form while the indexer writes none for any module (novox/hq
// ADR 0465 clause 2; issue 1444). Toolchains missing is delivery.ToolchainNoReport.
// ADR 0465 clause 2; issue 1444). Toolchains missing is sdkstep.ToolchainNoReport.
const (
// None2NoManifest is for a module whose directory the merged message carries no manifest of (an older bridge).
None2NoManifest = "the message carries no manifest"
@@ -34,14 +37,14 @@ var lockFile = map[string]string{"go": "go.sum", "typescript": "package-lock.jso
// moduleInputs is what a module's manifest and lock hashes give the indexer at a merge (issue 1450).
type moduleInputs struct {
// Kept is the new-form inputs the merge gave the module; nil when the manifest could not be read or gave none.
Kept *delivery.KeptInputs2
Kept *meshcatalog.ModulesByModuleValueInputs2
// Languages is the language of each artifact that has a toolchain (a bundle or a package of go or typescript), by
// artifact name. The toolchain versions are joined to them when the build modules report them (issue 1444, the
// watching half); the SDK's KeptInputs2 keeps no languages, so they are read from the manifest again then.
Languages map[string]string
// None2 is why the module has no fingerprint of the new form.
None2 string
// ArtifactPaths are the paths each artifact naming them names, as its manifest names them (delivery.Module.ArtifactPaths).
// ArtifactPaths are the paths each artifact naming them names, as its manifest names them (meshcatalog.ModulesByModuleValue.ArtifactPaths).
ArtifactPaths map[string][]string
}
@@ -105,14 +108,14 @@ func inputsOfNamed(module, raw, tree string, locks, objects map[string]string, p
if len(missing) > 0 {
return moduleInputs{None2: "the message carries no git object of the named paths " + strings.Join(missing, ", "), ArtifactPaths: artifactPaths}
}
if tree, err = fingerprint.NamedTree(delivery.ManifestDigest([]byte(raw)), named); err != nil {
if tree, err = fingerprint.NamedTree(meshcatalogclient.ManifestDigest([]byte(raw)), named); err != nil {
return moduleInputs{None2: err.Error(), ArtifactPaths: artifactPaths}
}
}
if tree == "" {
return moduleInputs{None2: "the merge names no tree for the module's directory", ArtifactPaths: artifactPaths}
}
kept := delivery.KeptInputs2{Tree: tree, Paths: named}
kept := meshcatalog.ModulesByModuleValueInputs2{Tree: tree, Paths: named}
contexts := false
var missing []string
langs := languagesOf(m)
@@ -155,7 +158,7 @@ func inputsOfNamed(module, raw, tree string, locks, objects map[string]string, p
case len(missing) > 0:
out.None2 = strings.Join(missing, "; ")
case len(langs) > 0:
out.None2 = delivery.ToolchainNoReport
out.None2 = sdkstep.ToolchainNoReport
default:
out.None2 = None2Pending
}
modules/mesh-delivery-indexer/internal/index/inputs_test.go
+9 -8
View File
@@ -6,7 +6,8 @@ import (
"strings"
"testing"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
"mesh-delivery-indexer/internal/fingerprint"
)
@@ -25,7 +26,7 @@ func noBases(string) (string, string) { return "", "none are held in this test"
func TestAGoModuleKeepsItsTreeAndTheHashOfItsLockFile(t *testing.T) { // 1450/AC1
in := inputsOf("avahi", catalogueManifest(t, "avahi"), "avahitree", map[string]string{"go.sum": avahiGoSum}, noBases)
want := &delivery.KeptInputs2{Tree: "avahitree", Locks: map[string]string{"tools": avahiGoSum}}
want := &meshcatalog.ModulesByModuleValueInputs2{Tree: "avahitree", Locks: map[string]string{"tools": avahiGoSum}}
if !reflect.DeepEqual(in.Kept, want) {
t.Errorf("kept = %+v, want %+v", in.Kept, want)
}
@@ -40,7 +41,7 @@ func TestAGoModuleKeepsItsTreeAndTheHashOfItsLockFile(t *testing.T) { // 1450/AC
t.Fatal(err)
}
m, ok := moduleOf(t, mods, "avahi")
if !ok || !reflect.DeepEqual(m.Inputs2, want) || m.Manifest != delivery.ManifestDigest([]byte(catalogueManifest(t, "avahi"))) {
if !ok || !reflect.DeepEqual(m.Inputs2, want) || m.Manifest != meshcatalogclient.ManifestDigest([]byte(catalogueManifest(t, "avahi"))) {
t.Errorf("module = %+v (%v), want inputs %+v and the manifest digest", m, ok, want)
}
}
@@ -55,7 +56,7 @@ func TestAnArtifactWhoseLockFileTheMergeDidNotCarryHasNoLocksEntry(t *testing.T)
t.Errorf("languages = %v, want tools: go (the toolchain still applies)", in.Languages)
}
// With the toolchain known, mesh-sdk gives such inputs a src2: value, never 'no lock'.
if fp, none := fingerprint.Of(inputsFrom("avahi", *in.Kept, map[string]delivery.Toolchain2{"tools": {Language: "go", Version: "1.25.0", Recipe: "1"}})); fp == "" || none != "" {
if fp, none := fingerprint.Of(inputsFrom("avahi", *in.Kept, map[string]meshcatalogclient.Toolchain2{"tools": {Language: "go", Version: "1.25.0", Recipe: "1"}})); fp == "" || none != "" {
t.Errorf("fingerprint = %q none %q, want a value", fp, none)
}
// A TypeScript bundle without package-lock.json (go.sum of another language does not count).
@@ -113,8 +114,8 @@ func TestABaseEntersByTheFingerprintOfItsModuleOrNamesItsAbsence(t *testing.T) {
}
// Through the indexer: modules.mesh-tools holds a fingerprint of the new form (a record written by a later release).
ix, mods, _ := manifestsIndexer(t, rec(t, "build-1", "avahi", catalogueRepo, "modules/avahi", "main", "", "2026-10-19T09:00:00Z", aInputs))
mods.put("mesh-tools", mustEncode(t, delivery.Module{Repository: "novox/mesh-tools", // a base held of another repository stays when the catalogue merges (issue 1900)
Commit: merge, None: "x", FP2: fp, Manifest: strings.Repeat("a", 64)}))
mods.put("mesh-tools", mustEncode(t, meshcatalog.ModulesByModuleValue{Repository: "novox/mesh-tools", // a base held of another repository stays when the catalogue merges (issue 1900)
Commit: merge, None: "x", Fp2: fp, Manifest: strings.Repeat("a", 64)}))
f := withManifests(mergedMsg(catalogueRepo, map[string]string{"modules/avahi": "t"}), map[string]string{"modules/avahi": aModuleWithABase(t)}, nil)
if err := ix.Apply(context.Background(), f); err != nil {
t.Fatal(err)
@@ -124,9 +125,9 @@ func TestABaseEntersByTheFingerprintOfItsModuleOrNamesItsAbsence(t *testing.T) {
}
}
func mustEncode(t *testing.T, m delivery.Module) []byte {
func mustEncode(t *testing.T, m meshcatalog.ModulesByModuleValue) []byte {
t.Helper()
b, err := delivery.EncodeModule(m)
b, err := meshcatalogclient.EncodeModule(m)
if err != nil {
t.Fatal(err)
}
modules/mesh-delivery-indexer/internal/index/live_builds_test.go
+3 -3
View File
@@ -4,7 +4,7 @@ import (
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
)
// The build records below are the live ones, as `mesh-controller.command builds` showed them on 2026-10-19 at about
@@ -25,8 +25,8 @@ const (
func liveRecord(id, module, repository, ref, commit, path, at, source string) [2]any {
when, _ := time.Parse(time.RFC3339, at)
raw, _ := delivery.EncodeBuildLogEntry(delivery.BuildLogEntry{Event: delivery.BuildEndedEvent, Module: module, FP: source, Repository: repository,
Directory: path, Commit: commit, Ref: ref, Node: "n1", At: when, Outcome: delivery.BuildDone})
raw, _ := builds.EncodeBuildLogEntry(builds.BuildLogEntry{Event: builds.BuildEndedEvent, Module: module, FP: source, Repository: repository,
Directory: path, Commit: commit, Ref: ref, Node: "n1", At: when, Outcome: builds.BuildDone})
return [2]any{module, raw}
}
modules/mesh-delivery-indexer/internal/index/manifests.go
+7 -5
View File
@@ -5,8 +5,10 @@ import (
"errors"
"fmt"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/keys"
)
// keepManifest writes manifests.<digest> for a manifest as the forge answered it, create-once (issue 1449): a key that
@@ -15,15 +17,15 @@ import (
// digest. A manifest that cannot be a value (empty, not UTF-8, not JSON) is an error the caller says; it is never tried
// again. With no manifests store nothing is written and the digest is still answered.
func (ix *Indexer) keepManifest(ctx context.Context, module, raw string) (string, error) {
digest := delivery.ManifestDigest([]byte(raw))
digest := meshcatalogclient.ManifestDigest([]byte(raw))
if ix.Manifests == nil {
return digest, nil
}
key, err := delivery.ManifestsKey(digest)
key, err := meshcatalogclient.ManifestsKey(digest)
if err != nil {
return "", err
}
data, err := delivery.EncodeManifest(delivery.Manifest{Module: module, JSON: raw})
data, err := meshcatalogclient.EncodeManifest(meshcatalog.ManifestsByDigestValue{Module: module, Manifest: raw})
if err != nil {
return "", fmt.Errorf("%w: manifests.%s: %v", errCannotBeAValue, key, err)
}
@@ -41,7 +43,7 @@ func (ix *Indexer) keepManifest(ctx context.Context, module, raw string) (string
if !found {
return "", fmt.Errorf("manifests.%s was there and is not: written again at the next delivery", key)
}
if cur, err := delivery.DecodeManifest(held); err == nil && delivery.SameManifest(cur, delivery.Manifest{V: delivery.Version, Module: module, JSON: raw}) {
if cur, err := meshcatalogclient.DecodeManifest(held); err == nil && cur == (meshcatalog.ManifestsByDigestValue{V: keys.Version, Module: module, Manifest: raw}) {
return digest, nil
}
return "", fmt.Errorf("manifests.%s holds another value than the manifest of %s, which hashes to it", key, module)
modules/mesh-delivery-indexer/internal/index/manifests_test.go
+6 -5
View File
@@ -5,8 +5,9 @@ import (
"os"
"testing"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/keys"
)
// The manifests these tests carry are real files, copied unchanged into testdata: the module.json of modules of
@@ -57,13 +58,13 @@ func mergedManifestIsKeptByItsDigestAndNamedByTheModule(t *testing.T) { // 1449/
if err := ix.Apply(context.Background(), f); err != nil {
t.Fatal(err)
}
digest := delivery.ManifestDigest([]byte(avahi))
digest := meshcatalogclient.ManifestDigest([]byte(avahi))
raw, ok := manifests.vals[digest]
if !ok || len(manifests.vals) != 1 {
t.Fatalf("manifests holds %d keys, want only %s", len(manifests.vals), digest)
}
m, err := delivery.DecodeManifest(raw)
if err != nil || m.Module != "avahi" || m.JSON != avahi {
m, err := meshcatalogclient.DecodeManifest(raw)
if err != nil || m.Module != "avahi" || m.Manifest != avahi {
t.Errorf("manifest = %+v err %v, want avahi and the bytes of the file", m, err)
}
if mod, ok := moduleOf(t, mods, "avahi"); !ok || mod.Manifest != digest {
@@ -83,7 +84,7 @@ func sameManifestAppliedAgainIsWrittenOnce(t *testing.T) { // 1449/AC2
if err := ix.Apply(context.Background(), f); err != nil {
t.Errorf("a second merge: %v", err)
}
pull := forgemsg.Forge{V: delivery.Version, Kind: forgemsg.ForgePull, Repository: catalogueRepo, Branch: "feat/x", Number: 301, Head: h1,
pull := forgemsg.Forge{V: keys.Version, Kind: forgemsg.ForgePull, Repository: catalogueRepo, Branch: "feat/x", Number: 301, Head: h1,
Action: forgemsg.PullUpdated, Dirs: []string{"modules/avahi"}, DirsKnown: true, Manifests: map[string]string{"modules/avahi": avahi}}
if err := ix.Apply(context.Background(), pull); err != nil {
t.Errorf("a pull message: %v", err)
modules/mesh-delivery-indexer/internal/index/modules.go
+32 -26
View File
@@ -7,9 +7,12 @@ import (
"slices"
"strings"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
"git.novox.be/novox/mesh-delivery/sdk/trunk"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/keys"
"mesh-delivery-indexer/internal/fingerprint"
"mesh-delivery-indexer/judge"
@@ -61,7 +64,7 @@ func (ix *Indexer) applyMoving(ctx context.Context, f forgemsg.Forge, drop bool,
return w, err
}
objects, namedErr := ix.namedObjects(ctx, f)
overlay := map[string]delivery.Module{}
overlay := map[string]meshcatalog.ModulesByModuleValue{}
dirOf := map[string]string{}
named, every := map[string]bool{}, true // every module the message's manifests name, and whether each names one
for _, dir := range slices.Sorted(maps.Keys(f.Manifests)) {
@@ -72,7 +75,7 @@ func (ix *Indexer) applyMoving(ctx context.Context, f forgemsg.Forge, drop bool,
every = false
continue
}
key, err := delivery.ModulesKey(name)
key, err := meshcatalogclient.ModulesKey(name)
if err != nil {
every = false
ix.logf("%s: the module %q has no usable key, skipped: %v", f.Repository, name, err)
@@ -80,7 +83,7 @@ func (ix *Indexer) applyMoving(ctx context.Context, f forgemsg.Forge, drop bool,
}
named[key] = true
cur, has := held[key]
if has && cur.Repository != "" && !delivery.SameRepository(cur.Repository, f.Repository) {
if has && cur.Repository != "" && !trunk.SameRepository(cur.Repository, f.Repository) {
if !moves[key] {
ix.waitToMove(key, f, dir)
ix.logf("%s at %s: %q names the module %s, which the indexer holds of %s; left as it is until %s's trunk drops it, and then moved here (a move between repositories is the merge of the group that moves it, ADR 0420 clause 7)", f.Repository, f.Merge, dir, name, cur.Repository, cur.Repository)
@@ -96,7 +99,7 @@ func (ix *Indexer) applyMoving(ctx context.Context, f forgemsg.Forge, drop bool,
ix.logf("%s at %s: %q and %q both name the module %s; %q, which the indexer holds, is written", f.Repository, f.Merge, other, dir, name, dir)
}
dirOf[name] = dir
next := delivery.Module{Repository: f.Repository, Commit: f.Merge, Src: ix.srcOf(f.Repository, f.Merge), Manifest: digests[dir], Path: pathOf(dir)}
next := meshcatalog.ModulesByModuleValue{Repository: f.Repository, Commit: f.Merge, Src: int64(ix.srcOf(f.Repository, f.Merge)), Manifest: digests[dir], Path: pathOf(dir)}
// The reader capabilities a changed manifest needs are written with it, before the judge's pass names them with its
// verdict, so no reader takes a new manifest as needing nothing (ADR 0614 §4).
next.Needs, _ = catalogue.ManifestNeeds([]byte(raw))
@@ -108,7 +111,7 @@ func (ix *Indexer) applyMoving(ctx context.Context, f forgemsg.Forge, drop bool,
} else {
packages := ix.goSumPackages(ctx, f, dir, cur, has)
in := inputsOfNamed(name, raw, f.Trees[dir], f.Locks[dir], objects[dir], packages, ix.baseFP(ctx))
next.Inputs2, next.None2, next.ArtifactPaths = in.Kept, in.None2, in.ArtifactPaths
next.Inputs2, next.None2, next.Paths = in.Kept, in.None2, in.ArtifactPaths
if in.Kept != nil && has && cur.Inputs2 != nil {
// The imports its last trunk build reported stand until its next build reports again (ADR 0593 §2, §4).
in.Kept.Imports, in.Kept.ImportForm = cur.Inputs2.Imports, cur.Inputs2.ImportForm
@@ -138,7 +141,7 @@ func (ix *Indexer) applyMoving(ctx context.Context, f forgemsg.Forge, drop bool,
// repository goes (issue 1126 note 94: a repository that removed its last module). A message with trees but no
// manifests (an older writer) or one that cannot be read whole says nothing.
if drop && (len(f.Manifests) > 0 || len(f.Trees) == 0) && every {
if err := ix.dropRemoved(ctx, f.Repository, held, func(key string, _ delivery.Module) bool { return named[key] }); err != nil {
if err := ix.dropRemoved(ctx, f.Repository, held, func(key string, _ meshcatalog.ModulesByModuleValue) bool { return named[key] }); err != nil {
return w, err
}
}
@@ -156,7 +159,7 @@ func (ix *Indexer) applyMoving(ctx context.Context, f forgemsg.Forge, drop bool,
// baseFP answers the new-form fingerprint of a base module as modules.<base> holds it, or why it has none.
func (ix *Indexer) baseFP(ctx context.Context) baseFP {
return func(module string) (string, string) {
key, err := delivery.ModulesKey(module)
key, err := meshcatalogclient.ModulesKey(module)
if err != nil {
return "", err.Error()
}
@@ -167,12 +170,12 @@ func (ix *Indexer) baseFP(ctx context.Context) baseFP {
if !found {
return "", "the indexer holds no record of it"
}
cur, err := delivery.DecodeModule(value)
cur, err := meshcatalogclient.DecodeModule(value)
switch {
case err != nil:
return "", "its record cannot be read: " + err.Error()
case cur.FP2 != "":
return cur.FP2, ""
case cur.Fp2 != "":
return cur.Fp2, ""
case cur.None2 != "":
return "", cur.None2
}
@@ -183,12 +186,15 @@ func (ix *Indexer) baseFP(ctx context.Context) baseFP {
// pathOf is the module's directory as Module.Path holds it: "." for the repository's root, since empty means not known.
func pathOf(dir string) string {
if dir == "" {
return delivery.RootPath
return meshcatalogclient.RootPath
}
return dir
}
func withV(m delivery.Module) delivery.Module { m.V = delivery.Version; return m }
func withV(m meshcatalog.ModulesByModuleValue) meshcatalog.ModulesByModuleValue {
m.V = keys.Version
return m
}
// namedObjects are, by module directory, the git objects of the paths each manifest of the merge names for every artifact
// (catalogue Manifest.NamedPaths): those the message carries, else asked of the forge at the merge commit in one call
@@ -231,12 +237,12 @@ func (ix *Indexer) namedObjects(ctx context.Context, f forgemsg.Forge) (map[stri
// dropRemoved deletes, from the modules the bucket holds of the repository, each one the trunk no longer holds (issue 1900):
// held(key, module) says whether the trunk holds it. A module of another repository, or of none known, is never touched.
func (ix *Indexer) dropRemoved(ctx context.Context, repository string, held map[string]delivery.Module, trunkHolds func(key string, m delivery.Module) bool) error {
func (ix *Indexer) dropRemoved(ctx context.Context, repository string, held map[string]meshcatalog.ModulesByModuleValue, trunkHolds func(key string, m meshcatalog.ModulesByModuleValue) bool) error {
var handed []string
elsewhere := map[string]forgemsg.Forge{}
for _, key := range slices.Sorted(maps.Keys(held)) {
cur := held[key]
if cur.Repository == "" || !delivery.SameRepository(cur.Repository, repository) || trunkHolds(key, cur) {
if cur.Repository == "" || !trunk.SameRepository(cur.Repository, repository) || trunkHolds(key, cur) {
continue
}
if ix.movingTo(key, repository) {
@@ -254,7 +260,7 @@ func (ix *Indexer) dropRemoved(ctx context.Context, repository string, held map[
if !found {
return nil, false, nil
}
if m, err := delivery.DecodeModule(value); err != nil || !delivery.SameRepository(m.Repository, repository) {
if m, err := meshcatalogclient.DecodeModule(value); err != nil || !trunk.SameRepository(m.Repository, repository) {
return nil, false, nil
}
return nil, true, nil
@@ -297,9 +303,9 @@ func (ix *Indexer) Reconcile(ctx context.Context) error {
return err
}
for _, repository := range slices.Sorted(maps.Keys(heads)) {
mine := map[string]delivery.Module{}
mine := map[string]meshcatalog.ModulesByModuleValue{}
for key, m := range held {
if m.Repository != "" && delivery.SameRepository(m.Repository, repository) {
if m.Repository != "" && trunk.SameRepository(m.Repository, repository) {
mine[key] = m
}
}
@@ -322,7 +328,7 @@ func (ix *Indexer) Reconcile(ctx context.Context) error {
head := heads[repository]
// The source can lag a merge already applied: a key written at a commit the head read does not hold (a merge heard
// first) is not read as removed, only one whose commit the head is, or contains.
stays := func(_ string, m delivery.Module) bool {
stays := func(_ string, m meshcatalog.ModulesByModuleValue) bool {
if m.Path == "" || there[m.Path] {
return true
}
@@ -353,11 +359,11 @@ const None2Unreadable = "the manifest cannot be read: "
// hashes by file name (the artifacts that own them are not known without the manifest) and the objects of the named
// paths the message carried; nil when the message carried no tree. Judging the manifest again (rejudged) turns them into
// the inputs of the module (issue 1923).
func unreadInputs(tree string, locks, paths map[string]string, packages []string) *delivery.KeptInputs2 {
func unreadInputs(tree string, locks, paths map[string]string, packages []string) *meshcatalog.ModulesByModuleValueInputs2 {
if tree == "" {
return nil
}
in := &delivery.KeptInputs2{Tree: tree}
in := &meshcatalog.ModulesByModuleValueInputs2{Tree: tree}
if len(locks) > 0 {
in.Locks = maps.Clone(locks)
}
@@ -371,7 +377,7 @@ func unreadInputs(tree string, locks, paths map[string]string, packages []string
// goSumPackages are the version line's packages of the merge's directory dir (ADR 0573 §2, ADR 0593 §2): the mesh Go
// modules of other repositories its go.sum names, and those of its own repository its go.mod replaces by a directory
// (siblingPackages), which no go.sum line names. A directory with no go.sum is no Go module and has none.
func (ix *Indexer) goSumPackages(ctx context.Context, f forgemsg.Forge, dir string, cur delivery.Module, has bool) []string {
func (ix *Indexer) goSumPackages(ctx context.Context, f forgemsg.Forge, dir string, cur meshcatalog.ModulesByModuleValue, has bool) []string {
others := ix.otherPackages(ctx, f, dir, cur, has)
if f.Locks[dir]["go.sum"] == "" && len(f.Packages[dir]) == 0 {
return others
@@ -385,14 +391,14 @@ func (ix *Indexer) goSumPackages(ctx context.Context, f forgemsg.Forge, dir stri
// siblingPackages are the mesh Go modules of the merge's own repository the go.mod of dir replaces by a directory, read
// from the forge (GoMod); where it cannot be read, the held module's own-repository packages stand, said.
func (ix *Indexer) siblingPackages(ctx context.Context, f forgemsg.Forge, dir string, cur delivery.Module, has bool) []string {
func (ix *Indexer) siblingPackages(ctx context.Context, f forgemsg.Forge, dir string, cur meshcatalog.ModulesByModuleValue, has bool) []string {
held := func() []string {
if !has || cur.Inputs2 == nil {
return nil
}
var out []string
for _, p := range cur.Inputs2.Packages {
if repo, _, ok := delivery.PackageRepository(p); ok && delivery.SameRepository(repo, f.Repository) {
if repo, _, ok := trunk.PackageRepository(p); ok && trunk.SameRepository(repo, f.Repository) {
out = append(out, p)
}
}
@@ -412,7 +418,7 @@ func (ix *Indexer) siblingPackages(ctx context.Context, f forgemsg.Forge, dir st
// otherPackages are the mesh Go modules of other repositories the go.sum of the merge's directory dir names: those the
// message carries (an older bridge), else the held module's while its go.sum's hash is unchanged, else the go.sum read
// from the forge (GoSum).
func (ix *Indexer) otherPackages(ctx context.Context, f forgemsg.Forge, dir string, cur delivery.Module, has bool) []string {
func (ix *Indexer) otherPackages(ctx context.Context, f forgemsg.Forge, dir string, cur meshcatalog.ModulesByModuleValue, has bool) []string {
if carried := f.Packages[dir]; len(carried) > 0 {
return fingerprint.OtherPackages(carried, f.Repository) // its own repository's come with its tree
}
@@ -420,7 +426,7 @@ func (ix *Indexer) otherPackages(ctx context.Context, f forgemsg.Forge, dir stri
if hash == "" {
return nil
}
if has && cur.Inputs2 != nil && delivery.SameRepository(cur.Repository, f.Repository) && ix.requirementsKept(f.Repository, dir) {
if has && cur.Inputs2 != nil && trunk.SameRepository(cur.Repository, f.Repository) && ix.requirementsKept(f.Repository, dir) {
for _, h := range cur.Inputs2.Locks {
if h == hash {
return fingerprint.OtherPackages(cur.Inputs2.Packages, f.Repository)
modules/mesh-delivery-indexer/internal/index/modules_test.go
+29 -22
View File
@@ -10,9 +10,16 @@ import (
"testing"
"time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
"git.novox.be/novox/mesh-delivery/sdk/trunk"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/keys"
"git.novox.be/novox/mesh-sdk/go/process"
)
@@ -28,18 +35,18 @@ func logged(t *testing.T, id, module, repo, path, ref, outcome, at string) []byt
t.Fatal(err)
}
if path == "" {
path = delivery.RootPath
path = meshcatalogclient.RootPath
}
e := delivery.BuildLogEntry{Event: delivery.BuildEndedEvent, Module: module, FP: "src2:" + id, Repository: repo, Directory: path,
Commit: "7a74cee83f88e6585a5511c3cc53bdf0f5303d20", Ref: ref, Node: "n1", At: when, Outcome: delivery.BuildDone}
e := builds.BuildLogEntry{Event: builds.BuildEndedEvent, Module: module, FP: "src2:" + id, Repository: repo, Directory: path,
Commit: "7a74cee83f88e6585a5511c3cc53bdf0f5303d20", Ref: ref, Node: "n1", At: when, Outcome: builds.BuildDone}
switch outcome {
case "":
case "failed":
e.Outcome = delivery.BuildFailed
e.Outcome = builds.BuildFailed
default:
e.Event, e.Outcome = delivery.BuildStartedEvent, ""
e.Event, e.Outcome = builds.BuildStartedEvent, ""
}
raw, err := delivery.EncodeBuildLogEntry(e)
raw, err := builds.EncodeBuildLogEntry(e)
if err != nil {
t.Fatal(err)
}
@@ -77,7 +84,7 @@ func mergedMsg(repo string, trees map[string]string) forgemsg.Forge {
for dir := range trees {
manifests[dir] = `{"module":"` + nameOfDir(dir) + `","version":"1"}`
}
return forgemsg.Forge{V: delivery.Version, Kind: forgemsg.ForgeMerged, Repository: repo, Branch: "feat/x", Number: 12, Head: h1, Merge: merge, Root: "r0", Trees: trees, Manifests: manifests}
return forgemsg.Forge{V: keys.Version, Kind: forgemsg.ForgeMerged, Repository: repo, Branch: "feat/x", Number: 12, Head: h1, Merge: merge, Root: "r0", Trees: trees, Manifests: manifests}
}
func nameOfDir(dir string) string {
@@ -87,13 +94,13 @@ func nameOfDir(dir string) string {
return path.Base(dir)
}
func moduleOf(t *testing.T, st *memStore, key string) (delivery.Module, bool) {
func moduleOf(t *testing.T, st *memStore, key string) (meshcatalog.ModulesByModuleValue, bool) {
t.Helper()
v, ok := st.vals[key]
if !ok {
return delivery.Module{}, false
return meshcatalog.ModulesByModuleValue{}, false
}
m, err := delivery.DecodeModule(v)
m, err := meshcatalogclient.DecodeModule(v)
if err != nil {
t.Fatal(err)
}
@@ -145,7 +152,7 @@ func TestAnUnchangedModuleWritesNothing(t *testing.T) { // 1239 (unchanged value
{name: "a merge heard while the taken state was missing is filled when the state appears", writes: 2, commit: map[string]string{"a": next, "b": merge, "c": merge}, taken: next,
then: func(ix *Indexer) error { // issue 1280: the key was missing, the merge was taken in without it
on := ix.Taken.(*memStore)
key, _ := delivery.RepositoryKey(repo)
key, _ := trunk.RepositoryKey(repo)
delete(on.vals, key)
ix.Taken = &memStore{unbound: process.ErrNotYet}
m := mergedMsg(repo, map[string]string{"modules/a": "treeA2", "modules/b": "treeB", "modules/c": "treeC"})
@@ -293,7 +300,7 @@ func TestAnUnchangedModuleWritesNothing(t *testing.T) { // 1239 (unchanged value
}},
{name: "a later source revision of another repository's merge", writes: 0, commit: map[string]string{"a": merge, "b": merge, "c": merge}, taken: merge,
then: func(ix *Indexer) error {
return ix.ApplySource(context.Background(), 9, delivery.Source{Group: "g", Heads: map[string]string{repo: merge, "novox/mesh-catalog": otherMerge}})
return ix.ApplySource(context.Background(), 9, meshdelivery.SourceByMainValue{Group: "g", Heads: map[string]string{repo: merge, "novox/mesh-catalog": otherMerge}})
}},
} {
t.Run(tc.name, func(t *testing.T) {
@@ -338,11 +345,11 @@ func TestAnUnchangedModuleWritesNothing(t *testing.T) { // 1239 (unchanged value
t.Errorf("%s = %+v (%v), want commit %.8s", module, m, ok, want)
}
}
key, _ := delivery.RepositoryKey(repo)
key, _ := trunk.RepositoryKey(repo)
if _, held := taken.vals[key]; !held && tc.taken == "" {
return
}
if got, err := delivery.DecodeTaken(taken.vals[key]); err != nil || got.Merge != tc.taken || got.Repository != repo {
if got, err := takein.DecodeTaken(taken.vals[key]); err != nil || got.Merge != tc.taken || got.Repository != repo {
t.Errorf("taken.%s = %+v (%v), want merge %.8s", key, got, err, tc.taken)
}
})
@@ -367,7 +374,7 @@ func TestAMergeWritesTheModulesDirectoryAndKeepsItThroughSourceAndRecompute(t *t
if err := ix.Apply(ctx, mergedMsg("novox/mesh-delivery", map[string]string{"modules/a": "newtree", "": "roottree"})); err != nil {
t.Fatal(err)
}
for module, want := range map[string]string{"a": "modules/a", "r": delivery.RootPath} {
for module, want := range map[string]string{"a": "modules/a", "r": meshcatalogclient.RootPath} {
if m, ok := moduleOf(t, mods, module); !ok || m.Path != want {
t.Errorf("after the merge %s = %+v (%v), want path %q", module, m, ok, want)
}
@@ -392,8 +399,8 @@ func TestARootModuleHasTheDirectoryDot(t *testing.T) { // 1615/AC3: empty means
if err := ix.Apply(context.Background(), mergedMsg("novox/mesh-sdk", map[string]string{"": "roottree"})); err != nil {
t.Fatal(err)
}
if m, ok := moduleOf(t, mods, "r"); !ok || m.Path != delivery.RootPath {
t.Errorf("module = %+v (%v), want path %q", m, ok, delivery.RootPath)
if m, ok := moduleOf(t, mods, "r"); !ok || m.Path != meshcatalogclient.RootPath {
t.Errorf("module = %+v (%v), want path %q", m, ok, meshcatalogclient.RootPath)
}
}
@@ -406,13 +413,13 @@ func TestAMergeAddingAModuleDirectoryWritesItsValueWithNoBuildRecordAnywhere(t *
t.Fatal(err)
}
m, ok := moduleOf(t, mods, "avahi")
if !ok || m.Path != "modules/avahi" || m.Repository != catalogueRepo || m.Commit != merge || m.Manifest != delivery.ManifestDigest([]byte(avahi)) || m.Inputs2 == nil || m.Inputs2.Tree != "newtree" {
if !ok || m.Path != "modules/avahi" || m.Repository != catalogueRepo || m.Commit != merge || m.Manifest != meshcatalogclient.ManifestDigest([]byte(avahi)) || m.Inputs2 == nil || m.Inputs2.Tree != "newtree" {
t.Fatalf("module = %+v (%v), want its directory, merge, manifest and inputs from the message alone", m, ok)
}
if m.FP != "" || m.None != "" {
if m.Fp != "" || m.None != "" {
t.Errorf("module = %+v, want no value of today's form (I4: it came from the build records)", m)
}
if m.FP2 == "" && m.None2 == "" {
if m.Fp2 == "" && m.None2 == "" {
t.Errorf("module = %+v, want a new-form fingerprint or its reason", m)
}
// ADR 0614 §4: the merge writes the reader capabilities its manifest needs, so no reader takes it as needing nothing
@@ -567,6 +574,6 @@ func TestTwoDirectoriesNamingOneModuleWriteOneAndSayTheOther(t *testing.T) { //
}
func takenKey(repo string) string {
key, _ := delivery.RepositoryKey(repo)
key, _ := trunk.RepositoryKey(repo)
return key
}
modules/mesh-delivery-indexer/internal/index/moves.go
+5 -4
View File
@@ -8,7 +8,8 @@ import (
"strings"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-delivery/sdk/trunk"
"git.novox.be/novox/mesh-sdk/go/keys"
)
// heldByAnotherTrunk reads, for a module the trunk of repository no longer holds, whether the trunk of another repository
@@ -24,7 +25,7 @@ func (ix *Indexer) heldByAnotherTrunk(ctx context.Context, module, repository st
ix.mu.Unlock()
for _, other := range slices.Sorted(maps.Keys(heads)) {
head := heads[other]
if head == "" || delivery.SameRepository(other, repository) {
if head == "" || trunk.SameRepository(other, repository) {
continue
}
dirs, err := ix.Trunk(ctx, other, head)
@@ -46,7 +47,7 @@ func (ix *Indexer) heldByAnotherTrunk(ctx context.Context, module, repository st
if read.Trees[dir] == "" || manifestModule(raw) != module {
continue
}
f := forgemsg.Forge{V: delivery.Version, Kind: forgemsg.ForgeMerged, Repository: other, Merge: head, Root: read.Root,
f := forgemsg.Forge{V: keys.Version, Kind: forgemsg.ForgeMerged, Repository: other, Merge: head, Root: read.Root,
Trees: map[string]string{dir: read.Trees[dir]}, Manifests: map[string]string{dir: raw}}
if l := read.Locks[dir]; len(l) > 0 {
f.Locks = map[string]map[string]string{dir: l}
@@ -102,7 +103,7 @@ func (ix *Indexer) movingTo(key, from string) bool {
ix.mu.Lock()
defer ix.mu.Unlock()
f, ok := ix.moving[key]
return ok && !delivery.SameRepository(f.Repository, from)
return ok && !trunk.SameRepository(f.Repository, from)
}
// handOver writes the module at key from the merge waiting to take it, and forgets the wait.
modules/mesh-delivery-indexer/internal/index/named_test.go
+11 -10
View File
@@ -7,7 +7,8 @@ import (
"strings"
"testing"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdeliverychecker"
"mesh-delivery-indexer/internal/fingerprint"
)
@@ -31,15 +32,15 @@ func TestAModuleNamingItsPathsTakesItsTreeFromThemAndKeepsTheirObjects(t *testin
if !ok || m.Inputs2 == nil {
t.Fatalf("module = %+v (%v)", m, ok)
}
want, err := fingerprint.NamedTree(delivery.ManifestDigest([]byte(raw)), sdkObjects)
want, err := fingerprint.NamedTree(meshcatalogclient.ManifestDigest([]byte(raw)), sdkObjects)
if err != nil {
t.Fatal(err)
}
if m.Inputs2.Tree != want || !reflect.DeepEqual(m.Inputs2.Paths, sdkObjects) {
t.Errorf("inputs = %+v, want the named tree %s and the objects kept", m.Inputs2, want)
}
if got := m.ArtifactPaths["lib"]; strings.Join(got, ",") != "src,test,package.json,package-lock.json,tsconfig.json" {
t.Errorf("artifact paths = %v, want lib's paths as its manifest names them", m.ArtifactPaths)
if got := m.Paths["lib"]; strings.Join(got, ",") != "src,test,package.json,package-lock.json,tsconfig.json" {
t.Errorf("artifact paths = %v, want lib's paths as its manifest names them", m.Paths)
}
// Recomputed from what is kept, with no merge, the module is the same (ADR 0499 clause 2).
before := string(mods.vals["mesh-sdk"])
@@ -59,7 +60,7 @@ func TestAMessageWithoutTheNamedPathsObjectsGivesTheModuleNoneNamingThem(t *test
t.Fatal(err)
}
m, _ := moduleOf(t, mods, "mesh-sdk")
if m.FP2 != "" || !strings.Contains(m.None2, "named paths") || !strings.Contains(m.None2, "tsconfig.json") {
if m.Fp2 != "" || !strings.Contains(m.None2, "named paths") || !strings.Contains(m.None2, "tsconfig.json") {
t.Errorf("module = %+v, want none naming the paths the message did not carry", m)
}
}
@@ -79,8 +80,8 @@ func TestACandidatesModuleTakesTheNamedPathsObjectsFromThePreview(t *testing.T)
t.Fatal(err)
}
named, _ := fingerprint.NamedTree(digest, sdkObjects)
with := delivery.Preview{Modules: map[string]delivery.PreviewModule{".": {Tree: named, Manifest: digest, Paths: sdkObjects}}}
without := delivery.Preview{Modules: map[string]delivery.PreviewModule{".": {Tree: "roottree", Manifest: digest}}}
with := meshdeliverychecker.PreviewsByRepositoryHeadTrunkValue{Modules: map[string]meshdeliverychecker.PreviewsByRepositoryHeadTrunkValueModulesValue{".": {Tree: named, Manifest: digest, Paths: sdkObjects}}}
without := meshdeliverychecker.PreviewsByRepositoryHeadTrunkValue{Modules: map[string]meshdeliverychecker.PreviewsByRepositoryHeadTrunkValueModulesValue{".": {Tree: "roottree", Manifest: digest}}}
if got := ix.candidateModule(context.Background(), "mesh-sdk", ".", without, nil, nil); !strings.Contains(got.None2, "named paths") {
t.Errorf("without the objects: %+v, want none naming the named paths", got)
}
@@ -88,7 +89,7 @@ func TestACandidatesModuleTakesTheNamedPathsObjectsFromThePreview(t *testing.T)
if strings.Contains(got.None2, "named paths") || got.Manifest != digest {
t.Errorf("with the objects: %+v, want them taken (none only for what else is missing)", got)
}
if got.FP2 != "" && !reflect.DeepEqual(got.Paths, sdkObjects) {
if got.Fp2 != "" && !reflect.DeepEqual(got.Paths, sdkObjects) {
t.Errorf("with a fingerprint: %+v, want the named paths' objects carried for the builder's fetch", got)
}
}
@@ -114,7 +115,7 @@ func TestTheIndexerAsksTheForgeForTheNamedPathsObjectsOfAMerge(t *testing.T) { /
t.Fatalf("asked %+v, want the root's named paths of the merge commit, once, and nothing for a module naming none", asked)
}
m, _ := moduleOf(t, mods, "mesh-sdk")
want, _ := fingerprint.NamedTree(delivery.ManifestDigest([]byte(raw)), sdkObjects)
want, _ := fingerprint.NamedTree(meshcatalogclient.ManifestDigest([]byte(raw)), sdkObjects)
if m.Inputs2 == nil || m.Inputs2.Tree != want {
t.Errorf("module = %+v, want the named tree %s", m, want)
}
@@ -129,7 +130,7 @@ func TestAForgeThatCannotSayTheObjectsGivesTheModuleNoneWithItsWords(t *testing.
if err := ix.Apply(context.Background(), withManifests(mergedMsg("novox/mesh-sdk", map[string]string{"": "roottree"}), map[string]string{"": raw}, nil)); err != nil {
t.Fatal(err)
}
if m, _ := moduleOf(t, mods, "mesh-sdk"); m.FP2 != "" || !strings.Contains(m.None2, "is not at the commit") {
if m, _ := moduleOf(t, mods, "mesh-sdk"); m.Fp2 != "" || !strings.Contains(m.None2, "is not at the commit") {
t.Errorf("module = %+v, want none with the forge's words", m)
}
}
modules/mesh-delivery-indexer/internal/index/names.go
+46
View File
@@ -0,0 +1,46 @@
package index
// The holders, states and buckets the indexer names, taken from its own manifest (module.json: its name, its `state`, its
// `reads`) and from the manifests of the holders it reads; a unit row checks them against the manifest
// (TestTheNamesAreTheManifests). No contract names a holder (novox/hq ADR 0625 §4, ADR 0634 §1).
const (
// Module is the indexer's own name; the catalog client's Module is the readers' name for it.
Module = "mesh-delivery-indexer"
// The indexer's own states, with the revisions each keeps per key (module.json `state`).
CandidatesState = "candidates"
CandidatesBucket = Module + "_" + CandidatesState
CandidatesHistory = 10
ModulesState = "modules"
ModulesBucket = Module + "_" + ModulesState
ModulesHistory = 20
ManifestsState = "manifests"
ManifestsBucket = Module + "_" + ManifestsState
ManifestsHistory = 1
// VerdictsState and VerdictsBucket are in verdicts.go.
VerdictsHistory = 5
PredefinedState = "predefined"
ConfigCandidatesState = "config-candidates"
PackagesState = "packages"
PackagesBucket = Module + "_" + PackagesState
PackagesHistory = 5
TakenState = "taken"
TakenBucket = Module + "_" + TakenState
PredefinedListKey = "list"
ForgeBridgeModule = "mesh-delivery-forge-bridge"
AssignerModule = "mesh-delivery-assigner"
GateModule = "mesh-delivery-gate"
CheckerModule = "mesh-delivery-checker"
BuildGoModule = "mesh-build-go"
BuildNodeModule = "mesh-build-node"
PlacementState = "placement"
PlacementBucket = AssignerModule + "_" + PlacementState
PlacementHistory = 5
SourceBucket = GateModule + "_source"
PreviewsBucket = CheckerModule + "_previews"
PreviewsHistory = 5
toolchainsState = "toolchains"
)
// ToolchainsBucket is the toolchains state of a build module (reads: mesh-build-go.toolchains, mesh-build-node.toolchains).
func ToolchainsBucket(module string) string { return module + "_" + toolchainsState }
modules/mesh-delivery-indexer/internal/index/newform_test.go
+7 -7
View File
@@ -4,7 +4,7 @@ import (
"context"
"testing"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
)
func TestAGoModuleAtAMergeHasItsManifestItsInputsAndNoFingerprintOfTheNewForm(t *testing.T) { // 1444/AC1
@@ -20,14 +20,14 @@ func TestAGoModuleAtAMergeHasItsManifestItsInputsAndNoFingerprintOfTheNewForm(t
if !ok {
t.Fatal("no module avahi")
}
if m.Manifest != delivery.ManifestDigest([]byte(avahi)) || m.Inputs2 == nil || m.Inputs2.Tree != "newtree" || m.Inputs2.Locks["tools"] != avahiGoSum {
if m.Manifest != meshcatalogclient.ManifestDigest([]byte(avahi)) || m.Inputs2 == nil || m.Inputs2.Tree != "newtree" || m.Inputs2.Locks["tools"] != avahiGoSum {
t.Errorf("module = %+v, want its manifest digest and inputs", m)
}
if m.None2 != "toolchain not yet reported" || m.FP2 != "" {
t.Errorf("none2 = %q fp2 = %q, want none2 'toolchain not yet reported'", m.None2, m.FP2)
if m.None2 != "toolchain not yet reported" || m.Fp2 != "" {
t.Errorf("none2 = %q fp2 = %q, want none2 'toolchain not yet reported'", m.None2, m.Fp2)
}
if m.FP != "" || m.None != "" {
t.Errorf("fp = %q none = %q, want no value of today's form (I4)", m.FP, m.None)
if m.Fp != "" || m.None != "" {
t.Errorf("fp = %q none = %q, want no value of today's form (I4)", m.Fp, m.None)
}
}
@@ -39,7 +39,7 @@ func TestAModuleWithNoToolchainArtifactGetsItsFingerprintAtOnce(t *testing.T) {
if err := ix.Apply(context.Background(), f); err != nil {
t.Fatal(err)
}
if m, ok := moduleOf(t, mods, "ca-trust"); !ok || m.FP2 == "" || m.None2 != "" || m.Inputs2 == nil || m.Inputs2.Builder != "" {
if m, ok := moduleOf(t, mods, "ca-trust"); !ok || m.Fp2 == "" || m.None2 != "" || m.Inputs2 == nil || m.Inputs2.Builder != "" {
t.Errorf("module = %+v (%v), want its fingerprint with no builder input", m, ok)
}
}
modules/mesh-delivery-indexer/internal/index/oldsections.go
+1 -3
View File
@@ -6,8 +6,6 @@ import (
"fmt"
"maps"
"git.novox.be/novox/mesh-sdk/go/delivery"
"mesh-delivery-indexer/judge"
)
@@ -19,7 +17,7 @@ const (
OldSectionsTool = "indexer_old_sections"
// OldSectionsState is the indexer's state of the count: the bucket mesh-delivery-indexer_oldsections.
OldSectionsState = "oldsections"
OldSectionsBucket = delivery.IndexerModule + "_" + OldSectionsState
OldSectionsBucket = Module + "_" + OldSectionsState
// OldSectionsKey is the one key it holds.
OldSectionsKey = "counts"
)
modules/mesh-delivery-indexer/internal/index/order.go
+5 -3
View File
@@ -7,8 +7,10 @@ import (
"slices"
"strings"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-delivery/sdk/trunk"
"git.novox.be/novox/mesh-sdk/go/process"
)
@@ -81,7 +83,7 @@ func (ix *Indexer) lateMerge(ctx context.Context, w *mergeWrite) error {
if ix.Taken == nil || w.merge == "" {
return nil
}
key, err := delivery.RepositoryKey(w.repository)
key, err := trunk.RepositoryKey(w.repository)
if err != nil {
return nil // writeTaken says it
}
@@ -92,7 +94,7 @@ func (ix *Indexer) lateMerge(ctx context.Context, w *mergeWrite) error {
if err != nil {
return fmt.Errorf("taken.%s: %w", key, err)
}
cur, err := delivery.DecodeTaken(value)
cur, err := takein.DecodeTaken(value)
if err != nil || cur.Merge == "" || cur.Merge == w.merge {
return nil
}
modules/mesh-delivery-indexer/internal/index/own.go
+9 -7
View File
@@ -7,8 +7,10 @@ import (
"path"
"strings"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-delivery/sdk/trunk"
)
// From R4e the indexer lists a repository's modules from what it holds itself (novox/hq ADR 0501 clause 6, issue 1851):
@@ -30,8 +32,8 @@ func manifestModule(raw string) string {
// heldModules are the values the modules bucket holds, by key (the module's name). A value that is not a module is left
// out and said.
func (ix *Indexer) heldModules(ctx context.Context) (map[string]delivery.Module, error) {
out := map[string]delivery.Module{}
func (ix *Indexer) heldModules(ctx context.Context) (map[string]meshcatalog.ModulesByModuleValue, error) {
out := map[string]meshcatalog.ModulesByModuleValue{}
if ix.Modules == nil {
return out, nil
}
@@ -47,7 +49,7 @@ func (ix *Indexer) heldModules(ctx context.Context) (map[string]delivery.Module,
if !found {
continue
}
m, err := delivery.DecodeModule(value)
m, err := meshcatalogclient.DecodeModule(value)
if err != nil {
ix.logf("modules.%s holds what is not a module, left: %v", key, err)
continue
@@ -58,10 +60,10 @@ func (ix *Indexer) heldModules(ctx context.Context) (map[string]delivery.Module,
}
// moduleAt is the module whose directory in the repository is dir, as the modules bucket holds it.
func moduleAt(held map[string]delivery.Module, repository, dir string) (string, bool) {
func moduleAt(held map[string]meshcatalog.ModulesByModuleValue, repository, dir string) (string, bool) {
want := pathOf(strings.Trim(dir, "/"))
for name, m := range held {
if m.Path == want && delivery.SameRepository(m.Repository, repository) {
if m.Path == want && trunk.SameRepository(m.Repository, repository) {
return name, true
}
}
@@ -71,7 +73,7 @@ func moduleAt(held map[string]delivery.Module, repository, dir string) (string,
// moduleOfDir is the module of a changed directory of a pull request: the module its manifest in the message names, else
// the one the modules bucket holds at that directory of the repository, else the directory's last element (the
// catalogue's rule for nearly every manifest).
func moduleOfDir(held map[string]delivery.Module, f forgemsg.Forge, dir string) string {
func moduleOfDir(held map[string]meshcatalog.ModulesByModuleValue, f forgemsg.Forge, dir string) string {
if name := manifestModule(f.Manifests[dir]); name != "" {
return name
}
modules/mesh-delivery-indexer/internal/index/package_users.go
+13 -11
View File
@@ -6,8 +6,10 @@ import (
"slices"
"strings"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/sdk/trunk"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/views"
)
@@ -62,7 +64,7 @@ func PackageUsersOf(modules, manifests, packages views.Source, only string) ([]P
if !ok {
continue
}
s, err := delivery.DecodeSeriesPackage(e.Value)
s, err := DecodeSeriesPackage(e.Value)
if err != nil {
return nil, fmt.Errorf("packages.%s holds what is not a series: %w", key, err)
}
@@ -75,12 +77,12 @@ func PackageUsersOf(modules, manifests, packages views.Source, only string) ([]P
p.Series[s.Series] = s.Newest.Version
}
}
type trunk struct {
mod delivery.Module
type trunkMod struct {
mod meshcatalog.ModulesByModuleValue
manifest catalogue.Manifest
}
var mods []string
held := map[string]trunk{}
held := map[string]trunkMod{}
names, err := modules.Keys()
if err != nil {
return nil, fmt.Errorf("the modules state cannot be listed: %w", err)
@@ -93,7 +95,7 @@ func PackageUsersOf(modules, manifests, packages views.Source, only string) ([]P
if !ok {
continue
}
mod, err := delivery.DecodeModule(e.Value)
mod, err := meshcatalogclient.DecodeModule(e.Value)
if err != nil || mod.Manifest == "" {
continue // no manifest held for it
}
@@ -104,25 +106,25 @@ func PackageUsersOf(modules, manifests, packages views.Source, only string) ([]P
if !ok {
continue
}
kept, err := delivery.DecodeManifest(me.Value)
kept, err := meshcatalogclient.DecodeManifest(me.Value)
if err != nil {
return nil, fmt.Errorf("the manifest of %s: %w", name, err)
}
m, err := catalogue.Decode([]byte(kept.JSON))
m, err := catalogue.Decode([]byte(kept.Manifest))
if err != nil {
return nil, fmt.Errorf("the manifest of %s does not read: %w", name, err)
}
for pkg := range m.Packages {
get(pkg)
}
held[name], mods = trunk{mod, m}, append(mods, name)
held[name], mods = trunkMod{mod, m}, append(mods, name)
}
slices.Sort(mods)
reaches := func(mod delivery.Module, goModule string) bool {
reaches := func(mod meshcatalog.ModulesByModuleValue, goModule string) bool {
if goModule == "" || mod.Inputs2 == nil {
return false
}
if repo, _, ok := delivery.PackageRepository(goModule); !ok || delivery.SameRepository(repo, mod.Repository) {
if repo, _, ok := trunk.PackageRepository(goModule); !ok || trunk.SameRepository(repo, mod.Repository) {
return false
}
for _, p := range append(slices.Clone(mod.Inputs2.Packages), mod.Inputs2.Imports...) {
modules/mesh-delivery-indexer/internal/index/packages.go
+30 -26
View File
@@ -9,14 +9,18 @@ import (
"slices"
"strings"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/sdk/trunk"
"mesh-delivery-indexer/internal/fingerprint"
)
// The published packages (novox/hq ADR 0593 §2): the indexer alone reads, from the forge's listing, the content digest
// of each package of every mesh Go module a held module reaches, and writes the module's latest published version and
// digests in its state `packages` (delivery.PackagesBucket), one key per Go module. The builder, the checker and the
// digests in its state `packages` (PackagesBucket), one key per Go module. The builder, the checker and the
// build modules read it there; nobody else computes a digest or a version.
//
// A Go module is reached when a held module's go.sum names it (KeptInputs2.Packages) or holds an import a build
@@ -25,8 +29,8 @@ import (
// reached module the state does not hold yet (FillPackages).
// publishedAll is every value of the packages state, by Go module path; empty with no state or one that does not answer.
func (ix *Indexer) publishedAll(ctx context.Context) map[string]delivery.PublishedPackages {
out := map[string]delivery.PublishedPackages{}
func (ix *Indexer) publishedAll(ctx context.Context) map[string]meshcatalog.PackagesByModuleValue {
out := map[string]meshcatalog.PackagesByModuleValue{}
if ix.Packages == nil {
return out
}
@@ -43,7 +47,7 @@ func (ix *Indexer) publishedAll(ctx context.Context) map[string]delivery.Publish
if err != nil || !found {
continue
}
p, err := delivery.DecodePublishedPackages(value)
p, err := DecodePublishedPackages(value)
if err != nil {
ix.logf("packages.%s holds what is not a published module, left out: %v", key, err)
continue
@@ -55,10 +59,10 @@ func (ix *Indexer) publishedAll(ctx context.Context) map[string]delivery.Publish
// reached are, by repository, the mesh Go module paths the held modules' go.sum names and the imports their builds
// reported.
func reached(held map[string]delivery.Module) (modules, imports map[string][]string) {
func reached(held map[string]meshcatalog.ModulesByModuleValue) (modules, imports map[string][]string) {
modules, imports = map[string][]string{}, map[string][]string{}
add := func(to map[string][]string, p string) {
repo, _, ok := delivery.PackageRepository(p)
repo, _, ok := trunk.PackageRepository(p)
if ok && !slices.Contains(to[repo], p) {
to[repo] = append(to[repo], p)
}
@@ -86,13 +90,13 @@ func reached(held map[string]delivery.Module) (modules, imports map[string][]str
// computed from the forge's listing (Files), by module path. An import is held by a known module when one holds its
// directory, else by the Go module whose go.mod is at its directory or the nearest above it (asked of the forge
// directory by directory). A module the listing has no go.mod for is said and left out.
func (ix *Indexer) packagesAt(ctx context.Context, repository, commit string, modules, imports []string) (map[string]delivery.PublishedPackages, error) {
func (ix *Indexer) packagesAt(ctx context.Context, repository, commit string, modules, imports []string) (map[string]meshcatalog.PackagesByModuleValue, error) {
if ix.Files == nil || commit == "" {
return nil, nil
}
dirOf := map[string]string{} // module path to its directory in the repository
for _, m := range modules {
if _, dir, ok := delivery.PackageRepository(m); ok {
if _, dir, ok := trunk.PackageRepository(m); ok {
dirOf[m] = dir
}
}
@@ -128,13 +132,13 @@ func (ix *Indexer) packagesAt(ctx context.Context, repository, commit string, mo
if holds(imp) {
continue
}
_, dir, _ := delivery.PackageRepository(imp)
_, dir, _ := trunk.PackageRepository(imp)
for d := dir; ; d = parent(d) {
if err := list([]string{d}); err != nil {
return nil, err
}
if _, ok := listings[d]["go.mod"]; ok {
m := strings.TrimSuffix(delivery.ForgeModulePrefix+strings.TrimPrefix(repository, "novox/")+"/"+d, "/")
m := strings.TrimSuffix(trunk.ForgeModulePrefix+strings.TrimPrefix(repository, "novox/")+"/"+d, "/")
dirOf[m] = d
break
}
@@ -147,7 +151,7 @@ func (ix *Indexer) packagesAt(ctx context.Context, repository, commit string, mo
if err := list(slices.Sorted(maps.Values(dirOf))); err != nil {
return nil, err
}
out := map[string]delivery.PublishedPackages{}
out := map[string]meshcatalog.PackagesByModuleValue{}
for _, m := range slices.Sorted(maps.Keys(dirOf)) {
dir := dirOf[m]
requires, pkgs, err := fingerprint.Packages(m, listings[dir])
@@ -157,9 +161,9 @@ func (ix *Indexer) packagesAt(ctx context.Context, repository, commit string, mo
}
d := dir
if d == "" {
d = delivery.RootPath
d = meshcatalogclient.RootPath
}
out[m] = delivery.PublishedPackages{Module: m, Repository: delivery.RepositoryName(repository), Dir: d, Commit: commit,
out[m] = meshcatalog.PackagesByModuleValue{Module: m, Repository: trunk.RepositoryName(repository), Dir: d, Commit: commit,
Requires: requires, Packages: pkgs}
}
return out, nil
@@ -179,12 +183,12 @@ func parent(d string) string {
// publish writes the packages state for the Go modules of repository the held modules (and the merge's overlay) reach,
// as the merge commit holds them: a key whose commit, requirements and digests are what it holds already is left.
func (ix *Indexer) publish(ctx context.Context, repository, commit string, held map[string]delivery.Module, onlyMissing bool) error {
func (ix *Indexer) publish(ctx context.Context, repository, commit string, held map[string]meshcatalog.ModulesByModuleValue, onlyMissing bool) error {
if ix.Packages == nil || ix.Files == nil {
return nil
}
modules, imports := reached(held)
repo := delivery.RepositoryName(repository)
repo := trunk.RepositoryName(repository)
if len(modules[repo]) == 0 && len(imports[repo]) == 0 {
return nil
}
@@ -209,7 +213,7 @@ func (ix *Indexer) publish(ctx context.Context, repository, commit string, held
}
// An import a published module of the repository already holds is resolved by its module's directory.
for m, p := range have {
if delivery.SameRepository(p.Repository, repo) && !slices.Contains(modules[repo], m) && !onlyMissing {
if trunk.SameRepository(p.Repository, repo) && !slices.Contains(modules[repo], m) && !onlyMissing {
modules[repo] = append(modules[repo], m)
}
}
@@ -219,20 +223,20 @@ func (ix *Indexer) publish(ctx context.Context, repository, commit string, held
}
for _, m := range slices.Sorted(maps.Keys(got)) {
next := got[m]
key, err := delivery.PackagesKey(m)
key, err := PackagesKey(m)
if err != nil {
ix.logf("%s: %v", m, err)
continue
}
err = write(ctx, ix.Packages, key, func(value []byte, found bool) ([]byte, bool, error) {
if found {
if cur, err := delivery.DecodePublishedPackages(value); err == nil && cur.Commit == next.Commit &&
if cur, err := DecodePublishedPackages(value); err == nil && cur.Commit == next.Commit &&
cur.Requires == next.Requires && reflect.DeepEqual(cur.Packages, next.Packages) {
return nil, false, nil
}
}
next.At = ix.now().UTC()
data, err := delivery.EncodePublishedPackages(next)
next.At = ptr(ix.now().UTC())
data, err := EncodePublishedPackages(next)
return data, false, err
})
if err != nil {
@@ -262,7 +266,7 @@ func (ix *Indexer) FillPackages(ctx context.Context) error {
repos[r] = true
}
for _, repo := range slices.Sorted(maps.Keys(repos)) {
key, err := delivery.RepositoryKey(repo)
key, err := trunk.RepositoryKey(repo)
if err != nil {
continue
}
@@ -270,7 +274,7 @@ func (ix *Indexer) FillPackages(ctx context.Context) error {
if err != nil || !found {
continue
}
t, err := delivery.DecodeTaken(value)
t, err := takein.DecodeTaken(value)
if err != nil {
continue
}
@@ -285,7 +289,7 @@ func (ix *Indexer) FillPackages(ctx context.Context) error {
// member's repository publishes at the member's head (members, by repository), every other at its latest published
// version. A head's modules are read from the forge once and kept while the head is a member somewhere; one the forge
// cannot list is said and left at its published version.
func (ix *Indexer) candidatePublished(ctx context.Context, members map[string]string, held map[string]delivery.Module) map[string]delivery.PublishedPackages {
func (ix *Indexer) candidatePublished(ctx context.Context, members map[string]string, held map[string]meshcatalog.ModulesByModuleValue) map[string]meshcatalog.PackagesByModuleValue {
out := ix.publishedAll(ctx)
if len(members) == 0 {
return out
@@ -304,7 +308,7 @@ func (ix *Indexer) candidatePublished(ctx context.Context, members map[string]st
var ms []string
ms = append(ms, modules[repo]...)
for m, p := range out {
if delivery.SameRepository(p.Repository, repo) && !slices.Contains(ms, m) {
if trunk.SameRepository(p.Repository, repo) && !slices.Contains(ms, m) {
ms = append(ms, m)
}
}
@@ -315,7 +319,7 @@ func (ix *Indexer) candidatePublished(ctx context.Context, members map[string]st
}
ix.mu.Lock()
if ix.atHeads == nil || len(ix.atHeads) >= maxHeadsKept {
ix.atHeads = map[string]map[string]delivery.PublishedPackages{}
ix.atHeads = map[string]map[string]meshcatalog.PackagesByModuleValue{}
}
ix.atHeads[at] = got
ix.mu.Unlock()
modules/mesh-delivery-indexer/internal/index/packages_values.go
+189
View File
@@ -0,0 +1,189 @@
package index
import (
"encoding/json"
"fmt"
"regexp"
"strings"
"time"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/sdk/trunk"
"git.novox.be/novox/mesh-sdk/go/keys"
)
// The values of the state packages (novox/hq ADR 0593 §2, ADR 0606 §1): the contract's types, written by the indexer
// alone, so their writing rules are the indexer's (ADR 0634 §1: a holder's own logic stays with it).
// PackagesKey is the key of a mesh Go module (its module path) in the packages state: the path as a key part. It refuses
// a path that is no mesh Go module's.
func PackagesKey(module string) (string, error) {
if _, _, ok := trunk.PackageRepository(module); !ok {
return "", fmt.Errorf("index: %q is no mesh Go module's path (it starts with %s)", module, trunk.ForgeModulePrefix)
}
return keys.EscapeName(module), nil
}
func checkPublished(p meshcatalog.PackagesByModuleValue) error {
pkgs := make([][2]string, len(p.Packages))
for i, d := range p.Packages {
pkgs[i] = [2]string{d.Package, d.Digest}
}
return checkPublishedFields(p.Module, p.Repository, p.Dir, p.Commit, p.Requires, p.At, pkgs)
}
var commitFull = regexp.MustCompile(`^[0-9a-f]{40}$`)
// checkPublishedFields is the rule of a published module's value, over its fields (a series' contents hold the same).
func checkPublishedFields(module, repository, dir, commit, requires string, at *time.Time, packages [][2]string) error {
want, _, ok := trunk.PackageRepository(module)
switch {
case !ok:
return fmt.Errorf("index: %q is no mesh Go module's path (it starts with %s)", module, trunk.ForgeModulePrefix)
case !trunk.SameRepository(repository, want):
return fmt.Errorf("index: the module %s is published from %s, and the value names the repository %q", module, want, repository)
case dir == "":
return fmt.Errorf("index: the published module %s names no directory (%q for the repository's root)", module, meshcatalogclient.RootPath)
case !commitFull.MatchString(commit):
return fmt.Errorf("index: the published module %s names the commit %q, which is no full commit", module, commit)
case requires == "":
return fmt.Errorf("index: the published module %s names no digest of its requirements", module)
case at == nil || at.IsZero():
return fmt.Errorf("index: the published module %s names no time", module)
}
for i, d := range packages {
if d[0] != module && !strings.HasPrefix(d[0], module+"/") {
return fmt.Errorf("index: the package %q is not in the module %s", d[0], module)
}
if d[1] == "" {
return fmt.Errorf("index: the package %s names no digest", d[0])
}
if i > 0 && packages[i-1][0] >= d[0] {
return fmt.Errorf("index: the packages of %s are not sorted and each once (%s after %s)", module, d[0], packages[i-1][0])
}
}
return nil
}
// EncodePublishedPackages is p as JSON at the kept values' version; it refuses what a reader could not take.
func EncodePublishedPackages(p meshcatalog.PackagesByModuleValue) ([]byte, error) {
if err := checkPublished(p); err != nil {
return nil, err
}
p.V = keys.Version
return json.Marshal(p)
}
// DecodePublishedPackages reads a per-module packages value; it refuses an unknown version (keys.ErrUnknownVersion) and
// what EncodePublishedPackages refuses.
func DecodePublishedPackages(data []byte) (meshcatalog.PackagesByModuleValue, error) {
if err := keys.CheckVersion(data); err != nil {
return meshcatalog.PackagesByModuleValue{}, err
}
var p meshcatalog.PackagesByModuleValue
if err := json.Unmarshal(data, &p); err != nil {
return meshcatalog.PackagesByModuleValue{}, fmt.Errorf("index: a packages value does not fit its shape: %w", err)
}
if err := checkPublished(p); err != nil {
return meshcatalog.PackagesByModuleValue{}, err
}
return p, nil
}
// SeriesPackageKey is the key of a package's series in the packages state: its ecosystem name as a key part and the series.
func SeriesPackageKey(name, series string) string { return keys.EscapeName(name) + "." + series }
// IsSeriesPackageKey says whether a key of the packages state is a series key, not a per-module one (a series key always
// holds a dot, which keys.EscapeName never leaves in a per-module key).
func IsSeriesPackageKey(key string) bool { return strings.Contains(key, ".") }
// builtFrom says whether pkg is `<module>/<artifact>`: a module name (no slash) and an artifact named after it.
func builtFrom(pkg string) bool {
module, artifact, ok := strings.Cut(pkg, "/")
return ok && module != "" && artifact != ""
}
var seriesForm = regexp.MustCompile(`^[0-9]+\.[0-9]+$`)
func checkSeries(s meshcatalog.PackagesByPackageSeriesValue) error {
switch {
case s.Name == "":
return fmt.Errorf("index: a series of %q names no package in its ecosystem", s.Package)
case !seriesForm.MatchString(s.Series):
return fmt.Errorf("index: the series %q of %s is no <major>.<minor>", s.Series, s.Name)
case s.Package != "" && !builtFrom(s.Package):
return fmt.Errorf("index: the series %s of %s is built from %q, which is no <module>/<artifact>", s.Series, s.Name, s.Package)
}
if v := s.Newest; v != nil {
if v.Version == "" {
return fmt.Errorf("index: the newest version of %s %s names no version", s.Name, s.Series)
}
if c := v.Contents; c != nil {
pkgs := make([][2]string, len(c.Packages))
for i, d := range c.Packages {
pkgs[i] = [2]string{d.Package, d.Digest}
}
if err := checkPublishedFields(c.Module, c.Repository, c.Dir, c.Commit, c.Requires, c.At, pkgs); err != nil {
return err
}
}
}
if v := s.Numbered; v != nil {
if v.Version == "" {
return fmt.Errorf("index: the numbered version of %s %s names no version", s.Name, s.Series)
}
if c := v.Contents; c != nil {
pkgs := make([][2]string, len(c.Packages))
for i, d := range c.Packages {
pkgs[i] = [2]string{d.Package, d.Digest}
}
if err := checkPublishedFields(c.Module, c.Repository, c.Dir, c.Commit, c.Requires, c.At, pkgs); err != nil {
return err
}
}
}
return nil
}
// EncodeSeriesPackage is s as JSON at the kept values' version; it refuses what a reader could not take.
func EncodeSeriesPackage(s meshcatalog.PackagesByPackageSeriesValue) ([]byte, error) {
if err := checkSeries(s); err != nil {
return nil, err
}
s.V = keys.Version
return json.Marshal(s)
}
// DecodeSeriesPackage reads a series key's value; it refuses an unknown version and what EncodeSeriesPackage refuses.
func DecodeSeriesPackage(data []byte) (meshcatalog.PackagesByPackageSeriesValue, error) {
if err := keys.CheckVersion(data); err != nil {
return meshcatalog.PackagesByPackageSeriesValue{}, err
}
var s meshcatalog.PackagesByPackageSeriesValue
if err := json.Unmarshal(data, &s); err != nil {
return meshcatalog.PackagesByPackageSeriesValue{}, fmt.Errorf("index: a series value does not fit its shape: %w", err)
}
if err := checkSeries(s); err != nil {
return meshcatalog.PackagesByPackageSeriesValue{}, err
}
return s, nil
}
// publishedOfContents is a series version's contents as the value of a per-module packages key (the same fields).
func publishedOfContents(c meshcatalog.PackagesByPackageSeriesValueNewestContents) meshcatalog.PackagesByModuleValue {
p := meshcatalog.PackagesByModuleValue{At: c.At, Commit: c.Commit, Dir: c.Dir, Module: c.Module, Repository: c.Repository, Requires: c.Requires, V: c.V}
for _, d := range c.Packages {
p.Packages = append(p.Packages, meshcatalog.PackagesByModuleValuePackagesItem{Package: d.Package, Digest: d.Digest})
}
return p
}
// contentsOfPublished is the value of a per-module packages key as a series version's contents.
func contentsOfPublished(p meshcatalog.PackagesByModuleValue) *meshcatalog.PackagesByPackageSeriesValueNewestContents {
c := &meshcatalog.PackagesByPackageSeriesValueNewestContents{At: p.At, Commit: p.Commit, Dir: p.Dir, Module: p.Module, Repository: p.Repository, Requires: p.Requires, V: p.V}
for _, d := range p.Packages {
c.Packages = append(c.Packages, meshcatalog.PackagesByPackageSeriesValueNewestContentsPackagesItem{Package: d.Package, Digest: d.Digest})
}
return c
}
modules/mesh-delivery-indexer/internal/index/paths_backfill_test.go
+10 -8
View File
@@ -4,7 +4,9 @@ import (
"context"
"testing"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/sdk/buildrecord"
)
// Issue 1614: a modules key written before the indexer learned paths (mesh-tools and node-tools live) gets its directory
@@ -13,8 +15,8 @@ import (
func heldValue(t *testing.T, st *memStore, name, path string) {
t.Helper()
data, err := delivery.EncodeModule(delivery.Module{Repository: "novox/mesh-tools", Commit: "e0fadc43d9dda0bc24bd19bb05b021dd87da8582",
FP2: "src2:" + hex64('2'), Manifest: hex64('9'), Path: path})
data, err := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Repository: "novox/mesh-tools", Commit: "e0fadc43d9dda0bc24bd19bb05b021dd87da8582",
Fp2: "src2:" + hex64('2'), Manifest: hex64('9'), Path: path})
if err != nil {
t.Fatal(err)
}
@@ -31,17 +33,17 @@ func hex64(c byte) string {
func TestAnEmptyPathIsFilledOnceFromTheModulesNewestBuildRecord(t *testing.T) { // 1614
ix, mods := modulesIndexer(t,
[2]any{delivery.BuildKey("mesh-tools", "build-1"), buildRecord(t, "novox/mesh-tools", "mesh-tools", "", "2026-10-19T09:00:00Z")},
[2]any{delivery.BuildKey("node-tools", "build-2"), buildRecord(t, "novox/mesh-tools", "node-tools", "node-tools", "2026-10-19T09:00:00Z")})
[2]any{buildrecord.BuildKey("mesh-tools", "build-1"), buildRecord(t, "novox/mesh-tools", "mesh-tools", "", "2026-10-19T09:00:00Z")},
[2]any{buildrecord.BuildKey("node-tools", "build-2"), buildRecord(t, "novox/mesh-tools", "node-tools", "node-tools", "2026-10-19T09:00:00Z")})
heldValue(t, mods, "mesh-tools", "")
heldValue(t, mods, "node-tools", "")
before, _ := moduleOf(t, mods, "node-tools")
if err := ix.Recompute(context.Background()); err != nil {
t.Fatal(err)
}
for name, want := range map[string]string{"mesh-tools": delivery.RootPath, "node-tools": "node-tools"} {
for name, want := range map[string]string{"mesh-tools": meshcatalogclient.RootPath, "node-tools": "node-tools"} {
m, _ := moduleOf(t, mods, name)
if m.Path != want || m.FP2 != before.FP2 || m.Commit != before.Commit {
if m.Path != want || m.Fp2 != before.Fp2 || m.Commit != before.Commit {
t.Errorf("%s = %+v, want path %q and nothing else changed", name, m, want)
}
}
@@ -55,7 +57,7 @@ func TestAnEmptyPathIsFilledOnceFromTheModulesNewestBuildRecord(t *testing.T) {
}
func TestASetPathIsNeverOverwrittenByABuildRecord(t *testing.T) { // 1614
ix, mods := modulesIndexer(t, [2]any{delivery.BuildKey("node-tools", "build-2"), buildRecord(t, "novox/mesh-tools", "node-tools", "elsewhere", "2026-10-19T09:00:00Z")})
ix, mods := modulesIndexer(t, [2]any{buildrecord.BuildKey("node-tools", "build-2"), buildRecord(t, "novox/mesh-tools", "node-tools", "elsewhere", "2026-10-19T09:00:00Z")})
heldValue(t, mods, "node-tools", "node-tools")
writes := mods.writes
if err := ix.Recompute(context.Background()); err != nil {
modules/mesh-delivery-indexer/internal/index/pending.go
+22 -17
View File
@@ -10,9 +10,14 @@ import (
"strings"
"time"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/nodebuildgo"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-delivery/sdk/trunk"
"git.novox.be/novox/mesh-sdk/go/keys"
"git.novox.be/novox/mesh-sdk/go/process"
"git.novox.be/novox/mesh-delivery/sdk/graph"
@@ -26,7 +31,7 @@ const (
// takein.PendingKey, so the key order is the merge order. An old `state` entry (the indexer is foundation, ADR
// 0595 §11), history 1, reached by the indexer alone.
PendingState = "pending"
PendingBucket = delivery.IndexerModule + "_" + PendingState
PendingBucket = Module + "_" + PendingState
// PendingTool reads it (ADR 0633 §5); TakeInPendingTool and DropPendingTool steer it.
PendingTool = "indexer_pending"
TakeInPendingTool = "indexer_take_in_pending"
@@ -52,8 +57,8 @@ type PendingRef struct {
Number int `json:"number,omitempty"`
Merge string `json:"merge"`
// Branch is the pull request's branch, which names its candidate: a merge covered by a newer one is taken out of it.
Branch string `json:"branch,omitempty"`
Members []delivery.Head `json:"members,omitempty"`
Branch string `json:"branch,omitempty"`
Members []meshdeliveryclient.Head `json:"members,omitempty"`
// Holds are the modules the merge's manifests name, which indexer_pending says it holds back.
Holds []string `json:"holds,omitempty"`
}
@@ -107,7 +112,7 @@ func DecodePending(data []byte) (Pending, forgemsg.Forge, error) {
}
if len(p.Forge) == 0 && p.Ref != nil {
r := p.Ref
return p, forgemsg.Forge{V: delivery.Version, Kind: forgemsg.ForgeMerged, Repository: r.Repository, Number: r.Number,
return p, forgemsg.Forge{V: keys.Version, Kind: forgemsg.ForgeMerged, Repository: r.Repository, Number: r.Number,
Merge: r.Merge, Branch: r.Branch, Group: p.Group, Members: r.Members}, nil
}
f, err := forgemsg.DecodeForge(p.Forge)
@@ -462,7 +467,7 @@ func (ix *Indexer) EndGroup(ctx context.Context, end forgemsg.ForgeGroupEnd) (Ap
}
isHeld := func(m forgemsg.GroupMerge) bool {
return slices.ContainsFunc(held, func(e PendingEntry) bool {
return delivery.SameRepository(e.Forge.Repository, m.Repository) && e.Forge.Number == m.Number
return trunk.SameRepository(e.Forge.Repository, m.Repository) && e.Forge.Number == m.Number
})
}
var missing []string
@@ -505,7 +510,7 @@ func (ix *Indexer) takenAlready(ctx context.Context, m forgemsg.GroupMerge) (boo
if ix.Taken == nil {
return false, nil
}
key, err := delivery.RepositoryKey(m.Repository)
key, err := trunk.RepositoryKey(m.Repository)
if err != nil {
return false, err
}
@@ -516,7 +521,7 @@ func (ix *Indexer) takenAlready(ctx context.Context, m forgemsg.GroupMerge) (boo
if err != nil {
return false, fmt.Errorf("taken.%s: %w", key, err)
}
cur, err := delivery.DecodeTaken(value)
cur, err := takein.DecodeTaken(value)
if err != nil || cur.Merge == "" {
return false, nil
}
@@ -706,7 +711,7 @@ func (ix *Indexer) dry() *dryRun {
ix.mu.Lock()
x.source = sourceNow{rev: ix.source.rev, heads: maps.Clone(ix.source.heads)}
x.previews = maps.Clone(ix.previews)
x.toolchains = map[string]map[string]delivery.Toolchain{}
x.toolchains = map[string]map[string]nodebuildgo.Toolchain{}
for l, by := range ix.toolchains {
x.toolchains[l] = maps.Clone(by)
}
@@ -737,7 +742,7 @@ func (ix *Indexer) computeOver(ctx context.Context, ready []PendingEntry) (strin
return "", d, fmt.Errorf("computing the take-in of %s (%s#%d): %w", e.Key, e.Forge.Repository, e.Forge.Number, err)
}
}
key, _ := delivery.ModulesKey(delivery.IndexerModule)
key, _ := meshcatalogclient.ModulesKey(Module)
value, _, found, err := d.ix.Modules.Get(ctx, key)
if err != nil {
return "", d, err
@@ -745,14 +750,14 @@ func (ix *Indexer) computeOver(ctx context.Context, ready []PendingEntry) (strin
if !found {
return NoBuild + "the indexer holds no record of itself", d, nil
}
m, err := delivery.DecodeModule(value)
m, err := meshcatalogclient.DecodeModule(value)
if err != nil {
return "", d, nil
}
if m.FP2 == "" {
if m.Fp2 == "" {
return NoBuild + m.None2, d, nil
}
return m.FP2, d, nil
return m.Fp2, d, nil
}
// claim marks the entries as taken in by this take-in, by compare-and-set on the revisions read. A lost one (another
@@ -911,18 +916,18 @@ func saidWaits(entries []PendingEntry) []takein.TakeIn {
// manifest, from the dry run; the manifests and the packages are written by the dry run itself. The revision is said
// pending under its oldest merge's key, and a wait it was said under before by another key is said taken.
func (ix *Indexer) holdFor(ctx context.Context, ready []PendingEntry, names, own string, d *dryRun) error {
key, _ := delivery.ModulesKey(delivery.IndexerModule)
key, _ := meshcatalogclient.ModulesKey(Module)
if err := copyKey(ctx, d.modules, ix.Modules, key); err != nil {
return fmt.Errorf("the indexer's own module entry: %w", err)
}
if gk, err := graph.Key(delivery.IndexerModule); err == nil {
if gk, err := graph.Key(Module); err == nil {
if err := copyKey(ctx, d.graph, ix.Graph, gk); err != nil && !errors.Is(err, process.ErrNotYet) {
return fmt.Errorf("the indexer's key of the module graph: %w", err)
}
}
if value, _, found, err := d.ix.Modules.Get(ctx, key); err == nil && found {
if m, err := delivery.DecodeModule(value); err == nil && m.Manifest != "" {
if vk, err := delivery.VerdictsKey(m.Manifest); err == nil {
if m, err := meshcatalogclient.DecodeModule(value); err == nil && m.Manifest != "" {
if vk, err := meshcatalogclient.VerdictsKey(m.Manifest); err == nil {
if err := copyKey(ctx, d.verdicts, ix.Verdicts, vk); err != nil && !errors.Is(err, process.ErrNotYet) {
return fmt.Errorf("the verdict on the indexer's manifest: %w", err)
}
modules/mesh-delivery-indexer/internal/index/pending_test.go
+23 -23
View File
@@ -9,10 +9,10 @@ import (
"strings"
"testing"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein"
"git.novox.be/novox/mesh-sdk/go/buscompose"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/process"
)
@@ -31,7 +31,7 @@ type takeInBed struct {
func newTakeInBed(t *testing.T) *takeInBed {
t.Helper()
ix, mods, f := newFormIndexerUnmerged(t, mod{"avahi", catalogueManifest(t, "avahi")},
mod{delivery.IndexerModule, renamed(t, "avahi", delivery.IndexerModule)})
mod{Module, renamed(t, "avahi", Module)})
if err := ix.Apply(context.Background(), f); err != nil {
t.Fatal(err)
}
@@ -55,7 +55,7 @@ func newTakeInBed(t *testing.T) *takeInBed {
logf(format, args...)
}
}
own := b.fp(t, delivery.IndexerModule)
own := b.fp(t, Module)
if own == "" || b.fp(t, "avahi") == "" {
t.Fatalf("the bed's modules have no fingerprint")
}
@@ -66,7 +66,7 @@ func newTakeInBed(t *testing.T) *takeInBed {
func (b *takeInBed) fp(t *testing.T, module string) string {
t.Helper()
m, _ := moduleOf(t, b.mods, module)
return m.FP2
return m.Fp2
}
func (b *takeInBed) commit(t *testing.T, module string) string {
@@ -87,7 +87,7 @@ func (b *takeInBed) merge(number int, c, avahi, indexer string) forgemsg.Forge {
f.Trees["modules/avahi"] = avahi
}
if indexer != "" {
f.Trees["modules/"+delivery.IndexerModule] = indexer
f.Trees["modules/"+Module] = indexer
}
return f
}
@@ -147,7 +147,7 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633)
})
t.Run("T7 and T3 a merge alone, an indexer change, a merge behind it and a revert", func(t *testing.T) {
b := newTakeInBed(t)
own, avahi0 := b.fp(t, delivery.IndexerModule), b.fp(t, "avahi")
own, avahi0 := b.fp(t, Module), b.fp(t, "avahi")
// A merge with no change to the indexer and nothing pending is taken in alone, as before.
b.arrive(t, 1, b.merge(20, sha('b'), "tree-avahi-2", ""))
if avahi1 := b.fp(t, "avahi"); avahi1 == avahi0 || len(b.keys()) != 0 || len(b.said) != 0 {
@@ -156,11 +156,11 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633)
avahi1, verdicts := b.fp(t, "avahi"), len(b.ix.Verdicts.(*memStore).vals)
// T4: a change to the indexer writes only its entry (with its graph key and verdict); the rest is pending.
b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2"))
named := b.fp(t, delivery.IndexerModule)
named := b.fp(t, Module)
if named == own || b.fp(t, "avahi") != avahi1 || b.commit(t, "avahi") != sha('b') {
t.Fatalf("indexer change: indexer %s (own %s), avahi %s at %s: want only the indexer written", named, own, b.fp(t, "avahi"), b.commit(t, "avahi"))
}
if g := graphOf(t, b.ix, delivery.IndexerModule); g.FP != named {
if g := graphOf(t, b.ix, Module); g.FP != named {
t.Errorf("graph of the indexer names %s, want %s", g.FP, named)
}
if n := len(b.ix.Verdicts.(*memStore).vals); n > verdicts+1 {
@@ -180,9 +180,9 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633)
t.Fatalf("behind: pending %v, avahi at %s, said %v: want both pending and nothing said again", b.keys(), b.commit(t, "avahi"), b.events())
}
// A revert brings the indexer back to the running build: everything is taken in, oldest first.
b.arrive(t, 4, b.merge(23, sha('e'), "tree-avahi-5", b.base.Trees["modules/"+delivery.IndexerModule]))
if len(b.keys()) != 0 || b.fp(t, delivery.IndexerModule) != own || b.commit(t, "avahi") != sha('e') {
t.Fatalf("revert: pending %v, indexer %s, avahi at %s: want all taken in", b.keys(), b.fp(t, delivery.IndexerModule), b.commit(t, "avahi"))
b.arrive(t, 4, b.merge(23, sha('e'), "tree-avahi-5", b.base.Trees["modules/"+Module]))
if len(b.keys()) != 0 || b.fp(t, Module) != own || b.commit(t, "avahi") != sha('e') {
t.Fatalf("revert: pending %v, indexer %s, avahi at %s: want all taken in", b.keys(), b.fp(t, Module), b.commit(t, "avahi"))
}
if want := []string{"pending " + takein.PendingKey(2), "taken " + takein.PendingKey(2)}; !slices.Equal(b.events(), want) {
t.Errorf("said %v, want %v", b.events(), want)
@@ -193,9 +193,9 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633)
b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2"))
g := b.merge(30, sha('f'), "", "")
g.Repository, g.Trees, g.Manifests, g.Locks, g.Group = "novox/mesh-sdk", map[string]string{}, nil, nil, "feat/x"
g.Members = []delivery.Head{{Repository: "novox/mesh-sdk", Number: 30, Head: g.Head}, {Repository: "novox/mesh-host", Number: 4, Head: sha('9')}}
g.Members = []meshdeliveryclient.Head{{Repository: "novox/mesh-sdk", Number: 30, Head: g.Head}, {Repository: "novox/mesh-host", Number: 4, Head: sha('9')}}
b.arrive(t, 5, g)
named := b.fp(t, delivery.IndexerModule)
named := b.fp(t, Module)
// The new indexer starts; its build is not known at first: a hand ask is answered meanwhile (R5), and the start waits.
next := &Indexer{}
*next = Indexer{Store: b.ix.Store, Modules: b.ix.Modules, Manifests: b.ix.Manifests, Graph: b.ix.Graph, Verdicts: b.ix.Verdicts,
@@ -234,7 +234,7 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633)
member := func(repo string, number int, c string) forgemsg.Forge {
f := b.merge(number, c, "", "")
f.Group = "feat/take-in"
f.Members = []delivery.Head{{Repository: catalogueRepo, Number: 40, Head: sha('1')}, {Repository: "novox/mesh-sdk", Number: 41, Head: sha('2')}}
f.Members = []meshdeliveryclient.Head{{Repository: catalogueRepo, Number: 40, Head: sha('1')}, {Repository: "novox/mesh-sdk", Number: 41, Head: sha('2')}}
if repo != catalogueRepo {
f.Repository, f.Trees, f.Manifests, f.Locks = repo, map[string]string{}, nil, nil
}
@@ -287,7 +287,7 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633)
t.Run("T5 two take-ins of one pending revision: the one that loses the claim writes nothing", func(t *testing.T) {
b := newTakeInBed(t)
b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2"))
named := b.fp(t, delivery.IndexerModule)
named := b.fp(t, Module)
b.ix.ownMu.Lock()
b.ix.ownFP = named
b.ix.ownMu.Unlock()
@@ -343,7 +343,7 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633)
b := newTakeInBed(t)
b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2"))
b.arrive(t, 3, b.merge(22, sha('d'), "tree-avahi-4", "tree-indexer-2"))
named := b.fp(t, delivery.IndexerModule)
named := b.fp(t, Module)
b.ix.ownMu.Lock()
b.ix.ownFP = named
b.ix.ownMu.Unlock()
@@ -369,7 +369,7 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633)
// The build the revision names under the next toolchain, worked out on a dry indexer; this one runs it.
x := b.ix.dry().ix
for _, node := range []string{"novox", "ace"} {
key, _ := delivery.ToolchainKey(node, "go")
key, _ := ToolchainKey(node, "go")
x.PutToolchain("mesh-build-go", key, toolchainValue(t, "go", "go1.27.2", "1"), false)
}
ready, _, _ := b.ix.loadNewest(ctx, WaitsOf(entries).Ready)
@@ -403,7 +403,7 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633)
answer = out.(takein.PendingAnswer)
}
}
if len(answer.Waits) != 1 || answer.Waits[0].Key != takein.PendingKey(2) || !slices.Equal(answer.Waits[0].Holds, []string{"avahi", delivery.IndexerModule}) ||
if len(answer.Waits) != 1 || answer.Waits[0].Key != takein.PendingKey(2) || !slices.Equal(answer.Waits[0].Holds, []string{"avahi", Module}) ||
answer.Waits[0].Said != takein.TakeInPendingEvent || len(answer.Waits[0].Keys) != 2 {
t.Fatalf("indexer_pending = %+v", answer)
}
@@ -447,7 +447,7 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633)
t.Fatal(err)
}
}
key, _ := delivery.CandidatesKey("feat/g")
key, _ := CandidatesKey("feat/g")
if _, _, found, _ := b.ix.Store.Get(ctx, key); !found {
t.Fatal("the group has no candidate")
}
@@ -457,7 +457,7 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633)
if len(b.keys()) != 2 || len(raw) > 4096 || b.commit(t, "avahi") != sha('b') {
t.Fatalf("a big merge held: pending %v, value %d bytes, avahi at %s: want two small values", b.keys(), len(raw), b.commit(t, "avahi"))
}
named := b.fp(t, delivery.IndexerModule)
named := b.fp(t, Module)
// The new indexer starts with nothing in memory and the bridge down: the newest merge is said lost with the bridge's
// words (event, state), the older one is covered and not lost, and nothing is taken in.
next := &Indexer{Store: b.ix.Store, Modules: b.ix.Modules, Manifests: b.ix.Manifests, Graph: b.ix.Graph, Verdicts: b.ix.Verdicts,
@@ -497,7 +497,7 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633)
}
// A hand take-in of a merge by its commit asks the bridge and runs the message through Arrive.
b.ix.Message = answers
b.publish(t, big(b.merge(30, sha('e'), "tree-avahi-6", b.base.Trees["modules/"+delivery.IndexerModule])))
b.publish(t, big(b.merge(30, sha('e'), "tree-avahi-6", b.base.Trees["modules/"+Module])))
if r := b.ix.TakeInMerge(ctx, catalogueRepo, sha('e'), 9, "lead@shanks", "held by the cap"); !r.Stored || b.commit(t, "avahi") != sha('e') {
t.Fatalf("merge take-in: %+v, avahi at %s: want it taken in", r, b.commit(t, "avahi"))
}
@@ -505,9 +505,9 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633)
t.Errorf("merge take-in of a merge the bridge does not know: %+v", r)
}
// An old value that holds the message whole reads as before.
old := b.merge(31, sha('f'), "tree-avahi-7", b.base.Trees["modules/"+delivery.IndexerModule])
old := b.merge(31, sha('f'), "tree-avahi-7", b.base.Trees["modules/"+Module])
rawOld, _ := forgemsg.EncodeForge(old)
data, _ := encodePending(Pending{Forge: rawOld, Since: b.ix.now(), LastWriter: b.fp(t, delivery.IndexerModule)})
data, _ := encodePending(Pending{Forge: rawOld, Since: b.ix.now(), LastWriter: b.fp(t, Module)})
if err := b.pending.Create(ctx, takein.PendingKey(11), data); err != nil {
t.Fatal(err)
}
modules/mesh-delivery-indexer/internal/index/previews.go
+56 -52
View File
@@ -8,7 +8,11 @@ import (
"slices"
"strings"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdeliverychecker"
sdkstep "git.novox.be/novox/mesh-delivery/sdk/buildstep"
"git.novox.be/novox/mesh-delivery/sdk/trunk"
"mesh-delivery-indexer/internal/fingerprint"
)
@@ -27,7 +31,7 @@ const (
NonePreviewConflict = "does not merge: the head conflicts with the trunk in "
// NonePreviewNoModule is for a module whose directory the preview names no manifest for: mesh-sdk's words, which the
// assigner reads as the change removing the module (issue 1297).
NonePreviewNoModule = delivery.CandidateNoneNoModule
NonePreviewNoModule = CandidateNoneNoModule
// NonePreviewNoManifest is for a module whose manifest at the merge the indexer does not hold.
NonePreviewNoManifest = "the manifest of the merge is not kept"
// NoneNoDirectory is for a module whose directory in its repository neither the modules bucket nor the preview tells.
@@ -35,10 +39,10 @@ const (
)
// PutPreview takes a change of the previews bucket in without writing any candidate: a put of a preview (key
// delivery.PreviewKey) or its delete. It answers whether the previews held changed. A key or a value that is not a
// PreviewKey) or its delete. It answers whether the previews held changed. A key or a value that is not a
// preview is said in the log and left out.
func (ix *Indexer) PutPreview(key string, value []byte, deleted bool) bool {
if _, _, _, err := delivery.ParsePreviewKey(key); err != nil {
if _, _, _, err := ParsePreviewKey(key); err != nil {
ix.logf("previews.%s is not a preview key, left out: %v", key, err)
return false
}
@@ -51,15 +55,15 @@ func (ix *Indexer) PutPreview(key string, value []byte, deleted bool) bool {
delete(ix.previews, key)
return true
}
pv, err := delivery.DecodePreview(value)
pv, err := DecodePreview(value)
if err != nil {
ix.logf("previews.%s is not a preview, left out: %v", key, err)
return false
}
if ix.previews == nil {
ix.previews = map[string]delivery.Preview{}
ix.previews = map[string]meshdeliverychecker.PreviewsByRepositoryHeadTrunkValue{}
}
if cur, had := ix.previews[key]; had && delivery.SamePreview(cur, pv) {
if cur, had := ix.previews[key]; had && SamePreview(cur, pv) {
return false
}
ix.previews[key] = pv
@@ -96,7 +100,7 @@ func (ix *Indexer) RecomputeCandidates(ctx context.Context) error {
if !found {
return nil, false, nil
}
c, err := delivery.DecodeCandidate(value)
c, err := DecodeCandidate(value)
if err != nil {
ix.logf("candidates.%s holds what is not a candidate, left: %v", key, err)
return nil, false, nil
@@ -104,7 +108,7 @@ func (ix *Indexer) RecomputeCandidates(ctx context.Context) error {
if !ix.withPreviews(ctx, &c) {
return nil, false, nil
}
data, err := delivery.EncodeCandidate(c)
data, err := EncodeCandidate(c)
return data, false, err
})
if err != nil {
@@ -116,24 +120,24 @@ func (ix *Indexer) RecomputeCandidates(ctx context.Context) error {
// previewOf is the preview of a head merged onto the trunk head the source heads name: found, or stale when only a preview
// onto another trunk head exists, or neither (found false, stale false) when the source is not known or none exists.
func (ix *Indexer) previewOf(heads map[string]string, repository, head string) (pv delivery.Preview, found, stale bool) {
func (ix *Indexer) previewOf(heads map[string]string, repository, head string) (pv meshdeliverychecker.PreviewsByRepositoryHeadTrunkValue, found, stale bool) {
ix.mu.Lock()
defer ix.mu.Unlock()
trunk := heads[repository]
if trunk == "" {
return delivery.Preview{}, false, false
trunkHead := heads[repository]
if trunkHead == "" {
return meshdeliverychecker.PreviewsByRepositoryHeadTrunkValue{}, false, false
}
if key, err := delivery.PreviewKey(repository, head, trunk); err == nil {
if key, err := PreviewKey(repository, head, trunkHead); err == nil {
if pv, ok := ix.previews[key]; ok {
return pv, true, false
}
}
for _, pv := range ix.previews {
if pv.Repository == repository && pv.Head == head {
return delivery.Preview{}, false, true
return meshdeliverychecker.PreviewsByRepositoryHeadTrunkValue{}, false, true
}
}
return delivery.Preview{}, false, false
return meshdeliverychecker.PreviewsByRepositoryHeadTrunkValue{}, false, false
}
// withPreviews gives each touched module of the candidate's pull requests the fingerprint its preview gives it, or the
@@ -144,7 +148,7 @@ func (ix *Indexer) previewOf(heads map[string]string, repository, head string) (
// whenever the fingerprints are, so the builder merges a preview onto the trunk head its fingerprint was computed on (the head
// of the pull request's repository in that revision), never onto a module value's commit, which stays at the commit its
// fingerprint first appeared at.
func (ix *Indexer) withPreviews(ctx context.Context, c *delivery.Candidate) bool {
func (ix *Indexer) withPreviews(ctx context.Context, c *meshcatalog.CandidatesByGroupValue) bool {
ix.mu.Lock()
rev, heads := ix.source.rev, ix.source.heads
ix.mu.Unlock()
@@ -153,13 +157,13 @@ func (ix *Indexer) withPreviews(ctx context.Context, c *delivery.Candidate) bool
// latest published version (novox/hq ADR 0573 §4): a group's members build against the group's own package.
members := map[string]string{}
for _, p := range c.Pulls {
members[delivery.RepositoryName(p.Repository)] = p.Head.Head
members[trunk.RepositoryName(p.Repository)] = p.Head
}
var held map[string]delivery.Module
var held map[string]meshcatalog.ModulesByModuleValue
listed := false
for i := range c.Pulls {
p := &c.Pulls[i]
pv, found, stale := ix.previewOf(heads, p.Repository, p.Head.Head)
pv, found, stale := ix.previewOf(heads, p.Repository, p.Head)
if !found && !stale {
continue
}
@@ -175,7 +179,7 @@ func (ix *Indexer) withPreviews(ctx context.Context, c *delivery.Candidate) bool
// whose manifest is the one the pull request gave the module (a module the pull request adds; ADR 0501 clause 6).
dirs := map[string]string{}
for name, m := range held {
if delivery.SameRepository(m.Repository, p.Repository) && m.Path != "" {
if trunk.SameRepository(m.Repository, p.Repository) && m.Path != "" {
dirs[name] = m.Path
}
}
@@ -191,7 +195,7 @@ func (ix *Indexer) withPreviews(ctx context.Context, c *delivery.Candidate) bool
}
fps := maps.Clone(p.Fps)
for _, m := range p.Modules {
next := delivery.CandidateModule{Manifest: p.Fps[m].Manifest}
next := meshcatalog.CandidatesByGroupValuePullsItemFpsValue{Manifest: p.Fps[m].Manifest}
switch {
case stale:
next.None2 = NonePreviewStale
@@ -206,22 +210,22 @@ func (ix *Indexer) withPreviews(ctx context.Context, c *delivery.Candidate) bool
next = ix.candidateModule(ctx, m, dir, pv, held, members)
}
if fps == nil {
fps = map[string]delivery.CandidateModule{}
fps = map[string]meshcatalog.CandidatesByGroupValuePullsItemFpsValue{}
}
if !reflect.DeepEqual(fps[m], next) {
fps[m] = next
changed = true
}
}
if changed || !maps.EqualFunc(fps, p.Fps, func(a, b delivery.CandidateModule) bool { return reflect.DeepEqual(a, b) }) {
if changed || !maps.EqualFunc(fps, p.Fps, func(a, b meshcatalog.CandidatesByGroupValuePullsItemFpsValue) bool { return reflect.DeepEqual(a, b) }) {
p.Fps = fps
}
}
if changed {
c.FPSet = delivery.FingerprintSet(c.Pulls)
c.Fpset = meshcatalogclient.FingerprintSet(c.Pulls)
}
if read && rev != 0 && c.Base != rev && (changed || c.Base == 0) {
c.Base, changed = rev, true
if read && rev != 0 && c.Base != int64(rev) && (changed || c.Base == 0) {
c.Base, changed = int64(rev), true
}
return changed
}
@@ -230,18 +234,18 @@ func (ix *Indexer) withPreviews(ctx context.Context, c *delivery.Candidate) bool
// merge (tree, manifest, locks), its bases' and the builder's fingerprints as the trunk's modules hold them, the toolchains
// the build modules agree on, and the version of each mesh package it uses (candidatePackages), by the rules Recompute
// applies to a module of the trunk.
func (ix *Indexer) candidateModule(ctx context.Context, m, dir string, pv delivery.Preview, held map[string]delivery.Module, members map[string]string) delivery.CandidateModule {
func (ix *Indexer) candidateModule(ctx context.Context, m, dir string, pv meshdeliverychecker.PreviewsByRepositoryHeadTrunkValue, held map[string]meshcatalog.ModulesByModuleValue, members map[string]string) meshcatalog.CandidatesByGroupValuePullsItemFpsValue {
pm, ok := pv.Modules[dir]
if !ok && dir == "" {
pm, ok = pv.Modules[delivery.RootPath]
pm, ok = pv.Modules[meshcatalogclient.RootPath]
}
if !ok && dir == delivery.RootPath {
if !ok && dir == meshcatalogclient.RootPath {
pm, ok = pv.Modules[""]
}
if !ok {
return delivery.CandidateModule{None2: NonePreviewNoModule}
return meshcatalog.CandidatesByGroupValuePullsItemFpsValue{None2: NonePreviewNoModule}
}
out := delivery.CandidateModule{Manifest: pm.Manifest}
out := meshcatalog.CandidatesByGroupValuePullsItemFpsValue{Manifest: pm.Manifest}
if ix.Manifests == nil {
out.None2 = NonePreviewNoManifest
return out
@@ -251,7 +255,7 @@ func (ix *Indexer) candidateModule(ctx context.Context, m, dir string, pv delive
out.None2 = NonePreviewNoManifest
return out
}
kept, err := delivery.DecodeManifest(value)
kept, err := meshcatalogclient.DecodeManifest(value)
if err != nil {
out.None2 = NonePreviewNoManifest
return out
@@ -259,31 +263,31 @@ func (ix *Indexer) candidateModule(ctx context.Context, m, dir string, pv delive
// The preview carries the directory's raw inputs (its tree, the named paths' objects, the lock hashes): the indexer
// computes the named tree (ADR 0593 §2). The version line's packages are the trunk module's, unless an older build
// module still names them.
trunk := held[m]
trunkMod := held[m]
packages := fingerprint.OtherPackages(pm.Packages, pv.Repository)
if trunk.Inputs2 != nil {
if trunkMod.Inputs2 != nil {
if len(pm.Packages) == 0 {
packages = trunk.Inputs2.Packages
packages = trunkMod.Inputs2.Packages
} else {
// The siblings its go.mod replaces by a directory are the trunk module's: no go.sum names them.
for _, p := range trunk.Inputs2.Packages {
if repo, _, ok := delivery.PackageRepository(p); ok && delivery.SameRepository(repo, pv.Repository) {
for _, p := range trunkMod.Inputs2.Packages {
if repo, _, ok := trunk.PackageRepository(p); ok && trunk.SameRepository(repo, pv.Repository) {
packages = append(packages, p)
}
}
}
}
in := inputsOfNamed(m, kept.JSON, pm.Tree, pm.Locks, pm.Paths, packages, ix.baseFP(ctx))
in := inputsOfNamed(m, kept.Manifest, pm.Tree, pm.Locks, pm.Paths, packages, ix.baseFP(ctx))
if in.Kept == nil {
out.None2 = in.None2
return out
}
if in.None2 != "" && in.None2 != delivery.ToolchainNoReport && in.None2 != None2Pending {
if in.None2 != "" && in.None2 != sdkstep.ToolchainNoReport && in.None2 != None2Pending {
out.None2 = in.None2 // a context, or a base with no fingerprint
return out
}
tools := ix.agreed()
toolchains := map[string]delivery.Toolchain2{}
toolchains := map[string]meshcatalogclient.Toolchain2{}
for _, artifact := range slices.Sorted(maps.Keys(in.Languages)) {
t := tools[in.Languages[artifact]]
if t.none != "" {
@@ -294,27 +298,27 @@ func (ix *Indexer) candidateModule(ctx context.Context, m, dir string, pv delive
}
inputs := *in.Kept // no builder input (ADR 0530 §1)
var fp, none string
var versions delivery.PackageVersions
if trunk.Inputs2 != nil && len(trunk.Inputs2.Imports) > 0 {
inputs.Imports, inputs.ImportForm = trunk.Inputs2.Imports, trunk.Inputs2.ImportForm
var versions sdkstep.PackageVersions
if trunkMod.Inputs2 != nil && len(trunkMod.Inputs2.Imports) > 0 {
inputs.Imports, inputs.ImportForm = trunkMod.Inputs2.Imports, trunkMod.Inputs2.ImportForm
}
// The import form against the group's own heads of the packages it publishes (ADR 0573 §4, ADR 0593 §2), as the trunk
// computes it once merged, the trunk's held fingerprint standing where nothing the build was built from changed.
published := ix.candidatePublished(ctx, members, held)
// A package of a repository outside the group by the series the candidate's manifest accepts (DLV-I6); a member's by the
// group's head.
var pins delivery.PackageVersions
var pins sdkstep.PackageVersions
if _, man, err := ix.manifestRaw(ctx, pm.Manifest); err == nil {
series := ix.SeriesAll(ctx)
for key, s := range series {
if repo, _, _ := delivery.PackageRepository(s.GoModule); members[repo] != "" {
if repo, _, _ := trunk.PackageRepository(s.GoModule); members[repo] != "" {
delete(series, key)
}
}
published, pins = acceptedSeries(series, man, pv.Repository, published)
}
if len(inputs.Imports) > 0 && len(fingerprint.Resolve(pv.Repository, inputs.Imports, published).Missing) == 0 {
f := formOf(m, pv.Repository, &inputs, toolchains, trunk, held, published, pins, nil)
f := formOf(m, pv.Repository, &inputs, toolchains, trunkMod, held, published, pins, nil)
fp, none, versions = f.fp, f.none, f.versions
} else {
inputs.Imports, inputs.ImportForm = nil, ""
@@ -322,7 +326,7 @@ func (ix *Indexer) candidateModule(ctx context.Context, m, dir string, pv delive
for p, v := range pins {
if _, named := versions[p]; named || slices.Contains(inputs.Packages, p) {
if versions == nil {
versions = delivery.PackageVersions{}
versions = sdkstep.PackageVersions{}
}
versions[p] = v
}
@@ -335,7 +339,7 @@ func (ix *Indexer) candidateModule(ctx context.Context, m, dir string, pv delive
out.None2 = none
return out
}
out.FP2, out.Packages = fp, versions
out.Fp2, out.Packages = fp, versions
if len(inputs.Paths) > 0 {
out.Paths = inputs.Paths // the builder's fetch for the candidate names them (ADR 0499 clause 4)
}
@@ -345,13 +349,13 @@ func (ix *Indexer) candidateModule(ctx context.Context, m, dir string, pv delive
// candidatePackages is the version each package of a candidate's module is built against (novox/hq ADR 0573 §2, §4): the
// head of the candidate's pull request whose repository publishes it (members, by repository), else its latest published
// version, as the trunk's modules hold it. A package with neither is left out, and the fingerprint is then none.
func candidatePackages(packages []string, held map[string]delivery.Module, members map[string]string) delivery.PackageVersions {
func candidatePackages(packages []string, held map[string]meshcatalog.ModulesByModuleValue, members map[string]string) sdkstep.PackageVersions {
versions, _ := fingerprint.HeldVersionsOf(packages, held)
for _, p := range packages {
repository, _, _ := delivery.PackageRepository(p)
repository, _, _ := trunk.PackageRepository(p)
if head := members[repository]; head != "" {
if versions == nil {
versions = delivery.PackageVersions{}
versions = sdkstep.PackageVersions{}
}
versions[p] = head
}
modules/mesh-delivery-indexer/internal/index/previews_test.go
+35 -30
View File
@@ -7,8 +7,13 @@ import (
"strings"
"testing"
sdkstep "git.novox.be/novox/mesh-delivery/sdk/buildstep"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdeliverychecker"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
"git.novox.be/novox/mesh-sdk/go/delivery"
"mesh-delivery-indexer/internal/fingerprint"
)
@@ -27,7 +32,7 @@ func previewBed(t *testing.T, report bool) (*Indexer, *memStore, *memStore) {
if report {
reportToolchain(t, ix, "mesh-build-go", "novox", "go", "go1.27.1")
}
if err := ix.ApplySource(context.Background(), 5, delivery.Source{Group: "g", Heads: map[string]string{catalogueRepo: trunkNow}}); err != nil {
if err := ix.ApplySource(context.Background(), 5, meshdelivery.SourceByMainValue{Group: "g", Heads: map[string]string{catalogueRepo: trunkNow}}); err != nil {
t.Fatal(err)
}
return ix, mods, ix.Store.(*memStore)
@@ -46,18 +51,18 @@ func avahiPull(t *testing.T, head string) forgemsg.Forge {
return f
}
func avahiPreview(t *testing.T, trunk, head string) delivery.Preview {
return delivery.Preview{Repository: catalogueRepo, Head: head, Trunk: trunk, Merge: mergeSha, Modules: map[string]delivery.PreviewModule{
"modules/avahi": {Tree: "tree-avahi", Manifest: delivery.ManifestDigest([]byte(avahiOnTools(t))), Locks: map[string]string{"go.sum": avahiGoSum}}}}
func avahiPreview(t *testing.T, trunk, head string) meshdeliverychecker.PreviewsByRepositoryHeadTrunkValue {
return meshdeliverychecker.PreviewsByRepositoryHeadTrunkValue{Repository: catalogueRepo, Head: head, Trunk: trunk, Merge: mergeSha, Modules: map[string]meshdeliverychecker.PreviewsByRepositoryHeadTrunkValueModulesValue{
"modules/avahi": {Tree: "tree-avahi", Manifest: meshcatalogclient.ManifestDigest([]byte(avahiOnTools(t))), Locks: map[string]string{"go.sum": avahiGoSum}}}}
}
func applyPreview(t *testing.T, ix *Indexer, pv delivery.Preview) {
func applyPreview(t *testing.T, ix *Indexer, pv meshdeliverychecker.PreviewsByRepositoryHeadTrunkValue) {
t.Helper()
key, err := delivery.PreviewKey(pv.Repository, pv.Head, pv.Trunk)
key, err := PreviewKey(pv.Repository, pv.Head, pv.Trunk)
if err != nil {
t.Fatal(err)
}
data, err := delivery.EncodePreview(pv)
data, err := EncodePreview(pv)
if err != nil {
t.Fatal(err)
}
@@ -66,7 +71,7 @@ func applyPreview(t *testing.T, ix *Indexer, pv delivery.Preview) {
}
}
func fpOf(t *testing.T, st *memStore) delivery.CandidateModule {
func fpOf(t *testing.T, st *memStore) meshcatalog.CandidatesByGroupValuePullsItemFpsValue {
t.Helper()
c := candidateOf(t, st)
if len(c.Pulls) != 1 {
@@ -78,7 +83,7 @@ func fpOf(t *testing.T, st *memStore) delivery.CandidateModule {
func TestACandidatesModuleGetsTheFingerprintATrunkModuleOfTheSameTreeHas(t *testing.T) { // 1445/AC2
ix, mods, cands := previewBed(t, true)
trunk, _ := moduleOf(t, mods, "avahi")
if trunk.FP2 == "" {
if trunk.Fp2 == "" {
t.Fatalf("the trunk's avahi has no fingerprint of the new form: %+v", trunk)
}
if err := ix.Apply(context.Background(), avahiPull(t, h1)); err != nil {
@@ -89,12 +94,12 @@ func TestACandidatesModuleGetsTheFingerprintATrunkModuleOfTheSameTreeHas(t *test
}
applyPreview(t, ix, avahiPreview(t, trunkNow, h1))
got := fpOf(t, cands)
if got.FP2 != trunk.FP2 || got.None2 != "" || got.Manifest != delivery.ManifestDigest([]byte(avahiOnTools(t))) {
t.Fatalf("candidate avahi = %+v, want the trunk's fingerprint %s", got, trunk.FP2)
if got.Fp2 != trunk.Fp2 || got.None2 != "" || got.Manifest != meshcatalogclient.ManifestDigest([]byte(avahiOnTools(t))) {
t.Fatalf("candidate avahi = %+v, want the trunk's fingerprint %s", got, trunk.Fp2)
}
c := candidateOf(t, cands)
if c.FPSet != delivery.FingerprintSet(c.Pulls) || c.FPSet == "" {
t.Errorf("fpset = %q", c.FPSet)
if c.Fpset != meshcatalogclient.FingerprintSet(c.Pulls) || c.Fpset == "" {
t.Errorf("fpset = %q", c.Fpset)
}
// The candidate names the source revision the preview was read onto (5, whose catalogue head is the preview's trunk):
// the builder merges the preview onto that head, never onto the module value's commit.
@@ -109,14 +114,14 @@ func TestACandidatesModuleGetsTheFingerprintATrunkModuleOfTheSameTreeHas(t *test
pv.Modules["modules/avahi"] = m
pv.Merge = "7777777777777777777777777777777777777777"
applyPreview(t, ix, pv)
if other := fpOf(t, cands); other.FP2 == "" || other.FP2 == trunk.FP2 {
if other := fpOf(t, cands); other.Fp2 == "" || other.Fp2 == trunk.Fp2 {
t.Errorf("a changed tree: %+v", other)
}
// Issue 1585 (ADR 0573 §2, §4): a candidate's module is built against the head of the candidate's pull request that
// publishes a package it uses, else against the package's latest published version; a preview naming a package of
// mesh-sdk that nothing published gives no fingerprint, and one of its own repository's comes with its tree.
const sdk, own = "git.novox.be/novox/mesh-sdk/go", "git.novox.be/novox/mesh-catalog/contract"
held := map[string]delivery.Module{"mesh-sdk": {Repository: "novox/mesh-sdk", Path: delivery.RootPath, Commit: trunkOld}}
held := map[string]meshcatalog.ModulesByModuleValue{"mesh-sdk": {Repository: "novox/mesh-sdk", Path: meshcatalogclient.RootPath, Commit: trunkOld}}
if got := candidatePackages([]string{sdk}, held, map[string]string{catalogueRepo: h1}); got[sdk] != trunkOld {
t.Errorf("outside a group: %v, want mesh-sdk at its trunk %s", got, trunkOld)
}
@@ -127,7 +132,7 @@ func TestACandidatesModuleGetsTheFingerprintATrunkModuleOfTheSameTreeHas(t *test
pv.Modules["modules/avahi"] = m
pv.Merge = "8888888888888888888888888888888888888888"
applyPreview(t, ix, pv)
if got := fpOf(t, cands); got.FP2 != "" || got.None2 != fingerprint.NoPackageVersion+sdk {
if got := fpOf(t, cands); got.Fp2 != "" || got.None2 != fingerprint.NoPackageVersion+sdk {
t.Errorf("a package nothing published: %+v", got)
}
}
@@ -138,8 +143,8 @@ func TestAnUnknownToolchainLeavesTheCandidatesModuleNamingIt(t *testing.T) { //
t.Fatal(err)
}
applyPreview(t, ix, avahiPreview(t, trunkNow, h1))
if got := fpOf(t, cands); got.FP2 != "" || !strings.Contains(got.None2, delivery.ToolchainNoReport) {
t.Errorf("candidate avahi = %+v, want none2 naming %q", got, delivery.ToolchainNoReport)
if got := fpOf(t, cands); got.Fp2 != "" || !strings.Contains(got.None2, sdkstep.ToolchainNoReport) {
t.Errorf("candidate avahi = %+v, want none2 naming %q", got, sdkstep.ToolchainNoReport)
}
}
@@ -149,15 +154,15 @@ func TestAPreviewMadeOntoAnOldTrunkIsOutOfDate(t *testing.T) { // 1445/AC3
t.Fatal(err)
}
applyPreview(t, ix, avahiPreview(t, trunkOld, h1))
if got := fpOf(t, cands); got.FP2 != "" || got.None2 != NonePreviewStale {
if got := fpOf(t, cands); got.Fp2 != "" || got.None2 != NonePreviewStale {
t.Errorf("candidate avahi = %+v, want it out of date", got)
}
// A preview onto the trunk head now makes it good again, and the trunk moving again makes it stale.
applyPreview(t, ix, avahiPreview(t, trunkNow, h1))
if got := fpOf(t, cands); got.FP2 == "" {
if got := fpOf(t, cands); got.Fp2 == "" {
t.Fatalf("after the right preview: %+v", got)
}
if err := ix.ApplySource(context.Background(), 6, delivery.Source{Group: "g", Heads: map[string]string{catalogueRepo: "8888888888888888888888888888888888888888"}}); err != nil {
if err := ix.ApplySource(context.Background(), 6, meshdelivery.SourceByMainValue{Group: "g", Heads: map[string]string{catalogueRepo: "8888888888888888888888888888888888888888"}}); err != nil {
t.Fatal(err)
}
if got := fpOf(t, cands); got.None2 != NonePreviewStale {
@@ -172,14 +177,14 @@ func TestAPreviewHeardBeforeItsPullRequestGivesTheFingerprintWhenTheCandidateIsW
t.Fatal(err)
}
trunk, _ := moduleOf(t, mods, "avahi")
if got := fpOf(t, cands); got.FP2 != trunk.FP2 {
t.Errorf("candidate avahi = %+v, want %s", got, trunk.FP2)
if got := fpOf(t, cands); got.Fp2 != trunk.Fp2 {
t.Errorf("candidate avahi = %+v, want %s", got, trunk.Fp2)
}
// The same pull request at a new head has no preview of it yet.
if err := ix.Apply(context.Background(), avahiPull(t, h2)); err != nil {
t.Fatal(err)
}
if got := fpOf(t, cands); got.FP2 != "" || got.None2 != NoneNoPreview {
if got := fpOf(t, cands); got.Fp2 != "" || got.None2 != NoneNoPreview {
t.Errorf("a new head: %+v, want no preview", got)
}
}
@@ -191,7 +196,7 @@ func TestAPreviewNamingNoManifestOfTheModulesDirectoryGivesAReason(t *testing.T)
t.Fatal(err)
}
pv := avahiPreview(t, trunkNow, h1)
pv.Modules = map[string]delivery.PreviewModule{"modules/other": pv.Modules["modules/avahi"]}
pv.Modules = map[string]meshdeliverychecker.PreviewsByRepositoryHeadTrunkValueModulesValue{"modules/other": pv.Modules["modules/avahi"]}
applyPreview(t, ix, pv)
if got := fpOf(t, cands); got.None2 != NonePreviewNoModule || !strings.Contains(NonePreviewNoModule, "manifest") {
t.Errorf("candidate avahi = %+v", got)
@@ -216,9 +221,9 @@ func TestAPreviewOfConflictsGivesTheCandidatesModuleTheReasonItDoesNotMerge(t *t
if err := ix.Apply(context.Background(), avahiPull(t, h1)); err != nil {
t.Fatal(err)
}
applyPreview(t, ix, delivery.Preview{Repository: catalogueRepo, Head: h1, Trunk: trunkNow, Conflicts: []string{"modules/avahi/main.go", "go.mod"}})
applyPreview(t, ix, meshdeliverychecker.PreviewsByRepositoryHeadTrunkValue{Repository: catalogueRepo, Head: h1, Trunk: trunkNow, Conflicts: []string{"modules/avahi/main.go", "go.mod"}})
got := fpOf(t, cands)
if got.FP2 != "" || !strings.HasPrefix(got.None2, "does not merge") || !strings.Contains(got.None2, "go.mod, modules/avahi/main.go") && !strings.Contains(got.None2, "modules/avahi/main.go, go.mod") {
if got.Fp2 != "" || !strings.HasPrefix(got.None2, "does not merge") || !strings.Contains(got.None2, "go.mod, modules/avahi/main.go") && !strings.Contains(got.None2, "modules/avahi/main.go, go.mod") {
t.Errorf("candidate avahi = %+v, want it not merging and naming the files", got)
}
}
@@ -227,8 +232,8 @@ func TestAPreviewOfConflictsGivesTheCandidatesModuleTheReasonItDoesNotMerge(t *t
// indexer writes it and the gate's source revision it names, kept in testdata/candidate-written.json so the builder is tested
// on what the indexer writes, never on a candidate made by hand. MESH_WRITE_TESTDATA=1 writes it again.
type writtenCandidate struct {
Source sourceRev `json:"source"`
Candidate delivery.Candidate `json:"candidate"`
Source sourceRev `json:"source"`
Candidate meshcatalog.CandidatesByGroupValue `json:"candidate"`
}
type sourceRev struct {
modules/mesh-delivery-indexer/internal/index/previews_values.go
+75
View File
@@ -0,0 +1,75 @@
package index
import (
"encoding/json"
"fmt"
"reflect"
"strings"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliverycheckerclient"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdeliverychecker"
"git.novox.be/novox/mesh-sdk/go/keys"
)
// A preview as the checker writes it into its state previews, read here (reads: mesh-delivery-checker.previews): the
// seat's type and the client's rules; the key is the form the seat names, `<repository>.<head>.<trunk>`, the repository
// as a key part (keys.EscapeName).
// PreviewKey is the key of a head merged onto a trunk commit.
func PreviewKey(repository, head, trunk string) (string, error) {
if err := meshdeliverycheckerclient.CheckRepository(repository); err != nil {
return "", err
}
if err := keys.CheckName("head", head); err != nil {
return "", err
}
if err := keys.CheckName("trunk commit", trunk); err != nil {
return "", err
}
return meshdeliverychecker.PreviewsByRepositoryHeadTrunkKey(keys.EscapeName(repository), head, trunk), nil
}
// ParsePreviewKey is the inverse of PreviewKey.
func ParsePreviewKey(key string) (repository, head, trunk string, err error) {
parts := strings.Split(key, ".")
if len(parts) != 3 {
return "", "", "", fmt.Errorf("index: %q is not a preview key, <repository>.<head>.<trunk>", key)
}
repository, err = keys.UnescapeName(parts[0])
if err != nil {
return "", "", "", fmt.Errorf("index: %q is not a preview key: %w", key, err)
}
if _, err := PreviewKey(repository, parts[1], parts[2]); err != nil {
return "", "", "", fmt.Errorf("index: %q is not a preview key: %w", key, err)
}
return repository, parts[1], parts[2], nil
}
// DecodePreview reads a preview.
func DecodePreview(data []byte) (meshdeliverychecker.PreviewsByRepositoryHeadTrunkValue, error) {
if err := keys.CheckVersion(data); err != nil {
return meshdeliverychecker.PreviewsByRepositoryHeadTrunkValue{}, err
}
p, err := meshdeliverychecker.DecodePreviewsByRepositoryHeadTrunkValue(data)
if err != nil {
return p, fmt.Errorf("index: a preview does not fit its shape: %w", err)
}
if err := meshdeliverycheckerclient.CheckPreview(p); err != nil {
return meshdeliverychecker.PreviewsByRepositoryHeadTrunkValue{}, err
}
return p, nil
}
// EncodePreview is p as JSON at the kept values' version (the checker writes previews; the indexer's tests do too).
func EncodePreview(p meshdeliverychecker.PreviewsByRepositoryHeadTrunkValue) ([]byte, error) {
if err := meshdeliverycheckerclient.CheckPreview(p); err != nil {
return nil, err
}
p.V = keys.Version
return json.Marshal(p)
}
// SamePreview says whether two previews are the same value.
func SamePreview(a, b meshdeliverychecker.PreviewsByRepositoryHeadTrunkValue) bool {
return reflect.DeepEqual(a, b)
}
modules/mesh-delivery-indexer/internal/index/registered.go
+15 -13
View File
@@ -7,8 +7,10 @@ import (
"slices"
"strings"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-delivery/sdk/trunk"
"git.novox.be/novox/mesh-sdk/go/keys"
)
// A module registered in the catalogue from its own repository (the operator registers it with the repository and the
@@ -57,7 +59,7 @@ func (ix *Indexer) indexAt(ctx context.Context, r Registration, force, reread bo
if ix.Modules == nil || ix.Read == nil {
return taken{why: "the indexer reads no forge or writes no modules"}, nil
}
key, err := delivery.ModulesKey(r.Module)
key, err := meshcatalogclient.ModulesKey(r.Module)
if err != nil {
ix.logf("%s: the registered module has no usable key, skipped: %v", r.Module, err)
return taken{why: fmt.Sprintf("the module has no usable key: %v", err)}, nil
@@ -68,17 +70,17 @@ func (ix *Indexer) indexAt(ctx context.Context, r Registration, force, reread bo
ix.markRead(r.Module)
return taken{held: true, why: "the modules state holds it already: a merge to its repository writes it from now on"}, nil
}
repository := delivery.RepositoryName(r.Repository)
repository := trunk.RepositoryName(r.Repository)
commit := r.Commit
dir := strings.Trim(r.Path, "/")
if dir == delivery.RootPath {
if dir == meshcatalogclient.RootPath {
dir = ""
}
// The head the gate's source names for the repository, when it names one, else the registration's commit. A module a
// merge removed is absent at the head and is said and left.
ix.mu.Lock()
for repo, head := range ix.source.heads {
if delivery.SameRepository(repo, r.Repository) && head != "" {
if trunk.SameRepository(repo, r.Repository) && head != "" {
commit = head
}
}
@@ -126,7 +128,7 @@ func (ix *Indexer) indexAt(ctx context.Context, r Registration, force, reread bo
t.why = fmt.Sprintf("the manifest at %q of %s at %s names the module %q", dir, repository, commit, name)
return t, nil
}
f := forgemsg.Forge{V: delivery.Version, Kind: forgemsg.ForgeMerged, Repository: repository, Merge: commit, Root: read.Root,
f := forgemsg.Forge{V: keys.Version, Kind: forgemsg.ForgeMerged, Repository: repository, Merge: commit, Root: read.Root,
Trees: map[string]string{dir: read.Trees[dir]},
Manifests: map[string]string{dir: raw}}
if l := read.Locks[dir]; len(l) > 0 {
@@ -234,7 +236,7 @@ func (ix *Indexer) TakeIn(ctx context.Context, module string) TakenIn {
}
regs := ix.Builds.RegistrationsOf(module)
if len(regs) == 0 {
if key, err := delivery.ModulesKey(module); err == nil && ix.Modules != nil {
if key, err := meshcatalogclient.ModulesKey(module); err == nil && ix.Modules != nil {
if _, _, found, err := ix.Modules.Get(ctx, key); err == nil && found {
out.Held, out.Answer = true, "the modules state holds it already: a merge to its repository writes it from now on"
return out
@@ -305,7 +307,7 @@ func (ix *Indexer) IndexRegisteredAll(ctx context.Context) error {
// apply: no such module held, or no head named, or the record at that head already.
func (ix *Indexer) takeInHeld(ctx context.Context, module string) (TakenIn, bool) {
out := TakenIn{Module: module}
key, err := delivery.ModulesKey(module)
key, err := meshcatalogclient.ModulesKey(module)
if err != nil || ix.Modules == nil {
return out, false
}
@@ -313,14 +315,14 @@ func (ix *Indexer) takeInHeld(ctx context.Context, module string) (TakenIn, bool
if err != nil || !found {
return out, false
}
cur, err := delivery.DecodeModule(value)
cur, err := meshcatalogclient.DecodeModule(value)
if err != nil || cur.Repository == "" || cur.Path == "" {
return out, false
}
head := ""
ix.mu.Lock()
for repo, h := range ix.source.heads {
if delivery.SameRepository(repo, cur.Repository) && h != "" {
if trunk.SameRepository(repo, cur.Repository) && h != "" {
head = h
}
}
@@ -373,7 +375,7 @@ func (ix *Indexer) TakeInRepository(ctx context.Context, repository string) Take
ix.mu.Lock()
head := ""
for repo, h := range ix.source.heads {
if delivery.SameRepository(repo, repository) && h != "" {
if trunk.SameRepository(repo, repository) && h != "" {
head, out.Repository = h, repo
}
}
@@ -397,7 +399,7 @@ func (ix *Indexer) TakeInRepository(ctx context.Context, repository string) Take
out.Answer = "the modules state cannot be read: " + err.Error()
return out
}
f := forgemsg.Forge{V: delivery.Version, Kind: forgemsg.ForgeMerged, Repository: out.Repository, Merge: head}
f := forgemsg.Forge{V: keys.Version, Kind: forgemsg.ForgeMerged, Repository: out.Repository, Merge: head}
if _, err := ix.ApplyMessage(ctx, f); err != nil {
out.Answer = fmt.Sprintf("the merge of %s at %.12s could not be taken in: %v", repository, head, err)
return out
@@ -408,7 +410,7 @@ func (ix *Indexer) TakeInRepository(ctx context.Context, repository string) Take
return out
}
for _, key := range slices.Sorted(maps.Keys(before)) {
if !delivery.SameRepository(before[key].Repository, repository) {
if !trunk.SameRepository(before[key].Repository, repository) {
continue
}
if now, ok := after[key]; ok {
modules/mesh-delivery-indexer/internal/index/registered_test.go
+11 -8
View File
@@ -7,7 +7,10 @@ import (
"strings"
"testing"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery"
"git.novox.be/novox/mesh-sdk/go/keys"
)
func TestARegisteredModuleIsIndexedFromItsRepository(t *testing.T) { // the grants part's gaps: modules registered from their own repositories had no manifest here
@@ -50,12 +53,12 @@ func TestARegisteredModuleIsIndexedFromItsRepository(t *testing.T) { // the gran
t.Run(tc.name, func(t *testing.T) {
ix, mods := modulesIndexer(t, rec(t, "build-1", "snake", forge, tc.path, tc.ref, "", "2026-10-19T09:00:00Z", ""))
if tc.held {
if err := mods.Create(context.Background(), "snake", mustModule(t, delivery.Module{Repository: "jschoubben/snake", Commit: h1, Path: ".", None2: "x"})); err != nil {
if err := mods.Create(context.Background(), "snake", mustModule(t, meshcatalog.ModulesByModuleValue{Repository: "jschoubben/snake", Commit: h1, Path: ".", None2: "x"})); err != nil {
t.Fatal(err)
}
}
if tc.other {
if err := mods.Create(context.Background(), "other", mustModule(t, delivery.Module{Repository: "jschoubben/snake", Commit: h1, Path: "modules/other", None2: "x"})); err != nil {
if err := mods.Create(context.Background(), "other", mustModule(t, meshcatalog.ModulesByModuleValue{Repository: "jschoubben/snake", Commit: h1, Path: "modules/other", None2: "x"})); err != nil {
t.Fatal(err)
}
}
@@ -172,9 +175,9 @@ func TestARegisteredModuleIsIndexedFromItsRepository(t *testing.T) { // the gran
}
}
func mustModule(t *testing.T, m delivery.Module) []byte {
func mustModule(t *testing.T, m meshcatalog.ModulesByModuleValue) []byte {
t.Helper()
b, err := delivery.EncodeModule(withV(m))
b, err := meshcatalogclient.EncodeModule(withV(m))
if err != nil {
t.Fatal(err)
}
@@ -258,7 +261,7 @@ func TestAForgeThatCannotSayLeavesTheModuleUnwrittenAndAsksAgain(t *testing.T) {
t.Fatal(err)
}
case "a new source revision":
if err := ix.ApplySource(ctx, 2, delivery.Source{Heads: map[string]string{"jschoubben/snake": h2, "novox/other": h1}}); err != nil {
if err := ix.ApplySource(ctx, 2, meshdelivery.SourceByMainValue{Heads: map[string]string{"jschoubben/snake": h2, "novox/other": h1}}); err != nil {
t.Fatal(err)
}
ix.retryWG.Wait() // read again in the background
@@ -278,7 +281,7 @@ func TestAForgeThatCannotSayLeavesTheModuleUnwrittenAndAsksAgain(t *testing.T) {
}
// Taken in, it is read no more: a further event asks the forge nothing.
n := len(asked)
if err := ix.ApplySource(ctx, 3, delivery.Source{Heads: map[string]string{"jschoubben/snake": h2}}); err != nil {
if err := ix.ApplySource(ctx, 3, meshdelivery.SourceByMainValue{Heads: map[string]string{"jschoubben/snake": h2}}); err != nil {
t.Fatal(err)
}
ix.retryWG.Wait()
@@ -317,7 +320,7 @@ func TestARecordOfAPullRequestDoesNotReplaceTheRegisteredRef(t *testing.T) { //
// An entry of another kind in the builder's log (the old reference form's close, design 51), of this form or another,
// is no build: passed over, never refused.
for _, other := range []string{
fmt.Sprintf(`{"v":%d,"event":"old-form-closed","module":"node-engine","at":"2026-10-04T09:00:00Z"}`, delivery.Version),
fmt.Sprintf(`{"v":%d,"event":"old-form-closed","module":"node-engine","at":"2026-10-04T09:00:00Z"}`, keys.Version),
`{"v":99,"event":"old-form-closed","at":"2026-10-04T09:00:00Z","nodes":["ace"]}`,
} {
if err := b.Put([]byte(other)); err != nil {
modules/mesh-delivery-indexer/internal/index/remove_test.go
+8 -7
View File
@@ -6,7 +6,8 @@ import (
"strings"
"testing"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
)
func twoModules(t *testing.T) (*Indexer, *memStore) {
@@ -26,9 +27,9 @@ func twoModules(t *testing.T) (*Indexer, *memStore) {
func TestAMergeThatRemovesAModuleDeletesItsKey(t *testing.T) { // 1900/AC1
ix, mods := twoModules(t)
// DLV-I9 (design 25 § 12.2): the module leaving the trunk is said as an empty manifest-accepted, under an id naming it.
var said []delivery.ManifestAccepted
var said []meshcatalog.ManifestAcceptedEvent
var ids []string
ix.SayAccepted = func(_ context.Context, a delivery.ManifestAccepted, id string) error {
ix.SayAccepted = func(_ context.Context, a meshcatalog.ManifestAcceptedEvent, id string) error {
said, ids = append(said, a), append(ids, id)
return nil
}
@@ -37,9 +38,9 @@ func TestAMergeThatRemovesAModuleDeletesItsKey(t *testing.T) { // 1900/AC1
for i, a := range said {
if a.Digest == "" {
left = append(left, a.Module)
// Join item 3: the id is the event's own (delivery.ManifestAccepted.EventID), naming the module and the
// Join item 3: the id is the event's own (acceptedEventID), naming the module and the
// modules state's revision of the value that left, so two leaves of one module are two events.
if a.BusState == nil || len(a.BusState) != 0 || a.Revision == 0 || ids[i] != a.EventID() || !strings.Contains(ids[i], "-left-b-") {
if a.BusState == nil || len(a.BusState) != 0 || a.Revision == 0 || ids[i] != acceptedEventID(a) || !strings.Contains(ids[i], "-left-b-") {
t.Errorf("the empty manifest-accepted = %+v, id %s", a, ids[i])
}
}
@@ -138,7 +139,7 @@ func TestStartingDeletesNothingWhenTheTrunkCannotBeRead(t *testing.T) { // 1900/
t.Errorf("%s: %d keys held, want 2 untouched", name, len(mods.vals))
}
}
_ = delivery.RootPath
_ = meshcatalogclient.RootPath
}
func TestAKeyWrittenAheadOfTheSourceHeadStaysAndOneWithoutAPathIsSkipped(t *testing.T) { // 1900/AC3 (review): the source lags a merge already applied
@@ -148,7 +149,7 @@ func TestAKeyWrittenAheadOfTheSourceHeadStaysAndOneWithoutAPathIsSkipped(t *test
if err := ix.ApplySource(ctx, 3, sourceOf(catalogueRepo, older)); err != nil {
t.Fatal(err)
}
mods.put("nopath", mustEncode(t, delivery.Module{Repository: catalogueRepo, Commit: older, None2: "x", Manifest: strings.Repeat("a", 64)}))
mods.put("nopath", mustEncode(t, meshcatalog.ModulesByModuleValue{Repository: catalogueRepo, Commit: older, None2: "x", Manifest: strings.Repeat("a", 64)}))
ix.Trunk = func(context.Context, string, string) ([]string, error) { return []string{"modules/a"}, nil }
ix.Contains = func(_ context.Context, _, commit, in string) (bool, error) { return commit == in, nil } // C is not in B
if err := ix.Reconcile(ctx); err != nil {
modules/mesh-delivery-indexer/internal/index/requirements.go
+11 -10
View File
@@ -9,7 +9,8 @@ import (
"strconv"
"strings"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-delivery/sdk/trunk"
"git.novox.be/novox/mesh-sdk/go/keys"
)
// The requirements state (DLV-I7, the lead's ruling on I7; REG S15, ADR 0606 §1): per mesh Go module, the highest
@@ -18,7 +19,7 @@ import (
// already requires.
const (
RequirementsState = "requirements"
RequirementsBucket = delivery.IndexerModule + "_" + RequirementsState
RequirementsBucket = Module + "_" + RequirementsState
RequirementsTool = "indexer_requirements"
)
@@ -47,10 +48,10 @@ func RequiredOf(goSum []byte) map[string]string {
out := map[string]string{}
for _, line := range strings.Split(string(goSum), "\n") {
f := strings.Fields(line)
if len(f) < 2 || !strings.HasPrefix(f[0], delivery.ForgeModulePrefix) {
if len(f) < 2 || !strings.HasPrefix(f[0], trunk.ForgeModulePrefix) {
continue
}
if _, _, ok := delivery.PackageRepository(f[0]); !ok {
if _, _, ok := trunk.PackageRepository(f[0]); !ok {
continue
}
v := strings.TrimSuffix(f[1], "/go.mod")
@@ -163,22 +164,22 @@ func (ix *Indexer) keepRequirements(ctx context.Context, repository, dir string,
ix.reqKept[consumer] = true
ix.reqMu.Unlock()
required := RequiredOf(goSum)
keys, err := ix.Requirements.Keys(ctx)
held, err := ix.Requirements.Keys(ctx)
if err != nil {
ix.logf("requirements: the state cannot be listed, so what %s requires is not kept: %v", consumer, err)
return
}
for _, pkg := range slices.Sorted(maps.Keys(required)) {
if !slices.Contains(keys, pkg) {
keys = append(keys, pkg)
if !slices.Contains(held, pkg) {
held = append(held, pkg)
}
}
for _, pkg := range keys {
for _, pkg := range held {
version, names := required[pkg]
err := write(ctx, ix.Requirements, pkg, func(value []byte, found bool) ([]byte, bool, error) {
r := Requirement{V: delivery.Version, Package: pkg, By: map[string]string{}}
r := Requirement{V: keys.Version, Package: pkg, By: map[string]string{}}
if found && json.Unmarshal(value, &r) != nil {
r = Requirement{V: delivery.Version, Package: pkg, By: map[string]string{}}
r = Requirement{V: keys.Version, Package: pkg, By: map[string]string{}}
}
if r.By == nil {
r.By = map[string]string{}
modules/mesh-delivery-indexer/internal/index/seatdefs.go
+18 -14
View File
@@ -25,10 +25,12 @@ import (
"strings"
"time"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/health"
"git.novox.be/novox/mesh-sdk/go/keys"
"git.novox.be/novox/mesh-sdk/go/seatdef"
"git.novox.be/novox/mesh-sdk/go/views"
)
@@ -36,7 +38,7 @@ import (
const (
// SeatDefinitionsState is the indexer's state of the seat definitions (design 26), one key per seat and source.
SeatDefinitionsState = "seat-definitions"
SeatDefinitionsBucket = delivery.IndexerModule + "_" + SeatDefinitionsState
SeatDefinitionsBucket = Module + "_" + SeatDefinitionsState
// The seat mesh-catalog's verb and events (ADR 0636 §4): the indexer, which holds the seat, serves the verb and says
// the events on the seat's own subject, for the trunks' definitions only, changed when a digest moved and removed when
// a trunk's definition went.
@@ -109,12 +111,12 @@ func SeatDefinitionKey(seat, source, repository string, pull int) string {
}
func pullSuffix(repository string, pull int) string {
return SourcePull + "." + delivery.EscapeName(repository) + "." + strconv.Itoa(pull)
return SourcePull + "." + keys.EscapeName(repository) + "." + strconv.Itoa(pull)
}
// OutOfStepKey is the key of a pull request's out-of-step files.
func OutOfStepKey(repository string, pull int) string {
return outOfStepPrefix + delivery.EscapeName(repository) + "." + strconv.Itoa(pull)
return outOfStepPrefix + keys.EscapeName(repository) + "." + strconv.Itoa(pull)
}
// errForgeRead is a read the forge did not answer (never a file it does not hold): the message is left for redelivery.
@@ -138,7 +140,7 @@ func contractsModules(manifests map[string]string) []contractsModule {
continue
}
base := dir
if base == delivery.RootPath {
if base == meshcatalogclient.RootPath {
base = ""
}
out = append(out, contractsModule{module: m.Module, dir: dir, defs: path.Clean(path.Join(base, a.GeneratedFrom))})
@@ -323,7 +325,7 @@ func (ix *Indexer) SeatDefinitionsAgain(ctx context.Context, module string, pull
out.Answer = "the indexer holds no seat-definitions state or forge reader yet"
return out
}
key, err := delivery.ModulesKey(module)
key, err := meshcatalogclient.ModulesKey(module)
if err != nil {
out.Answer = err.Error()
return out
@@ -333,7 +335,7 @@ func (ix *Indexer) SeatDefinitionsAgain(ctx context.Context, module string, pull
out.Answer = fmt.Sprintf("the trunk holds no module %s (%v)", module, err)
return out
}
m, err := delivery.DecodeModule(raw)
m, err := meshcatalogclient.DecodeModule(raw)
if err != nil {
out.Answer = err.Error()
return out
@@ -345,13 +347,13 @@ func (ix *Indexer) SeatDefinitionsAgain(ctx context.Context, module string, pull
}
dir := m.Path
if dir == "" {
dir = delivery.RootPath
dir = meshcatalogclient.RootPath
}
whose := "its trunk's"
if pull > 0 {
// A pull request is read with its head's own manifest, so a change of generated-from is read as the head has it.
file := "module.json"
if dir != delivery.RootPath {
if dir != meshcatalogclient.RootPath {
file = path.Join(dir, file)
}
raw, err := ix.ReadFile(ctx, m.Repository, head, file)
@@ -572,7 +574,9 @@ func (ix *Indexer) sayChanged(ctx context.Context, c SeatDefinitionChanged) {
}
// findingID is the finding's subject id for a contracts module.
func findingID(what, module string) string { return delivery.IndexerModule + "." + what + "." + module }
func findingID(what, module string) string {
return Module + "." + what + "." + module
}
// unparsedFinding writes the contracts module's seat-definition-unparsed finding for its trunk's seats that did not
// parse (by seat), or clears it when every one parsed.
@@ -608,7 +612,7 @@ func (ix *Indexer) finding(ctx context.Context, id, kind string, raise bool, sum
at := ix.now().UTC()
data, err := health.EncodeFinding(health.Finding{Key: key, Kind: kind, Subject: health.Subject{Scope: health.ScopeModule, ID: id},
Severity: health.Warning, Summary: summary, Headline: headline, Explanation: explanation, Resolved: resolved,
Evidence: []health.Evidence{{At: at, Said: summary}}, Source: kind, Observer: delivery.IndexerModule, At: at})
Evidence: []health.Evidence{{At: at, Said: summary}}, Source: kind, Observer: Module, At: at})
return data, false, err
})
if err != nil {
@@ -629,7 +633,7 @@ func (ix *Indexer) ApplyOutOfStepBuild(ctx context.Context, value []byte) {
Commit string `json:"commit"`
OutOfStep []string `json:"out-of-step"`
}
if json.Unmarshal(value, &e) != nil || e.Event != delivery.BuildEndedEvent || e.Module == "" {
if json.Unmarshal(value, &e) != nil || e.Event != builds.BuildEndedEvent || e.Module == "" {
return
}
if !trunkLike[e.Ref] {
@@ -645,7 +649,7 @@ func (ix *Indexer) ApplyOutOfStepBuild(ctx context.Context, value []byte) {
"Its build's test step regenerated the types and found the committed files differ, missing or not written by the "+
"generator (a generator's change does this). A change regenerating them clears this once a build of "+e.Module+" ends clean.",
"A build of "+e.Module+" ended clean: its generated types match their definitions.")
case e.Outcome == delivery.BuildDone:
case e.Outcome == builds.BuildDone:
ix.finding(ctx, findingID("out-of-step", e.Module), OutOfStepKind, false, "", "", "", "")
}
}
@@ -762,7 +766,7 @@ func OutOfStepOf(defs, findings views.Source) ([]OutOfStepRow, error) {
if err != nil || f.Kind != OutOfStepKind {
continue
}
out = append(out, OutOfStepRow{Source: SourceTrunk, Module: strings.TrimPrefix(f.Subject.ID, delivery.IndexerModule+".out-of-step."), Summary: f.Summary})
out = append(out, OutOfStepRow{Source: SourceTrunk, Module: strings.TrimPrefix(f.Subject.ID, Module+".out-of-step."), Summary: f.Summary})
}
keys, err := defs.Keys()
if err != nil {
modules/mesh-delivery-indexer/internal/index/series.go
+49 -34
View File
@@ -9,9 +9,12 @@ import (
"slices"
"strings"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
sdkstep "git.novox.be/novox/mesh-delivery/sdk/buildstep"
"git.novox.be/novox/mesh-delivery/sdk/trunk"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/keys"
"git.novox.be/novox/mesh-delivery/sdk/packagename"
)
@@ -25,19 +28,33 @@ import (
// A series key always holds a dot (`mesh-sdk/go.0.1`), which EscapeName never leaves in a per-module key: a reader tells the
// two forms apart by IsSeriesKey.
// SeriesPackage is the value of a series key, and SeriesVersion one version of a series: mesh-sdk's
// delivery.SeriesPackage and SeriesVersion (J4), the contract every reader of the state decodes with
// delivery.DecodeSeriesPackage.
// SeriesPackage is the value of a series key, and SeriesVersion one version of a series: the seat mesh-catalog's
// types (the newest published version's; the numbered one's is the same fields, numberedOf), which every reader of the
// state decodes.
type (
SeriesPackage = delivery.SeriesPackage
SeriesVersion = delivery.SeriesVersion
SeriesPackage = meshcatalog.PackagesByPackageSeriesValue
SeriesVersion = meshcatalog.PackagesByPackageSeriesValueNewest
)
// SeriesKey is the key of a package's series in the packages state (delivery.SeriesPackageKey).
func SeriesKey(pkg, series string) string { return delivery.SeriesPackageKey(pkg, series) }
// numberedOf is a series version as the series' numbered one: the same fields in the contract's other type.
func numberedOf(v SeriesVersion) *meshcatalog.PackagesByPackageSeriesValueNumbered {
n := &meshcatalog.PackagesByPackageSeriesValueNumbered{Commit: v.Commit, Digest: v.Digest, Fingerprint: v.Fingerprint, Version: v.Version}
if c := v.Contents; c != nil {
p := publishedOfContents(*c)
nc := &meshcatalog.PackagesByPackageSeriesValueNumberedContents{At: p.At, Commit: p.Commit, Dir: p.Dir, Module: p.Module, Repository: p.Repository, Requires: p.Requires, V: p.V}
for _, d := range p.Packages {
nc.Packages = append(nc.Packages, meshcatalog.PackagesByPackageSeriesValueNumberedContentsPackagesItem{Package: d.Package, Digest: d.Digest})
}
n.Contents = nc
}
return n
}
// IsSeriesKey says whether a key of the packages state is a series key, not a per-module one (delivery.IsSeriesPackageKey).
func IsSeriesKey(key string) bool { return delivery.IsSeriesPackageKey(key) }
// SeriesKey is the key of a package's series in the packages state (SeriesPackageKey).
func SeriesKey(pkg, series string) string { return SeriesPackageKey(pkg, series) }
// IsSeriesKey says whether a key of the packages state is a series key, not a per-module one (IsSeriesPackageKey).
func IsSeriesKey(key string) bool { return IsSeriesPackageKey(key) }
// SeriesOf is the series of a trunk version (`0.1.15` is of `0.1`); false for a version that is no `<major>.<minor>.<patch>`
// or that carries a tag (a pre-release).
@@ -66,11 +83,11 @@ func (ix *Indexer) HearVersion(ctx context.Context, h HeardVersion) (bool, error
return false, nil
}
v := &SeriesVersion{Version: strings.TrimPrefix(h.Version, "v"), Commit: h.Commit, Digest: h.Digest, Fingerprint: h.Fingerprint}
numbered := *v
numbered := numberedOf(*v)
if h.Published && h.GoModule != "" && h.Commit != "" && ix.Files != nil {
// The digests of the published version's commit, read before the write (DLV-I6): a user accepting the series is
// fingerprinted against what this version holds, not against the latest merge.
repo, _, _ := delivery.PackageRepository(h.GoModule)
repo, _, _ := trunk.PackageRepository(h.GoModule)
got, err := ix.packagesAt(ctx, repo, h.Commit, []string{h.GoModule}, nil)
switch p, ok := got[h.GoModule]; {
case err != nil:
@@ -78,32 +95,30 @@ func (ix *Indexer) HearVersion(ctx context.Context, h HeardVersion) (bool, error
case !ok:
ix.logf("%s %s: the forge's listing at %s holds no Go module %s, so its users keep the commit form", h.Package, h.Version, h.Commit, h.GoModule)
default:
// When the contents were read: delivery.SeriesVersion.Contents is a PublishedPackages, which names its time
// When the contents were read: a series version's Contents is a published module, which names its time
// (DecodeSeriesPackage refuses one with none).
p.At = ix.now().UTC()
v.Contents = &p
p.At = ptr(ix.now().UTC())
v.Contents = contentsOfPublished(p)
}
}
newer := func(held *SeriesVersion) bool {
return held == nil || CompareVersions("v"+held.Version, "v"+v.Version) < 0
}
newer := func(held string, _ bool) bool { return CompareVersions("v"+held, "v"+v.Version) < 0 }
kept := false
err := write(ctx, ix.Packages, SeriesKey(h.Package, series), func(value []byte, found bool) ([]byte, bool, error) {
s := SeriesPackage{V: delivery.Version, Name: h.Package, Series: series}
s := SeriesPackage{V: keys.Version, Name: h.Package, Series: series}
if found {
if err := json.Unmarshal(value, &s); err != nil {
ix.logf("packages.%s holds what is not a series, written anew: %v", SeriesKey(h.Package, series), err)
s = SeriesPackage{V: delivery.Version, Name: h.Package, Series: series}
s = SeriesPackage{V: keys.Version, Name: h.Package, Series: series}
}
}
kept = false
if s.Name == "" {
s.Name, kept = h.Package, true
}
if newer(s.Numbered) {
s.Numbered, kept = &numbered, true
if s.Numbered == nil || newer(s.Numbered.Version, true) {
s.Numbered, kept = numbered, true
}
if h.Published && newer(s.Newest) {
if h.Published && (s.Newest == nil || newer(s.Newest.Version, true)) {
s.Newest, kept = v, true
}
if h.Seat != "" && s.Seat != h.Seat {
@@ -115,7 +130,7 @@ func (ix *Indexer) HearVersion(ctx context.Context, h HeardVersion) (bool, error
if !kept {
return nil, false, nil
}
s.At = ix.now()
s.At = ptr(ix.now())
next, err := json.Marshal(s)
return next, false, err
})
@@ -141,7 +156,7 @@ func (ix *Indexer) SeriesAll(ctx context.Context) map[string]SeriesPackage {
if err != nil || !found {
continue
}
s, err := delivery.DecodeSeriesPackage(value)
s, err := DecodeSeriesPackage(value)
if err != nil {
ix.logf("packages.%s holds what is not a series, left out: %v", key, err)
continue
@@ -158,21 +173,21 @@ func (ix *Indexer) SeriesAll(ctx context.Context) map[string]SeriesPackage {
// `v0.1.15`). A series with no published version yet, or whose contents the forge did not list, leaves the module's
// latest published commit (the commit form, milestone § 2 item 4). A module of the same repository takes its sibling by
// path, never by a series. With nothing to pin, published is answered as it is.
func acceptedSeries(series map[string]SeriesPackage, m catalogue.Manifest, repository string, published map[string]delivery.PublishedPackages) (map[string]delivery.PublishedPackages, delivery.PackageVersions) {
var out map[string]delivery.PublishedPackages
var pins delivery.PackageVersions
func acceptedSeries(series map[string]SeriesPackage, m catalogue.Manifest, repository string, published map[string]meshcatalog.PackagesByModuleValue) (map[string]meshcatalog.PackagesByModuleValue, sdkstep.PackageVersions) {
var out map[string]meshcatalog.PackagesByModuleValue
var pins sdkstep.PackageVersions
for _, key := range slices.Sorted(maps.Keys(series)) {
s := series[key]
if s.GoModule == "" || s.Newest == nil || s.Newest.Contents == nil || m.AcceptedSeries(s.Name) != s.Series {
continue
}
if repo, _, ok := delivery.PackageRepository(s.GoModule); !ok || delivery.SameRepository(repo, repository) {
if repo, _, ok := trunk.PackageRepository(s.GoModule); !ok || trunk.SameRepository(repo, repository) {
continue
}
if out == nil {
out, pins = maps.Clone(published), delivery.PackageVersions{}
out, pins = maps.Clone(published), sdkstep.PackageVersions{}
}
p := *s.Newest.Contents
p := publishedOfContents(*s.Newest.Contents)
version := "v" + s.Newest.Version
p.Commit = version
out[s.GoModule], pins[s.GoModule] = p, version
@@ -209,9 +224,9 @@ func (ix *Indexer) KeepPackageNames(ctx context.Context, f forgemsg.Forge) error
}
series, built := a.SeriesOf(), m.Module+"/"+a.Name
errs = append(errs, write(ctx, ix.Packages, SeriesKey(name, series), func(value []byte, found bool) ([]byte, bool, error) {
s := SeriesPackage{V: delivery.Version, Series: series}
s := SeriesPackage{V: keys.Version, Series: series}
if found && json.Unmarshal(value, &s) != nil {
s = SeriesPackage{V: delivery.Version, Series: series}
s = SeriesPackage{V: keys.Version, Series: series}
}
if s.Name == name && s.Package == built && (a.Language != "go" || s.GoModule == name) {
return nil, false, nil
@@ -220,7 +235,7 @@ func (ix *Indexer) KeepPackageNames(ctx context.Context, f forgemsg.Forge) error
if a.Language == "go" {
s.GoModule = name
}
s.At = ix.now()
s.At = ptr(ix.now())
next, err := json.Marshal(s)
return next, false, err
}))
modules/mesh-delivery-indexer/internal/index/source.go
+10 -8
View File
@@ -6,7 +6,9 @@ import (
"maps"
"slices"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery"
"git.novox.be/novox/mesh-delivery/sdk/trunk"
)
// The gate's source revision (novox/hq ADR 0437 §5, ADR 0450 §1): source.main names every repository's merged head, and its
@@ -36,7 +38,7 @@ func (ix *Indexer) srcOf(repository, merge string) uint64 {
// that carries no Src yet, is written again with Src the revision (once: a later revision leaves it); a module computed at another commit is left (a merge heard later is stamped
// when it is written, applyMerged), and a repository none of the indexer's modules is built from writes nothing. The
// modules of a repository are those the modules bucket holds of it (ADR 0501 clause 6).
func (ix *Indexer) ApplySource(ctx context.Context, rev uint64, s delivery.Source) error {
func (ix *Indexer) ApplySource(ctx context.Context, rev uint64, s meshdelivery.SourceByMainValue) error {
ix.kickJudge(ctx)
ix.mu.Lock()
if rev < ix.source.rev {
@@ -59,7 +61,7 @@ func (ix *Indexer) ApplySource(ctx context.Context, rev uint64, s delivery.Sourc
for _, key := range slices.Sorted(maps.Keys(held)) {
repository := ""
for r := range s.Heads {
if delivery.SameRepository(r, held[key].Repository) {
if trunk.SameRepository(r, held[key].Repository) {
repository = r
}
}
@@ -71,7 +73,7 @@ func (ix *Indexer) ApplySource(ctx context.Context, rev uint64, s delivery.Sourc
if !found {
return nil, false, nil
}
cur, err := delivery.DecodeModule(value)
cur, err := meshcatalogclient.DecodeModule(value)
if err != nil {
ix.logf("modules.%s holds what is not a module, left: %v", key, err)
return nil, false, nil
@@ -81,11 +83,11 @@ func (ix *Indexer) ApplySource(ctx context.Context, rev uint64, s delivery.Sourc
if cur.Repository != repository || cur.Commit != head || cur.Src != 0 {
return nil, false, nil
}
cur.Src = rev
if cur.FPSrc == 0 {
cur.FPSrc = rev // its fingerprint moved at this merge, whose revision is heard only now (ADR 0601 §1)
cur.Src = int64(rev)
if cur.FpSrc == 0 {
cur.FpSrc = int64(rev) // its fingerprint moved at this merge, whose revision is heard only now (ADR 0601 §1)
}
data, err := delivery.EncodeModule(cur)
data, err := meshcatalogclient.EncodeModule(cur)
return data, false, err
})
if err != nil {
modules/mesh-delivery-indexer/internal/index/source_test.go
+5 -5
View File
@@ -5,13 +5,13 @@ import (
"reflect"
"testing"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery"
)
const otherMerge = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"
func sourceOf(repo, head string) delivery.Source {
return delivery.Source{Group: "g", Heads: map[string]string{repo: head}}
func sourceOf(repo, head string) meshdelivery.SourceByMainValue {
return meshdelivery.SourceByMainValue{Group: "g", Heads: map[string]string{repo: head}}
}
func TestASourceRevisionStampsTheModulesOfTheRepositoriesHeadItNames(t *testing.T) { // 1344/AC1
@@ -25,13 +25,13 @@ func TestASourceRevisionStampsTheModulesOfTheRepositoriesHeadItNames(t *testing.
t.Fatalf("module before the source = %+v, want Src 0", before)
}
// The source names the head merged for the delivery repository, and another repository's head no module of ours is built from.
s := delivery.Source{Group: "g", Heads: map[string]string{"novox/mesh-delivery": merge, "novox/mesh-catalog": otherMerge}}
s := meshdelivery.SourceByMainValue{Group: "g", Heads: map[string]string{"novox/mesh-delivery": merge, "novox/mesh-catalog": otherMerge}}
if err := ix.ApplySource(ctx, 7, s); err != nil {
t.Fatal(err)
}
m, ok := moduleOf(t, mods, "a")
want := before
want.Src, want.FPSrc = 7, 7 // its fingerprint moved at that merge (ADR 0601 §1)
want.Src, want.FpSrc = 7, 7 // its fingerprint moved at that merge (ADR 0601 §1)
if !ok || !reflect.DeepEqual(m, want) {
t.Errorf("module a = %+v (%v), want %+v: the fingerprint and the commit as the merge wrote them, with Src 7", m, ok, want)
}
modules/mesh-delivery-indexer/internal/index/stale_none2_test.go
+4 -4
View File
@@ -20,7 +20,7 @@ func unreadableAvahi(t *testing.T) string {
func TestAMergeOfAnUnreadableManifestKeepsTheInputsBesideTheReason(t *testing.T) { // 1923/AC1
_, mods := newFormIndexer(t, mod{"avahi", unreadableAvahi(t)})
m, ok := moduleOf(t, mods, "avahi")
if !ok || !strings.HasPrefix(m.None2, "the manifest cannot be read: ") || m.FP2 != "" {
if !ok || !strings.HasPrefix(m.None2, "the manifest cannot be read: ") || m.Fp2 != "" {
t.Fatalf("module = %+v, want the reason 'the manifest cannot be read' and no fingerprint", m)
}
if m.Inputs2 == nil || m.Inputs2.Tree != "tree-avahi" || m.Inputs2.Locks["go.sum"] != avahiGoSum {
@@ -35,7 +35,7 @@ func TestAStaleUnreadableVerdictIsJudgedAgainAtStartAndAnotherReasonIsLeft(t *te
reportToolchain(t, ixGood, "mesh-build-go", "novox", "go", "go1.27.1")
reportToolchain(t, ixGood, "mesh-build-go", "ace", "go", "go1.27.1")
want, _ := moduleOf(t, good, "avahi")
if want.FP2 == "" {
if want.Fp2 == "" {
t.Fatalf("the readable manifest gives no fingerprint: %+v", want)
}
@@ -51,8 +51,8 @@ func TestAStaleUnreadableVerdictIsJudgedAgainAtStartAndAnotherReasonIsLeft(t *te
t.Fatal(err)
}
got, _ := moduleOf(t, mods, "avahi")
if got.FP2 != want.FP2 || got.None2 != "" {
t.Errorf("after the start: fp2 %q none2 %q, want %q and no reason", got.FP2, got.None2, want.FP2)
if got.Fp2 != want.Fp2 || got.None2 != "" {
t.Errorf("after the start: fp2 %q none2 %q, want %q and no reason", got.Fp2, got.None2, want.Fp2)
}
// A manifest still unreadable keeps its value as it was.
modules/mesh-delivery-indexer/internal/index/takein.go
+5 -4
View File
@@ -8,7 +8,8 @@ import (
"strings"
"time"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/sdk/trunk"
)
// The indexer's doing tool (novox/hq issue 1945; the instruction "tools are how a module is spoken to"): indexer_take_in
@@ -20,7 +21,7 @@ const (
TakeInTool = "indexer_take_in"
// TakeInsState is the indexer's state of take-in asks: the bucket mesh-delivery-indexer_takeins, keyed as the modules state.
TakeInsState = "takeins"
TakeInsBucket = delivery.IndexerModule + "_" + TakeInsState
TakeInsBucket = Module + "_" + TakeInsState
)
// TakeInAsk is a value of the takeins state: which module a person asked to be taken in, why, who and when, and, once the
@@ -84,7 +85,7 @@ func NewTakeIn(module, why, by string, at time.Time) (string, []byte, error) {
if module == "" {
return "", nil, errors.New("name the module to take in (module)")
}
key, err := delivery.ModulesKey(module)
key, err := meshcatalogclient.ModulesKey(module)
if err != nil {
return "", nil, err
}
@@ -95,7 +96,7 @@ func NewTakeIn(module, why, by string, at time.Time) (string, []byte, error) {
// NewTakeInRepository is the value indexer_take_in writes for a repository: its key and the value.
func NewTakeInRepository(repository, why, by string, at time.Time) (string, []byte, error) {
repository = strings.TrimSpace(repository)
key, err := delivery.RepositoryKey(repository)
key, err := trunk.RepositoryKey(repository)
if err != nil {
return "", nil, err
}
modules/mesh-delivery-indexer/internal/index/taken.go
+6 -4
View File
@@ -8,8 +8,10 @@ import (
"slices"
"time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-delivery/sdk/trunk"
"git.novox.be/novox/mesh-sdk/go/process"
)
@@ -30,14 +32,14 @@ func (ix *Indexer) writeTakenKnown(ctx context.Context, f forgemsg.Forge, notOld
if ix.Taken == nil || f.Merge == "" {
return nil
}
key, err := delivery.RepositoryKey(f.Repository)
key, err := trunk.RepositoryKey(f.Repository)
if err != nil {
ix.logf("%s at %s: no taken key is written: %v", f.Repository, f.Merge, err)
return nil
}
err = write(ctx, ix.Taken, key, func(value []byte, found bool) ([]byte, bool, error) {
if found {
if cur, err := delivery.DecodeTaken(value); err == nil {
if cur, err := takein.DecodeTaken(value); err == nil {
if cur.Merge == f.Merge {
return nil, false, nil
}
@@ -57,7 +59,7 @@ func (ix *Indexer) writeTakenKnown(ctx context.Context, f forgemsg.Forge, notOld
if ix.Now != nil {
now = ix.Now
}
data, err := delivery.EncodeTaken(delivery.Taken{Repository: delivery.RepositoryName(f.Repository), Merge: f.Merge, At: now().UTC()})
data, err := takein.EncodeTaken(takein.Taken{Repository: trunk.RepositoryName(f.Repository), Merge: f.Merge, At: now().UTC()})
return data, false, err
})
if errors.Is(err, process.ErrNotYet) {
modules/mesh-delivery-indexer/internal/index/testdata/candidate-written.json
Vendored
+10 -10
View File
@@ -6,26 +6,26 @@
}
},
"candidate": {
"v": 1,
"group": "feat/x",
"base": 5,
"fpset": "01c840ec276b",
"group": "feat/x",
"heads": "dfe9fb8f79e8",
"pulls": [
{
"repository": "novox/mesh-catalog",
"number": 301,
"head": "0123456789abcdef0123456789abcdef01234567",
"modules": [
"avahi"
],
"fps": {
"avahi": {
"fp2": "src2:e8d595aea1b13f1f5578ef1f245dab0d28a351c28079ecdcd2276c63a932d7be",
"manifest": "8a3856b6c0566d76f2556fc4f46c5251dc7935517bb6bd15809ded7e5546a0d8"
}
}
},
"head": "0123456789abcdef0123456789abcdef01234567",
"modules": [
"avahi"
],
"number": 301,
"repository": "novox/mesh-catalog"
}
],
"fpset": "01c840ec276b"
"v": 1
}
}
modules/mesh-delivery-indexer/internal/index/toolchains.go
+67 -62
View File
@@ -8,8 +8,12 @@ import (
"slices"
"strings"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/nodebuildgo"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
sdkstep "git.novox.be/novox/mesh-delivery/sdk/buildstep"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
"mesh-delivery-indexer/judge"
)
@@ -23,7 +27,7 @@ import (
// records builds and makes no artifact's bytes (ADR 0498 §1), so its Builder input is empty for every module, the builder's
// own included; a change of how an artifact is built is a change of its build module's recipe (ADR 0530 §2).
// ToolchainModules are the build module of each language, whose state toolchains (delivery.ToolchainsBucket) the
// ToolchainModules are the build module of each language, whose state toolchains (ToolchainsBucket) the
// indexer reads: the language's agreed toolchain is the one every node's entry in it agrees on.
var ToolchainModules = map[string]string{"go": "mesh-build-go", "typescript": "mesh-build-node"}
@@ -33,7 +37,7 @@ var ToolchainModules = map[string]string{"go": "mesh-build-go", "typescript": "m
var ToolchainStreams = map[string]string{"mesh-build-go": "TOOLCHAINS_GO", "mesh-build-node": "TOOLCHAINS_NODE"}
// PutToolchain takes one change of a build module's toolchains bucket in, without writing any module: a put of a node's
// toolchain of a language (key <node>.<language>, delivery.ToolchainKey), or its delete. It answers whether what the
// toolchain of a language (key <node>.<language>, ToolchainKey), or its delete. It answers whether what the
// language's toolchain is of (version and recipe) changed. A key, or a value, that is not a toolchain of the module's
// language is said in the log and left out.
func (ix *Indexer) PutToolchain(module, key string, value []byte, deleted bool) bool {
@@ -47,7 +51,7 @@ func (ix *Indexer) PutToolchain(module, key string, value []byte, deleted bool)
ix.logf("toolchains of %q are not read: it is no build module the indexer knows", module)
return false
}
node, keyLanguage, err := delivery.ParseToolchainKey(key)
node, keyLanguage, err := ParseToolchainKey(key)
if err != nil || keyLanguage != language {
ix.logf("%s_toolchains.%s is not a %s toolchain's key, left out: %v", module, key, language, err)
return false
@@ -61,16 +65,16 @@ func (ix *Indexer) PutToolchain(module, key string, value []byte, deleted bool)
delete(ix.toolchains[language], node)
return true
}
tc, err := delivery.DecodeToolchain(value)
tc, err := DecodeToolchain(value)
if err != nil || tc.Language != language {
ix.logf("%s_toolchains.%s is not a %s toolchain, left out: %v", module, key, language, err)
return false
}
if ix.toolchains == nil {
ix.toolchains = map[string]map[string]delivery.Toolchain{}
ix.toolchains = map[string]map[string]nodebuildgo.Toolchain{}
}
if ix.toolchains[language] == nil {
ix.toolchains[language] = map[string]delivery.Toolchain{}
ix.toolchains[language] = map[string]nodebuildgo.Toolchain{}
}
cur, had := ix.toolchains[language][node]
ix.toolchains[language][node] = tc
@@ -102,32 +106,33 @@ func (ix *Indexer) agreed() map[string]toolchainOrNone {
defer ix.mu.Unlock()
out := map[string]toolchainOrNone{}
for language := range ToolchainModules {
var entries []delivery.NodeToolchain
var entries []sdkstep.NodeToolchain
for _, node := range slices.Sorted(maps.Keys(ix.toolchains[language])) {
entries = append(entries, delivery.NodeToolchain{Node: node, Toolchain: ix.toolchains[language][node]})
t := ix.toolchains[language][node]
entries = append(entries, sdkstep.NodeToolchain{Node: node, Toolchain: sdkstep.Toolchain{Language: t.Language, Version: t.Version, Recipe: t.Recipe}})
}
tc, none := delivery.AgreedToolchain(entries)
out[language] = toolchainOrNone{tc, none}
tc, none := sdkstep.AgreedToolchain(entries)
out[language] = toolchainOrNone{meshcatalogclient.Toolchain2{Language: tc.Language, Version: tc.Version, Recipe: tc.Recipe}, none}
}
return out
}
type toolchainOrNone struct {
tc delivery.Toolchain2
tc meshcatalogclient.Toolchain2
none string
}
// known is a module the indexer holds the inputs of: its record as read, the manifest's shape and the key it is written at.
type known struct {
key string
cur delivery.Module
cur meshcatalog.ModulesByModuleValue
manifest catalogue.Manifest
languages map[string]string
bases []string
contexts bool
// held is the fingerprint the bucket holds for the module and the toolchains it was computed with (for a merge, the
// value before it): what stillComputed keeps while the builders agree on no toolchain.
held delivery.Module
held meshcatalog.ModulesByModuleValue
}
// Recompute writes again every module whose new-form fingerprint, or reason for none, is no longer what it holds: from
@@ -142,7 +147,7 @@ func (ix *Indexer) Recompute(ctx context.Context) error { return ix.recomputeWit
// of what the bucket holds, are computed with the others, and are written once, with their final fingerprint or reason
// (issue 1444: a merge writes each module once). A record of the overlay that has no kept inputs is written as it is. w is
// the merge's write (order.go), nil for none; it is told each module written.
func (ix *Indexer) recomputeWith(ctx context.Context, overlay map[string]delivery.Module, w *mergeWrite) error {
func (ix *Indexer) recomputeWith(ctx context.Context, overlay map[string]meshcatalog.ModulesByModuleValue, w *mergeWrite) error {
if err := ix.fillPaths(ctx); err != nil {
return err
}
@@ -153,7 +158,7 @@ func (ix *Indexer) recomputeWith(ctx context.Context, overlay map[string]deliver
return ix.RecomputeCandidates(ctx)
}
func (ix *Indexer) recomputeModules(ctx context.Context, overlay map[string]delivery.Module, w *mergeWrite) error {
func (ix *Indexer) recomputeModules(ctx context.Context, overlay map[string]meshcatalog.ModulesByModuleValue, w *mergeWrite) error {
if ix.Modules == nil {
return nil
}
@@ -166,7 +171,7 @@ func (ix *Indexer) recomputeModules(ctx context.Context, overlay map[string]deli
}
// recomputeLocked is recomputeModules with ix.recompute held by the caller (a hold that moves, holds.go).
func (ix *Indexer) recomputeLocked(ctx context.Context, overlay map[string]delivery.Module, w *mergeWrite) error {
func (ix *Indexer) recomputeLocked(ctx context.Context, overlay map[string]meshcatalog.ModulesByModuleValue, w *mergeWrite) error {
if ix.Manifests == nil {
return ix.writeOverlay(ctx, overlay, nil, w)
}
@@ -180,18 +185,18 @@ func (ix *Indexer) recomputeLocked(ctx context.Context, overlay map[string]deliv
}
}
slices.Sort(keys)
held := map[string]delivery.Module{}
held := map[string]meshcatalog.ModulesByModuleValue{}
stale := map[string]bool{} // values an SDK that could not read their manifest wrote, judged again (issue 1923)
all := map[string]*known{}
for _, key := range keys {
cur, merged := overlay[key]
var before delivery.Module
var before meshcatalog.ModulesByModuleValue
value, _, found, err := ix.Modules.Get(ctx, key)
if err != nil {
return fmt.Errorf("modules.%s: %w", key, err)
}
if found {
if before, err = delivery.DecodeModule(value); err != nil && !merged {
if before, err = meshcatalogclient.DecodeModule(value); err != nil && !merged {
ix.logf("modules.%s holds what is not a module, left: %v", key, err)
continue
}
@@ -222,8 +227,8 @@ func (ix *Indexer) recomputeLocked(ctx context.Context, overlay map[string]deliv
series := ix.SeriesAll(ctx) // each package's series, the newest a user accepting it takes (DLV-I6)
type result struct {
fp, none string
tcs map[string]delivery.ToolchainUsed // by language, when there is a fingerprint
pkgs delivery.PackageVersions // the package versions, when there is a fingerprint
tcs map[string]meshcatalog.ToolchainUsed // by language, when there is a fingerprint
pkgs sdkstep.PackageVersions // the package versions, when there is a fingerprint
}
done := map[string]result{}
visiting := map[string]bool{}
@@ -238,8 +243,8 @@ func (ix *Indexer) recomputeLocked(ctx context.Context, overlay map[string]deliv
switch {
case !has:
return result{fp: "", none: "the indexer holds no record of it"}
case cur.FP2 != "":
return result{fp: cur.FP2, none: ""}
case cur.Fp2 != "":
return result{fp: cur.Fp2, none: ""}
case cur.None2 != "":
return result{fp: "", none: cur.None2}
}
@@ -263,7 +268,7 @@ func (ix *Indexer) recomputeLocked(ctx context.Context, overlay map[string]deliv
}
slices.Sort(kept.Bases)
slices.Sort(missing)
toolchains := map[string]delivery.Toolchain2{}
toolchains := map[string]meshcatalogclient.Toolchain2{}
var noToolchain string
for _, artifact := range slices.Sorted(maps.Keys(k.languages)) {
t := tools[k.languages[artifact]]
@@ -298,7 +303,7 @@ func (ix *Indexer) recomputeLocked(ctx context.Context, overlay map[string]deliv
k.cur.Inputs2 = &kept
return r
}
finals := map[string]delivery.Module{}
finals := map[string]meshcatalog.ModulesByModuleValue{}
edges := map[string]graphEdges{} // the module graph's nodes of this recompute (graph.go)
for _, key := range slices.Sorted(maps.Keys(all)) {
k := all[key]
@@ -306,11 +311,11 @@ func (ix *Indexer) recomputeLocked(ctx context.Context, overlay map[string]deliv
edges[key] = graphEdges{module: key, cur: k.cur, manifest: k.manifest, fp: r.fp, none: r.none, tcs: r.tcs, pkgs: r.pkgs}
if _, merged := overlay[key]; merged {
final := overlay[key]
final.FP2, final.None2, final.Inputs2, final.Toolchains, final.Packages = r.fp, r.none, k.cur.Inputs2, r.tcs, r.pkgs
final.Fp2, final.None2, final.Inputs2, final.Toolchains, final.Packages = r.fp, r.none, k.cur.Inputs2, r.tcs, r.pkgs
// What the bases gave is kept as it was while the value it gave does not change, so a merge
// that moves nothing for the module writes nothing (as Recompute does).
if value, _, found, err := ix.Modules.Get(ctx, key); err == nil && found {
if old, err := delivery.DecodeModule(value); err == nil && old.Inputs2 != nil && old.FP2 == r.fp && old.None2 == r.none {
if old, err := meshcatalogclient.DecodeModule(value); err == nil && old.Inputs2 != nil && old.Fp2 == r.fp && old.None2 == r.none {
in := *final.Inputs2
in.Bases, in.Builder = old.Inputs2.Bases, old.Inputs2.Builder
final.Inputs2 = &in
@@ -319,32 +324,32 @@ func (ix *Indexer) recomputeLocked(ctx context.Context, overlay map[string]deliv
finals[key] = final
continue
}
if !stale[key] && k.cur.FP2 == r.fp && k.cur.None2 == r.none && reflect.DeepEqual(k.cur.Toolchains, r.tcs) && reflect.DeepEqual(k.cur.Packages, r.pkgs) {
if !stale[key] && k.cur.Fp2 == r.fp && k.cur.None2 == r.none && reflect.DeepEqual(k.cur.Toolchains, r.tcs) && reflect.DeepEqual(k.cur.Packages, r.pkgs) {
continue
}
digest, inputs, paths := k.cur.Manifest, k.cur.Inputs2, k.cur.ArtifactPaths
digest, inputs, paths := k.cur.Manifest, k.cur.Inputs2, k.cur.Paths
wrote := false
err := write(ctx, ix.Modules, key, func(value []byte, found bool) ([]byte, bool, error) {
if !found {
return nil, false, nil
}
cur, err := delivery.DecodeModule(value)
cur, err := meshcatalogclient.DecodeModule(value)
if err != nil || cur.Manifest != digest || cur.Inputs2 == nil {
return nil, false, nil // a merge changed it meanwhile; its own recompute follows
}
if cur.FP2 == r.fp && cur.None2 == r.none && reflect.DeepEqual(cur.Toolchains, r.tcs) && reflect.DeepEqual(cur.Packages, r.pkgs) {
if cur.Fp2 == r.fp && cur.None2 == r.none && reflect.DeepEqual(cur.Toolchains, r.tcs) && reflect.DeepEqual(cur.Packages, r.pkgs) {
return nil, false, nil
}
if cur.FP2 != r.fp {
if cur.Fp2 != r.fp {
// Moved with no merge (a toolchain, a package, a base): no release carries this build, and the source revision
// moves only at a merge, so it is not known (0) and the move is asked of a person; never the revision of now,
// which may be the very release that last merged the module (ADR 0601 §1). The source stamp leaves it, since
// Src is set.
cur.FPSrc = 0
cur.FpSrc = 0
}
cur.FP2, cur.None2, cur.Toolchains, cur.Packages = r.fp, r.none, r.tcs, r.pkgs
cur.Inputs2, cur.ArtifactPaths = inputs, paths
data, err := delivery.EncodeModule(cur)
cur.Fp2, cur.None2, cur.Toolchains, cur.Packages = r.fp, r.none, r.tcs, r.pkgs
cur.Inputs2, cur.Paths = inputs, paths
data, err := meshcatalogclient.EncodeModule(cur)
wrote = err == nil
return data, false, err
})
@@ -371,7 +376,7 @@ func (ix *Indexer) recomputeLocked(ctx context.Context, overlay map[string]deliv
// the repository, and the controller's take-in hears only that one. The repository's trunk head is the gate's source.main.
//
// A merged message older than the repository's taken merge never reaches here (order.go): it is asked before any write.
func (ix *Indexer) writeOverlay(ctx context.Context, overlay, finals map[string]delivery.Module, w *mergeWrite) error {
func (ix *Indexer) writeOverlay(ctx context.Context, overlay, finals map[string]meshcatalog.ModulesByModuleValue, w *mergeWrite) error {
for _, key := range slices.Sorted(maps.Keys(overlay)) {
next := overlay[key]
if f, ok := finals[key]; ok {
@@ -381,7 +386,7 @@ func (ix *Indexer) writeOverlay(ctx context.Context, overlay, finals map[string]
err := write(ctx, ix.Modules, key, func(value []byte, found bool) ([]byte, bool, error) {
wrote = false
next := next
cur, err := delivery.DecodeModule(value)
cur, err := meshcatalogclient.DecodeModule(value)
held := found && err == nil
if held && cur.Manifest == next.Manifest {
// The judge's verdict stands while the digest does (ADR 0578 §3, ADR 0579 §1): a recompute never writes a
@@ -391,10 +396,10 @@ func (ix *Indexer) writeOverlay(ctx context.Context, overlay, finals map[string]
}
// The source revision at which FP2 last changed (ADR 0601 §1): this merge's when it moved it (0 until the
// source revision naming the merge is heard, which stamps it), else the one held.
if held && cur.FP2 == next.FP2 {
next.FPSrc = cur.FPSrc
if held && cur.Fp2 == next.Fp2 {
next.FpSrc = cur.FpSrc
} else {
next.FPSrc = next.Src
next.FpSrc = next.Src
}
if held && sameButWhere(cur, next) {
return nil, false, nil
@@ -402,7 +407,7 @@ func (ix *Indexer) writeOverlay(ctx context.Context, overlay, finals map[string]
if next.Manifest != "" && (!held || cur.Manifest != next.Manifest) {
w.taken(next.Manifest) // a new module, or a changed manifest: fresh, judged in full (ADR 0579 §1)
}
data, err := delivery.EncodeModule(next)
data, err := meshcatalogclient.EncodeModule(next)
wrote = err == nil
return data, false, err
})
@@ -418,9 +423,9 @@ func (ix *Indexer) writeOverlay(ctx context.Context, overlay, finals map[string]
// sameButWhere says the held value cur is next but for where it was computed (Commit and Src) and the judge's verdict,
// which only the judge writes: the module is unchanged.
func sameButWhere(cur, next delivery.Module) bool {
func sameButWhere(cur, next meshcatalog.ModulesByModuleValue) bool {
next = withV(next)
next.Commit, next.Src, next.FPSrc = cur.Commit, cur.Src, cur.FPSrc
next.Commit, next.Src, next.FpSrc = cur.Commit, cur.Src, cur.FpSrc
next.Verdict, next.Problems, next.Unknown, next.Warnings = cur.Verdict, cur.Problems, cur.Unknown, cur.Warnings
next.Needs = cur.Needs
return reflect.DeepEqual(cur, next)
@@ -428,7 +433,7 @@ func sameButWhere(cur, next delivery.Module) bool {
// manifestRaw is manifests.<digest>: the manifest as stored and parsed by the running SDK.
func (ix *Indexer) manifestRaw(ctx context.Context, digest string) (string, catalogue.Manifest, error) {
key, err := delivery.ManifestsKey(digest)
key, err := meshcatalogclient.ManifestsKey(digest)
if err != nil {
return "", catalogue.Manifest{}, err
}
@@ -439,20 +444,20 @@ func (ix *Indexer) manifestRaw(ctx context.Context, digest string) (string, cata
if !found {
return "", catalogue.Manifest{}, fmt.Errorf("manifests.%s is not there", key)
}
kept, err := delivery.DecodeManifest(value)
kept, err := meshcatalogclient.DecodeManifest(value)
if err != nil {
return "", catalogue.Manifest{}, err
}
m, err := judge.ForInputs([]byte(kept.JSON))
return kept.JSON, m, err
m, err := judge.ForInputs([]byte(kept.Manifest))
return kept.Manifest, m, err
}
// rejudged is a value an SDK that could not read its manifest wrote, as the running SDK judges it: its kept inputs (the
// tree, the lock hashes by file name, the named paths' objects) taken through the manifest as a merge takes them, and the
// reason that gives. It answers false for any other value, and for one whose manifest the running SDK still cannot make
// inputs of (issue 1923).
func (ix *Indexer) rejudged(ctx context.Context, m catalogue.Manifest, cur delivery.Module, raw string) (delivery.Module, bool) {
if cur.FP2 != "" || cur.Inputs2 == nil || !strings.HasPrefix(cur.None2, None2Unreadable) {
func (ix *Indexer) rejudged(ctx context.Context, m catalogue.Manifest, cur meshcatalog.ModulesByModuleValue, raw string) (meshcatalog.ModulesByModuleValue, bool) {
if cur.Fp2 != "" || cur.Inputs2 == nil || !strings.HasPrefix(cur.None2, None2Unreadable) {
return cur, false
}
in := inputsOfNamed(m.Module, raw, cur.Inputs2.Tree, cur.Inputs2.Locks, cur.Inputs2.Paths, cur.Inputs2.Packages, ix.baseFP(ctx))
@@ -460,7 +465,7 @@ func (ix *Indexer) rejudged(ctx context.Context, m catalogue.Manifest, cur deliv
return cur, false
}
in.Kept.Imports, in.Kept.ImportForm = cur.Inputs2.Imports, cur.Inputs2.ImportForm
cur.Inputs2, cur.None2, cur.ArtifactPaths = in.Kept, in.None2, in.ArtifactPaths
cur.Inputs2, cur.None2, cur.Paths = in.Kept, in.None2, in.ArtifactPaths
return cur, true
}
@@ -484,11 +489,11 @@ func hasContext(m catalogue.Manifest) bool {
// with the toolchains it was computed with in place of the languages agreed on none, give it again, in its form. Any
// other input that moved (a merge, a base, an agreed toolchain, an imported package) gives another fingerprint, and the
// module has none until they agree.
func stillComputed(module string, kept delivery.KeptInputs2, k *known, agreed map[string]delivery.Toolchain2, held map[string]delivery.Module, published map[string]delivery.PublishedPackages, pins delivery.PackageVersions) (string, delivery.PackageVersions, bool) {
if k.held.FP2 == "" {
func stillComputed(module string, kept meshcatalog.ModulesByModuleValueInputs2, k *known, agreed map[string]meshcatalogclient.Toolchain2, held map[string]meshcatalog.ModulesByModuleValue, published map[string]meshcatalog.PackagesByModuleValue, pins sdkstep.PackageVersions) (string, sdkstep.PackageVersions, bool) {
if k.held.Fp2 == "" {
return "", nil, false
}
toolchains := map[string]delivery.Toolchain2{}
toolchains := map[string]meshcatalogclient.Toolchain2{}
for artifact, language := range k.languages {
if tc, ok := agreed[artifact]; ok {
toolchains[artifact] = tc
@@ -498,20 +503,20 @@ func stillComputed(module string, kept delivery.KeptInputs2, k *known, agreed ma
if !ok {
return "", nil, false
}
toolchains[artifact] = delivery.Toolchain2{Language: language, Version: used.Version, Recipe: used.Recipe}
toolchains[artifact] = meshcatalogclient.Toolchain2{Language: language, Version: used.Version, Recipe: used.Recipe}
}
f := formOf(module, k.held.Repository, &kept, toolchains, k.held, held, published, pins, nil)
return f.fp, f.versions, f.none == "" && f.fp == k.held.FP2
return f.fp, f.versions, f.none == "" && f.fp == k.held.Fp2
}
// toolchainsUsed are the toolchains a fingerprint was computed with, by language (an artifact's toolchain is its language's).
func toolchainsUsed(by map[string]delivery.Toolchain2) map[string]delivery.ToolchainUsed {
func toolchainsUsed(by map[string]meshcatalogclient.Toolchain2) map[string]meshcatalog.ToolchainUsed {
if len(by) == 0 {
return nil
}
out := map[string]delivery.ToolchainUsed{}
out := map[string]meshcatalog.ToolchainUsed{}
for _, t := range by {
out[t.Language] = delivery.ToolchainUsed{Version: t.Version, Recipe: t.Recipe}
out[t.Language] = meshcatalog.ToolchainUsed{Version: t.Version, Recipe: t.Recipe}
}
return out
}
@@ -554,12 +559,12 @@ func (ix *Indexer) fillPaths(ctx context.Context) error {
if !found {
return nil, false, nil
}
cur, err := delivery.DecodeModule(value)
cur, err := meshcatalogclient.DecodeModule(value)
if err != nil || cur.Path != "" {
return nil, false, nil
}
cur.Path = dir
data, err := delivery.EncodeModule(cur)
data, err := meshcatalogclient.EncodeModule(cur)
return data, false, err
})
if err != nil {
modules/mesh-delivery-indexer/internal/index/toolchains_test.go
+70 -63
View File
@@ -10,8 +10,15 @@ import (
"testing"
"time"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/nodebuildgo"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
"git.novox.be/novox/mesh-sdk/go/delivery"
sdkstep "git.novox.be/novox/mesh-delivery/sdk/buildstep"
"git.novox.be/novox/mesh-sdk/go/keys"
"git.novox.be/novox/mesh-sdk/go/process"
"git.novox.be/novox/mesh-delivery/sdk/graph"
@@ -56,8 +63,8 @@ const (
func agentBuild(id, src, at string) [2]any {
when, _ := time.Parse(time.RFC3339, at)
raw, _ := delivery.EncodeBuildLogEntry(delivery.BuildLogEntry{Event: delivery.BuildEndedEvent, Module: "build-agent", FP: src, Repository: agentCloneURL,
Directory: "modules/build-agent", Commit: "cada7e3c", Node: "n1", At: when, Outcome: delivery.BuildDone})
raw, _ := builds.EncodeBuildLogEntry(builds.BuildLogEntry{Event: builds.BuildEndedEvent, Module: "build-agent", FP: src, Repository: agentCloneURL,
Directory: "modules/build-agent", Commit: "cada7e3c", Node: "n1", At: when, Outcome: builds.BuildDone})
return [2]any{"build-agent", raw}
}
@@ -111,7 +118,7 @@ func newFormIndexerUnmerged(t *testing.T, mods ...mod) (*Indexer, *memStore, for
// toolchainValue is a toolchains value as the build module's writer encodes it (mesh-sdk go/delivery EncodeToolchain).
func toolchainValue(t *testing.T, language, version, recipe string) []byte {
t.Helper()
b, err := delivery.EncodeToolchain(delivery.Toolchain{Language: language, Version: version, Recipe: recipe, At: time.Now()})
b, err := EncodeToolchain(nodebuildgo.Toolchain{Language: language, Version: version, Recipe: recipe, At: ptr(time.Now())})
if err != nil {
t.Fatal(err)
}
@@ -125,7 +132,7 @@ func reportToolchain(t *testing.T, ix *Indexer, module, node, language, version
func reportToolchainRecipe(t *testing.T, ix *Indexer, module, node, language, version, recipe string) {
t.Helper()
key, err := delivery.ToolchainKey(node, language)
key, err := ToolchainKey(node, language)
if err != nil {
t.Fatal(err)
}
@@ -136,27 +143,27 @@ func reportToolchainRecipe(t *testing.T, ix *Indexer, module, node, language, ve
func TestAgreedToolchainChangesAGoModulesReasonWithNoMerge(t *testing.T) { // 1444/AC5
ix, mods := newFormIndexer(t, mod{"avahi", catalogueManifest(t, "avahi")})
if m, _ := moduleOf(t, mods, "avahi"); m.None2 != delivery.ToolchainNoReport {
if m, _ := moduleOf(t, mods, "avahi"); m.None2 != sdkstep.ToolchainNoReport {
t.Run("a recipe change builds a Go module at once and again once the agreement moves", aRecipeChangeBuildsAGoModuleAtOnceAndAgainOnceTheAgreementMoves)
t.Fatalf("before: none2 = %q", m.None2)
}
// The release that last merged avahi: its source revision names avahi's commit, so Src (and FPSrc) are 7.
merged, _ := moduleOf(t, mods, "avahi")
src := delivery.Source{Group: "g", Heads: map[string]string{merged.Repository: merged.Commit}}
src := meshdelivery.SourceByMainValue{Group: "g", Heads: map[string]string{merged.Repository: merged.Commit}}
if err := ix.ApplySource(context.Background(), 7, src); err != nil {
t.Fatal(err)
}
if m, _ := moduleOf(t, mods, "avahi"); m.Src != 7 || m.FPSrc != 7 {
t.Fatalf("before: src %d fp-src %d, want 7 and 7", m.Src, m.FPSrc)
if m, _ := moduleOf(t, mods, "avahi"); m.Src != 7 || m.FpSrc != 7 {
t.Fatalf("before: src %d fp-src %d, want 7 and 7", m.Src, m.FpSrc)
}
if g := graphOf(t, ix, "avahi"); g.None != delivery.ToolchainNoReport || g.Toolchains != nil {
if g := graphOf(t, ix, "avahi"); g.None != sdkstep.ToolchainNoReport || g.Toolchains != nil {
t.Errorf("before: graph.avahi = %+v, want no toolchain edge and the reason", g)
}
reportToolchain(t, ix, "mesh-build-go", "novox", "go", "go1.27.1")
reportToolchain(t, ix, "mesh-build-go", "ace", "go", "go1.27.1")
m, _ := moduleOf(t, mods, "avahi")
// L3 (ADR 0603 §2, §3): the agreement moving rewrites the module's toolchain edge, and again when it moves on.
if g := graphOf(t, ix, "avahi"); g.FP != m.FP2 || g.Toolchains["go"].Version != "go1.27.1" || g.Toolchains["go"].Module != "mesh-build-go" {
if g := graphOf(t, ix, "avahi"); g.FP != m.Fp2 || g.Toolchains["go"].Version != "go1.27.1" || g.Toolchains["go"].Module != "mesh-build-go" {
t.Errorf("after: graph.avahi = %+v, want the toolchain edge of go1.27.1", g)
}
reportToolchain(t, ix, "mesh-build-go", "novox", "go", "go1.28.0")
@@ -167,19 +174,19 @@ func TestAgreedToolchainChangesAGoModulesReasonWithNoMerge(t *testing.T) { // 14
reportToolchain(t, ix, "mesh-build-go", "novox", "go", "go1.27.1")
reportToolchain(t, ix, "mesh-build-go", "ace", "go", "go1.27.1")
m, _ = moduleOf(t, mods, "avahi")
if m.None2 != "" || m.FP2 == "" {
t.Errorf("none2 = %q fp2 = %q, want a fingerprint: no builder input is waited for (ADR 0530 §1)", m.None2, m.FP2)
if m.None2 != "" || m.Fp2 == "" {
t.Errorf("none2 = %q fp2 = %q, want a fingerprint: no builder input is waited for (ADR 0530 §1)", m.None2, m.Fp2)
}
// ADR 0601 §1 (review round 2): a fingerprint moved with no merge has no release that carries it: fp-src 0, not known,
// though the source revision is still the release's at the module's own Src; a source revision heard again stamps nothing.
if m.FPSrc != 0 {
t.Errorf("a fingerprint moved by a recompute: fp-src %d, want 0", m.FPSrc)
if m.FpSrc != 0 {
t.Errorf("a fingerprint moved by a recompute: fp-src %d, want 0", m.FpSrc)
}
if err := ix.ApplySource(context.Background(), 8, src); err != nil {
t.Fatal(err)
}
if again, _ := moduleOf(t, mods, "avahi"); again.FPSrc != 0 {
t.Errorf("a source revision stamped the recomputed fingerprint: fp-src %d", again.FPSrc)
if again, _ := moduleOf(t, mods, "avahi"); again.FpSrc != 0 {
t.Errorf("a source revision stamped the recomputed fingerprint: fp-src %d", again.FpSrc)
}
}
@@ -192,19 +199,19 @@ func aRecipeChangeBuildsAGoModuleAtOnceAndAgainOnceTheAgreementMoves(t *testing.
reportToolchain(t, ix, "mesh-build-go", node, "go", "go1.27.1")
}
first, _ := moduleOf(t, mods, "avahi")
if first.FP2 == "" || graphOf(t, ix, "avahi").Toolchains["go"].Recipe != "1" {
t.Fatalf("with the running recipe: fp2 %q, graph %+v", first.FP2, graphOf(t, ix, "avahi"))
if first.Fp2 == "" || graphOf(t, ix, "avahi").Toolchains["go"].Recipe != "1" {
t.Fatalf("with the running recipe: fp2 %q, graph %+v", first.Fp2, graphOf(t, ix, "avahi"))
}
// The release carrying the new recipe is merged, but the nodes still run the old one: one node reporting the new recipe is
// no strict majority, so the fingerprint stays, nothing waits.
reportToolchainRecipe(t, ix, "mesh-build-go", "novox", "go", "go1.27.1", "2")
if held, _ := moduleOf(t, mods, "avahi"); held.FP2 != first.FP2 || held.None2 != "" {
t.Errorf("one node on the new recipe: fp2 %q none2 %q, want the fingerprint of the running recipe, %q", held.FP2, held.None2, first.FP2)
if held, _ := moduleOf(t, mods, "avahi"); held.Fp2 != first.Fp2 || held.None2 != "" {
t.Errorf("one node on the new recipe: fp2 %q none2 %q, want the fingerprint of the running recipe, %q", held.Fp2, held.None2, first.Fp2)
}
reportToolchainRecipe(t, ix, "mesh-build-go", "ace", "go", "go1.27.1", "2")
again, _ := moduleOf(t, mods, "avahi")
if again.FP2 == "" || again.FP2 == first.FP2 || again.None2 != "" || graphOf(t, ix, "avahi").Toolchains["go"].Recipe != "2" {
t.Errorf("the agreement moved: fp2 %q (was %q) none2 %q graph %+v, want a second fingerprint at recipe 2", again.FP2, first.FP2, again.None2, graphOf(t, ix, "avahi"))
if again.Fp2 == "" || again.Fp2 == first.Fp2 || again.None2 != "" || graphOf(t, ix, "avahi").Toolchains["go"].Recipe != "2" {
t.Errorf("the agreement moved: fp2 %q (was %q) none2 %q graph %+v, want a second fingerprint at recipe 2", again.Fp2, first.Fp2, again.None2, graphOf(t, ix, "avahi"))
}
}
@@ -213,7 +220,7 @@ func TestNodesReportingDifferentVersionsLeaveTheReasonNamingBoth(t *testing.T) {
// disagree is kept: TestAModuleKeepsTheToolchainsItsFingerprintWasComputedWith).
ix, mods := newFormIndexer(t, mod{"avahi", catalogueManifest(t, "avahi")})
for node, version := range map[string]string{"novox": "go1.27.1", "ace": "go1.26.0"} {
key, _ := delivery.ToolchainKey(node, "go")
key, _ := ToolchainKey(node, "go")
ix.PutToolchain("mesh-build-go", key, toolchainValue(t, "go", version, "1"), false)
}
if err := ix.Recompute(context.Background()); err != nil {
@@ -241,7 +248,7 @@ func goInputs(t *testing.T, ix *Indexer, mods *memStore, module, builder string,
t.Helper()
return fingerprint.Inputs{Module: module, Tree: "tree-" + module, Builder: builder, Bases: bases,
Locks: map[string]string{"tools": avahiGoSum},
Toolchains: map[string]delivery.Toolchain2{"tools": {Language: "go", Version: "go1.27.1", Recipe: "1"}}}
Toolchains: map[string]meshcatalogclient.Toolchain2{"tools": {Language: "go", Version: "go1.27.1", Recipe: "1"}}}
}
func fp2(t *testing.T, in fingerprint.Inputs) string {
@@ -262,12 +269,12 @@ func TestBasesAreComputedFirstAndNoFingerprintTakesTheBuilder(t *testing.T) { //
toolsFP := fp2(t, goInputs(t, ix, mods, "mesh-tools", ""))
want := map[string]string{"mesh-delivery-builder": builderFP, "mesh-tools": toolsFP, "avahi": fp2(t, goInputs(t, ix, mods, "avahi", "", toolsFP))}
for name, fp := range want {
if m, _ := moduleOf(t, mods, name); m.FP2 != fp || m.None2 != "" || m.Inputs2 == nil || m.Inputs2.Builder != "" {
t.Errorf("%s: fp2 = %q none2 = %q inputs %+v, want %s with no builder input", name, m.FP2, m.None2, m.Inputs2, fp)
if m, _ := moduleOf(t, mods, name); m.Fp2 != fp || m.None2 != "" || m.Inputs2 == nil || m.Inputs2.Builder != "" {
t.Errorf("%s: fp2 = %q none2 = %q inputs %+v, want %s with no builder input", name, m.Fp2, m.None2, m.Inputs2, fp)
}
}
if m, _ := moduleOf(t, mods, "other"); m.FP2 != "" || !strings.Contains(m.None2, "ts-base") || !strings.Contains(m.None2, "base") {
t.Errorf("other: fp2 = %q none2 = %q, want none2 naming the base ts-base", m.FP2, m.None2)
if m, _ := moduleOf(t, mods, "other"); m.Fp2 != "" || !strings.Contains(m.None2, "ts-base") || !strings.Contains(m.None2, "base") {
t.Errorf("other: fp2 = %q none2 = %q, want none2 naming the base ts-base", m.Fp2, m.None2)
}
// L3 (ADR 0603 §2): the module graph holds each module's edges as this recompute computed them.
goTool := map[string]graph.Toolchain{"go": {Module: "mesh-build-go", Version: "go1.27.1", Recipe: "1"}}
@@ -328,8 +335,8 @@ func TestBasesAreComputedFirstAndNoFingerprintTakesTheBuilder(t *testing.T) { //
t.Fatal(err)
}
for name, fp := range want {
if m, _ := moduleOf(t, mods, name); m.FP2 != fp {
t.Errorf("%s moved with a build of build-agent: fp2 = %q, want %s", name, m.FP2, fp)
if m, _ := moduleOf(t, mods, name); m.Fp2 != fp {
t.Errorf("%s moved with a build of build-agent: fp2 = %q, want %s", name, m.Fp2, fp)
}
}
// ADR 0614 §2, §3: the graph names what each module invokes and serves, and leaves out, over the whole graph, the
@@ -356,7 +363,7 @@ func TestABuildHeardBeforeAnyToolchainIsReadDropsNoFingerprint(t *testing.T) { /
ix, mods := newFormIndexer(t, builderModules(t)...)
reportToolchain(t, ix, "mesh-build-go", "novox", "go", "go1.27.1")
before, _ := moduleOf(t, mods, "mesh-tools")
if before.FP2 == "" {
if before.Fp2 == "" {
t.Fatalf("mesh-tools has no fp2 to begin with: %+v", before)
}
// a restarted indexer has read no toolchains bucket yet
@@ -366,8 +373,8 @@ func TestABuildHeardBeforeAnyToolchainIsReadDropsNoFingerprint(t *testing.T) { /
if err := ix.RecomputeAtStart(context.Background()); err != nil {
t.Fatal(err)
}
if m, _ := moduleOf(t, mods, "mesh-tools"); m.FP2 != before.FP2 || m.None2 != "" {
t.Errorf("mesh-tools: fp2 = %q none2 = %q, want fp2 %q kept", m.FP2, m.None2, before.FP2)
if m, _ := moduleOf(t, mods, "mesh-tools"); m.Fp2 != before.Fp2 || m.None2 != "" {
t.Errorf("mesh-tools: fp2 = %q none2 = %q, want fp2 %q kept", m.Fp2, m.None2, before.Fp2)
}
}
@@ -388,8 +395,8 @@ func TestModulesThatStandOnEachOtherAreNamedAsACycle(t *testing.T) { // 1444 (th
mod{"bb", standingOn(t, renamed(t, "avahi", "bb"), "aa")})
reportToolchain(t, ix, "mesh-build-go", "novox", "go", "go1.27.1")
for _, name := range []string{"aa", "bb"} {
if m, _ := moduleOf(t, mods, name); m.FP2 != "" || !strings.Contains(m.None2, "cycle") {
t.Errorf("%s: fp2 = %q none2 = %q, want a reason naming the cycle", name, m.FP2, m.None2)
if m, _ := moduleOf(t, mods, name); m.Fp2 != "" || !strings.Contains(m.None2, "cycle") {
t.Errorf("%s: fp2 = %q none2 = %q, want a reason naming the cycle", name, m.Fp2, m.None2)
}
}
}
@@ -402,8 +409,8 @@ func TestAMergeWritesEachModuleOnceWithItsFinalValueAndAReplayWritesNothing(t *t
t.Fatal(err)
}
m, _ := moduleOf(t, mods, "avahi")
if mods.writes != 1 || m.FP2 == "" {
t.Errorf("writes = %d fp2 = %q after one merge, want one write with its fingerprint", mods.writes, m.FP2)
if mods.writes != 1 || m.Fp2 == "" {
t.Errorf("writes = %d fp2 = %q after one merge, want one write with its fingerprint", mods.writes, m.Fp2)
}
if err := ix.Apply(ctx, f); err != nil {
t.Fatal(err)
@@ -418,7 +425,7 @@ func TestAMergeKeepsAFingerprintTheModuleStillHas(t *testing.T) { // 1444 (revie
ix, mods := newFormIndexer(t, builderModules(t)...)
reportToolchain(t, ix, "mesh-build-go", "novox", "go", "go1.27.1")
before, _ := moduleOf(t, mods, "mesh-tools")
if before.FP2 == "" {
if before.Fp2 == "" {
t.Fatalf("mesh-tools has no fp2: %+v", before)
}
w := mods.writes
@@ -429,8 +436,8 @@ func TestAMergeKeepsAFingerprintTheModuleStillHas(t *testing.T) { // 1444 (revie
t.Fatal(err)
}
// and the commit stays the one the fingerprint first appeared at (the operator, 2026-10-25)
if after, _ := moduleOf(t, mods, "mesh-tools"); after.FP2 != before.FP2 || after.Commit != before.Commit || mods.writes != w {
t.Errorf("fp2 = %q at %.8s (was %q at %.8s), writes %d more", after.FP2, after.Commit, before.FP2, before.Commit, mods.writes-w)
if after, _ := moduleOf(t, mods, "mesh-tools"); after.Fp2 != before.Fp2 || after.Commit != before.Commit || mods.writes != w {
t.Errorf("fp2 = %q at %.8s (was %q at %.8s), writes %d more", after.Fp2, after.Commit, before.Fp2, before.Commit, mods.writes-w)
}
}
@@ -441,18 +448,18 @@ func TestAModuleKeepsTheToolchainsItsFingerprintWasComputedWith(t *testing.T) {
}
reportToolchain(t, ix, "mesh-build-go", "novox", "go", "go1.27.1")
m, _ := moduleOf(t, mods, "avahi")
want := map[string]delivery.ToolchainUsed{"go": {Version: "go1.27.1", Recipe: "1"}}
if m.FP2 == "" || !reflect.DeepEqual(m.Toolchains, want) {
t.Fatalf("avahi: fp2 %q toolchains %+v, want %+v", m.FP2, m.Toolchains, want)
want := map[string]meshcatalog.ToolchainUsed{"go": {Version: "go1.27.1", Recipe: "1"}}
if m.Fp2 == "" || !reflect.DeepEqual(m.Toolchains, want) {
t.Fatalf("avahi: fp2 %q toolchains %+v, want %+v", m.Fp2, m.Toolchains, want)
}
if o, _ := moduleOf(t, mods, "other"); o.FP2 != "" || o.Toolchains != nil {
if o, _ := moduleOf(t, mods, "other"); o.Fp2 != "" || o.Toolchains != nil {
t.Errorf("a module with no fingerprint carries toolchains %+v", o.Toolchains)
}
// An upgrade moves the fingerprint and the toolchains together, with no merge.
reportToolchain(t, ix, "mesh-build-go", "novox", "go", "go1.27.2")
n, _ := moduleOf(t, mods, "avahi")
if n.FP2 == m.FP2 || n.Toolchains["go"].Version != "go1.27.2" {
t.Errorf("after the upgrade: fp2 %q (was %q) toolchains %+v, want the new version with the new fingerprint", n.FP2, m.FP2, n.Toolchains)
if n.Fp2 == m.Fp2 || n.Toolchains["go"].Version != "go1.27.2" {
t.Errorf("after the upgrade: fp2 %q (was %q) toolchains %+v, want the new version with the new fingerprint", n.Fp2, m.Fp2, n.Toolchains)
}
// Issue 1585 (ADR 0573 §2): a module whose go.sum names mesh-sdk takes mesh-sdk's latest published version into its
// fingerprint and keeps it beside its toolchains; a mesh-sdk trunk merge moves it with no merge of its own, and a module
@@ -463,50 +470,50 @@ func TestAModuleKeepsTheToolchainsItsFingerprintWasComputedWith(t *testing.T) {
if err := ix.Apply(context.Background(), uses); err != nil {
t.Fatal(err)
}
if u, _ := moduleOf(t, mods, "avahi"); u.FP2 != "" || u.None2 != fingerprint.NoPackageVersion+sdk {
t.Errorf("no published mesh-sdk: fp2 %q none2 %q", u.FP2, u.None2)
if u, _ := moduleOf(t, mods, "avahi"); u.Fp2 != "" || u.None2 != fingerprint.NoPackageVersion+sdk {
t.Errorf("no published mesh-sdk: fp2 %q none2 %q", u.Fp2, u.None2)
}
tools, _ := moduleOf(t, mods, "mesh-tools")
sdkMerge := func(commit, tree string) delivery.Module {
sdkMerge := func(commit, tree string) meshcatalog.ModulesByModuleValue {
t.Helper()
f := forgemsg.Forge{V: delivery.Version, Kind: forgemsg.ForgeMerged, Repository: "novox/mesh-sdk", Branch: "feat/x", Number: 3, Head: h1,
f := forgemsg.Forge{V: keys.Version, Kind: forgemsg.ForgeMerged, Repository: "novox/mesh-sdk", Branch: "feat/x", Number: 3, Head: h1,
Merge: commit, Root: tree, Trees: map[string]string{"": tree}, Manifests: map[string]string{"": `{"module":"mesh-sdk","version":"1"}`}}
if err := ix.Apply(context.Background(), f); err != nil {
t.Fatal(err)
}
a, _ := moduleOf(t, mods, "avahi")
if a.FP2 == "" || !reflect.DeepEqual(a.Packages, delivery.PackageVersions{sdk: commit}) {
t.Fatalf("after mesh-sdk's merge %.8s: fp2 %q none2 %q packages %v", commit, a.FP2, a.None2, a.Packages)
if a.Fp2 == "" || !reflect.DeepEqual(a.Packages, map[string]string{sdk: commit}) {
t.Fatalf("after mesh-sdk's merge %.8s: fp2 %q none2 %q packages %v", commit, a.Fp2, a.None2, a.Packages)
}
if o, _ := moduleOf(t, mods, "mesh-tools"); o.FP2 != tools.FP2 || o.Packages != nil {
t.Errorf("mesh-tools uses no package and moved: fp2 %q (was %q) packages %v", o.FP2, tools.FP2, o.Packages)
if o, _ := moduleOf(t, mods, "mesh-tools"); o.Fp2 != tools.Fp2 || o.Packages != nil {
t.Errorf("mesh-tools uses no package and moved: fp2 %q (was %q) packages %v", o.Fp2, tools.Fp2, o.Packages)
}
return a
}
first := sdkMerge(strings.Repeat("1", 40), "tree-sdk-1")
second := sdkMerge(strings.Repeat("2", 40), "tree-sdk-2")
if second.FP2 == first.FP2 {
t.Errorf("a mesh-sdk trunk merge kept avahi's fingerprint %q", first.FP2)
if second.Fp2 == first.Fp2 {
t.Errorf("a mesh-sdk trunk merge kept avahi's fingerprint %q", first.Fp2)
}
n = second
// Nodes that disagree keep the fingerprint already computed, with its toolchains (issue 1940, 2026-10-25 20:56: a
// stale report blanked every module and the controller's take-in refused modules already built); only a new
// fingerprint needs the agreement: a merge that moves the module's tree gives it none, naming the disagreement.
reportToolchain(t, ix, "mesh-build-go", "ace", "go", "go1.26.0")
if d, _ := moduleOf(t, mods, "avahi"); d.FP2 != n.FP2 || !reflect.DeepEqual(d.Toolchains, n.Toolchains) || d.None2 != "" {
t.Errorf("disagreeing nodes: fp2 %q toolchains %+v none2 %q, want the fingerprint kept (%q) with %+v", d.FP2, d.Toolchains, d.None2, n.FP2, n.Toolchains)
if d, _ := moduleOf(t, mods, "avahi"); d.Fp2 != n.Fp2 || !reflect.DeepEqual(d.Toolchains, n.Toolchains) || d.None2 != "" {
t.Errorf("disagreeing nodes: fp2 %q toolchains %+v none2 %q, want the fingerprint kept (%q) with %+v", d.Fp2, d.Toolchains, d.None2, n.Fp2, n.Toolchains)
}
_, _, f := newFormIndexerUnmerged(t, builderModules(t)...)
f.Trees["modules/avahi"] = "tree-avahi-moved"
if err := ix.Apply(context.Background(), f); err != nil {
t.Fatal(err)
}
if d, _ := moduleOf(t, mods, "avahi"); d.FP2 != "" || d.Toolchains != nil || !strings.HasPrefix(d.None2, "builders disagree") {
t.Errorf("a new tree while nodes disagree: fp2 %q toolchains %+v none2 %q, want no fingerprint and the disagreement", d.FP2, d.Toolchains, d.None2)
if d, _ := moduleOf(t, mods, "avahi"); d.Fp2 != "" || d.Toolchains != nil || !strings.HasPrefix(d.None2, "builders disagree") {
t.Errorf("a new tree while nodes disagree: fp2 %q toolchains %+v none2 %q, want no fingerprint and the disagreement", d.Fp2, d.Toolchains, d.None2)
}
// They agree again: the new tree gets its fingerprint.
reportToolchain(t, ix, "mesh-build-go", "ace", "go", "go1.27.2")
if a, _ := moduleOf(t, mods, "avahi"); a.FP2 == "" || a.FP2 == n.FP2 {
t.Errorf("agreed again: fp2 %q (was %q), want a new fingerprint", a.FP2, n.FP2)
if a, _ := moduleOf(t, mods, "avahi"); a.Fp2 == "" || a.Fp2 == n.Fp2 {
t.Errorf("agreed again: fp2 %q (was %q), want a new fingerprint", a.Fp2, n.Fp2)
}
}
modules/mesh-delivery-indexer/internal/index/toolchains_values.go
+81
View File
@@ -0,0 +1,81 @@
package index
import (
"encoding/json"
"fmt"
"strings"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/nodebuildgo"
"git.novox.be/novox/mesh-sdk/go/keys"
)
// The toolchains the build modules keep in their state toolchains, one key per node and language, `<node>.<language>`
// (reads: mesh-build-go.toolchains, mesh-build-node.toolchains). Both build modules state it in the one form of a
// node-build seat; the indexer reads it as nodebuildgo.Toolchain and takes the agreed one with the delivery sdk's rule
// (buildstep.AgreedToolchain, novox/hq ADR 0584).
func checkToolchainLanguage(language string) error {
switch language {
case "go", "typescript":
return nil
}
return fmt.Errorf("index: the toolchain language %q is not go or typescript", language)
}
// ToolchainKey is the key of a node's toolchain of one language. It refuses a node that is not a name and a language
// other than go or typescript.
func ToolchainKey(node, language string) (string, error) {
if err := keys.CheckName("node", node); err != nil {
return "", err
}
if err := checkToolchainLanguage(language); err != nil {
return "", err
}
return nodebuildgo.ToolchainsByNodeLanguageKey(node, language), nil
}
// ParseToolchainKey is the inverse of ToolchainKey.
func ParseToolchainKey(key string) (node, language string, err error) {
node, language, ok := strings.Cut(key, ".")
if !ok {
return "", "", fmt.Errorf("index: %q is not a toolchain key, <node>.<language>", key)
}
if _, err := ToolchainKey(node, language); err != nil {
return "", "", fmt.Errorf("index: %q is not a toolchain key: %w", key, err)
}
return node, language, nil
}
func checkToolchain(t nodebuildgo.Toolchain) error {
if err := checkToolchainLanguage(t.Language); err != nil {
return err
}
if t.Version == "" || t.Recipe == "" {
return fmt.Errorf("index: a %s toolchain needs its version and its recipe", t.Language)
}
return nil
}
// EncodeToolchain is t as JSON at the kept values' version (the build modules write it; the indexer's tests do too).
func EncodeToolchain(t nodebuildgo.Toolchain) ([]byte, error) {
if err := checkToolchain(t); err != nil {
return nil, err
}
t.V = keys.Version
return json.Marshal(t)
}
// DecodeToolchain reads a toolchains value; it refuses an unknown version and what EncodeToolchain refuses.
func DecodeToolchain(data []byte) (nodebuildgo.Toolchain, error) {
if err := keys.CheckVersion(data); err != nil {
return nodebuildgo.Toolchain{}, err
}
t, err := nodebuildgo.DecodeToolchainsByNodeLanguageValue(data)
if err != nil {
return nodebuildgo.Toolchain{}, fmt.Errorf("index: a toolchain does not fit its shape: %w", err)
}
if err := checkToolchain(t); err != nil {
return nodebuildgo.Toolchain{}, err
}
return t, nil
}
modules/mesh-delivery-indexer/internal/index/verdicts.go
+64 -53
View File
@@ -10,22 +10,23 @@ import (
"strings"
"time"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/process"
"mesh-delivery-indexer/judge"
)
// The indexer judges every manifest it holds, once, with its own judge (novox/hq ADR 0578 §1): it writes each verdict in
// its state verdicts (delivery.VerdictsBucket), one key per manifest digest, only when the verdict differs from the one
// its state verdicts (VerdictsBucket), one key per manifest digest, only when the verdict differs from the one
// held; a module's value in the modules state carries a refusal (Verdict, Problems), so a reader of modules takes it
// without a second read; a new refusal is said as the event manifest-refused; a manifest with a field the judge does not know waits instead (ownBuildIsNewest). It judges at its start, so a judge that
// changed (a new indexer) judges every manifest again, and after every merge it takes in, since a rule between manifests
// can turn on any of them. Nothing polls.
// VerdictsBucket is the indexer's state of verdicts; VerdictsState is its local name, as the tools bundle reads it.
var VerdictsBucket = delivery.VerdictsBucket
const VerdictsBucket = Module + "_" + VerdictsState
const VerdictsState = "verdicts"
@@ -113,12 +114,12 @@ func (ix *Indexer) judgeOver(ctx context.Context, fresh map[string]bool, overMod
ix.logf("judge: the manifest of %s (digest %s) is not in the manifests state, so it is not judged", name, digest)
continue
}
kept, err := delivery.DecodeManifest(value)
kept, err := meshcatalogclient.DecodeManifest(value)
if err != nil {
ix.logf("judge: manifests.%s of %s cannot be read, so it is not judged: %v", digest, name, err)
continue
}
raws[name] = []byte(kept.JSON)
raws[name] = []byte(kept.Manifest)
digests[name] = digest
}
predefined := ix.predefinedList(ctx)
@@ -134,13 +135,13 @@ func (ix *Indexer) judgeOver(ctx context.Context, fresh map[string]bool, overMod
continue
}
value, _, found, err := ix.Manifests.Get(ctx, digest)
kept, derr := delivery.DecodeManifest(value)
kept, derr := meshcatalogclient.DecodeManifest(value)
if err != nil || !found || derr != nil {
ix.logf("judge: the trunk's manifest %s of %s cannot be read, so its renames are not judged against it", digest, module)
continue
}
head, herr := catalogue.Decode(raws[module])
trunk, terr := catalogue.Decode([]byte(kept.JSON))
trunk, terr := catalogue.Decode([]byte(kept.Manifest))
if herr != nil || terr != nil {
continue // the manifest's own problems say why
}
@@ -163,11 +164,11 @@ func (ix *Indexer) judgeOver(ctx context.Context, fresh map[string]bool, overMod
continue
}
out.Judged++
v := delivery.ManifestVerdict{Module: name, State: delivery.VerdictAccepted, Judge: id, At: ix.now()}
v := meshcatalog.VerdictsByDigestValue{Module: name, State: meshcatalogclient.VerdictAccepted, Judge: id, At: ptr(ix.now())}
if fields := waits[name]; len(fields) > 0 {
accepted := ix.acceptedBefore(ctx, digests[name], fresh)
if !newest && !accepted {
v.State, v.Unknown = delivery.VerdictManifestWaits, fields
v.State, v.Unknown = meshcatalogclient.VerdictManifestWaits, fields
out.Waiting = append(out.Waiting, name)
} else {
_, problems := judge.Manifest(raws[name], ix.predefinedList(ctx))
@@ -175,7 +176,7 @@ func (ix *Indexer) judgeOver(ctx context.Context, fresh map[string]bool, overMod
v.Warnings = problems
out.Outgrown = append(out.Outgrown, name)
} else {
v.State, v.Problems = delivery.VerdictRefused, problems
v.State, v.Problems = meshcatalogclient.VerdictRefused, problems
}
}
} else if problems := refused[name]; len(problems) > 0 {
@@ -183,10 +184,10 @@ func (ix *Indexer) judgeOver(ctx context.Context, fresh map[string]bool, overMod
v.Warnings = problems
out.Outgrown = append(out.Outgrown, name)
} else {
v.State, v.Problems = delivery.VerdictRefused, problems
v.State, v.Problems = meshcatalogclient.VerdictRefused, problems
}
}
if !v.Refused() {
if !meshcatalogclient.VerdictRefuses(v) {
if m, err := catalogue.Decode(raws[name]); err == nil {
m.Module = name
accepted = append(accepted, m)
@@ -205,21 +206,21 @@ func (ix *Indexer) judgeOver(ctx context.Context, fresh map[string]bool, overMod
ix.logf("judge: %s judged again by %s: %s", digests[name], id, v.State)
} else if wrote {
out.Written++
if v.Waits() {
ix.logf("judge: %s", v.Why())
if meshcatalogclient.VerdictWaits(v) {
ix.logf("judge: %s", meshcatalogclient.VerdictWhy(v))
}
if v.Refused() {
ix.logf("judge: %s", v.Why())
if meshcatalogclient.VerdictRefuses(v) {
ix.logf("judge: %s", meshcatalogclient.VerdictWhy(v))
if ix.SayRefused != nil {
r := delivery.RefusedOf(digests[name], v)
r.Revision = revision
r := refusedOf(digests[name], v)
r.Revision = int64(revision)
ix.SayRefused(ctx, r)
}
} else if !v.Waits() {
} else if !meshcatalogclient.VerdictWaits(v) {
// Every manifest newly accepted is said whole to the bus module, its entries empty when it asks for none
// (issue 1966 D3, ADR 0595 §4, design 25 § 12.2); a manifest set aside to wait gets none (DLV-I9).
a := acceptedOf(digests[name], v, revision, raws[name])
_ = ix.sayAccepted(ctx, a, a.EventID())
_ = ix.sayAccepted(ctx, a, acceptedEventID(a))
}
}
// The reader capabilities the manifest's own words, forms and seat names need, named with the verdict (ADR 0614 §4).
@@ -270,8 +271,8 @@ func (ix *Indexer) staleVerdict(ctx context.Context, digest string) bool {
if !found {
return true
}
v, err := delivery.DecodeManifestVerdict(value)
return err != nil || (v.Held() && v.Judge != ix.judgeID())
v, err := meshcatalogclient.DecodeVerdict(value)
return err != nil || (meshcatalogclient.VerdictHeld(v) && v.Judge != ix.judgeID())
}
// judgeDigests judges each digest of over (module to digest) whose verdict is stale (staleVerdict), one at a time in the
@@ -307,7 +308,7 @@ func (ix *Indexer) judgeStale(ctx context.Context) {
if err != nil || !found {
continue
}
if v, err := delivery.DecodeManifestVerdict(value); err == nil && v.Held() && v.Judge != ix.judgeID() {
if v, err := meshcatalogclient.DecodeVerdict(value); err == nil && meshcatalogclient.VerdictHeld(v) && v.Judge != ix.judgeID() {
byModule[v.Module] = append(byModule[v.Module], digest)
}
}
@@ -351,7 +352,7 @@ func (ix *Indexer) knownPackages(ctx context.Context) judge.KnownPackages {
ix.logf("judge: packages.%s cannot be read, so the names of packages are not judged: %v", key, err)
return nil
}
if s, err := delivery.DecodeSeriesPackage(value); found && err == nil {
if s, err := DecodeSeriesPackage(value); found && err == nil {
known[s.Name] = true
}
}
@@ -380,35 +381,35 @@ func (ix *Indexer) acceptedBefore(ctx context.Context, digest string, fresh map[
if !found {
return !fresh[digest]
}
prev, err := delivery.DecodeManifestVerdict(value)
return err == nil && prev.State == delivery.VerdictAccepted
prev, err := meshcatalogclient.DecodeVerdict(value)
return err == nil && prev.State == meshcatalogclient.VerdictAccepted
}
// writeVerdict writes v under the digest's key unless the verdict held there says the same (ManifestVerdict.Same); wrote
// says it was written, and revision is the verdicts state's revision of that write, which the event saying it carries
// (novox/hq ADR 0622 §2). It is read back after the write: when another write came between, the revision read holds
// another verdict and revision is 0, so a reader takes the later write's own event and never pairs v with its revision.
func (ix *Indexer) writeVerdict(ctx context.Context, digest string, v delivery.ManifestVerdict) (wrote bool, revision uint64, err error) {
key, err := delivery.VerdictsKey(digest)
func (ix *Indexer) writeVerdict(ctx context.Context, digest string, v meshcatalog.VerdictsByDigestValue) (wrote bool, revision uint64, err error) {
key, err := meshcatalogclient.VerdictsKey(digest)
if err != nil {
return false, 0, err
}
err = write(ctx, ix.Verdicts, key, func(value []byte, found bool) ([]byte, bool, error) {
wrote = false
if found {
if cur, err := delivery.DecodeManifestVerdict(value); err == nil && cur.Same(v) {
if cur, err := meshcatalogclient.DecodeVerdict(value); err == nil && meshcatalogclient.SameVerdict(cur, v) {
return nil, false, nil
}
}
wrote = true
next, err := delivery.EncodeManifestVerdict(v)
next, err := meshcatalogclient.EncodeVerdict(v)
return next, false, err
})
if err != nil || !wrote {
return wrote, 0, err
}
if value, rev, found, err := ix.Verdicts.Get(ctx, key); err == nil && found {
if cur, err := delivery.DecodeManifestVerdict(value); err == nil && cur.Same(v) {
if cur, err := meshcatalogclient.DecodeVerdict(value); err == nil && meshcatalogclient.SameVerdict(cur, v) {
revision = rev
}
}
@@ -435,8 +436,8 @@ func (ix *Indexer) learnOwnBuild(ctx context.Context) {
// ownBuildIsNewest says whether the build of the indexer this program runs (ownFP, from the node-engine) is the newest the
// trunk holds: the indexer's own modules value names it. A newer build merged and not yet delivered ends it at the trunk's
// merge, as one that pins a changed mesh-sdk always is; false when either is unknown.
func (ix *Indexer) ownBuildIsNewest(held map[string]delivery.Module) bool {
fp := held[delivery.IndexerModule].FP2
func (ix *Indexer) ownBuildIsNewest(held map[string]meshcatalog.ModulesByModuleValue) bool {
fp := held[Module].Fp2
if fp == "" {
return false
}
@@ -448,9 +449,9 @@ func (ix *Indexer) ownBuildIsNewest(held map[string]delivery.Module) bool {
// carryVerdict puts a refusal into the module's value, and takes one out once the manifest is accepted, while the value
// still names the digest judged, and the reader capabilities the manifest needs beside it (needs, ADR 0614 §4); an
// accepted module's value carries no verdict, so judging writes no value of a module the
// judge accepts (a reader takes a value without a verdict as accepted, delivery.TakeManifestVerdict).
func (ix *Indexer) carryVerdict(ctx context.Context, name, digest string, v delivery.ManifestVerdict, needs []string) error {
key, err := delivery.ModulesKey(name)
// judge accepts (a reader takes a value without a verdict as accepted, meshcatalogclient.TakeVerdict).
func (ix *Indexer) carryVerdict(ctx context.Context, name, digest string, v meshcatalog.VerdictsByDigestValue, needs []string) error {
key, err := meshcatalogclient.ModulesKey(name)
if err != nil {
return err
}
@@ -458,17 +459,17 @@ func (ix *Indexer) carryVerdict(ctx context.Context, name, digest string, v deli
if !found {
return nil, false, nil
}
cur, err := delivery.DecodeModule(value)
cur, err := meshcatalogclient.DecodeModule(value)
if err != nil || cur.Manifest != digest {
return nil, false, nil // gone, unreadable, or moved to another manifest since: the next pass judges it
}
// A wait is written as a refusal naming the fields, so a reader pinned before the verdict waits holds it as refused.
state, problems := "", v.Problems
switch {
case v.Waits():
state, problems = delivery.VerdictRefused, []string{v.WaitsOn()}
case v.Refused():
state = delivery.VerdictRefused
case meshcatalogclient.VerdictWaits(v):
state, problems = meshcatalogclient.VerdictRefused, []string{meshcatalogclient.WaitsOn(v)}
case meshcatalogclient.VerdictRefuses(v):
state = meshcatalogclient.VerdictRefused
}
// The judge reads the manifest alone; the forms its templates' text writes came with its merge (ADR 0632 §2) and
// stand, cur being of the same manifest.
@@ -481,7 +482,7 @@ func (ix *Indexer) carryVerdict(ctx context.Context, name, digest string, v deli
if state == "" {
cur.Problems = nil
}
next, err := delivery.EncodeModule(cur)
next, err := meshcatalogclient.EncodeModule(cur)
return next, false, err
})
}
@@ -492,11 +493,13 @@ func (ix *Indexer) predefinedList(ctx context.Context) catalogue.PredefinedList
if ix.Predefined == nil {
return nil
}
value, _, found, err := ix.Predefined.Get(ctx, delivery.PredefinedListKey)
value, _, found, err := ix.Predefined.Get(ctx, PredefinedListKey)
if err != nil || !found {
return nil
}
var kept delivery.PredefinedKept
var kept struct {
List json.RawMessage `json:"list"`
}
if json.Unmarshal(value, &kept) != nil {
return nil
}
@@ -542,16 +545,16 @@ func ListVerdicts(keys []string, get func(key string) ([]byte, error), module st
}
continue
}
v, err := delivery.DecodeManifestVerdict(value)
v, err := meshcatalogclient.DecodeVerdict(value)
if err != nil {
unread[key] = err.Error()
continue
}
if (module != "" && v.Module != module) || (!all && !v.Held()) {
if (module != "" && v.Module != module) || (!all && !meshcatalogclient.VerdictHeld(v)) {
continue
}
rows = append(rows, VerdictRow{Module: v.Module, Digest: key, State: v.State, Words: v.Why(), Unknown: v.Unknown,
Problems: v.Problems, Warnings: v.Warnings, Judge: v.Judge, At: v.At})
rows = append(rows, VerdictRow{Module: v.Module, Digest: key, State: v.State, Words: meshcatalogclient.VerdictWhy(v), Unknown: v.Unknown,
Problems: v.Problems, Warnings: v.Warnings, Judge: v.Judge, At: timeOf(v.At)})
}
slices.SortFunc(rows, func(a, b VerdictRow) int {
if c := strings.Compare(a.Module, b.Module); c != 0 {
@@ -578,7 +581,7 @@ func (ix *Indexer) JudgeAgain(ctx context.Context, module, digest string) TakenI
if digest != "" {
// One digest (a candidate's, or one the trunk no longer names), judged by this judge in the catalogue the trunk holds.
value, _, found, err := ix.Manifests.Get(ctx, digest)
kept, derr := delivery.DecodeManifest(value)
kept, derr := meshcatalogclient.DecodeManifest(value)
switch {
case err != nil || !found:
out.Answer = "the indexer keeps no manifest of digest " + digest
@@ -617,15 +620,15 @@ func (ix *Indexer) answerVerdict(ctx context.Context, out TakenIn, digest string
out.Answer = "judged, but the verdict on " + module + " cannot be read"
return out
}
v, err := delivery.DecodeManifestVerdict(value)
v, err := meshcatalogclient.DecodeVerdict(value)
if err != nil {
out.Answer = "judged, but the verdict on " + module + " cannot be read: " + err.Error()
return out
}
out.Stored = true
out.Answer = "judged again by " + v.Judge + ": " + v.State
if v.Held() {
out.Answer += " (" + v.Why() + ")"
if meshcatalogclient.VerdictHeld(v) {
out.Answer += " (" + meshcatalogclient.VerdictWhy(v) + ")"
}
return out
}
@@ -634,7 +637,7 @@ func (ix *Indexer) answerVerdict(ctx context.Context, out TakenIn, digest string
// the takeins state and the value.
func NewJudgeAgain(module, digest, why, by string, at time.Time) (string, []byte, error) {
if digest != "" {
if _, err := delivery.ManifestsKey(digest); err != nil {
if _, err := meshcatalogclient.ManifestsKey(digest); err != nil {
return "", nil, err
}
}
@@ -663,3 +666,11 @@ func withTemplateForms(needs, had []string) []string {
slices.Sort(out)
return out
}
// timeOf is the time a pointer holds, the zero time for none.
func timeOf(t *time.Time) time.Time {
if t == nil {
return time.Time{}
}
return *t
}
modules/mesh-delivery-indexer/internal/index/verdicts_test.go
+49 -48
View File
@@ -9,9 +9,10 @@ import (
"testing"
"time"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
"mesh-delivery-indexer/judge"
)
@@ -23,14 +24,14 @@ func TestTheIndexerJudgesEveryManifestOnce(t *testing.T) {
ctx := context.Background()
ix := &Indexer{Modules: newMem(), Manifests: newMem(), Verdicts: newMem(), JudgeID: "judge-1",
Now: func() time.Time { return time.Date(2026, 11, 1, 3, 0, 0, 0, time.UTC) }}
var said []delivery.ManifestRefused
ix.SayRefused = func(_ context.Context, r delivery.ManifestRefused) { said = append(said, r) }
var said []meshcatalog.ManifestRefusedEvent
ix.SayRefused = func(_ context.Context, r meshcatalog.ManifestRefusedEvent) { said = append(said, r) }
hold := func(module, raw string) string {
digest, err := ix.keepManifest(ctx, module, raw)
if err != nil {
t.Fatal(err)
}
value, err := delivery.EncodeModule(delivery.Module{FP2: "src2:" + strings.Repeat("a", 64), Manifest: digest,
value, err := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: "src2:" + strings.Repeat("a", 64), Manifest: digest,
Repository: "novox/mesh-catalog", Commit: "c"})
if err != nil {
t.Fatal(err)
@@ -45,29 +46,29 @@ func TestTheIndexerJudgesEveryManifestOnce(t *testing.T) {
if err != nil || j.Judged != 2 || j.Written != 2 || len(j.Refused) != 1 {
t.Fatalf("the first pass: %+v %v", j, err)
}
verdict := func(digest string) delivery.ManifestVerdict {
verdict := func(digest string) meshcatalog.VerdictsByDigestValue {
value, _, found, _ := ix.Verdicts.Get(ctx, digest)
v, err := delivery.DecodeManifestVerdict(value)
v, err := meshcatalogclient.DecodeVerdict(value)
if !found || err != nil {
t.Fatalf("no verdict under %s: %v", digest, err)
}
return v
}
if v := verdict(good); v.Refused() || v.Judge != "judge-1" {
if v := verdict(good); meshcatalogclient.VerdictRefuses(v) || v.Judge != "judge-1" {
t.Fatalf("good: %+v", v)
}
if v := verdict(bad); !v.Refused() || !strings.Contains(v.Problems[0], `requires "amqp"`) {
if v := verdict(bad); !meshcatalogclient.VerdictRefuses(v) || !strings.Contains(v.Problems[0], `requires "amqp"`) {
t.Fatalf("bad: %+v", v)
}
module := func(name string) delivery.Module {
module := func(name string) meshcatalog.ModulesByModuleValue {
value, _, _, _ := ix.Modules.Get(ctx, name)
m, err := delivery.DecodeModule(value)
m, err := meshcatalogclient.DecodeModule(value)
if err != nil {
t.Fatal(err)
}
return m
}
if m := module("bad"); m.Verdict != delivery.VerdictRefused || len(m.Problems) != 1 {
if m := module("bad"); m.Verdict != meshcatalogclient.VerdictRefused || len(m.Problems) != 1 {
t.Fatalf("modules.bad carries %+v", m)
}
if m := module("good"); m.Verdict != "" {
@@ -75,7 +76,7 @@ func TestTheIndexerJudgesEveryManifestOnce(t *testing.T) {
}
// ADR 0622 §2: the event carries the verdicts state's revision of the verdict it says.
if _, rev, _, _ := ix.Verdicts.Get(ctx, bad); len(said) != 1 || said[0].Module != "bad" || said[0].Digest != bad ||
said[0].Judge != "judge-1" || said[0].Revision == 0 || said[0].Revision != rev {
said[0].Judge != "judge-1" || said[0].Revision == 0 || said[0].Revision != int64(rev) {
t.Fatalf("said %+v (the verdict's revision %d)", said, rev)
}
// A new judge that judges alike writes nothing and says nothing again.
@@ -103,7 +104,7 @@ func TestTheIndexerJudgesEveryManifestOnce(t *testing.T) {
if j, err := ix.JudgeAll(ctx, nil); err != nil || len(j.Outgrown) != 1 || j.Outgrown[0] != "old" {
t.Fatalf("an outgrown manifest: %+v %v", j, err)
}
if v := verdict(old); v.Refused() || len(v.Warnings) != 1 || !strings.Contains(v.Warnings[0], `requires "amqp"`) {
if v := verdict(old); meshcatalogclient.VerdictRefuses(v) || len(v.Warnings) != 1 || !strings.Contains(v.Warnings[0], `requires "amqp"`) {
t.Fatalf("old: %+v", v)
}
if m := module("old"); m.Verdict != "" || len(m.Warnings) != 1 || len(said) != 1 {
@@ -114,18 +115,18 @@ func TestTheIndexerJudgesEveryManifestOnce(t *testing.T) {
if j, err := ix.JudgeAll(ctx, map[string]bool{changed: true, fresh: true}); err != nil || len(j.Outgrown) != 0 {
t.Fatalf("a changed and a new manifest: %+v %v", j, err)
}
if !verdict(changed).Refused() || !verdict(fresh).Refused() || module("old").Verdict != delivery.VerdictRefused || len(module("old").Warnings) != 0 {
if !meshcatalogclient.VerdictRefuses(verdict(changed)) || !meshcatalogclient.VerdictRefuses(verdict(fresh)) || module("old").Verdict != meshcatalogclient.VerdictRefused || len(module("old").Warnings) != 0 {
t.Fatalf("the changed and the new manifest are not refused: %+v %+v", verdict(changed), verdict(fresh))
}
// A merge's write of a module whose digest it leaves as it is carries the verdict over, and nothing is fresh; a changed
// digest is fresh, judged in full (review R1, R2).
w := &mergeWrite{}
oldNeeds := module("old").Needs
same := delivery.Module{FP2: "src2:" + strings.Repeat("c", 64), Manifest: changed, Repository: "novox/mesh-catalog", Commit: "c2"}
if err := ix.writeOverlay(ctx, map[string]delivery.Module{"old": same}, nil, w); err != nil {
same := meshcatalog.ModulesByModuleValue{Fp2: "src2:" + strings.Repeat("c", 64), Manifest: changed, Repository: "novox/mesh-catalog", Commit: "c2"}
if err := ix.writeOverlay(ctx, map[string]meshcatalog.ModulesByModuleValue{"old": same}, nil, w); err != nil {
t.Fatal(err)
}
if m := module("old"); m.Verdict != delivery.VerdictRefused || len(m.Problems) == 0 || len(w.fresh) != 0 {
if m := module("old"); m.Verdict != meshcatalogclient.VerdictRefused || len(m.Problems) == 0 || len(w.fresh) != 0 {
t.Fatalf("a merge rewrote a refused module as %+v, fresh %v", m, w.fresh)
}
if got := module("old").Needs; len(oldNeeds) == 0 || !slices.Equal(got, oldNeeds) {
@@ -133,15 +134,15 @@ func TestTheIndexerJudgesEveryManifestOnce(t *testing.T) {
}
other := strings.Repeat("d", 64)
same.Manifest = other
if err := ix.writeOverlay(ctx, map[string]delivery.Module{"old": same}, nil, w); err != nil || !w.fresh[other] {
if err := ix.writeOverlay(ctx, map[string]meshcatalog.ModulesByModuleValue{"old": same}, nil, w); err != nil || !w.fresh[other] {
t.Fatalf("a changed digest is not fresh: %v %v", w.fresh, err)
}
// 1966/AC10: a newly accepted manifest asking for bus state is said once as manifest-accepted, carrying both entries
// whole; a verdict already held says nothing.
var accepted []delivery.ManifestAccepted
var accepted []meshcatalog.ManifestAcceptedEvent
var ids []string
var failing int // publishes still to fail
ix.SayAccepted = func(_ context.Context, a delivery.ManifestAccepted, id string) error {
ix.SayAccepted = func(_ context.Context, a meshcatalog.ManifestAcceptedEvent, id string) error {
if failing > 0 {
failing--
return errors.New("no stream answered")
@@ -166,12 +167,12 @@ func TestTheIndexerJudgesEveryManifestOnce(t *testing.T) {
if needs := module("good").Needs; !slices.Contains(needs, "word:module") || len(catalogue.MissingManifestReads(needs, nil)) != 0 {
t.Fatalf("modules.good needs %v", needs)
}
if _, rev, _, _ := ix.Verdicts.Get(ctx, moved); len(accepted) != 1 || accepted[0].Revision == 0 || accepted[0].Revision != rev {
if _, rev, _, _ := ix.Verdicts.Get(ctx, moved); len(accepted) != 1 || accepted[0].Revision == 0 || accepted[0].Revision != int64(rev) {
t.Fatalf("said accepted %+v, want the verdict's revision %d (ADR 0622 §2)", accepted, rev)
}
if len(accepted) != 1 || accepted[0].Module != "moved" || accepted[0].Digest != moved || accepted[0].Judge != "judge-2" ||
len(accepted[0].BusState) != 2 || accepted[0].BusState[0].Name != "cache" || accepted[0].BusState[1].Template != "log" ||
string(accepted[0].BusState[0].Options["history"]) != "8" {
fmt.Sprint(accepted[0].BusState[0].Options["history"]) != "8" {
t.Fatalf("said accepted %+v", accepted)
}
// DLV-I9 (design 25 § 12.2): a manifest asking for none, newly accepted, is said too, its entries an empty list.
@@ -237,7 +238,7 @@ func TestTheIndexerJudgesEveryManifestOnce(t *testing.T) {
t.Fatal(err)
}
ix.JudgeAgain(ctx, "renamer", head)
if v := verdict(head); (want == "") != !v.Refused() || (want != "" && !strings.Contains(strings.Join(v.Problems, " "), want)) {
if v := verdict(head); (want == "") != !meshcatalogclient.VerdictRefuses(v) || (want != "" && !strings.Contains(strings.Join(v.Problems, " "), want)) {
t.Errorf("the change %s: %+v, want refused naming %q", raw, v, want)
}
}
@@ -246,7 +247,7 @@ func TestTheIndexerJudgesEveryManifestOnce(t *testing.T) {
// takes in is, though its candidate was accepted; a name the state knows is accepted at both.
ix.Packages = newMem()
sdkGo := "git.novox.be/novox/mesh-sdk/go"
if value, err := delivery.EncodeSeriesPackage(delivery.SeriesPackage{V: 1, Name: sdkGo, Package: "mesh-sdk/go", Series: "0.1"}); err != nil {
if value, err := EncodeSeriesPackage(SeriesPackage{V: 1, Name: sdkGo, Package: "mesh-sdk/go", Series: "0.1"}); err != nil {
t.Fatal(err)
} else {
ix.Packages.(*memStore).put(SeriesKey(sdkGo, "0.1"), value)
@@ -260,14 +261,14 @@ func TestTheIndexerJudgesEveryManifestOnce(t *testing.T) {
t.Fatal(err)
}
ix.judgeDigests(ctx, map[string]string{"pkguser": head}, "novox/mesh-catalog#1")
if v := verdict(head); v.Refused() {
if v := verdict(head); meshcatalogclient.VerdictRefuses(v) {
t.Errorf("the candidate %s is refused: %+v", raw, v)
}
merged := hold("pkguser", raw)
if _, err := ix.JudgeAll(ctx, map[string]bool{merged: true}); err != nil {
t.Fatal(err)
}
if v := verdict(merged); v.Refused() != refusedAtMerge || refusedAtMerge && !strings.Contains(strings.Join(v.Problems, " "), `name "@novox/ui", which no package`) {
if v := verdict(merged); meshcatalogclient.VerdictRefuses(v) != refusedAtMerge || refusedAtMerge && !strings.Contains(strings.Join(v.Problems, " "), `name "@novox/ui", which no package`) {
t.Errorf("the merge of %s: %+v, want refused %v for the name", raw, v, refusedAtMerge)
}
}
@@ -310,7 +311,7 @@ func TestTheIndexerJudgesEveryManifestOnce(t *testing.T) {
if _, err := ix.JudgeAll(ctx, map[string]bool{cfg: true}); err != nil {
t.Fatal(err)
}
if v := verdict(cfg); !v.Refused() || !strings.Contains(strings.Join(v.Problems, " "), "part of the foundation") {
if v := verdict(cfg); !meshcatalogclient.VerdictRefuses(v) || !strings.Contains(strings.Join(v.Problems, " "), "part of the foundation") {
t.Fatalf("mesh-config asking new-form bus state: %+v", v)
}
if !slices.Equal(judge.Foundation, []string{"nats", "mesh-grants", "mesh-delivery-indexer", "mesh-delivery-assigner", "node-tools",
@@ -322,10 +323,10 @@ func TestTheIndexerJudgesEveryManifestOnce(t *testing.T) {
if _, err := ix.JudgeAll(ctx, map[string]bool{late: true, grants: true}); err != nil {
t.Fatal(err)
}
if v := verdict(late); !v.Refused() || !strings.Contains(v.Problems[0], "late uses the old section logs, which no other module uses any more") {
if v := verdict(late); !meshcatalogclient.VerdictRefuses(v) || !strings.Contains(v.Problems[0], "late uses the old section logs, which no other module uses any more") {
t.Fatalf("late: %+v", v)
}
if v := verdict(grants); v.Refused() {
if v := verdict(grants); meshcatalogclient.VerdictRefuses(v) {
t.Fatalf("the foundation's manifest was refused: %+v", v)
}
// 1966/AC11: the sole user of an old section changes its manifest and keeps the section: accepted, since it was accepted
@@ -338,7 +339,7 @@ func TestTheIndexerJudgesEveryManifestOnce(t *testing.T) {
if _, err := ix.JudgeAll(ctx, map[string]bool{solo2: true}); err != nil {
t.Fatal(err)
}
if v := verdict(solo2); v.Refused() {
if v := verdict(solo2); meshcatalogclient.VerdictRefuses(v) {
t.Fatalf("the sole user of objects, changing its manifest and keeping objects, was refused: %+v", v)
}
// A count that cannot be read is never taken as 0: the zero rule is not applied in that pass and the log says so.
@@ -350,7 +351,7 @@ func TestTheIndexerJudgesEveryManifestOnce(t *testing.T) {
if _, err := ix.JudgeAll(ctx, map[string]bool{retry: true}); err != nil {
t.Fatal(err)
}
if v := verdict(retry); v.Refused() || !strings.Contains(strings.Join(logged, "\n"), "the zero rule is not applied") {
if v := verdict(retry); meshcatalogclient.VerdictRefuses(v) || !strings.Contains(strings.Join(logged, "\n"), "the zero rule is not applied") {
t.Fatalf("a count that could not be read: %+v, logged %q", v, logged)
}
ix.OldSections = held
@@ -358,7 +359,7 @@ func TestTheIndexerJudgesEveryManifestOnce(t *testing.T) {
// event) while a newer build of the indexer is on the trunk, and is judged again at the next judge; once this program's
// build is the trunk's newest it is refused; a wait is written into the module's value as a refusal naming the fields.
ownFP := "src2:" + strings.Repeat("e", 64)
hold(delivery.IndexerModule, `{"module":"mesh-delivery-indexer","version":"1"}`)
hold(Module, `{"module":"mesh-delivery-indexer","version":"1"}`)
ix.ownFP = ownFP // this program started when the trunk held another build of the indexer than the one it holds now
later := hold("later", `{"module":"later","version":"1","a-field-from-later":1,"another":{"x":1}}`)
said = nil
@@ -372,10 +373,10 @@ func TestTheIndexerJudgesEveryManifestOnce(t *testing.T) {
t.Fatalf("a manifest set aside to wait was said accepted: %+v (DLV-I9)", a)
}
}
if v := verdict(later); !v.Waits() || v.Refused() || strings.Join(v.Unknown, ",") != "a-field-from-later,another" {
if v := verdict(later); !meshcatalogclient.VerdictWaits(v) || meshcatalogclient.VerdictRefuses(v) || strings.Join(v.Unknown, ",") != "a-field-from-later,another" {
t.Fatalf("later: %+v", v)
}
if m := module("later"); m.Verdict != delivery.VerdictRefused || len(m.Unknown) != 2 || !m.VerdictOf("later").Waits() ||
if m := module("later"); m.Verdict != meshcatalogclient.VerdictRefused || len(m.Unknown) != 2 || !meshcatalogclient.VerdictWaits(*meshcatalogclient.ModuleVerdict("later", m)) ||
!strings.Contains(m.Problems[0], "does not know yet") {
t.Fatalf("modules.later carries %+v", m)
}
@@ -383,33 +384,33 @@ func TestTheIndexerJudgesEveryManifestOnce(t *testing.T) {
// trunk's value of the indexer names a newer build than the one the engine says runs, so the field still waits.
ix.ownFP = ""
ix.OwnBuild = func(context.Context) (string, error) { return ownFP, nil }
if _, err := ix.JudgeAtStart(ctx); err != nil || !verdict(later).Waits() {
if _, err := ix.JudgeAtStart(ctx); err != nil || !meshcatalogclient.VerdictWaits(verdict(later)) {
t.Fatalf("the trunk moved before the start pass: %v %+v", err, verdict(later))
}
ix.OwnBuild = func(context.Context) (string, error) { return "", errors.New("the engine does not answer") }
ix.ownFP = ""
if _, err := ix.JudgeAtStart(ctx); err != nil || !verdict(later).Waits() {
if _, err := ix.JudgeAtStart(ctx); err != nil || !meshcatalogclient.VerdictWaits(verdict(later)) {
t.Fatalf("an unknown build refused: %v %+v", err, verdict(later))
}
ix.ownFP = ownFP
// A renamed field is a known mistake: refused at once.
renamed := hold("renamed", `{"module":"renamed","version":"1","needs":{"x":"y"}}`)
if j, err := ix.JudgeAll(ctx, map[string]bool{renamed: true}); err != nil || len(j.Waiting) != 1 || !verdict(renamed).Refused() {
if j, err := ix.JudgeAll(ctx, map[string]bool{renamed: true}); err != nil || len(j.Waiting) != 1 || !meshcatalogclient.VerdictRefuses(verdict(renamed)) {
t.Fatalf("a renamed field: %+v %v", j, err)
}
// The trunk's newest build of the indexer is this program's: the field is one no judge to come knows.
ix.ownFP = "src2:" + strings.Repeat("a", 64)
if j, err := ix.JudgeAll(ctx, nil); err != nil || len(j.Waiting) != 0 || !verdict(later).Refused() || !strings.Contains(verdict(later).Problems[0], "a-field-from-later") {
if j, err := ix.JudgeAll(ctx, nil); err != nil || len(j.Waiting) != 0 || !meshcatalogclient.VerdictRefuses(verdict(later)) || !strings.Contains(verdict(later).Problems[0], "a-field-from-later") {
t.Fatalf("the newest build refuses: %+v %v %+v", j, err, verdict(later))
}
if m := module("later"); m.Verdict != delivery.VerdictRefused || m.VerdictOf("later").Waits() || len(m.Unknown) != 0 {
if m := module("later"); m.Verdict != meshcatalogclient.VerdictRefused || meshcatalogclient.VerdictWaits(*meshcatalogclient.ModuleVerdict("later", m)) || len(m.Unknown) != 0 {
t.Fatalf("modules.later after the refusal carries %+v", m)
}
// The tools: indexer_verdicts lists the verdicts that wait or are refused (and every one with all), indexer_judge
// judges now and says how the module's verdict stands.
keys, _ := ix.Verdicts.Keys(ctx)
get := func(key string) ([]byte, error) { v, _, _, err := ix.Verdicts.Get(ctx, key); return v, err }
if rows, unread := ListVerdicts(keys, get, "later", false); len(rows) != 1 || rows[0].State != delivery.VerdictRefused || len(unread) != 0 {
if rows, unread := ListVerdicts(keys, get, "later", false); len(rows) != 1 || rows[0].State != meshcatalogclient.VerdictRefused || len(unread) != 0 {
t.Fatalf("indexer_verdicts of later: %+v %v", rows, unread)
}
if rows, _ := ListVerdicts(keys, get, "", true); len(rows) <= len(keys)-3 || rows[0].Module > rows[len(rows)-1].Module {
@@ -446,32 +447,32 @@ func TestTheIndexerJudgesEveryManifestOnce(t *testing.T) {
if err != nil {
t.Fatal(err)
}
if _, _, err := ix.writeVerdict(ctx, digest, delivery.ManifestVerdict{Module: module, State: delivery.VerdictRefused,
Problems: []string{module + " says tool-runner, which this judge does not know"}, Judge: "judge-0", At: time.Now()}); err != nil {
if _, _, err := ix.writeVerdict(ctx, digest, meshcatalog.VerdictsByDigestValue{Module: module, State: meshcatalogclient.VerdictRefused,
Problems: []string{module + " says tool-runner, which this judge does not know"}, Judge: "judge-0", At: ptr(time.Now())}); err != nil {
t.Fatal(err)
}
return digest
}
atStart := stale("tools", `{"module":"tools","version":"7"}`)
ownRefusal := stale("client", `{"module":"client","version":"7"}`)
if _, _, err := ix.writeVerdict(ctx, ownRefusal, delivery.ManifestVerdict{Module: "client", State: delivery.VerdictRefused,
Problems: []string{"x"}, Judge: ix.judgeID(), At: time.Now()}); err != nil {
if _, _, err := ix.writeVerdict(ctx, ownRefusal, meshcatalog.VerdictsByDigestValue{Module: "client", State: meshcatalogclient.VerdictRefused,
Problems: []string{"x"}, Judge: ix.judgeID(), At: ptr(time.Now())}); err != nil {
t.Fatal(err)
}
if _, err := ix.JudgeAtStart(ctx); err != nil || verdict(atStart).Refused() || verdict(atStart).Judge != ix.judgeID() || !verdict(ownRefusal).Refused() {
if _, err := ix.JudgeAtStart(ctx); err != nil || meshcatalogclient.VerdictRefuses(verdict(atStart)) || verdict(atStart).Judge != ix.judgeID() || !meshcatalogclient.VerdictRefuses(verdict(ownRefusal)) {
t.Fatalf("the start pass: %+v, this judge's own refusal %+v (%v)", verdict(atStart), verdict(ownRefusal), err)
}
byPull := stale("tools", `{"module":"tools","version":"8"}`)
pull := forgemsg.Forge{Kind: forgemsg.ForgePull, Repository: "novox/mesh-tools", Number: 3, Branch: "main",
Manifests: map[string]string{"modules/tools": `{"module":"tools","version":"8"}`}}
if _, err := ix.ApplyMessage(ctx, pull); err != nil || verdict(byPull).Refused() || verdict(byPull).Judge != ix.judgeID() {
if _, err := ix.ApplyMessage(ctx, pull); err != nil || meshcatalogclient.VerdictRefuses(verdict(byPull)) || verdict(byPull).Judge != ix.judgeID() {
t.Fatalf("a pull carrying a digest an older judge refused: %+v %v", verdict(byPull), err)
}
byTool := stale("tools", `{"module":"tools","version":"9"}`)
if got := ix.JudgeAgain(ctx, "client", byTool); got.Stored || !strings.Contains(got.Answer, "tools's, not client's") || !verdict(byTool).Refused() {
if got := ix.JudgeAgain(ctx, "client", byTool); got.Stored || !strings.Contains(got.Answer, "tools's, not client's") || !meshcatalogclient.VerdictRefuses(verdict(byTool)) {
t.Fatalf("indexer_judge with another module's digest: %+v", got)
}
if got := ix.JudgeAgain(ctx, "tools", byTool); !got.Stored || !strings.Contains(got.Answer, "accepted") || verdict(byTool).Refused() {
if got := ix.JudgeAgain(ctx, "tools", byTool); !got.Stored || !strings.Contains(got.Answer, "accepted") || meshcatalogclient.VerdictRefuses(verdict(byTool)) {
t.Fatalf("indexer_judge with a digest: %+v %+v", got, verdict(byTool))
}
if key, value, err := NewJudgeAgain("tools", byTool, "why", "me", time.Now()); err != nil || key != "judge.tools" {
modules/mesh-delivery-indexer/internal/index/views.go
+16 -17
View File
@@ -1,7 +1,6 @@
package index
import (
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/health"
"git.novox.be/novox/mesh-delivery/sdk/graph"
@@ -9,7 +8,7 @@ import (
)
// MarksBucket is the indexer's state of what its views family said each state up to.
const MarksBucket = delivery.IndexerModule + "_" + views.MarksState
const MarksBucket = Module + "_" + views.MarksState
// FirstViewTools are the indexer's first-view tools (ADR 0621 §1, ADR 0622 §1, ADR 0631 §1): the tool, the state it answers, and
// the words of what that state holds. Each answers only its callers list (CallersEnv), which the manifest writes.
@@ -24,10 +23,10 @@ var FirstViewTools = []ViewedTool{
var ReadTools = []ViewedTool{
{Tool: "indexer_read_manifests", State: "manifests", Words: "every manifest the indexer keeps, by its digest."},
{Tool: "indexer_read_modules", State: "modules", Words: "every module the indexer holds, its fingerprint, inputs and manifest digest."},
{Tool: "indexer_read_predefined", State: delivery.IndexerPredefinedState, Words: "the predefined config values of each module on the trunk."},
{Tool: "indexer_read_config_candidates", State: delivery.IndexerConfigCandidatesState, Words: "the config declarations of each open pull request."},
{Tool: "indexer_read_predefined", State: PredefinedState, Words: "the predefined config values of each module on the trunk."},
{Tool: "indexer_read_config_candidates", State: ConfigCandidatesState, Words: "the config declarations of each open pull request."},
{Tool: "indexer_read_candidates", State: "candidates", Words: "the candidate groups: the pull requests sharing a branch, their heads and the modules each touches."},
{Tool: "indexer_read_packages", State: delivery.PackagesState, Words: "the published packages per series."},
{Tool: "indexer_read_packages", State: PackagesState, Words: "the published packages per series."},
{Tool: "indexer_read_graph", State: graph.State, Words: "the module graph: each module's build and provision edges."},
{Tool: "indexer_read_heard", State: HeardState, Words: "the forge messages the indexer heard and what it did with each."},
{Tool: "indexer_read_taken", State: "taken", Words: "each repository's merge the indexer took in last."},
@@ -42,16 +41,16 @@ type ViewedTool struct {
// ViewedStates are the states the indexer's views family says, each by its bucket: the ones its readers read.
var ViewedStates = map[string]string{
"manifests": delivery.ManifestsBucket,
"modules": delivery.ModulesBucket,
VerdictsState: VerdictsBucket,
delivery.IndexerPredefinedState: PredefinedBucket,
delivery.IndexerConfigCandidatesState: ConfigCandidatesBucket,
"candidates": delivery.CandidatesBucket,
delivery.PackagesState: delivery.PackagesBucket,
graph.State: graph.Bucket,
HeardState: HeardBucket,
"taken": delivery.TakenBucket,
health.FindingsState: health.FindingsBucket(delivery.IndexerModule),
SeatDefinitionsState: SeatDefinitionsBucket,
"manifests": ManifestsBucket,
"modules": ModulesBucket,
VerdictsState: VerdictsBucket,
PredefinedState: PredefinedBucket,
ConfigCandidatesState: ConfigCandidatesBucket,
"candidates": CandidatesBucket,
PackagesState: PackagesBucket,
graph.State: graph.Bucket,
HeardState: HeardBucket,
"taken": TakenBucket,
health.FindingsState: health.FindingsBucket(Module),
SeatDefinitionsState: SeatDefinitionsBucket,
}
modules/mesh-delivery-indexer/judge/judge.go
+3 -2
View File
@@ -1,6 +1,6 @@
// Package judge is the mesh's one judge of module manifests (novox/hq ADR 0578 §1, §2): strict decoding, the seat and
// namespace rules, the bus and stream limits, and the rules between the manifests of one catalogue. Only the indexer,
// which judges every manifest it takes in and writes the verdicts (delivery.VerdictsBucket), and the checker, which
// which judges every manifest it takes in and writes the verdicts (index.VerdictsBucket), and the checker, which
// judges every manifest a change touches by running this judge built at the trunk or a group's head (cmd/mesh-judge,
// as the composer is run), use it. Every other part decodes a manifest leniently (catalogue.Decode) and takes the
// verdict; it never judges (ADR 0578 §3).
@@ -15,7 +15,6 @@ import (
"encoding/hex"
json "encoding/json"
"fmt"
"git.novox.be/novox/mesh-sdk/go/seatdef"
"io"
"maps"
"os"
@@ -26,6 +25,8 @@ import (
"strings"
"sync"
"git.novox.be/novox/mesh-sdk/go/seatdef"
"git.novox.be/novox/mesh-delivery/sdk/composer"
"git.novox.be/novox/mesh-sdk/go/catalogue"
)
modules/mesh-delivery-indexer/judge/packages.go
+1 -1
View File
@@ -11,7 +11,7 @@ import (
// KnownPackages are the names of the packages the mesh knows, each by its name in its own ecosystem (design 32: a Go
// package's go.mod module path, a TypeScript package's package.json name): the names the indexer's packages state holds
// (its series keys, delivery.SeriesPackage's Name), read at each trunk merge, and, at the merge check, the names a member
// (its series keys, index.SeriesPackage's Name), read at each trunk merge, and, at the merge check, the names a member
// of the change's delivery group names at its head for a package new in the group. Both read a name with mesh-delivery's
// sdk/packagename. Nil is no list at hand: the names are then not judged, only their series.
type KnownPackages map[string]bool
modules/mesh-delivery-indexer/manifest_test.go
+18 -2
View File
@@ -14,7 +14,6 @@ import (
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-delivery/sdk/graph"
"git.novox.be/novox/mesh-sdk/go/health"
@@ -114,12 +113,29 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build
t.Errorf("emits = %v, want the two take-in events (the verdict events are the seat mesh-catalog's)", m.Emits)
}
// 1126 note 76: heard keeps the last 64 FORGE messages the indexer handled, for indexer_heard.
if want := []state{{Name: "candidates", History: 10}, {Name: "heard", History: 64}, {Name: "manifests", History: 1}, {Name: "modules", History: 20}, {Name: "predefined", History: 5}, {Name: "config-candidates", History: 5}, {Name: "taken", History: 1}, {Name: "takeins", History: 1}, {Name: "verdicts", History: 5}, {Name: "oldsections", History: 1}, {Name: delivery.PackagesState, History: delivery.PackagesHistory},
if want := []state{{Name: "candidates", History: 10}, {Name: "heard", History: 64}, {Name: "manifests", History: 1}, {Name: "modules", History: 20}, {Name: "predefined", History: 5}, {Name: "config-candidates", History: 5}, {Name: "taken", History: 1}, {Name: "takeins", History: 1}, {Name: "verdicts", History: 5}, {Name: "oldsections", History: 1}, {Name: index.PackagesState, History: index.PackagesHistory},
{Name: graph.State, History: graph.History}, {Name: views.MarksState, History: 1}, {Name: health.FindingsState, History: 5}, {Name: views.StatusState, History: 1},
{Name: index.RequirementsState, History: 5}, {Name: index.SeatDefinitionsState, History: 5},
{Name: index.HoldsState, History: 5}, {Name: index.EngineReadsState, History: 1}, {Name: index.HoldBoundsState, History: 1, KeyTTL: true}, {Name: index.PendingState, History: 1}}; !reflect.DeepEqual(m.State, want) { // DLV-I7, DLV-I17, ADR 0632 §2
t.Errorf("state = %+v, want %+v", m.State, want)
}
// D-11 (ADR 0634 §1): the holders and buckets the indexer names (internal/index/names.go) are its manifest's: its own
// name and states with their revisions, and the states it reads, each as `<holder>.<state>`.
if index.Module != m.Module {
t.Errorf("the indexer names itself %q, its manifest's module is %q", index.Module, m.Module)
}
for bucket, history := range map[string]int{index.CandidatesBucket: index.CandidatesHistory, index.ModulesBucket: index.ModulesHistory, index.ManifestsBucket: index.ManifestsHistory,
index.VerdictsBucket: index.VerdictsHistory, index.PackagesBucket: index.PackagesHistory, index.TakenBucket: 1, index.PredefinedBucket: 5, index.ConfigCandidatesBucket: 5} {
if !slices.ContainsFunc(m.State, func(st state) bool { return m.Module+"_"+st.Name == bucket && st.History == history }) {
t.Errorf("the indexer names its bucket %s with %d revisions, its manifest's states are %+v", bucket, history, m.State)
}
}
for bucket, read := range map[string]string{index.SourceBucket: "mesh-delivery-gate.source", index.PreviewsBucket: "mesh-delivery-checker.previews",
index.ToolchainsBucket(index.BuildGoModule): "mesh-build-go.toolchains", index.ToolchainsBucket(index.BuildNodeModule): "mesh-build-node.toolchains"} {
if !slices.Contains(m.Reads, read) || strings.Replace(read, ".", "_", 1) != bucket {
t.Errorf("the indexer reads the bucket %s, its manifest's reads are %v (want %s)", bucket, m.Reads, read)
}
}
// DLV-F1r (ADR 0615 §1): the assigner's placement is read through its first view and views family, never its state.
if want := []string{"mesh-build-go.toolchains", "mesh-build-node.toolchains", "mesh-delivery-builder.build-log", "mesh-delivery-checker.previews", "mesh-delivery-gate.source"}; !reflect.DeepEqual(sorted(m.Reads), want) {
t.Errorf("reads = %v, want %v", m.Reads, want)
modules/mesh-delivery-indexer/takein/takein.go
+30 -29
View File
@@ -11,7 +11,8 @@ import (
"strings"
"time"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient"
"git.novox.be/novox/mesh-sdk/go/keys"
)
// Merge is one merge a take-in event or a wait holds: the repository as `<owner>/<name>`, its pull request and the
@@ -31,7 +32,7 @@ func checkRepository(repository string) error {
return nil
}
// decode reads a value of delivery.Version, refusing another or none with delivery.ErrUnknownVersion, as mesh-sdk
// decode reads a value of keys.Version, refusing another or none with keys.ErrUnknownVersion, as mesh-sdk
// delivery's values are read.
func decode(data []byte, into any) error {
var probe struct {
@@ -40,8 +41,8 @@ func decode(data []byte, into any) error {
if err := json.Unmarshal(data, &probe); err != nil {
return fmt.Errorf("delivery: value is not JSON: %w", err)
}
if probe.V == nil || *probe.V != delivery.Version {
return fmt.Errorf("%w: this reader reads %d", delivery.ErrUnknownVersion, delivery.Version)
if probe.V == nil || *probe.V != keys.Version {
return fmt.Errorf("%w: this reader reads %d", keys.ErrUnknownVersion, keys.Version)
}
if err := json.Unmarshal(data, into); err != nil {
return fmt.Errorf("delivery: value does not fit its shape: %w", err)
@@ -121,18 +122,18 @@ func checkTakeInKey(key string) error {
// it holds, oldest first; Since is when the wait began. A `taken` event names the same once the wait ended; By and Why
// name a hand act (a hand take-in, or a drop, Dropped), and are carried by nothing else.
type TakeIn struct {
V int `json:"v"`
Kind string `json:"kind"`
Key string `json:"key"`
What string `json:"what"`
Merges []Merge `json:"merges"`
Group string `json:"group,omitempty"`
Members []delivery.Head `json:"members,omitempty"`
WaitsFor string `json:"waits-for,omitempty"`
Since time.Time `json:"since"`
By string `json:"by,omitempty"`
Why string `json:"why,omitempty"`
Dropped bool `json:"dropped,omitempty"`
V int `json:"v"`
Kind string `json:"kind"`
Key string `json:"key"`
What string `json:"what"`
Merges []Merge `json:"merges"`
Group string `json:"group,omitempty"`
Members []meshdeliveryclient.Head `json:"members,omitempty"`
WaitsFor string `json:"waits-for,omitempty"`
Since time.Time `json:"since"`
By string `json:"by,omitempty"`
Why string `json:"why,omitempty"`
Dropped bool `json:"dropped,omitempty"`
// Lost are the keys of merges whose message the forge bridge's stream no longer holds: they are not taken in, and the
// revision waits until indexer_drop_pending drops them (issue 1585).
Lost []string `json:"lost,omitempty"`
@@ -194,7 +195,7 @@ func EncodeTakeIn(t TakeIn) ([]byte, error) {
if err := t.check(); err != nil {
return nil, err
}
t.V = delivery.Version
t.V = keys.Version
return json.Marshal(t)
}
@@ -221,18 +222,18 @@ func SameTakeIn(a, b TakeIn) bool { return reflect.DeepEqual(a, b) }
// group being gathered names that group's key in Behind. Keys are the pending keys of its merges, in Merges' order (indexer_drop_pending drops one). Holds are the modules it holds back: every module of its
// merges' trees. Said is the last take-in event said of it (`take-in.<kind>`, of its Key) and SaidAt when.
type PendingWait struct {
Key string `json:"key"`
What string `json:"what"`
Merges []Merge `json:"merges"`
Keys []string `json:"keys"`
Group string `json:"group,omitempty"`
Members []delivery.Head `json:"members,omitempty"`
Behind string `json:"behind,omitempty"`
WaitsFor string `json:"waits-for,omitempty"`
Since time.Time `json:"since"`
Holds []string `json:"holds"`
Said string `json:"said,omitempty"`
SaidAt time.Time `json:"said-at,omitzero"`
Key string `json:"key"`
What string `json:"what"`
Merges []Merge `json:"merges"`
Keys []string `json:"keys"`
Group string `json:"group,omitempty"`
Members []meshdeliveryclient.Head `json:"members,omitempty"`
Behind string `json:"behind,omitempty"`
WaitsFor string `json:"waits-for,omitempty"`
Since time.Time `json:"since"`
Holds []string `json:"holds"`
Said string `json:"said,omitempty"`
SaidAt time.Time `json:"said-at,omitzero"`
// Lost are the keys of its merges whose message the forge bridge's stream no longer holds (TakeIn.Lost).
Lost []string `json:"lost,omitempty"`
}
modules/mesh-delivery-indexer/takein/takein_test.go
+6 -5
View File
@@ -8,7 +8,8 @@ import (
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient"
"git.novox.be/novox/mesh-sdk/go/keys"
)
// sampleTakeIns are a pending revision, a group being gathered and a hand take-in's taken (1585/T2).
@@ -17,7 +18,7 @@ func sampleTakeIns(since time.Time) []TakeIn {
return []TakeIn{
{Kind: TakeInPending, Key: PendingKey(12), What: TakeInRevision, Merges: m, WaitsFor: "src2:abc", Since: since},
{Kind: TakeInPending, Key: GroupTakeInKey("feat/take-in"), What: TakeInGroup, Merges: m, Group: "feat/take-in",
Members: []delivery.Head{{Repository: "novox/mesh-delivery", Number: 9, Head: "ffff"}, {Repository: "novox/mesh-sdk", Number: 37, Head: "dddd"}}, Since: since},
Members: []meshdeliveryclient.Head{{Repository: "novox/mesh-delivery", Number: 9, Head: "ffff"}, {Repository: "novox/mesh-sdk", Number: 37, Head: "dddd"}}, Since: since},
{Kind: TakeInTaken, Key: PendingKey(12), What: TakeInRevision, Merges: m, WaitsFor: "src2:abc", Since: since,
By: "lead-47@shanks", Why: "the named indexer can never run"},
}
@@ -50,14 +51,14 @@ func TestTakeInEvents(t *testing.T) {
if err != nil {
t.Fatalf("take-in %s %s: %v", ti.Kind, ti.What, err)
}
ti.V = delivery.Version
ti.V = keys.Version
if got, err := DecodeTakeIn(data); err != nil || !SameTakeIn(got, ti) {
t.Errorf("take-in %s round trip: %+v %v", ti.Key, got, err)
}
}
for _, in := range []string{`{"key":"m.1"}`, `{"v":2,"key":"m.1"}`} {
if _, err := DecodeTakeIn([]byte(in)); !errors.Is(err, delivery.ErrUnknownVersion) {
t.Errorf("decode of %s = %v, want delivery.ErrUnknownVersion", in, err)
if _, err := DecodeTakeIn([]byte(in)); !errors.Is(err, keys.ErrUnknownVersion) {
t.Errorf("decode of %s = %v, want keys.ErrUnknownVersion", in, err)
}
}
// A body that does not hold is refused, naming why.