Merge branch 'ix16-wip' into feat/47-G-build-indexer
This commit is contained in:
@@ -126,7 +126,8 @@ func TestManifest(t *testing.T) {
|
||||
if !slices.Equal(m.Consumes, []string{"mesh-catalog.seat-definition-changed", "mesh-catalog.seat-definition-removed", "mesh-delivery-indexer.views.graph", "mesh-delivery-indexer.views.manifests", "mesh-delivery-indexer.views.modules", buckets.TellerModule + "." + tellerevents.AskEnded, "mesh-pipeline-runner.pipeline-ended", "mesh-pipeline-runner.group-ended", "node-engine.manifest-reads-changed"}) {
|
||||
t.Errorf("consumes = %v, want the indexer's views of graph, manifests and modules, the teller's ask-ended and the engines' reads changed", m.Consumes)
|
||||
}
|
||||
if want := []string{"mesh-delivery-indexer.graph", "mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules",
|
||||
if want := []string{"mesh-delivery-indexer.graph", "mesh-delivery-indexer.manifests",
|
||||
"seat:mesh-catalog.revision", // ADR 0663 §2: which modules exist
|
||||
"seat:mesh-catalog.seat-definitions", // ADR 0636 §4: the seat list
|
||||
"seat:mesh-hand-acts.hand-act", "seat:mesh-hand-acts.hand-acts", "seat:mesh-hand-acts.warranted", // ADR 0565 §9, §11: the seat alone
|
||||
"seat:node-backup.backed-up", "seat:node-backup.now",
|
||||
|
||||
@@ -52,7 +52,7 @@
|
||||
"invokes": [
|
||||
"mesh-delivery-indexer.graph",
|
||||
"mesh-delivery-indexer.manifests",
|
||||
"mesh-delivery-indexer.modules",
|
||||
"seat:mesh-catalog.revision",
|
||||
"seat:mesh-catalog.seat-definitions",
|
||||
"seat:mesh-hand-acts.hand-act",
|
||||
"seat:mesh-hand-acts.hand-acts",
|
||||
|
||||
@@ -13,8 +13,9 @@ import (
|
||||
"mesh-delivery-indexer/internal/index"
|
||||
)
|
||||
|
||||
// catalogRead reads the indexer's modules, manifests and graph states for the seat's verbs.
|
||||
func catalogRead(modules, manifests, nodes stdio.KeptState, seatDefinitions, marks views.Source) index.CatalogRead {
|
||||
// catalogRead reads the indexer's view of the trunk (its modules and module graph, from the newest catalog graph
|
||||
// revision) and its manifests state for the seat's verbs.
|
||||
func catalogRead(modules views.Source, manifests stdio.KeptState, nodes views.Source, seatDefinitions, marks views.Source) index.CatalogRead {
|
||||
return index.CatalogRead{
|
||||
SeatDefinitions: seatDefinitions, Marks: marks,
|
||||
Modules: func() (map[string]meshcatalog.ModulesByModuleValue, error) {
|
||||
@@ -24,11 +25,11 @@ func catalogRead(modules, manifests, nodes stdio.KeptState, seatDefinitions, mar
|
||||
}
|
||||
out := map[string]meshcatalog.ModulesByModuleValue{}
|
||||
for _, k := range keys {
|
||||
e, err := modules.Get(k)
|
||||
e, ok, err := modules.Get(k)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("modules.%s: %w", k, err)
|
||||
}
|
||||
if e == nil {
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if m, err := meshcatalogclient.DecodeModule(e.Value); err == nil {
|
||||
@@ -59,11 +60,11 @@ func catalogRead(modules, manifests, nodes stdio.KeptState, seatDefinitions, mar
|
||||
}
|
||||
g := graph.Graph{}
|
||||
for _, k := range keys {
|
||||
e, err := nodes.Get(k)
|
||||
e, ok, err := nodes.Get(k)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("graph.%s: %w", k, err)
|
||||
}
|
||||
if e == nil {
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if n, err := graph.Decode(e.Value); err == nil {
|
||||
@@ -76,7 +77,7 @@ func catalogRead(modules, manifests, nodes stdio.KeptState, seatDefinitions, mar
|
||||
}
|
||||
|
||||
// catalogTools are the seat's verbs over the indexer's own states, served beside the indexer's own tools, since the indexer's manifest claims the seat (ADR 0604 §2).
|
||||
func catalogTools() []stdio.Tool {
|
||||
return index.CatalogTools(catalogRead(stdio.State("modules"), stdio.State("manifests"), stdio.State(graph.State),
|
||||
func catalogTools(state func(string) views.Source) []stdio.Tool {
|
||||
return index.CatalogTools(catalogRead(state("modules"), stdio.State("manifests"), state(graph.State),
|
||||
keptSource(index.SeatDefinitionsState), keptSource(views.MarksState)))
|
||||
}
|
||||
|
||||
@@ -17,6 +17,7 @@ import (
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/busstate"
|
||||
|
||||
"mesh-delivery-indexer/internal/index"
|
||||
)
|
||||
@@ -56,7 +57,10 @@ func Tools(ask asker, now func() time.Time, read stdio.KeptState, counts func()
|
||||
"source names for that repository, for a merge whose merged message the indexer never heard (the forge bridge has no " +
|
||||
"tool): a head whose trunk holds no module.json is taken in as a merge that leaves none (every module of the repository " +
|
||||
"is dropped, one another repository's trunk holds handed over from there first, listed in handed and dropped); a head " +
|
||||
"that holds modules is answered \"take each module in by name\".",
|
||||
"that holds modules is answered \"take each module in by name\". With neither module nor repository it is the indexer's " +
|
||||
"catch-up tool, named in its manifest beside the catalog graph revisions: it takes in the head the gate's source names for " +
|
||||
"every repository, as merges, and keeps the view as the next revision when an entry moved; the node agent calls it so " +
|
||||
"after restoring them. Answers the repositories taken in and, in unread, those whose merged message was not read.",
|
||||
Input: map[string]any{"type": "object", "properties": map[string]any{
|
||||
"module": map[string]any{"type": "string", "description": "the module's name, as the catalogue registers it"},
|
||||
"repository": map[string]any{"type": "string", "description": "instead of module: the repository (owner/name) whose head is taken in"},
|
||||
@@ -65,9 +69,12 @@ func Tools(ask asker, now func() time.Time, read stdio.KeptState, counts func()
|
||||
var key string
|
||||
var value []byte
|
||||
var err error
|
||||
if str(args, "repository") != "" && str(args, "module") == "" {
|
||||
switch {
|
||||
case str(args, "repository") == "" && str(args, "module") == "":
|
||||
key, value, err = index.NewCatchUp(str(args, "why"), stdio.CallerOf(args), now())
|
||||
case str(args, "repository") != "" && str(args, "module") == "":
|
||||
key, value, err = index.NewTakeInRepository(str(args, "repository"), str(args, "why"), stdio.CallerOf(args), now())
|
||||
} else {
|
||||
default:
|
||||
key, value, err = index.NewTakeIn(str(args, "module"), str(args, "why"), stdio.CallerOf(args), now())
|
||||
}
|
||||
if err != nil {
|
||||
@@ -275,16 +282,26 @@ func main() {
|
||||
asks := askTakeIn(stdio.State(index.TakeInsState), 25*time.Second)
|
||||
acts := meshhandactsclient.Recorder{Ask: stdio.Ask}
|
||||
tools := Tools(asks, time.Now, stdio.State(index.VerdictsState), heldCounts(stdio.State(index.OldSectionsState)), acts)
|
||||
tools = append(tools, catalogTools()...)
|
||||
tools = append(tools, viewTools(os.Getenv, keptSource, time.Now)...)
|
||||
// The modules and the module graph are the newest catalog graph revision's (milestone 47/G): revision.go.
|
||||
revisions := bundleObjects{busstate.ObjectsOf(index.CatalogRevisionsObjects)}
|
||||
trunk := index.TrunkStates{Objects: revisions, Under: keptSource}
|
||||
tools = append(tools, catalogTools(trunk.State)...)
|
||||
// The seat mesh-catalog's verb revision, a first view of the catalog half answered in this runtime of its own to exactly
|
||||
// the parts whose entry declares it (novox/hq ADR 0663).
|
||||
tools = append(tools, index.RevisionTool(revisions, index.Module))
|
||||
tools = append(tools, index.CatalogRevisionTools(revisions, asks, time.Now, func(what, why string) error {
|
||||
_, err := acts.Record(meshhandactsclient.Request{What: what, Why: why, Cause: index.DropCandidateGraphTool, Verb: "drop-candidate-graph"})
|
||||
return err
|
||||
})...)
|
||||
tools = append(tools, viewTools(os.Getenv, trunk.State, time.Now)...)
|
||||
tools = append(tools, index.PendingTools(keptRead{stdio.State(index.PendingState)}, keptRead{stdio.State(index.OwnBuildStateName)}, asks, time.Now)...)
|
||||
tools = append(tools, index.FingerprintsTools(asks, time.Now, func(what, why string) error {
|
||||
_, err := acts.Record(meshhandactsclient.Request{What: what, Why: why, Cause: index.FingerprintsAgainTool, Verb: "fingerprints-again"})
|
||||
return err
|
||||
}, 10*time.Second)...)
|
||||
tools = append(tools, index.TryCandidateTools(keptRead{stdio.State("candidates")}, keptRead{stdio.State(index.TryRecoveryState)}, asks, time.Now)...)
|
||||
tools = append(tools, configReadersTool(stdio.NewWatchedStates()), busStateTool(keptSource), requirementsTool(keptSource),
|
||||
packageUsersTool(keptSource), seatDefinitionTool(keptSource), outOfStepTool(keptSource),
|
||||
tools = append(tools, configReadersTool(stdio.NewWatchedStates(), trunk.State("modules")), busStateTool(trunk.State), requirementsTool(keptSource),
|
||||
packageUsersTool(trunk.State), seatDefinitionTool(keptSource), outOfStepTool(keptSource),
|
||||
holdsTool(stdio.State(index.HoldsState), asks, time.Now, acts), engineReadsTool(time.Now))
|
||||
if err := stdio.Serve("", tools); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "[mesh-delivery-indexer-tools] %v\n", err)
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/busstate"
|
||||
|
||||
"mesh-delivery-indexer/internal/index"
|
||||
)
|
||||
|
||||
// bundleObjects is the objects catalog-revisions as the bundle reaches them through its runtime (the old objects section
|
||||
// of a foundation part, reached by the module's own name).
|
||||
type bundleObjects struct{ o busstate.Objects }
|
||||
|
||||
func (b bundleObjects) Put(_ context.Context, name string, data []byte) error {
|
||||
_, err := b.o.Put(name, data, "")
|
||||
return err
|
||||
}
|
||||
|
||||
func (b bundleObjects) Get(_ context.Context, name string) ([]byte, error) {
|
||||
d, err := b.o.Get(name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if d.Info.Name == "" || d.Info.Deleted {
|
||||
return nil, index.ErrNoObject
|
||||
}
|
||||
return d.Data, nil
|
||||
}
|
||||
|
||||
func (b bundleObjects) Delete(_ context.Context, name string) error { return b.o.Delete(name) }
|
||||
|
||||
func (b bundleObjects) Names(context.Context) ([]string, error) {
|
||||
infos, err := b.o.List()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
names := []string{}
|
||||
for _, i := range infos {
|
||||
if !i.Deleted {
|
||||
names = append(names, i.Name)
|
||||
}
|
||||
}
|
||||
return names, nil
|
||||
}
|
||||
|
||||
func (b bundleObjects) Fill(context.Context) (int64, int64, error) {
|
||||
f, err := b.o.Fill()
|
||||
return int64(f.Bytes), int64(f.MaxBytes), err
|
||||
}
|
||||
@@ -37,9 +37,9 @@ func viewTools(getenv func(string) string, state func(string) views.Source, now
|
||||
func keptSource(name string) views.Source { return views.KeptSource{State: stdio.State(name)} }
|
||||
|
||||
// configReadersTool is indexer_config_readers (design 32 *A value no longer read*; DLV-I10): read from one watched copy
|
||||
// of the indexer's modules, manifests and config-candidates states, so an ask on each manifest-accepted reads no state
|
||||
// of the indexer's manifests and config-candidates states and its view of the trunk (the newest catalog graph revision), so an ask on each manifest-accepted reads no state
|
||||
// whole again.
|
||||
func configReadersTool(watched *stdio.WatchedStates) stdio.Tool {
|
||||
func configReadersTool(watched *stdio.WatchedStates, modules views.Source) stdio.Tool {
|
||||
return stdio.Tool{Name: index.ConfigReadersTool,
|
||||
Description: "Who reads a module's config keys now (design 32): every value key and own secret any manifest of the module " +
|
||||
"the indexer keeps ever declared, each with its readers — the module itself where its trunk's manifest declares it, and " +
|
||||
@@ -53,13 +53,15 @@ func configReadersTool(watched *stdio.WatchedStates) stdio.Tool {
|
||||
return nil, fmt.Errorf("%s names the module", index.ConfigReadersTool)
|
||||
}
|
||||
trunk := map[string]meshcatalog.ModulesByModuleValue{}
|
||||
mods, err := watched.Values("modules")
|
||||
keys, err := modules.Keys()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the modules state cannot be read: %w", err)
|
||||
return nil, fmt.Errorf("the view of the trunk cannot be read: %w", err)
|
||||
}
|
||||
for k, v := range mods {
|
||||
if m, err := meshcatalogclient.DecodeModule(v); err == nil {
|
||||
trunk[k] = m
|
||||
for _, k := range keys {
|
||||
if e, ok, err := modules.Get(k); err == nil && ok {
|
||||
if m, err := meshcatalogclient.DecodeModule(e.Value); err == nil {
|
||||
trunk[k] = m
|
||||
}
|
||||
}
|
||||
}
|
||||
kept := map[string]meshcatalog.ManifestsByDigestValue{}
|
||||
|
||||
@@ -39,6 +39,7 @@ import (
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/sdk/graph"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/views"
|
||||
|
||||
"mesh-delivery-indexer/internal/index"
|
||||
@@ -86,7 +87,7 @@ func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10; 1585/T14, T15
|
||||
}, func(ctx context.Context, js jetstream.JetStream) {
|
||||
// The states and the stream the controller makes from the indexer's manifest (old sections, ADR 0595 §11).
|
||||
for bucket, history := range map[string]uint8{index.PendingBucket: 1, index.VerdictsBucket: index.VerdictsHistory,
|
||||
index.GraphBucket: 1, index.TakenBucket: 1, index.PackagesBucket: 1,
|
||||
index.TakenBucket: 1, index.PackagesBucket: 1,
|
||||
index.SeatDefinitionsBucket: 1, index.RequirementsBucket: 1, findingsBucket: 1, index.HoldsBucket: 1, index.UnjudgedBucket: 1} {
|
||||
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: bucket, History: history}); err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -262,6 +263,13 @@ func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10; 1585/T14, T15
|
||||
named := held.WaitsFor
|
||||
get := func(bucket, key string) ([]byte, bool) {
|
||||
t.Helper()
|
||||
if bucket == index.ModulesState { // the view of the trunk, from the newest catalog graph revision
|
||||
e, err := revisionView{js: js}.Get(ctx, key)
|
||||
if err != nil {
|
||||
return nil, false
|
||||
}
|
||||
return e.Value(), true
|
||||
}
|
||||
kv, err := js.KeyValue(ctx, bucket)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -273,7 +281,7 @@ func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10; 1585/T14, T15
|
||||
return e.Value(), true
|
||||
}
|
||||
// Only what building and moving the named indexer needs is written.
|
||||
raw, ok := get(index.ModulesBucket, ix)
|
||||
raw, ok := get(index.ModulesState, ix)
|
||||
own, err := meshcatalogclient.DecodeModule(raw)
|
||||
if !ok || err != nil || own.Fp2 != named || own.Commit != m1.Merge {
|
||||
t.Fatalf("modules.%s = %+v (%v), want the named build %s at the merge", ix, own, err, named)
|
||||
@@ -284,11 +292,8 @@ func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10; 1585/T14, T15
|
||||
if _, ok := get(index.VerdictsBucket, own.Manifest); !ok {
|
||||
t.Errorf("the verdict on the indexer's manifest %s is not written", own.Manifest)
|
||||
}
|
||||
if gk, _ := graph.Key(ix); !func() bool { _, ok := get(index.GraphBucket, gk); return ok }() {
|
||||
t.Errorf("the indexer's key %s of the module graph is not written", gk)
|
||||
}
|
||||
repoKey, _ := trunk.RepositoryKey(m1.Repository)
|
||||
if _, ok := get(index.ModulesBucket, "a"); ok {
|
||||
if _, ok := get(index.ModulesState, "a"); ok {
|
||||
t.Errorf("modules.a is written while the revision is pending")
|
||||
}
|
||||
if _, ok := get(index.TakenBucket, repoKey); ok {
|
||||
@@ -348,7 +353,7 @@ func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10; 1585/T14, T15
|
||||
break
|
||||
}
|
||||
}
|
||||
if raw, ok := get(index.ModulesBucket, "a"); !ok || !strings.Contains(string(raw), m1.Merge) {
|
||||
if raw, ok := get(index.ModulesState, "a"); !ok || !strings.Contains(string(raw), m1.Merge) {
|
||||
t.Errorf("modules.a = %s after the take-in at the start, want it at the merge %s", raw, m1.Merge)
|
||||
}
|
||||
if _, ok := get(index.TakenBucket, repoKey); !ok {
|
||||
@@ -357,7 +362,7 @@ func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10; 1585/T14, T15
|
||||
if len(w) != 1 || w[0].Key != groupKey || len(w[0].Keys) != 1 || w[0].Keys[0] != g1Key {
|
||||
t.Fatalf("indexer_pending after the start = %+v, want only the group gathered, its first member %s", w, g1Key)
|
||||
}
|
||||
if _, ok := get(index.ModulesBucket, "c"); ok {
|
||||
if _, ok := get(index.ModulesState, "c"); ok {
|
||||
t.Errorf("modules.c is written while its group is gathered")
|
||||
}
|
||||
|
||||
@@ -376,8 +381,8 @@ func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10; 1585/T14, T15
|
||||
}
|
||||
var hasC, hasK bool
|
||||
for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline) && !(hasC && hasK && len(w) == 0); time.Sleep(50 * time.Millisecond) {
|
||||
_, hasC = get(index.ModulesBucket, "c")
|
||||
_, hasK = get(index.ModulesBucket, "k")
|
||||
_, hasC = get(index.ModulesState, "c")
|
||||
_, hasK = get(index.ModulesState, "k")
|
||||
w = waits()
|
||||
}
|
||||
if !hasC || !hasK || len(w) != 0 {
|
||||
@@ -471,7 +476,75 @@ var acceptedHeard atomic.Int64
|
||||
|
||||
// startIndexerRun starts the bus, the stream FORGE with the indexer's durable consumer, the buckets, lets put fill
|
||||
// mesh-controller_builds, and runs the program; restart starts it again. mesh-delivery-gate_source is never created.
|
||||
func startIndexerRun(t *testing.T, put func(context.Context, buildLog), before ...func(context.Context, jetstream.JetStream)) (jetstream.JetStream, context.Context, jetstream.KeyValue, jetstream.KeyValue, func()) {
|
||||
// revisionView reads the indexer's view of the trunk as the indexer itself does at a restart (milestone 47/G): the newest
|
||||
// trunk revision its objects catalog-revisions keep, a module by its key.
|
||||
type revisionView struct{ js jetstream.JetStream }
|
||||
|
||||
func (v revisionView) Get(ctx context.Context, key string) (jetstream.KeyValueEntry, error) {
|
||||
view, _, _, err := index.TrunkView(ctx, newBusObjects(v.js, testAccount, index.CatalogRevisionsObjects))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
value, rev, found, err := view.Get(ctx, key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !found {
|
||||
return nil, jetstream.ErrKeyNotFound
|
||||
}
|
||||
return viewEntry{key: key, value: value, revision: rev}, nil
|
||||
}
|
||||
|
||||
// History is the module's value in every trunk revision the objects keep that holds it, oldest first.
|
||||
func (v revisionView) History(ctx context.Context, key string) ([]jetstream.KeyValueEntry, error) {
|
||||
o := newBusObjects(v.js, testAccount, index.CatalogRevisionsObjects)
|
||||
names, err := o.Names(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var revs []int64
|
||||
for _, n := range names {
|
||||
if r, ok := index.TrunkRevisionOf(n); ok {
|
||||
revs = append(revs, r)
|
||||
}
|
||||
}
|
||||
slices.Sort(revs)
|
||||
var out []jetstream.KeyValueEntry
|
||||
for _, r := range revs {
|
||||
_, entries, err := index.ReadRevision(ctx, o, fmt.Sprintf("trunk.%d", r))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, e := range entries {
|
||||
if e.Module != key {
|
||||
continue
|
||||
}
|
||||
data, err := meshcatalogclient.EncodeModule(index.ModuleValueOf(e))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, viewEntry{key: key, value: data, revision: uint64(r)})
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// viewEntry is one module's value of the view.
|
||||
type viewEntry struct {
|
||||
key string
|
||||
value []byte
|
||||
revision uint64
|
||||
}
|
||||
|
||||
func (e viewEntry) Bucket() string { return index.CatalogRevisionsObjects }
|
||||
func (e viewEntry) Key() string { return e.key }
|
||||
func (e viewEntry) Value() []byte { return e.value }
|
||||
func (e viewEntry) Revision() uint64 { return e.revision }
|
||||
func (e viewEntry) Created() time.Time { return time.Time{} }
|
||||
func (e viewEntry) Delta() uint64 { return 0 }
|
||||
func (e viewEntry) Operation() jetstream.KeyValueOp { return jetstream.KeyValuePut }
|
||||
|
||||
func startIndexerRun(t *testing.T, put func(context.Context, buildLog), before ...func(context.Context, jetstream.JetStream)) (jetstream.JetStream, context.Context, jetstream.KeyValue, revisionView, func()) {
|
||||
dir, err := os.MkdirTemp(os.Getenv("TMPDIR"), "i")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -526,10 +599,11 @@ func startIndexerRun(t *testing.T, put func(context.Context, buildLog), before .
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
modules, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: index.ModulesBucket, History: index.ModulesHistory})
|
||||
if err != nil {
|
||||
// The indexer's objects catalog-revisions, which the grants part creates from its manifest's old objects section.
|
||||
if _, err := js.CreateObjectStore(ctx, jetstream.ObjectStoreConfig{Bucket: newBusObjects(nil, testAccount, index.CatalogRevisionsObjects).bucket()}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
modules := revisionView{js: js}
|
||||
// The bucket of the manifests, made by the controller from the manifest's state entry manifests (history 1, issue 1449).
|
||||
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: index.ManifestsBucket, History: index.ManifestsHistory}); err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -583,7 +657,7 @@ func startIndexerRun(t *testing.T, put func(context.Context, buildLog), before .
|
||||
}
|
||||
|
||||
// startIndexer is startIndexerRun for a test that never starts the program again.
|
||||
func startIndexer(t *testing.T, put func(context.Context, buildLog), before ...func(context.Context, jetstream.JetStream)) (jetstream.JetStream, context.Context, jetstream.KeyValue, jetstream.KeyValue) {
|
||||
func startIndexer(t *testing.T, put func(context.Context, buildLog), before ...func(context.Context, jetstream.JetStream)) (jetstream.JetStream, context.Context, jetstream.KeyValue, revisionView) {
|
||||
js, ctx, candidates, modules, _ := startIndexerRun(t, put, before...)
|
||||
return js, ctx, candidates, modules
|
||||
}
|
||||
@@ -612,8 +686,14 @@ func (l buildLog) end(t *testing.T, repo, module, path string) {
|
||||
// message on FORGE makes modules.<m> within 20 s, the fingerprint of the merge's tree and the recorded inputs.
|
||||
func TestIndexerWritesModulesAtAMerge(t *testing.T) { // 1239/AC9; I1, I3: the modules are the merged message's, no build record names them (issue 1851)
|
||||
// The bucket holds only build-agent's record, which no fingerprint reads since R4g (ADR 0530 §1).
|
||||
var asks jetstream.KeyValue
|
||||
js, ctx, _, modules := startIndexer(t, func(ctx context.Context, builds buildLog) {
|
||||
builds.end(t, "novox/mesh-controller", "build-agent", "modules/build-agent")
|
||||
}, func(ctx context.Context, js jetstream.JetStream) {
|
||||
var err error
|
||||
if asks, err = js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: index.TakeInsBucket, History: 1}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
})
|
||||
const merge = "89abcdef0123456789abcdef0123456789abcdef"
|
||||
data, err := forgemsg.EncodeForge(forgemsg.Forge{Kind: forgemsg.ForgeMerged, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12,
|
||||
@@ -636,6 +716,60 @@ func TestIndexerWritesModulesAtAMerge(t *testing.T) { // 1239/AC9; I1, I3: the m
|
||||
if m.Repository != "novox/mesh-delivery" || m.Commit != merge || m.Path != "modules/a" || m.Manifest == "" || m.Fp != "" || m.None != "" { // 1606/AC2
|
||||
t.Errorf("module = %+v within 20 s, want path modules/a, its manifest, and no value of today's form", m)
|
||||
}
|
||||
// Milestone 47/G W4.13 (design 47, the indexer whole): the source revision taken in is one catalog graph revision kept
|
||||
// as objects, said as catalog-revision on the seat's subject, every entry it lists stored once by digest.
|
||||
objects := newBusObjects(js, testAccount, index.CatalogRevisionsObjects)
|
||||
list, err := index.ListRevisions(ctx, objects, false)
|
||||
if err != nil || len(list.Revisions) != 1 || list.Revisions[0].Name != "trunk.1" || !list.Revisions[0].Current ||
|
||||
list.Entries != list.Revisions[0].Entries || len(list.Unlisted) != 0 || list.Fill.Bytes == 0 {
|
||||
t.Fatalf("revisions kept = %+v (%v): want trunk.1 alone, current, each entry it lists stored once", list, err)
|
||||
}
|
||||
events, err := js.Stream(ctx, "EVENTS")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if msg, err := events.GetLastMsgForSubject(ctx, "mesh.seat.mesh-catalog.event."+meshcatalog.EventCatalogRevision); err != nil ||
|
||||
!strings.Contains(string(msg.Data), `"revision":1`) {
|
||||
t.Errorf("catalog-revision said: %v (%v): want revision 1", msg, err)
|
||||
}
|
||||
// W4.12 through the process: a recompute by hand moves nothing and answers equal; a candidate's half dropped by hand
|
||||
// leaves the trunk's revision and its entries.
|
||||
if err := index.KeepRevision(ctx, objects, meshcatalog.CatalogRevision{Kind: "candidate", Group: "g1", Heads: "h1", Revision: 1,
|
||||
Entries: []meshcatalog.EntryRef{{Module: "x", Digest: "dx"}}}, map[string][]byte{"dx": []byte(`{"module":"x"}`)}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
answer := func(key string, value []byte) index.TakenIn {
|
||||
t.Helper()
|
||||
w, err := asks.Watch(ctx, key, jetstream.UpdatesOnly())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer w.Stop()
|
||||
if _, err := asks.Put(ctx, key, value); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for e := range w.Updates() {
|
||||
if e == nil {
|
||||
continue
|
||||
}
|
||||
if a, err := index.DecodeTakeIn(e.Value()); err == nil && a.Done() {
|
||||
return *a.Result
|
||||
}
|
||||
}
|
||||
t.Fatalf("%s is not answered", key)
|
||||
return index.TakenIn{}
|
||||
}
|
||||
key, value, _ := index.NewRecomputeAsk("a check", "ctl", time.Now())
|
||||
if r := answer(key, value); !r.Stored || !strings.Contains(r.Answer, "equal to the kept revision 1") {
|
||||
t.Errorf("indexer_recompute answered %+v: want equal to the kept revision 1", r)
|
||||
}
|
||||
key, value, _ = index.NewDropCandidateGraphAsk("g1", "h1", "its gate event never came", "ctl", time.Now())
|
||||
if r := answer(key, value); !r.Stored || !slices.Equal(r.Dropped, []string{"entry.dx"}) {
|
||||
t.Errorf("indexer_drop_candidate_graph answered %+v: want candidate.g1.h1 dropped with entry.dx", r)
|
||||
}
|
||||
if after, err := index.ListRevisions(ctx, objects, false); err != nil || len(after.Revisions) != 1 || after.Entries != list.Entries {
|
||||
t.Errorf("revisions after the drop = %+v (%v): want trunk.1 and its entries alone", after, err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestIndexerKeepsManifestsAtAMerge: a merged message of mesh-catalog carrying the real manifest of the module avahi and the
|
||||
@@ -960,17 +1094,16 @@ func TestIndexerJudgesAnUnreadableVerdictAgainAtStart(t *testing.T) { // 1923/AC
|
||||
if _, err := manifests.Put(ctx, mkey, mdata); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
stale, err := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Repository: "novox/mesh-catalog", Commit: "89abcdef0123456789abcdef0123456789abcdef", Path: "modules/avahi",
|
||||
// The view the indexer starts from is the newest trunk revision its objects keep (milestone 47/G).
|
||||
stale := meshcatalog.ModulesByModuleValue{Repository: "novox/mesh-catalog", Commit: "89abcdef0123456789abcdef0123456789abcdef", Path: "modules/avahi",
|
||||
Manifest: digest, None2: "the manifest cannot be read: claims node-build-go, which is a seat in the mesh's own namespace that it does not define",
|
||||
Inputs2: &meshcatalog.ModulesByModuleValueInputs2{Tree: "tree-avahi", Locks: map[string]string{"go.sum": goSum}}})
|
||||
Inputs2: &meshcatalog.ModulesByModuleValueInputs2{Tree: "tree-avahi", Locks: map[string]string{"go.sum": goSum}}}
|
||||
rev, objects, err := index.CatalogRevisionOf(meshcatalog.CatalogRevision{Revision: 1, Kind: "trunk"},
|
||||
map[string]meshcatalog.CatalogEntry{"avahi": index.CatalogEntryOf("avahi", stale, catalogue.Manifest{}, graph.Node{None: stale.None2})})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
held, err := js.KeyValue(ctx, index.ModulesBucket)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := held.Put(ctx, "avahi", stale); err != nil {
|
||||
if err := index.KeepRevision(ctx, newBusObjects(js, testAccount, index.CatalogRevisionsObjects), rev, objects); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: index.ToolchainsBucket("mesh-build-go"), History: 1})
|
||||
|
||||
@@ -375,9 +375,21 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
modules, err := process.EssentialBucket(ctx, js, "indexer", index.ModulesBucket, "write a module's fingerprint")
|
||||
if err != nil {
|
||||
return err
|
||||
// The view of the trunk is the newest trunk revision of the catalog graph the objects keep (milestone 47/G, ADR 0656 §6:
|
||||
// one form of the fact), read once here and kept as a revision after every write; objects not on the bus yet, or none
|
||||
// kept, start it empty, and the first start's catch-up takes the trunk in once the gate's source is read (catchUp).
|
||||
revisions := newBusObjects(js, cred, index.CatalogRevisionsObjects)
|
||||
modules, graph, at, err := index.TrunkView(ctx, revisions)
|
||||
switch {
|
||||
case errors.Is(err, process.ErrNotYet):
|
||||
logger.Printf("the view of the trunk starts empty: %v; the trunk is taken in once they are there and the gate's source is read", err)
|
||||
modules, graph = index.MemoryStore(), index.MemoryStore()
|
||||
case err != nil:
|
||||
return fmt.Errorf("the view of the trunk cannot be read from the newest catalog graph revision: %w", err)
|
||||
case at == 0:
|
||||
logger.Printf("the view of the trunk starts empty: %s keeps no trunk revision yet; the trunk is taken in once the gate's source is read", index.CatalogRevisionsObjects)
|
||||
default:
|
||||
logger.Printf("the view of the trunk read from trunk revision %d", at)
|
||||
}
|
||||
manifests, err := process.EssentialBucket(ctx, js, "indexer", index.ManifestsBucket, "keep a module's manifest")
|
||||
if err != nil {
|
||||
@@ -413,8 +425,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
// What each consumer's go.sum requires of each mesh Go module (DLV-I7): a state this delivery brings, bound when it
|
||||
// appears; until then indexer_requirements answers none, and the builder's floor waits on its answer.
|
||||
requirements := process.AwaitBucket(ctx, js, conn, index.RequirementsBucket, "what consumers require is not kept", logger.Printf)
|
||||
// The module graph (ADR 0603 §2): a state this delivery brings, bound when it appears; until then no graph is written.
|
||||
graph := process.AwaitBucket(ctx, js, conn, index.GraphBucket, "no module graph is written", logger.Printf)
|
||||
// The module graph (ADR 0603 §2) is held in memory, read with the view: each revision's entries carry its nodes.
|
||||
ownBuildState := process.AwaitBucket(ctx, js, conn, index.OwnBuildBucket, "which build this indexer runs is not kept", logger.Printf)
|
||||
// The manifests a take-in wrote and no judge pass judged yet (issue 1966): a state this delivery brings, bound when it
|
||||
// appears; until then a take-in's new manifests are kept for their judging in its memory alone.
|
||||
@@ -427,14 +438,16 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
seatBus := &seatBus{defs: awaitedSource{ctx, seatDefs}, marks: awaitedSource{ctx, marks}}
|
||||
seatReader, saySeats := followSeatList(seatBus, saySeatDefinition(js, cred), logger.Printf)
|
||||
readWhenBound(seatReader, seatDefs, marks)
|
||||
ix := &index.Indexer{Findings: awaitedStore{b: findings}, OldSections: awaitedStore{b: oldSections}, Unjudged: awaitedStore{b: unjudged}, Requirements: awaitedStore{b: requirements}, Store: kvStore{candidates}, Modules: kvStore{modules}, Manifests: kvStore{manifests}, Taken: awaitedStore{b: taken}, Builds: known,
|
||||
ix := &index.Indexer{Findings: awaitedStore{b: findings}, OldSections: awaitedStore{b: oldSections}, Unjudged: awaitedStore{b: unjudged}, Requirements: awaitedStore{b: requirements}, Store: kvStore{candidates}, Modules: modules, Manifests: kvStore{manifests}, Taken: awaitedStore{b: taken}, Builds: known,
|
||||
Verdicts: awaitedStore{b: verdicts}, SayRefused: sayRefused(conn, cred, logger.Printf), SayAccepted: sayAccepted(conn, cred),
|
||||
Predefined: awaitedStore{b: predefined}, ConfigCandidates: awaitedStore{b: configCandidates}, Logf: logger.Printf,
|
||||
Named: namedVia(calls.Call), Trunk: trunkVia(calls.Call), Contains: containsVia(calls.Call), Read: readVia(calls.Call),
|
||||
ReadOpen: openVia(calls.Call), Packages: awaitedStore{b: packages, js: js, name: index.PackagesBucket}, Files: filesVia(calls.Call), GoSum: goSumVia(calls.Call), GoMod: goModVia(calls.Call),
|
||||
OwnBuild: ownBuildVia(calls.Call, cred.Node), OpenTries: openTriesVia(calls.Call), OwnBuildState: awaitedStore{b: ownBuildState}, TryRecovery: awaitedStore{b: tryRecovery}, Graph: awaitedStore{b: graph},
|
||||
OwnBuild: ownBuildVia(calls.Call, cred.Node), OpenTries: openTriesVia(calls.Call), OwnBuildState: awaitedStore{b: ownBuildState}, TryRecovery: awaitedStore{b: tryRecovery}, Graph: graph,
|
||||
SeatDefinitions: awaitedStore{b: seatDefs}, ReadFile: readFileVia(calls.Call), SaySeatDefinition: saySeats,
|
||||
Message: messageVia(calls.CallKept), // a kept call: a read, called again through a transport fault (ADR 0668 §1)
|
||||
Message: messageVia(calls.CallKept), Revisions: revisions, // Message a kept call: a read, called again through a transport fault (ADR 0668 §1)
|
||||
SayCatalogRevision: sayCatalogRevision(conn, cred),
|
||||
SayFill: sayFill(conn, cred),
|
||||
// The take-in step (ADR 0633): its state pending and its stream take-in, both this delivery's, bound when they appear;
|
||||
// until then every merge is taken in at once, as before.
|
||||
Pending: awaitedStore{b: process.AwaitBucket(ctx, js, conn, index.PendingBucket, "no merge is held for the indexer its revision names: each is taken in at once", logger.Printf)},
|
||||
@@ -744,10 +757,10 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
// said up to; a first view (the tools bundle) answers at that mark.
|
||||
startViews(ctx, js, cred, marks,
|
||||
findings,
|
||||
map[string]readyOf{"manifests": boundNow(manifests), "modules": boundNow(modules), "candidates": boundNow(candidates),
|
||||
map[string]readyOf{"manifests": boundNow(manifests), "candidates": boundNow(candidates),
|
||||
index.VerdictsState: awaited(verdicts), index.PredefinedState: awaited(predefined),
|
||||
index.ConfigCandidatesState: awaited(configCandidates), index.PackagesState: awaited(packages),
|
||||
graphState: awaited(graph), index.HeardState: awaited(heard.b), "taken": awaited(taken), "findings": awaited(findings),
|
||||
index.HeardState: awaited(heard.b), "taken": awaited(taken), "findings": awaited(findings),
|
||||
index.SeatDefinitionsState: awaited(seatDefs)}, errs, logger.Printf)
|
||||
cc, err := process.HearEvents(cons, func(m jetstream.Msg) {
|
||||
handle(ctx, m, ix, heard, logger)
|
||||
|
||||
@@ -79,6 +79,9 @@ func (m *memStore) Delete(_ context.Context, k string, _ uint64) error { delete(
|
||||
|
||||
const head = "0123456789abcdef0123456789abcdef01234567"
|
||||
|
||||
// testAccount is the bus account the tests run the indexer as.
|
||||
var testAccount = process.Account{Node: "n1", Module: "mesh-delivery-indexer"}
|
||||
|
||||
func TestAMessageIsIndexedAndAcked(t *testing.T) { // 1238/AC2 (unit part)
|
||||
// Review round 2: the indexer's views emitter writes its finding view-unsaid (Emitter.Findings set).
|
||||
if em := viewsEmitter(context.Background(), nil, process.Account{}, nil, &process.AwaitedBucket{}, nil, t.Logf); em.Findings == nil {
|
||||
@@ -143,6 +146,10 @@ func TestAMessageIsIndexedAndAcked(t *testing.T) { // 1238/AC2 (unit part)
|
||||
t.Errorf("answered leaves %v, presence asked %v", held.vals, asked)
|
||||
}
|
||||
})
|
||||
// Milestone 47/G: the object buckets are the module's own by the name its account runs as, never a fixed name.
|
||||
if b := newBusObjects(nil, process.Account{Module: "renamed-indexer"}, index.CatalogRevisionsObjects).bucket(); b != "renamed-indexer_catalog-revisions" {
|
||||
t.Errorf("the catalog graph revisions open as %s, want the account's module's", b)
|
||||
}
|
||||
t.Run("not a forge message: terminated and logged", func(t *testing.T) { // 1238 (never redelivered)
|
||||
var logs bytes.Buffer
|
||||
st := &memStore{vals: map[string][]byte{}}
|
||||
|
||||
@@ -0,0 +1,166 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
"mesh-delivery-indexer/internal/index"
|
||||
)
|
||||
|
||||
// busObjects is one of the indexer's own object buckets in the old `objects` section (novox/hq ADR 0619 §2), which the
|
||||
// grants part creates: it is opened at its first use and kept; until the bucket exists every call is an error saying so,
|
||||
// and the next call opens it again (no timer: the next revision or tool call asks).
|
||||
type busObjects struct {
|
||||
js jetstream.JetStream
|
||||
// module is the module this program runs as, read from its bus account (process.Account.Module), never a fixed name.
|
||||
module string
|
||||
local string
|
||||
mu sync.Mutex
|
||||
o jetstream.ObjectStore
|
||||
}
|
||||
|
||||
// newBusObjects is the object bucket of this local name of the module the account runs as.
|
||||
func newBusObjects(js jetstream.JetStream, cred process.Account, local string) *busObjects {
|
||||
return &busObjects{js: js, module: cred.Module, local: local}
|
||||
}
|
||||
|
||||
// bucket is the bus's name of the object bucket: the module's own name and the local name.
|
||||
func (b *busObjects) bucket() string { return catalogue.ObjectsBucket(b.module, b.local) }
|
||||
|
||||
func (b *busObjects) open(ctx context.Context) (jetstream.ObjectStore, error) {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
if b.o != nil {
|
||||
return b.o, nil
|
||||
}
|
||||
o, err := b.js.ObjectStore(ctx, b.bucket())
|
||||
if errors.Is(err, jetstream.ErrBucketNotFound) || errors.Is(err, jetstream.ErrStreamNotFound) {
|
||||
return nil, fmt.Errorf("the object bucket %s is not on the bus yet (the grants part creates it from the manifest's objects): %w", b.local, process.ErrNotYet)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the object bucket %s is not open: %w", b.local, err)
|
||||
}
|
||||
b.o = o
|
||||
return o, nil
|
||||
}
|
||||
|
||||
func (b *busObjects) Put(ctx context.Context, name string, data []byte) error {
|
||||
o, err := b.open(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = o.Put(ctx, jetstream.ObjectMeta{Name: name}, bytes.NewReader(data))
|
||||
return err
|
||||
}
|
||||
|
||||
func (b *busObjects) Get(ctx context.Context, name string) ([]byte, error) {
|
||||
o, err := b.open(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
data, err := o.GetBytes(ctx, name)
|
||||
if errors.Is(err, jetstream.ErrObjectNotFound) {
|
||||
return nil, index.ErrNoObject
|
||||
}
|
||||
return data, err
|
||||
}
|
||||
|
||||
func (b *busObjects) Delete(ctx context.Context, name string) error {
|
||||
o, err := b.open(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := o.Delete(ctx, name); err != nil && !errors.Is(err, jetstream.ErrObjectNotFound) {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *busObjects) Names(ctx context.Context) ([]string, error) {
|
||||
o, err := b.open(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
infos, err := o.List(ctx)
|
||||
if errors.Is(err, jetstream.ErrNoObjectsFound) {
|
||||
return []string{}, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
names := make([]string, 0, len(infos))
|
||||
for _, i := range infos {
|
||||
names = append(names, i.Name)
|
||||
}
|
||||
return names, nil
|
||||
}
|
||||
|
||||
func (b *busObjects) Fill(ctx context.Context) (int64, int64, error) {
|
||||
o, err := b.open(ctx)
|
||||
if err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
st, err := o.Status(ctx)
|
||||
if err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
info, err := b.js.Stream(ctx, "OBJ_"+st.Bucket())
|
||||
if err != nil {
|
||||
return int64(st.Size()), 0, err
|
||||
}
|
||||
return int64(st.Size()), info.CachedInfo().Config.MaxBytes, nil
|
||||
}
|
||||
|
||||
// sayCatalogRevision publishes the seat mesh-catalog's event catalog-revision under the duplicate id of its kind and
|
||||
// number, so a revision said twice is heard once (design 47, Two graphs).
|
||||
func sayCatalogRevision(conn *nats.Conn, cred process.Account) func(context.Context, meshcatalog.CatalogRevisionEvent) error {
|
||||
return func(_ context.Context, e meshcatalog.CatalogRevisionEvent) error {
|
||||
data, err := json.Marshal(e)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
msg := nats.NewMsg(seatEventSubject(meshcatalog.EventCatalogRevision))
|
||||
id := fmt.Sprintf("%s-%s-%d%s", meshcatalog.EventCatalogRevision, e.Kind, e.Revision, e.Heads)
|
||||
msg.Header.Set("x-event-id", id)
|
||||
msg.Header.Set("x-source", cred.Module)
|
||||
msg.Header.Set("x-node", cred.Node)
|
||||
msg.Header.Set("x-time", time.Now().UTC().Format(time.RFC3339Nano))
|
||||
msg.Header.Set("content-type", "application/json")
|
||||
msg.Header.Set(nats.MsgIdHdr, id)
|
||||
msg.Data = data
|
||||
return conn.PublishMsg(msg)
|
||||
}
|
||||
}
|
||||
|
||||
// sayFill publishes the seat mesh-catalog's event filling, full or room of the catalog graph revisions under the duplicate
|
||||
// id given (milestone 47/G W4.6), so one said twice is heard once.
|
||||
func sayFill(conn *nats.Conn, cred process.Account) func(context.Context, string, string, meshcatalog.Fill) error {
|
||||
return func(_ context.Context, event, id string, f meshcatalog.Fill) error {
|
||||
data, err := json.Marshal(f)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
msg := nats.NewMsg(seatEventSubject(event))
|
||||
msg.Header.Set("x-event-id", id)
|
||||
msg.Header.Set("x-source", cred.Module)
|
||||
msg.Header.Set("x-node", cred.Node)
|
||||
msg.Header.Set("x-time", time.Now().UTC().Format(time.RFC3339Nano))
|
||||
msg.Header.Set("content-type", "application/json")
|
||||
msg.Header.Set(nats.MsgIdHdr, id)
|
||||
msg.Data = data
|
||||
return conn.PublishMsg(msg)
|
||||
}
|
||||
}
|
||||
@@ -15,6 +15,7 @@ import (
|
||||
type takerIn interface {
|
||||
TakeIn(ctx context.Context, module string) index.TakenIn
|
||||
TakeInRepository(ctx context.Context, repository string) index.TakenIn
|
||||
CatchUp(ctx context.Context) index.TakenIn
|
||||
JudgeAgain(ctx context.Context, module, digest string) index.TakenIn
|
||||
SayAcceptedAgain(ctx context.Context, module string) index.TakenIn
|
||||
SeatDefinitionsAgain(ctx context.Context, module string, pull int, head string) index.TakenIn
|
||||
@@ -25,6 +26,8 @@ type takerIn interface {
|
||||
DropTryCandidate(ctx context.Context, id, by, why string) index.TakenIn
|
||||
RecoverTriesByHand(ctx context.Context, by, why string) index.TakenIn
|
||||
FingerprintsAgain(ctx context.Context, module string) index.TakenIn
|
||||
RecomputeByHand(ctx context.Context, by, why string) index.TakenIn
|
||||
DropCandidateGraph(ctx context.Context, group, heads, by, why string) index.TakenIn
|
||||
}
|
||||
|
||||
// takenIn answers one value of the takeins state (the tool indexer_take_in, issue 1945): a value not yet answered takes the
|
||||
@@ -43,7 +46,13 @@ func takenIn(ctx context.Context, ix takerIn, kv jetstream.KeyValue, e jetstream
|
||||
return
|
||||
}
|
||||
asked, r := "take-in", index.TakenIn{}
|
||||
if a.Fingerprints {
|
||||
if a.Recompute {
|
||||
asked, r = "recompute of the catalog graph", ix.RecomputeByHand(ctx, a.By, a.Why)
|
||||
a.Module = r.Module
|
||||
} else if a.CandidateGroup != "" && a.Drop {
|
||||
asked, r = "drop of a candidate's catalog half", ix.DropCandidateGraph(ctx, a.CandidateGroup, a.CandidateHeads, a.By, a.Why)
|
||||
a.Module = r.Module
|
||||
} else if a.Fingerprints {
|
||||
asked, r = "computing of fingerprints again", ix.FingerprintsAgain(ctx, a.Module)
|
||||
} else if a.RecoverTries {
|
||||
asked, r = "recovery of the tries", ix.RecoverTriesByHand(ctx, a.By, a.Why)
|
||||
@@ -60,6 +69,9 @@ func takenIn(ctx context.Context, ix takerIn, kv jetstream.KeyValue, e jetstream
|
||||
} else if a.Pending != "" {
|
||||
asked, r = "hand take-in of what waits", ix.TakeInPendingByHand(ctx, a.Pending, a.By, a.Why)
|
||||
a.Module = a.Pending
|
||||
} else if a.CatchUp {
|
||||
asked, r = "catch-up of every repository", ix.CatchUp(ctx)
|
||||
a.Module = r.Module
|
||||
} else if a.Repository != "" {
|
||||
asked, r = "take-in of a repository", ix.TakeInRepository(ctx, a.Repository)
|
||||
a.Module = a.Repository
|
||||
|
||||
@@ -97,6 +97,14 @@ type Indexer struct {
|
||||
ReadFile func(ctx context.Context, repository, sha, path string) ([]byte, error)
|
||||
// SaySeatDefinition says seat-definition-changed (seatdefs.go); nil says nothing.
|
||||
SaySeatDefinition func(ctx context.Context, c SeatDefinitionChanged) error
|
||||
// Revisions is the object bucket catalog-revisions (catalog_revisions.go; design 47, Two graphs); nil keeps no
|
||||
// catalog graph revision.
|
||||
Revisions Objects
|
||||
// SayCatalogRevision says the seat mesh-catalog's event catalog-revision for a revision kept; nil says nothing.
|
||||
SayCatalogRevision func(ctx context.Context, e meshcatalog.CatalogRevisionEvent) error
|
||||
// SayFill says the seat mesh-catalog's event filling, full or room of the catalog graph revisions, under the duplicate
|
||||
// id given (W4.6).
|
||||
SayFill func(ctx context.Context, event, id string, f meshcatalog.Fill) error
|
||||
// Graph is the module graph's state (GraphBucket, graph.go; novox/hq ADR 0603 §2); nil writes no graph.
|
||||
Graph Store
|
||||
// Files asks the forge for every blob under each directory at a commit, by directory then path relative to it, each
|
||||
@@ -164,12 +172,15 @@ type Indexer struct {
|
||||
failedAt map[string]string // the head of a repository whose tree lacks a directory asked, by repository: not asked again until the head changes
|
||||
// OwnBuild answers the fingerprint of the build this program runs, from the node-engine's seat; nil knows none.
|
||||
OwnBuild func(ctx context.Context) (string, error)
|
||||
underSeats bool // a dry run inside judge.WithSeats: its judge passes do not hold the seat list (verdicts.go)
|
||||
judgeMu sync.Mutex // one judge pass at a time (verdicts.go)
|
||||
ownMu sync.Mutex // guards ownFP
|
||||
ownFP string // the fingerprint of the build this program runs, fixed once at the start (FixOwnBuild)
|
||||
ownFixed bool // FixOwnBuild asked already: it asks once
|
||||
ownSince time.Time // when the build was asked and not known, with ownWhy
|
||||
underSeats bool // a dry run inside judge.WithSeats: its judge passes do not hold the seat list (verdicts.go)
|
||||
judgeMu sync.Mutex // one judge pass at a time (verdicts.go)
|
||||
ownMu sync.Mutex // guards ownFP
|
||||
keepMu sync.Mutex // one catalog graph revision kept at a time (keepEntries)
|
||||
catchingUp atomic.Bool // the first start's catch-up runs: its merges keep no revision until it ends (CatchUp)
|
||||
catchUpWG sync.WaitGroup // the first start's catch-up in the background (kickCatchUp), for tests to wait on
|
||||
ownFP string // the fingerprint of the build this program runs, fixed once at the start (FixOwnBuild)
|
||||
ownFixed bool // FixOwnBuild asked already: it asks once
|
||||
ownSince time.Time // when the build was asked and not known, with ownWhy
|
||||
ownWhy string
|
||||
// OwnBuildState keeps which build this process knows it runs, or since when and why not (FixOwnBuild); nil keeps none.
|
||||
OwnBuildState Store
|
||||
|
||||
@@ -0,0 +1,141 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"maps"
|
||||
"reflect"
|
||||
"slices"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-delivery/sdk/graph"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/keys"
|
||||
)
|
||||
|
||||
// The catalog graph (novox/hq ADR 0656; design 47, *Two graphs, one source of truth*): at each revision one entry per
|
||||
// module, as its manifest declares it, in the seat mesh-catalog's layout (its definition's CatalogEntry), and the
|
||||
// revision a list of its entries' digests, each entry kept once as an object named by its digest
|
||||
// (catalog-revisions: trunk.<revision>, candidate.<group>.<heads>, entry.<digest>).
|
||||
|
||||
// CatalogEntryLayout is the layout version an entry and a revision are written in.
|
||||
const CatalogEntryLayout = 1
|
||||
|
||||
// CatalogEntryOf is the catalog graph entry of one module: the module graph's node of it as a recompute computed it
|
||||
// (its build, provision and verb edges, the verb edges left out on a cycle, ADR 0614), with where the trunk holds it,
|
||||
// its manifest's digest and verdict, the main package its process runs (MainOf), the package versions it builds against
|
||||
// and what the indexer computes its next revision from (Inputs: the lead's ruling on milestone 47/G finding 1), so that
|
||||
// ModuleValueOf gives the module's value back from the entry alone. The acting rules are the revision's to add.
|
||||
func CatalogEntryOf(module string, cur meshcatalog.ModulesByModuleValue, m catalogue.Manifest, n graph.Node) meshcatalog.CatalogEntry {
|
||||
e := meshcatalog.CatalogEntry{V: CatalogEntryLayout, Module: module, Repository: cur.Repository, Path: cur.Path,
|
||||
Commit: cur.Commit, Manifest: cur.Manifest, Verdict: cur.Verdict, Fp: n.FP, None: n.None, Bases: n.Bases,
|
||||
Packages: maps.Clone(cur.Packages), Provides: n.Provides, Requires: n.Requires, Awaits: n.Awaits, Invokes: n.Invokes,
|
||||
Serves: n.Serves, LeftOut: n.LeftOut, Main: MainOf(m)}
|
||||
if n.FP == "" && n.None == "" {
|
||||
// A module the recompute computed no node for (its manifest cannot be read) keeps what its value holds.
|
||||
e.Fp, e.None = cur.Fp2, cur.None2
|
||||
}
|
||||
in := meshcatalog.EntryInputs{Toolchains: maps.Clone(cur.Toolchains), Paths: maps.Clone(cur.Paths), Problems: cur.Problems,
|
||||
Warnings: cur.Warnings, Unknown: cur.Unknown, Src: cur.Src, FpSrc: cur.FpSrc}
|
||||
if cur.Inputs2 != nil {
|
||||
from := meshcatalog.FingerprintInputs(*cur.Inputs2)
|
||||
in.From = &from
|
||||
}
|
||||
if !reflect.ValueOf(in).IsZero() {
|
||||
e.Inputs = &in
|
||||
}
|
||||
for language, t := range n.Toolchains {
|
||||
if e.Toolchains == nil {
|
||||
e.Toolchains = map[string]meshcatalog.BuildEdge{}
|
||||
}
|
||||
e.Toolchains[language] = meshcatalog.BuildEdge{Module: t.Module, Version: t.Version, Recipe: t.Recipe}
|
||||
}
|
||||
return e
|
||||
}
|
||||
|
||||
// ModuleValueOf is the module's value as the indexer computes with it, read back from its catalog graph entry: the
|
||||
// inverse of CatalogEntryOf, by which the indexer's view of the trunk is the newest trunk revision's entries (the lead's
|
||||
// ruling on milestone 47/G finding 1; ADR 0656 §6: one form of the fact). The old line's fields (the old-form
|
||||
// fingerprint, the reader needs) are not kept.
|
||||
func ModuleValueOf(e meshcatalog.CatalogEntry) meshcatalog.ModulesByModuleValue {
|
||||
v := meshcatalog.ModulesByModuleValue{V: keys.Version, Repository: e.Repository, Path: e.Path, Commit: e.Commit, Manifest: e.Manifest,
|
||||
Verdict: e.Verdict, Fp2: e.Fp, None2: e.None, Packages: maps.Clone(e.Packages)}
|
||||
if in := e.Inputs; in != nil {
|
||||
v.Toolchains, v.Paths, v.Problems, v.Warnings, v.Unknown = maps.Clone(in.Toolchains), maps.Clone(in.Paths), in.Problems, in.Warnings, in.Unknown
|
||||
v.Src, v.FpSrc = in.Src, in.FpSrc
|
||||
if in.From != nil {
|
||||
from := meshcatalog.ModulesByModuleValueInputs2(*in.From)
|
||||
v.Inputs2 = &from
|
||||
}
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
// NodeOf is the node of the module graph an entry carries, as CatalogEntryOf took it in: the graph the indexer holds in
|
||||
// memory is read back from the newest trunk revision at its start (TrunkView), so a revision kept before its first
|
||||
// recompute loses no edge.
|
||||
func NodeOf(e meshcatalog.CatalogEntry) graph.Node {
|
||||
n := graph.Node{V: graph.Version, FP: e.Fp, None: e.None, Repository: e.Repository, Path: e.Path, Bases: e.Bases,
|
||||
Provides: e.Provides, Requires: e.Requires, Awaits: e.Awaits, Invokes: e.Invokes, Serves: e.Serves, LeftOut: e.LeftOut}
|
||||
if len(e.Packages) > 0 {
|
||||
n.Packages = slices.Sorted(maps.Keys(e.Packages))
|
||||
}
|
||||
for language, t := range e.Toolchains {
|
||||
if n.Toolchains == nil {
|
||||
n.Toolchains = map[string]graph.Toolchain{}
|
||||
}
|
||||
n.Toolchains[language] = graph.Toolchain{Module: t.Module, Version: t.Version, Recipe: t.Recipe}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// EncodeEntry is an entry's bytes and their sha256 in hex, the name of its object: the same entry in two revisions is
|
||||
// one object (design 47, *An unchanged entry is stored once*).
|
||||
func EncodeEntry(e meshcatalog.CatalogEntry) (string, []byte, error) {
|
||||
data, err := json.Marshal(e)
|
||||
if err != nil {
|
||||
return "", nil, fmt.Errorf("the entry of %s: %w", e.Module, err)
|
||||
}
|
||||
sum := sha256.Sum256(data)
|
||||
return hex.EncodeToString(sum[:]), data, nil
|
||||
}
|
||||
|
||||
// CatalogRevisionOf is a revision of entries, listed by module name with each entry's digest, and the entry objects it
|
||||
// lists by digest, each once. The caller numbers the revision, names its kind (trunk or candidate) and what it was read
|
||||
// from.
|
||||
func CatalogRevisionOf(rev meshcatalog.CatalogRevision, entries map[string]meshcatalog.CatalogEntry) (meshcatalog.CatalogRevision, map[string][]byte, error) {
|
||||
rev.V, rev.Entries = CatalogEntryLayout, make([]meshcatalog.EntryRef, 0, len(entries))
|
||||
objects := map[string][]byte{}
|
||||
for _, module := range slices.Sorted(maps.Keys(entries)) {
|
||||
digest, data, err := EncodeEntry(entries[module])
|
||||
if err != nil {
|
||||
return meshcatalog.CatalogRevision{}, nil, err
|
||||
}
|
||||
objects[digest] = data
|
||||
rev.Entries = append(rev.Entries, meshcatalog.EntryRef{Module: module, Digest: digest})
|
||||
}
|
||||
return rev, objects, nil
|
||||
}
|
||||
|
||||
// MainOf is the Go main package the check's first tier builds when the module acts (the seat mesh-catalog's entry
|
||||
// field main): the from of the first Go artifact a process resource of the manifest runs, as the manifest names it;
|
||||
// empty when it runs none.
|
||||
func MainOf(m catalogue.Manifest) string {
|
||||
if m.Build == nil {
|
||||
return ""
|
||||
}
|
||||
for _, r := range m.Resources {
|
||||
if fmt.Sprint(r["type"]) != "process" {
|
||||
continue
|
||||
}
|
||||
name := fmt.Sprint(r["artifact"])
|
||||
for _, a := range m.Build.Artifacts {
|
||||
if a.Name == name && a.Language == "go" && a.From != "" {
|
||||
return a.From
|
||||
}
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -0,0 +1,483 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
|
||||
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-delivery/sdk/graph"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
"git.novox.be/novox/mesh-sdk/go/views"
|
||||
)
|
||||
|
||||
// The indexer's object bucket catalog-revisions (design 47, Two graphs; novox/hq ADR 0619 §2: declared in the old
|
||||
// `objects` section, its cap there): a revision is its list of entries' digests under trunk.<revision> or
|
||||
// candidate.<group>.<heads>, and each entry one object entry.<digest>, written once and shared by every revision that
|
||||
// lists it.
|
||||
|
||||
// CatalogRevisionsObjects is the bucket's local name in the indexer's manifest.
|
||||
const CatalogRevisionsObjects = "catalog-revisions"
|
||||
|
||||
// ErrNoObject is an object the bucket does not hold.
|
||||
var ErrNoObject = errors.New("no such object")
|
||||
|
||||
// Objects is an object bucket as the indexer keeps its revisions in it.
|
||||
type Objects interface {
|
||||
// Put stores data whole under name, replacing one of that name.
|
||||
Put(ctx context.Context, name string, data []byte) error
|
||||
// Get is the object's bytes; ErrNoObject when it holds none of that name.
|
||||
Get(ctx context.Context, name string) ([]byte, error)
|
||||
// Delete removes the object; one it does not hold is no error.
|
||||
Delete(ctx context.Context, name string) error
|
||||
// Names is every object's name; none is an empty list.
|
||||
Names(ctx context.Context) ([]string, error)
|
||||
// Fill is the bucket's bytes used and its cap in bytes.
|
||||
Fill(ctx context.Context) (used, capacity int64, err error)
|
||||
}
|
||||
|
||||
// EntryObject is the object name of the entry of that digest.
|
||||
func EntryObject(digest string) string { return "entry." + digest }
|
||||
|
||||
// RevisionObject is the object name of a revision: trunk.<revision>, or candidate.<group>.<heads> for a candidate,
|
||||
// which keeps its newest revision only.
|
||||
func RevisionObject(rev meshcatalog.CatalogRevision) string {
|
||||
if rev.Kind == "candidate" {
|
||||
return "candidate." + rev.Group + "." + rev.Heads
|
||||
}
|
||||
return "trunk." + strconv.FormatInt(rev.Revision, 10)
|
||||
}
|
||||
|
||||
// TrunkRevisionOf is the revision number a trunk object's name holds, and whether it is one.
|
||||
func TrunkRevisionOf(name string) (int64, bool) {
|
||||
n, ok := strings.CutPrefix(name, "trunk.")
|
||||
if !ok {
|
||||
return 0, false
|
||||
}
|
||||
r, err := strconv.ParseInt(n, 10, 64)
|
||||
return r, err == nil
|
||||
}
|
||||
|
||||
// KeepRevision stores a revision as CatalogRevisionOf made it: each entry object the bucket does not hold yet, then the
|
||||
// revision's list, so a list is never kept whose entries are not; an entry already held is not written again.
|
||||
func KeepRevision(ctx context.Context, o Objects, rev meshcatalog.CatalogRevision, entries map[string][]byte) error {
|
||||
held, err := o.Names(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("the objects %s: %w", CatalogRevisionsObjects, err)
|
||||
}
|
||||
have := make(map[string]bool, len(held))
|
||||
for _, n := range held {
|
||||
have[n] = true
|
||||
}
|
||||
for _, ref := range rev.Entries {
|
||||
name := EntryObject(ref.Digest)
|
||||
if have[name] {
|
||||
continue
|
||||
}
|
||||
data, ok := entries[ref.Digest]
|
||||
if !ok {
|
||||
return fmt.Errorf("revision %s lists the entry of %s (%s), which it does not carry", RevisionObject(rev), ref.Module, ref.Digest)
|
||||
}
|
||||
if err := o.Put(ctx, name, data); err != nil {
|
||||
return fmt.Errorf("the entry of %s in %s: %w", ref.Module, CatalogRevisionsObjects, err)
|
||||
}
|
||||
have[name] = true
|
||||
}
|
||||
list, err := json.Marshal(rev)
|
||||
if err != nil {
|
||||
return fmt.Errorf("revision %s: %w", RevisionObject(rev), err)
|
||||
}
|
||||
if err := o.Put(ctx, RevisionObject(rev), list); err != nil {
|
||||
return fmt.Errorf("revision %s in %s: %w", RevisionObject(rev), CatalogRevisionsObjects, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ReadRevision is the revision kept under name and its entries whole, in the revision's order; an entry it lists that
|
||||
// the bucket lost is an error naming it, never an entry left out.
|
||||
func ReadRevision(ctx context.Context, o Objects, name string) (meshcatalog.CatalogRevision, []meshcatalog.CatalogEntry, error) {
|
||||
var rev meshcatalog.CatalogRevision
|
||||
data, err := o.Get(ctx, name)
|
||||
if err != nil {
|
||||
return rev, nil, fmt.Errorf("revision %s: %w", name, err)
|
||||
}
|
||||
if err := json.Unmarshal(data, &rev); err != nil {
|
||||
return rev, nil, fmt.Errorf("revision %s: %w", name, err)
|
||||
}
|
||||
entries := make([]meshcatalog.CatalogEntry, 0, len(rev.Entries))
|
||||
for _, ref := range rev.Entries {
|
||||
data, err := o.Get(ctx, EntryObject(ref.Digest))
|
||||
if err != nil {
|
||||
return rev, nil, fmt.Errorf("revision %s, the entry of %s (%s): %w", name, ref.Module, ref.Digest, err)
|
||||
}
|
||||
var e meshcatalog.CatalogEntry
|
||||
if err := json.Unmarshal(data, &e); err != nil {
|
||||
return rev, nil, fmt.Errorf("revision %s, the entry of %s: %w", name, ref.Module, err)
|
||||
}
|
||||
entries = append(entries, e)
|
||||
}
|
||||
return rev, entries, nil
|
||||
}
|
||||
|
||||
// newestTrunk is the newest trunk revision the bucket keeps, its list only; none is a zero revision and false.
|
||||
func newestTrunk(ctx context.Context, o Objects) (meshcatalog.CatalogRevision, bool, error) {
|
||||
names, err := o.Names(ctx)
|
||||
if err != nil {
|
||||
return meshcatalog.CatalogRevision{}, false, err
|
||||
}
|
||||
var newest int64
|
||||
for _, n := range names {
|
||||
if r, ok := TrunkRevisionOf(n); ok && r > newest {
|
||||
newest = r
|
||||
}
|
||||
}
|
||||
if newest == 0 {
|
||||
return meshcatalog.CatalogRevision{}, false, nil
|
||||
}
|
||||
data, err := o.Get(ctx, "trunk."+strconv.FormatInt(newest, 10))
|
||||
if err != nil {
|
||||
return meshcatalog.CatalogRevision{}, false, err
|
||||
}
|
||||
var rev meshcatalog.CatalogRevision
|
||||
if err := json.Unmarshal(data, &rev); err != nil {
|
||||
return rev, false, fmt.Errorf("trunk.%d: %w", newest, err)
|
||||
}
|
||||
return rev, true, nil
|
||||
}
|
||||
|
||||
// changedEntries are the modules whose entry differs between two revisions' lists, one gone or added included, by name.
|
||||
func changedEntries(before, after []meshcatalog.EntryRef) []string {
|
||||
was := map[string]string{}
|
||||
for _, r := range before {
|
||||
was[r.Module] = r.Digest
|
||||
}
|
||||
var changed []string
|
||||
for _, r := range after {
|
||||
if d, ok := was[r.Module]; !ok || d != r.Digest {
|
||||
changed = append(changed, r.Module)
|
||||
}
|
||||
delete(was, r.Module)
|
||||
}
|
||||
for m := range was {
|
||||
changed = append(changed, m)
|
||||
}
|
||||
slices.Sort(changed)
|
||||
return changed
|
||||
}
|
||||
|
||||
// keepTrunkRevision keeps the catalog graph revision a merge's or a recompute's module graph makes (design 47, Two
|
||||
// graphs: a revision for each source revision taken in, and for a fingerprint moved with no merge), numbered after the
|
||||
// newest kept, and says catalog-revision; a graph whose entries are the newest revision's makes none. A revision not
|
||||
// kept is said in the log and returned, so the take-in that made it is not said taken.
|
||||
func (ix *Indexer) keepTrunkRevision(ctx context.Context, edges map[string]graphEdges, whole graph.Graph) error {
|
||||
if ix.Revisions == nil {
|
||||
return nil
|
||||
}
|
||||
// Each entry is the module's value as this recompute left it in modules (its fingerprint, toolchains, package versions
|
||||
// and inputs written), so the revision holds what the next one is computed from.
|
||||
held, err := ix.heldModules(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// Every module the view holds is an entry, one the recompute computed no edges for (a manifest it cannot read) too.
|
||||
entries := make(map[string]meshcatalog.CatalogEntry, len(held))
|
||||
for name, cur := range held {
|
||||
entries[name] = CatalogEntryOf(name, cur, edges[name].manifest, whole[name])
|
||||
}
|
||||
for name, e := range edges {
|
||||
if _, ok := held[name]; !ok {
|
||||
entries[name] = CatalogEntryOf(name, e.cur, e.manifest, whole[name])
|
||||
}
|
||||
}
|
||||
return ix.keepEntries(ctx, entries)
|
||||
}
|
||||
|
||||
// KeepView keeps the view of the trunk as a trunk revision after a write to it outside a recompute (the lead's ruling on
|
||||
// milestone 47/G: the judge pass's verdicts, a build's reported imports, the source stamp, the modules the trunk no longer
|
||||
// holds, the indexer's own entry a held revision writes): each entry is the module's value in the view with its node of
|
||||
// the module graph in memory, so what the view holds is never lost at a restart. A view whose entries are the newest
|
||||
// revision's keeps none.
|
||||
func (ix *Indexer) KeepView(ctx context.Context) error {
|
||||
if ix.Revisions == nil || ix.Modules == nil || ix.catchingUp.Load() {
|
||||
return nil
|
||||
}
|
||||
held, err := ix.heldModules(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
nodes := graph.Graph{}
|
||||
if ix.Graph != nil {
|
||||
keys, err := ix.Graph.Keys(ctx)
|
||||
if err != nil && !errors.Is(err, process.ErrNotYet) {
|
||||
return fmt.Errorf("the module graph: %w", err)
|
||||
}
|
||||
for _, k := range keys {
|
||||
if value, _, found, err := ix.Graph.Get(ctx, k); err == nil && found {
|
||||
if n, err := graph.Decode(value); err == nil {
|
||||
nodes[k] = n
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
entries := make(map[string]meshcatalog.CatalogEntry, len(held))
|
||||
for name, cur := range held {
|
||||
var m catalogue.Manifest
|
||||
if cur.Manifest != "" {
|
||||
if _, read, err := ix.manifestRaw(ctx, cur.Manifest); err == nil {
|
||||
m = read
|
||||
}
|
||||
}
|
||||
entries[name] = CatalogEntryOf(name, cur, m, nodes[name])
|
||||
}
|
||||
return ix.keepEntries(ctx, entries)
|
||||
}
|
||||
|
||||
// keepEntries keeps entries as the trunk revision after the newest, says catalog-revision, and keeps none when no entry
|
||||
// moved; one keep at a time, so two never take the same number.
|
||||
func (ix *Indexer) keepEntries(ctx context.Context, entries map[string]meshcatalog.CatalogEntry) error {
|
||||
if ix.catchingUp.Load() {
|
||||
return nil // the catch-up of a first start keeps its one revision at its end (CatchUp)
|
||||
}
|
||||
ix.keepMu.Lock()
|
||||
defer ix.keepMu.Unlock()
|
||||
ix.mu.Lock()
|
||||
src := int64(ix.source.rev)
|
||||
ix.mu.Unlock()
|
||||
newest, any, err := newestTrunk(ctx, ix.Revisions)
|
||||
if errors.Is(err, process.ErrNotYet) {
|
||||
ix.logf("catalog graph: no revision is kept: its objects %s are not on the bus yet", CatalogRevisionsObjects)
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
ix.logf("catalog graph: no revision is kept: the newest is not read: %v", err)
|
||||
return fmt.Errorf("the newest catalog graph revision: %w", err)
|
||||
}
|
||||
at := ix.now().UTC()
|
||||
rev, objects, err := CatalogRevisionOf(meshcatalog.CatalogRevision{Revision: newest.Revision + 1, Kind: "trunk", Source: src, At: &at}, entries)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
changed := changedEntries(newest.Entries, rev.Entries)
|
||||
if (any && len(changed) == 0) || (!any && len(entries) == 0) {
|
||||
return nil // nothing moved; an empty first revision is never kept, so the first start's catch-up still runs
|
||||
}
|
||||
used, capacity, fillErr := ix.Revisions.Fill(ctx)
|
||||
if fillErr == nil && capacity > 0 && used+newBytes(ctx, ix.Revisions, rev, objects) > capacity {
|
||||
// W4.6 (design 47, Two graphs): a full store refuses the next revision, said once per source revision, and the
|
||||
// take-in that made it holds its pending merge with the reason objects full until room is said.
|
||||
refused := fmt.Sprintf("the next revision; source revision %d held pending with the reason %s", src, HeldObjectsFull)
|
||||
ix.logf("catalog graph: revision %d of source revision %d is not kept: %s is full (%d of %d bytes)", rev.Revision, src, CatalogRevisionsObjects, used, capacity)
|
||||
ix.sayFill(ctx, meshcatalog.EventFull, fmt.Sprintf("%d", src), meshcatalog.Fill{Bytes: used, Cap: capacity, Refused: refused})
|
||||
return fmt.Errorf("revision %d of source revision %d: %w", rev.Revision, src, ErrObjectsFull)
|
||||
}
|
||||
if err := KeepRevision(ctx, ix.Revisions, rev, objects); err != nil {
|
||||
ix.logf("catalog graph: revision %d of source revision %d is not kept: %v", rev.Revision, src, err)
|
||||
return err
|
||||
}
|
||||
if after, capacity, err := ix.Revisions.Fill(ctx); fillErr == nil && err == nil && capacity > 0 && used*5 < capacity*4 && after*5 >= capacity*4 {
|
||||
ix.sayFill(ctx, meshcatalog.EventFilling, fmt.Sprintf("%d", rev.Revision), meshcatalog.Fill{Bytes: after, Cap: capacity})
|
||||
}
|
||||
ix.logf("catalog graph: revision %d of source revision %d kept, %d entries, %d changed", rev.Revision, src, len(rev.Entries), len(changed))
|
||||
if ix.SayCatalogRevision != nil {
|
||||
if err := ix.SayCatalogRevision(ctx, meshcatalog.CatalogRevisionEvent{V: CatalogEntryLayout, Revision: rev.Revision, Kind: rev.Kind,
|
||||
Source: src, At: &at, Changed: changed, Entries: int64(len(rev.Entries))}); err != nil {
|
||||
ix.logf("catalog graph: revision %d is not said: %v", rev.Revision, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// HeldObjectsFull is the reason a pending merge is held while the catalog graph revisions are full (Pending.Held).
|
||||
const HeldObjectsFull = "objects full"
|
||||
|
||||
// ErrObjectsFull is a revision refused because the catalog graph revisions are at their cap.
|
||||
var ErrObjectsFull = errors.New("the catalog graph revisions are full")
|
||||
|
||||
// newBytes is the size of what keeping rev adds: its list and each entry the bucket does not hold yet.
|
||||
func newBytes(ctx context.Context, o Objects, rev meshcatalog.CatalogRevision, entries map[string][]byte) int64 {
|
||||
held, _ := o.Names(ctx)
|
||||
have := make(map[string]bool, len(held))
|
||||
for _, n := range held {
|
||||
have[n] = true
|
||||
}
|
||||
list, _ := json.Marshal(rev)
|
||||
size := int64(len(list))
|
||||
for _, ref := range rev.Entries {
|
||||
if !have[EntryObject(ref.Digest)] {
|
||||
size += int64(len(entries[ref.Digest]))
|
||||
}
|
||||
}
|
||||
return size
|
||||
}
|
||||
|
||||
// sayFill says the seat's event filling, full or room of the catalog graph revisions, under the duplicate id of the
|
||||
// event and what it is about, so the same one said twice is heard once.
|
||||
func (ix *Indexer) sayFill(ctx context.Context, event, about string, f meshcatalog.Fill) {
|
||||
if ix.SayFill == nil {
|
||||
return
|
||||
}
|
||||
at := ix.now().UTC()
|
||||
f.At, f.Store = &at, CatalogRevisionsObjects
|
||||
if err := ix.SayFill(ctx, event, event+"-"+about, f); err != nil {
|
||||
ix.logf("catalog graph: %s is not said: %v", event, err)
|
||||
}
|
||||
}
|
||||
|
||||
// RoomAgain says room and evaluates pending again when a merge held with the reason objects full finds the store
|
||||
// below its filling mark (80 % of its cap): called after anything removes catalog graph revisions (no timer). It answers whether room was said.
|
||||
func (ix *Indexer) RoomAgain(ctx context.Context) (bool, error) {
|
||||
if ix.Revisions == nil || ix.Pending == nil {
|
||||
return false, nil
|
||||
}
|
||||
entries, err := ReadPending(ctx, ix.Pending, ix.logf)
|
||||
if err != nil || !slices.ContainsFunc(entries, func(e PendingEntry) bool { return e.Value.Held == HeldObjectsFull }) {
|
||||
return false, err
|
||||
}
|
||||
used, capacity, err := ix.Revisions.Fill(ctx)
|
||||
if err != nil || (capacity > 0 && used*5 >= capacity*4) {
|
||||
return false, err // room is said below the filling mark, 80 %, so one refusal is not followed by another at once
|
||||
}
|
||||
ix.sayFill(ctx, meshcatalog.EventRoom, fmt.Sprintf("%d", used), meshcatalog.Fill{Bytes: used, Cap: capacity})
|
||||
return true, ix.PendingAgain(ctx) // room in catalog-revisions: pending evaluated again
|
||||
}
|
||||
|
||||
// TrunkView is the indexer's view of the trunk read from the newest trunk revision the bucket keeps (the lead's ruling
|
||||
// on milestone 47/G finding 1; ADR 0656 §6, one form of the fact): a store in memory holding each module's value as
|
||||
// ModuleValueOf reads it back from its entry, under its modules key, a store in memory of each module's node of the module
|
||||
// graph (NodeOf), and the revision it was read at (0 when the bucket keeps none yet: both start empty and the first start's
|
||||
// catch-up fills them). A lost entry is an error naming the module (ReadRevision), never a module left out of the view.
|
||||
func TrunkView(ctx context.Context, o Objects) (modules, nodes Store, at int64, err error) {
|
||||
view, held := newOverlay(nil), newOverlay(nil)
|
||||
newest, any, err := newestTrunk(ctx, o)
|
||||
if err != nil || !any {
|
||||
return view, held, 0, err
|
||||
}
|
||||
_, entries, err := ReadRevision(ctx, o, RevisionObject(newest))
|
||||
if err != nil {
|
||||
return nil, nil, 0, err
|
||||
}
|
||||
for _, e := range entries {
|
||||
key, err := meshcatalogclient.ModulesKey(e.Module)
|
||||
if err != nil {
|
||||
return nil, nil, 0, fmt.Errorf("trunk.%d: the entry of %q: %w", newest.Revision, e.Module, err)
|
||||
}
|
||||
data, err := meshcatalogclient.EncodeModule(ModuleValueOf(e))
|
||||
if err != nil {
|
||||
return nil, nil, 0, fmt.Errorf("trunk.%d: the entry of %s: %w", newest.Revision, e.Module, err)
|
||||
}
|
||||
if err := view.Create(ctx, key, data); err != nil {
|
||||
return nil, nil, 0, fmt.Errorf("trunk.%d: the entry of %s: %w", newest.Revision, e.Module, err)
|
||||
}
|
||||
gk, err := graph.Key(e.Module)
|
||||
if err != nil {
|
||||
continue // a module the graph cannot name has no node, as writeGraph writes none
|
||||
}
|
||||
node, err := graph.Encode(NodeOf(e))
|
||||
if err != nil {
|
||||
return nil, nil, 0, fmt.Errorf("trunk.%d: the node of %s: %w", newest.Revision, e.Module, err)
|
||||
}
|
||||
if err := held.Create(ctx, gk, node); err != nil {
|
||||
return nil, nil, 0, fmt.Errorf("trunk.%d: the node of %s: %w", newest.Revision, e.Module, err)
|
||||
}
|
||||
}
|
||||
return view, held, newest.Revision, nil
|
||||
}
|
||||
|
||||
// MemoryStore is a store held in memory alone: the module graph's nodes, which each revision's entries carry, so the
|
||||
// indexer keeps no state of them on the bus (milestone 47/G).
|
||||
func MemoryStore() Store { return newOverlay(nil) }
|
||||
|
||||
// The view of the trunk is the newest catalog graph revision the indexer's objects catalog-revisions keep (milestone
|
||||
// 47/G, ADR 0656 §6): the tools bundle answers the modules and the module graph it once read from the states modules and
|
||||
// graph from that revision, at its number, and says it waits while the indexer keeps none yet (ADR 0615 §6), never an
|
||||
// empty catalog.
|
||||
|
||||
// ErrNoRevision is a view asked before the indexer kept its first trunk revision.
|
||||
var ErrNoRevision = errors.New("view-waits: the indexer keeps no catalog graph revision yet (its first start takes the trunk in); ask again on catalog-revision")
|
||||
|
||||
// TrunkStates reads the view of the trunk from the newest trunk revision; under answers every other state.
|
||||
type TrunkStates struct {
|
||||
Objects Objects
|
||||
Under func(string) views.Source
|
||||
}
|
||||
|
||||
// read is the view's modules and nodes, and the revision they were read at; none kept yet is ErrNoRevision.
|
||||
func (t TrunkStates) read(nodes bool) (Store, int64, error) {
|
||||
mods, held, at, err := TrunkView(context.Background(), t.Objects)
|
||||
if err == nil && at == 0 {
|
||||
err = ErrNoRevision
|
||||
}
|
||||
if nodes {
|
||||
return held, at, err
|
||||
}
|
||||
return mods, at, err
|
||||
}
|
||||
|
||||
// state is the source of a state by its name: modules and graph from the revision, their marks at its number, every
|
||||
// other the indexer's own state.
|
||||
func (t TrunkStates) State(name string) views.Source {
|
||||
switch name {
|
||||
case "modules", graph.State:
|
||||
return revisionSource{t: t, nodes: name == graph.State}
|
||||
case views.MarksState:
|
||||
return markSource{t: t, under: t.Under(name)}
|
||||
}
|
||||
return t.Under(name)
|
||||
}
|
||||
|
||||
// revisionSource is the modules or the module graph of the newest trunk revision, each value at its number.
|
||||
type revisionSource struct {
|
||||
t TrunkStates
|
||||
nodes bool
|
||||
}
|
||||
|
||||
func (r revisionSource) Keys() ([]string, error) {
|
||||
s, _, err := r.t.read(r.nodes)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return s.Keys(context.Background())
|
||||
}
|
||||
|
||||
func (r revisionSource) Get(key string) (*views.Entry, bool, error) {
|
||||
s, at, err := r.t.read(r.nodes)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
value, _, found, err := s.Get(context.Background(), key)
|
||||
if err != nil || !found {
|
||||
return nil, false, err
|
||||
}
|
||||
return &views.Entry{Value: value, Revision: uint64(at)}, true, nil
|
||||
}
|
||||
|
||||
// markSource answers the mark of modules and graph as the newest trunk revision's number, which catalog-revision says
|
||||
// the changes after; every other mark is the views family's.
|
||||
type markSource struct {
|
||||
t TrunkStates
|
||||
under views.Source
|
||||
}
|
||||
|
||||
func (m markSource) Keys() ([]string, error) { return m.under.Keys() }
|
||||
|
||||
func (m markSource) Get(state string) (*views.Entry, bool, error) {
|
||||
if state != "modules" && state != graph.State {
|
||||
return m.under.Get(state)
|
||||
}
|
||||
_, at, err := m.t.read(false)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
data, err := json.Marshal(views.Mark{V: views.Version, State: state, Revision: uint64(at), At: time.Now().UTC()})
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
return &views.Entry{Value: data, Revision: uint64(at)}, true, nil
|
||||
}
|
||||
@@ -81,22 +81,7 @@ func (ix *Indexer) writeGraph(ctx context.Context, edges map[string]graphEdges)
|
||||
GraphMost, len(names), names[GraphMost], len(names)-GraphMost-1)
|
||||
names = names[:GraphMost]
|
||||
}
|
||||
providers := func(provision string) []catalogue.Manifest {
|
||||
var out []catalogue.Manifest
|
||||
for _, name := range slices.Sorted(maps.Keys(edges)) {
|
||||
if slices.Contains(edges[name].manifest.Offers(), provision) {
|
||||
out = append(out, edges[name].manifest)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
// The verb edges on a cycle are left out over the whole graph (ADR 0614 §3), every module it knows, so a module past
|
||||
// GraphMost still closes the cycle it lies on.
|
||||
whole := graph.Graph{}
|
||||
for name, e := range edges {
|
||||
whole[name] = graphNode(e, providers)
|
||||
}
|
||||
whole.LeaveOut()
|
||||
whole := wholeGraph(edges)
|
||||
for _, name := range names {
|
||||
key, err := graph.Key(name)
|
||||
if err != nil {
|
||||
@@ -141,3 +126,23 @@ func (ix *Indexer) writeGraph(ctx context.Context, edges map[string]graphEdges)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// wholeGraph is the module graph of every module of edges, the verb edges on a cycle left out over the whole graph
|
||||
// (ADR 0614 §3), so a module past GraphMost still closes the cycle it lies on.
|
||||
func wholeGraph(edges map[string]graphEdges) graph.Graph {
|
||||
providers := func(provision string) []catalogue.Manifest {
|
||||
var out []catalogue.Manifest
|
||||
for _, name := range slices.Sorted(maps.Keys(edges)) {
|
||||
if slices.Contains(edges[name].manifest.Offers(), provision) {
|
||||
out = append(out, edges[name].manifest)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
whole := graph.Graph{}
|
||||
for name, e := range edges {
|
||||
whole[name] = graphNode(e, providers)
|
||||
}
|
||||
whole.LeaveOut()
|
||||
return whole
|
||||
}
|
||||
|
||||
@@ -1,6 +1,12 @@
|
||||
package index
|
||||
|
||||
import "encoding/json"
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"maps"
|
||||
"slices"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// rawList is the list a kept predefined value holds, as it was written.
|
||||
func rawList(value []byte) json.RawMessage {
|
||||
@@ -10,3 +16,69 @@ func rawList(value []byte) json.RawMessage {
|
||||
_ = json.Unmarshal(value, &kept)
|
||||
return kept.List
|
||||
}
|
||||
|
||||
// memObjects is an object bucket in memory with a cap: a put past the cap is refused as the bus's DiscardNew refuses it.
|
||||
type memObjects struct {
|
||||
mu sync.Mutex
|
||||
objs map[string][]byte
|
||||
puts int
|
||||
capacity int64
|
||||
}
|
||||
|
||||
func newMemObjects(capacity int64) *memObjects {
|
||||
return &memObjects{objs: map[string][]byte{}, capacity: capacity}
|
||||
}
|
||||
|
||||
func (m *memObjects) used() int64 {
|
||||
var n int64
|
||||
for _, d := range m.objs {
|
||||
n += int64(len(d))
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
func (m *memObjects) Put(_ context.Context, name string, data []byte) error {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
if m.capacity > 0 && m.used()-int64(len(m.objs[name]))+int64(len(data)) > m.capacity {
|
||||
return errObjectsFullForTest
|
||||
}
|
||||
m.objs[name] = slices.Clone(data)
|
||||
m.puts++
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *memObjects) Get(_ context.Context, name string) ([]byte, error) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
d, ok := m.objs[name]
|
||||
if !ok {
|
||||
return nil, ErrNoObject
|
||||
}
|
||||
return slices.Clone(d), nil
|
||||
}
|
||||
|
||||
func (m *memObjects) Delete(_ context.Context, name string) error {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
delete(m.objs, name)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *memObjects) Names(context.Context) ([]string, error) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
return slices.Sorted(maps.Keys(m.objs)), nil
|
||||
}
|
||||
|
||||
func (m *memObjects) Fill(context.Context) (int64, int64, error) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
return m.used(), m.capacity, nil
|
||||
}
|
||||
|
||||
var errObjectsFullForTest = &objectsFullError{}
|
||||
|
||||
type objectsFullError struct{}
|
||||
|
||||
func (*objectsFullError) Error() string { return "maximum bytes exceeded" }
|
||||
|
||||
@@ -27,6 +27,13 @@ import (
|
||||
// Imports that cannot be resolved (a package the forge cannot list) are kept with no ImportForm: the next recompute that
|
||||
// resolves them writes the import form, one rebuild too many, never a stale module.
|
||||
func (ix *Indexer) ApplyBuildEnd(ctx context.Context, value []byte) error {
|
||||
if err := ix.applyBuildEnd(ctx, value); err != nil {
|
||||
return err
|
||||
}
|
||||
return ix.KeepView(ctx) // the imports kept with the view (KeepView), so a restart reads them back
|
||||
}
|
||||
|
||||
func (ix *Indexer) applyBuildEnd(ctx context.Context, value []byte) error {
|
||||
if !builds.IsBuildEntry(value) {
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -292,6 +292,13 @@ func (ix *Indexer) dropRemoved(ctx context.Context, repository string, held map[
|
||||
// message (applyMerged). A repository whose trunk cannot be read, or reads as holding no module at all, is left as it is
|
||||
// and said: a failure never empties the bucket.
|
||||
func (ix *Indexer) Reconcile(ctx context.Context) error {
|
||||
if err := ix.reconcile(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
return ix.KeepView(ctx) // the modules dropped leave the view's next revision too
|
||||
}
|
||||
|
||||
func (ix *Indexer) reconcile(ctx context.Context) error {
|
||||
if ix.Modules == nil || ix.Trunk == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -1,7 +1,5 @@
|
||||
package index
|
||||
|
||||
import "git.novox.be/novox/mesh-delivery/sdk/graph"
|
||||
|
||||
// The holders, states and buckets the indexer names, taken from its own manifest (module.json: its name, its `state`, its
|
||||
// `reads`) and from the manifests of the holders it reads; a unit row checks them against the manifest
|
||||
// (TestTheNamesAreTheManifests). No contract names a holder (novox/hq ADR 0625 §4, ADR 0634 §1).
|
||||
@@ -14,13 +12,9 @@ const (
|
||||
CandidatesBucket = Module + "_" + CandidatesState
|
||||
CandidatesHistory = 10
|
||||
ModulesState = "modules"
|
||||
ModulesBucket = Module + "_" + ModulesState
|
||||
ModulesHistory = 20
|
||||
ManifestsState = "manifests"
|
||||
ManifestsBucket = Module + "_" + ManifestsState
|
||||
ManifestsHistory = 1
|
||||
// GraphBucket is the bucket of the state graph (sdk/graph names the state, not its holder).
|
||||
GraphBucket = Module + "_" + graph.State
|
||||
// VerdictsState and VerdictsBucket are in verdicts.go.
|
||||
VerdictsHistory = 5
|
||||
PredefinedState = "predefined"
|
||||
|
||||
@@ -95,6 +95,9 @@ type Pending struct {
|
||||
// UnreadSince is when the message was first found unread: past meshconditionsclient.CallUnansweredAfter, with the forge
|
||||
// heard since, it is the finding call-unanswered (unreadcall.go, ADR 0668 §3).
|
||||
UnreadSince time.Time `json:"unread-since,omitzero"`
|
||||
// Held is why a ready merge is not taken in although its revision names this build: objects full (HeldObjectsFull),
|
||||
// until room is said (RoomAgain).
|
||||
Held string `json:"held,omitempty"`
|
||||
}
|
||||
|
||||
// PendingEntry is one key of the pending state as read: its key, revision, value and the merged message decoded. For a
|
||||
@@ -782,11 +785,32 @@ func (ix *Indexer) evaluate(ctx context.Context, own string, start bool) (Applie
|
||||
if errors.Is(err, errVerdictsNotYet) {
|
||||
return a, "the verdicts state is not on the bus yet: the revision waits for it, taken in when it binds", nil
|
||||
}
|
||||
if errors.Is(err, ErrObjectsFull) {
|
||||
ix.holdFull(ctx, ready)
|
||||
return a, ix.waitingLine(w) + "; held: " + HeldObjectsFull, nil
|
||||
}
|
||||
return a, ix.waitingLine(w), err
|
||||
}
|
||||
return Applied{}, ix.waitingLine(w), ix.holdFor(ctx, ready, names, own, dry)
|
||||
}
|
||||
|
||||
// holdFull marks the ready merges held with the reason objects full (Pending.Held), read again at their revisions now
|
||||
// (the take-in's claim moved them), so indexer_pending shows why they wait.
|
||||
func (ix *Indexer) holdFull(ctx context.Context, ready []PendingEntry) {
|
||||
entries, err := ReadPending(ctx, ix.Pending, ix.logf)
|
||||
if err != nil {
|
||||
ix.logf("take-in: the merges held by a full %s are not marked: %v", CatalogRevisionsObjects, err)
|
||||
return
|
||||
}
|
||||
var held []PendingEntry
|
||||
for _, e := range entries {
|
||||
if slices.ContainsFunc(ready, func(r PendingEntry) bool { return r.Key == e.Key }) && e.Value.Held != HeldObjectsFull {
|
||||
held = append(held, e)
|
||||
}
|
||||
}
|
||||
ix.markSaid(ctx, held, nil, func(p *Pending) { p.Held = HeldObjectsFull })
|
||||
}
|
||||
|
||||
// holdLost holds a revision whose newest merge of a repository cannot be read (the forge bridge's stream no longer holds
|
||||
// it, or the bus refuses the read): said once on take-in.pending with the lost keys (TakeIn.Lost), and on each lost entry
|
||||
// (Pending.Lost, the reason, which indexer_pending shows), until a newer merge of its repository covers it or
|
||||
@@ -884,7 +908,9 @@ func (ix *Indexer) dry() *dryRun {
|
||||
// requirements, findings, seat definitions, holds and their bounds are not overlaid, nothing is said, and no hold is
|
||||
// weighed (Holds and Counted nil hold nothing); ReadFile only reads.
|
||||
Requirements: nil, Findings: nil, SeatDefinitions: nil, ReadFile: ix.ReadFile, SaySeatDefinition: nil,
|
||||
Holds: nil, Counted: nil, AskEngines: nil, HoldBounds: nil, OwnBuildState: nil, underSeats: true}
|
||||
Holds: nil, Counted: nil, AskEngines: nil, HoldBounds: nil, OwnBuildState: nil, underSeats: true,
|
||||
// A dry run keeps no catalog graph revision (catalog_revisions.go): its revision is the take-in's to keep.
|
||||
Revisions: nil, SayCatalogRevision: nil, SayFill: nil}
|
||||
x.Logf = func(format string, args ...any) { ix.logf("take-in, computed and not written: "+format, args...) }
|
||||
x.Store, d.candidates = over(ix.Store)
|
||||
x.Modules, d.modules = over(ix.Modules)
|
||||
@@ -1187,6 +1213,11 @@ func (ix *Indexer) holdFor(ctx context.Context, ready []PendingEntry, names, own
|
||||
}
|
||||
}
|
||||
}
|
||||
// The indexer's own entry the held revision names is kept as the trunk revision after the newest, every other entry as
|
||||
// it was (milestone 47/G W4.7: a merged revision naming a newer indexer replaces only the indexer's entry).
|
||||
if err := ix.KeepView(ctx); err != nil {
|
||||
return fmt.Errorf("the revision with the indexer's own entry: %w", err)
|
||||
}
|
||||
t := revisionWait(takein.TakeInPending, ready, names)
|
||||
t.Refused = refused
|
||||
var ended, unmarked []PendingEntry
|
||||
|
||||
@@ -18,6 +18,7 @@ import (
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein"
|
||||
"git.novox.be/novox/mesh-delivery/sdk/trunk"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/buscompose"
|
||||
"git.novox.be/novox/mesh-sdk/go/busretry"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
@@ -744,6 +745,141 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633)
|
||||
}
|
||||
}
|
||||
})
|
||||
t.Run("W4.6 a full store holds the merge with the reason objects full, says full once, and room takes it in", func(t *testing.T) {
|
||||
b := newTakeInBed(t)
|
||||
b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2"))
|
||||
named := b.fp(t, Module)
|
||||
b.ix.ownMu.Lock()
|
||||
b.ix.ownFP = named
|
||||
b.ix.ownMu.Unlock()
|
||||
store := newMemObjects(100) // 90 bytes held by another revision: full for this one
|
||||
_ = store.Put(ctx, "trunk.0", make([]byte, 90))
|
||||
b.ix.Revisions = store
|
||||
var said []string
|
||||
b.ix.SayFill = func(_ context.Context, event, id string, f meshcatalog.Fill) error {
|
||||
said = append(said, event+" "+f.Refused)
|
||||
return nil
|
||||
}
|
||||
if _, _, err := b.ix.TakeInAtStart(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var p Pending
|
||||
_ = json.Unmarshal(b.pending.vals[b.keys()[0]], &p)
|
||||
if len(b.keys()) != 1 || p.Held != HeldObjectsFull || len(said) != 1 || !strings.HasPrefix(said[0], "full ") || !strings.Contains(said[0], HeldObjectsFull) {
|
||||
t.Fatalf("pending %v held %q, said %q: want the merge held with %q and full said", b.keys(), p.Held, said, HeldObjectsFull)
|
||||
}
|
||||
if roomed, err := b.ix.RoomAgain(ctx); err != nil || roomed {
|
||||
t.Fatalf("room while still full: %v %v", roomed, err)
|
||||
}
|
||||
_ = store.Delete(ctx, "trunk.0") // the other revision removed: room
|
||||
store.capacity = 0
|
||||
if roomed, err := b.ix.RoomAgain(ctx); err != nil || !roomed || len(b.keys()) != 0 || said[len(said)-1] != "room " {
|
||||
t.Errorf("room said %v (%v), said %q, pending %v: want room said and the merge taken in", roomed, err, said, b.keys())
|
||||
}
|
||||
// The same take-in into a store it fills to 85 % of its cap says filling once, and is taken in.
|
||||
kept := store.used()
|
||||
b2 := newTakeInBed(t)
|
||||
b2.arrive(t, 2, b2.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2"))
|
||||
b2.ix.ownMu.Lock()
|
||||
b2.ix.ownFP = b2.fp(t, Module)
|
||||
b2.ix.ownMu.Unlock()
|
||||
b2.ix.Revisions = newMemObjects(kept * 100 / 85)
|
||||
var filling []meshcatalog.Fill
|
||||
b2.ix.SayFill = func(_ context.Context, event, _ string, f meshcatalog.Fill) error {
|
||||
if event == meshcatalog.EventFilling {
|
||||
filling = append(filling, f)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if _, _, err := b2.ix.TakeInAtStart(ctx); err != nil || len(b2.keys()) != 0 || len(filling) != 1 || filling[0].Store != CatalogRevisionsObjects {
|
||||
t.Errorf("a take-in to 85 %% said filling %+v (%v), pending %v: want it said once and the merge taken in", filling, err, b2.keys())
|
||||
}
|
||||
})
|
||||
t.Run("W4.12 a candidate's half dropped by hand frees its own entries, says room and takes the held merge in; the read tools list it; a recompute that moves nothing is equal", func(t *testing.T) {
|
||||
bed := func(store *memObjects) *takeInBed {
|
||||
b := newTakeInBed(t)
|
||||
b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2"))
|
||||
b.ix.ownMu.Lock()
|
||||
b.ix.ownFP = b.fp(t, Module)
|
||||
b.ix.ownMu.Unlock()
|
||||
b.ix.Revisions = store
|
||||
return b
|
||||
}
|
||||
free := newMemObjects(0)
|
||||
if _, _, err := bed(free).ix.TakeInAtStart(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
size := free.used() // what the merge's revision takes
|
||||
store := newMemObjects(2 * size)
|
||||
shared := []byte(`{"module":"b"}`)
|
||||
if err := KeepRevision(ctx, store, meshcatalog.CatalogRevision{Kind: "candidate", Group: "g1", Heads: "h1", Revision: 1,
|
||||
Entries: []meshcatalog.EntryRef{{Module: "a", Digest: "da"}, {Module: "b", Digest: "db"}}},
|
||||
map[string][]byte{"da": make([]byte, size+10), "db": shared}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := KeepRevision(ctx, store, meshcatalog.CatalogRevision{Kind: "candidate", Group: "g2", Heads: "h2", Revision: 1,
|
||||
Entries: []meshcatalog.EntryRef{{Module: "b", Digest: "db"}}}, map[string][]byte{"db": shared}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b := bed(store)
|
||||
var said []string
|
||||
b.ix.SayFill = func(_ context.Context, event, _ string, _ meshcatalog.Fill) error { said = append(said, event); return nil }
|
||||
if _, _, err := b.ix.TakeInAtStart(ctx); err != nil || len(b.keys()) != 1 {
|
||||
t.Fatalf("pending %v (%v): want the merge held for objects full", b.keys(), err)
|
||||
}
|
||||
var recorded []string
|
||||
asks := 0
|
||||
tools := map[string]stdio.Tool{}
|
||||
for _, tool := range CatalogRevisionTools(store, func(_ string, value []byte) (TakeInAsk, error) {
|
||||
asks++
|
||||
a, err := DecodeTakeIn(value)
|
||||
if err != nil {
|
||||
return a, err
|
||||
}
|
||||
var r TakenIn
|
||||
if a.Recompute {
|
||||
r = b.ix.RecomputeByHand(ctx, a.By, a.Why)
|
||||
} else {
|
||||
r = b.ix.DropCandidateGraph(ctx, a.CandidateGroup, a.CandidateHeads, a.By, a.Why)
|
||||
}
|
||||
a.Result = &r
|
||||
return a, nil
|
||||
}, time.Now, func(what, why string) error {
|
||||
recorded = append(recorded, what)
|
||||
if why == "unrecorded" {
|
||||
return errors.New("mesh-hand-acts did not answer")
|
||||
}
|
||||
return nil
|
||||
}) {
|
||||
tools[tool.Name] = tool
|
||||
}
|
||||
if _, err := tools[DropCandidateGraphTool].Run(map[string]any{"group": "g1", "heads": "h1", "why": "unrecorded"}); err == nil || asks != 0 {
|
||||
t.Fatalf("a drop whose hand act is not recorded: %v, %d asks: want refused before it is asked", err, asks)
|
||||
}
|
||||
got, err := tools[DropCandidateGraphTool].Run(map[string]any{"group": "g1", "heads": "h1", "why": "its gate event never came"})
|
||||
r, _ := got.(*TakenIn)
|
||||
if err != nil || r == nil || !r.Stored || !slices.Equal(r.Dropped, []string{"entry.da"}) || len(recorded) != 2 {
|
||||
t.Fatalf("drop answered %+v (%v), recorded %q: want candidate.g1.h1 dropped with entry.da alone, after its hand act", r, err, recorded)
|
||||
}
|
||||
if _, err := store.Get(ctx, "entry.db"); err != nil || len(b.keys()) != 0 || !slices.Contains(said, meshcatalog.EventRoom) {
|
||||
t.Errorf("entry.db %v, pending %v, said %q: want the shared entry kept, room said and the merge taken in", err, b.keys(), said)
|
||||
}
|
||||
list, err := tools[CatalogRevisionsTool].Run(map[string]any{})
|
||||
l, _ := list.(RevisionsAnswer)
|
||||
if err != nil || len(l.Revisions) != 2 || l.Revisions[0].Name != "candidate.g2.h2" || l.Revisions[1].Name != "trunk.1" || !l.Revisions[1].Current ||
|
||||
l.Fill.Cap != 2*size || l.Fill.Bytes != store.used() || len(l.Unlisted) != 0 {
|
||||
t.Errorf("indexer_catalog_revisions = %+v (%v): want candidate.g2.h2 and the current trunk.1, the fill against its cap", l, err)
|
||||
}
|
||||
if c, err := tools[CandidateGraphsTool].Run(map[string]any{}); err != nil || len(c.(RevisionsAnswer).Revisions) != 1 {
|
||||
t.Errorf("indexer_candidate_graphs = %+v (%v): want g2's alone", c, err)
|
||||
}
|
||||
if one, err := tools[CatalogRevisionTool].Run(map[string]any{}); err != nil || one.(map[string]any)["name"] != "trunk.1" {
|
||||
t.Errorf("indexer_catalog_revision = %v (%v): want the newest trunk revision whole", one, err)
|
||||
}
|
||||
if got, err := tools[RecomputeTool].Run(map[string]any{"why": "a check"}); err != nil || !strings.Contains(got.(*TakenIn).Answer, "equal to the kept revision 1") {
|
||||
t.Errorf("indexer_recompute = %+v (%v): want equal to the kept revision 1", got, err)
|
||||
}
|
||||
})
|
||||
t.Run("T7 a toolchain report evaluates pending again: the revision then names the running build and is taken in", func(t *testing.T) {
|
||||
b := newTakeInBed(t)
|
||||
b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2"))
|
||||
@@ -970,7 +1106,7 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633)
|
||||
for _, f := range []string{"Store", "Modules", "Manifests", "Predefined", "ConfigCandidates", "Taken", "Verdicts", "OldSections", "Unjudged",
|
||||
"JudgeID", "SayRefused", "SayAccepted", "Packages", "Graph", "Files", "GoSum", "GoMod", "Now", "Builds", "Named", "Trunk",
|
||||
"Read", "Contains", "Logf", "ReadOpen", "OwnBuild", "Message", "Pending", "SayTakeIn",
|
||||
"Requirements", "Findings", "SeatDefinitions", "ReadFile", "SaySeatDefinition", "Holds", "Counted", "AskEngines", "HoldBounds", "OpenTries", "TryRecovery", "OwnBuildState"} {
|
||||
"Requirements", "Findings", "SeatDefinitions", "ReadFile", "SaySeatDefinition", "Holds", "Counted", "AskEngines", "HoldBounds", "OpenTries", "TryRecovery", "OwnBuildState", "Revisions", "SayCatalogRevision", "SayFill"} {
|
||||
named[f] = true
|
||||
}
|
||||
ty := reflect.TypeOf(Indexer{})
|
||||
|
||||
@@ -220,6 +220,8 @@ type TakenIn struct {
|
||||
// and deleted.
|
||||
Handed []string `json:"handed,omitempty"`
|
||||
Dropped []string `json:"dropped,omitempty"`
|
||||
// Unread are the repositories a catch-up (CatchUp) could not take in: their merged message was not read.
|
||||
Unread []string `json:"unread,omitempty"`
|
||||
// Fingerprints are the modules whose fingerprint a recompute by hand moved (FingerprintsAgain), each old and new.
|
||||
Fingerprints []FingerprintMove `json:"fingerprints,omitempty"`
|
||||
}
|
||||
|
||||
@@ -0,0 +1,326 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
)
|
||||
|
||||
// The indexer's tools of its catalog graph revisions (design 47, the indexer's tools; the instruction "tools are how a
|
||||
// module is spoken to"): indexer_catalog_revision and indexer_catalog_revisions read the objects catalog-revisions,
|
||||
// indexer_candidate_graphs every candidate's catalog half; indexer_recompute and indexer_drop_candidate_graph steer the
|
||||
// indexer by an ask in takeins, as indexer_take_in does, since the indexer's process serves no request subject.
|
||||
const (
|
||||
CatalogRevisionTool = "indexer_catalog_revision"
|
||||
CatalogRevisionsTool = "indexer_catalog_revisions"
|
||||
CandidateGraphsTool = "indexer_candidate_graphs"
|
||||
RecomputeTool = "indexer_recompute"
|
||||
DropCandidateGraphTool = "indexer_drop_candidate_graph"
|
||||
)
|
||||
|
||||
// RecomputeKey is the takeins key of a recompute by hand: one at a time.
|
||||
const RecomputeKey = "recompute"
|
||||
|
||||
var candidateToken = regexp.MustCompile(`^[A-Za-z0-9_=-]+$`)
|
||||
|
||||
// NewRecomputeAsk is the value indexer_recompute writes: its key in takeins and the value.
|
||||
func NewRecomputeAsk(why, by string, at time.Time) (string, []byte, error) {
|
||||
data, err := json.Marshal(TakeInAsk{V: 1, Recompute: true, Why: strings.TrimSpace(why), By: by, At: at.UTC()})
|
||||
return RecomputeKey, data, err
|
||||
}
|
||||
|
||||
// NewDropCandidateGraphAsk is the value indexer_drop_candidate_graph writes: its key in takeins,
|
||||
// candidate-graph.<group>.<heads>, and the value.
|
||||
func NewDropCandidateGraphAsk(group, heads, why, by string, at time.Time) (string, []byte, error) {
|
||||
group, heads = strings.TrimSpace(group), strings.TrimSpace(heads)
|
||||
switch {
|
||||
case !candidateToken.MatchString(group) || !candidateToken.MatchString(heads):
|
||||
return "", nil, errors.New("name the candidate by its group and its heads, as indexer_candidate_graphs names them")
|
||||
case strings.TrimSpace(why) == "":
|
||||
return "", nil, errors.New("say why (why): a drop is a hand act, recorded through mesh-hand-acts")
|
||||
}
|
||||
data, err := json.Marshal(TakeInAsk{V: 1, CandidateGroup: group, CandidateHeads: heads, Drop: true, Why: strings.TrimSpace(why), By: by, At: at.UTC()})
|
||||
return "candidate-graph." + group + "." + heads, data, err
|
||||
}
|
||||
|
||||
// RevisionSummary is one kept catalog graph revision as indexer_catalog_revisions and indexer_candidate_graphs list it.
|
||||
type RevisionSummary struct {
|
||||
Name string `json:"name"`
|
||||
Kind string `json:"kind"`
|
||||
Revision int64 `json:"revision"`
|
||||
Source int64 `json:"source"`
|
||||
Group string `json:"group,omitempty"`
|
||||
Heads string `json:"heads,omitempty"`
|
||||
At *time.Time `json:"at,omitempty"`
|
||||
Entries int `json:"entries"`
|
||||
// Current says it is the trunk's newest revision, which is kept whatever pins it.
|
||||
Current bool `json:"current,omitempty"`
|
||||
}
|
||||
|
||||
// RevisionsAnswer is what indexer_catalog_revisions answers: every kept revision, the entries stored once by digest, the
|
||||
// store's fill against its cap, and each object it could not read.
|
||||
type RevisionsAnswer struct {
|
||||
Revisions []RevisionSummary `json:"revisions"`
|
||||
Entries int `json:"entries"`
|
||||
Unlisted []string `json:"unlisted,omitempty"`
|
||||
Fill meshcatalog.Fill `json:"fill"`
|
||||
FillUnread string `json:"fill-unread,omitempty"`
|
||||
Unread []string `json:"unread,omitempty"`
|
||||
}
|
||||
|
||||
// ListRevisions reads every revision the objects keep, candidates included (kind "candidate" alone when candidates).
|
||||
func ListRevisions(ctx context.Context, o Objects, candidates bool) (RevisionsAnswer, error) {
|
||||
out := RevisionsAnswer{Revisions: []RevisionSummary{}}
|
||||
names, err := o.Names(ctx)
|
||||
if err != nil {
|
||||
return out, fmt.Errorf("the objects %s cannot be listed: %w", CatalogRevisionsObjects, err)
|
||||
}
|
||||
listed := map[string]bool{}
|
||||
var newest int64
|
||||
entries := []string{}
|
||||
for _, n := range names {
|
||||
if strings.HasPrefix(n, "entry.") {
|
||||
entries = append(entries, n)
|
||||
continue
|
||||
}
|
||||
data, err := o.Get(ctx, n)
|
||||
var rev meshcatalog.CatalogRevision
|
||||
if err == nil {
|
||||
err = json.Unmarshal(data, &rev)
|
||||
}
|
||||
if err != nil {
|
||||
out.Unread = append(out.Unread, n+": "+err.Error())
|
||||
continue
|
||||
}
|
||||
for _, ref := range rev.Entries {
|
||||
listed[EntryObject(ref.Digest)] = true
|
||||
}
|
||||
if rev.Kind == "trunk" && rev.Revision > newest {
|
||||
newest = rev.Revision
|
||||
}
|
||||
if candidates && rev.Kind != "candidate" {
|
||||
continue
|
||||
}
|
||||
out.Revisions = append(out.Revisions, RevisionSummary{Name: n, Kind: rev.Kind, Revision: rev.Revision, Source: rev.Source,
|
||||
Group: rev.Group, Heads: rev.Heads, At: rev.At, Entries: len(rev.Entries)})
|
||||
}
|
||||
for i := range out.Revisions {
|
||||
out.Revisions[i].Current = out.Revisions[i].Kind == "trunk" && out.Revisions[i].Revision == newest
|
||||
}
|
||||
slices.SortFunc(out.Revisions, func(a, b RevisionSummary) int { return strings.Compare(a.Name, b.Name) })
|
||||
out.Entries = len(entries)
|
||||
for _, e := range entries {
|
||||
if !listed[e] {
|
||||
out.Unlisted = append(out.Unlisted, e)
|
||||
}
|
||||
}
|
||||
slices.Sort(out.Unlisted)
|
||||
used, capacity, err := o.Fill(ctx)
|
||||
out.Fill = meshcatalog.Fill{Bytes: used, Cap: capacity, Store: CatalogRevisionsObjects}
|
||||
if err != nil {
|
||||
out.FillUnread = err.Error()
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// removeUnlisted deletes every entry no kept revision lists any more: an entry is stored once and shared, so it goes
|
||||
// only with the last revision that lists it. It answers the entries removed.
|
||||
func removeUnlisted(ctx context.Context, o Objects) ([]string, error) {
|
||||
list, err := ListRevisions(ctx, o, false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(list.Unread) > 0 {
|
||||
// A revision that cannot be read may list any entry: none is removed rather than one it still needs.
|
||||
return nil, fmt.Errorf("no entry is removed while a revision cannot be read: %s", strings.Join(list.Unread, "; "))
|
||||
}
|
||||
var removed []string
|
||||
for _, e := range list.Unlisted {
|
||||
if err := o.Delete(ctx, e); err != nil {
|
||||
return removed, fmt.Errorf("%s: %w", e, err)
|
||||
}
|
||||
removed = append(removed, e)
|
||||
}
|
||||
return removed, nil
|
||||
}
|
||||
|
||||
// DropCandidateGraph drops a candidate's catalog half by hand (indexer_drop_candidate_graph, design 47: for a candidate
|
||||
// whose gate event never came), the entries no other revision lists with it, and says room when that takes the store
|
||||
// below its filling mark, which takes in a merge held for objects full (RoomAgain).
|
||||
func (ix *Indexer) DropCandidateGraph(ctx context.Context, group, heads, by, why string) TakenIn {
|
||||
name := RevisionObject(meshcatalog.CatalogRevision{Kind: "candidate", Group: group, Heads: heads})
|
||||
r := TakenIn{Module: name}
|
||||
if ix.Revisions == nil {
|
||||
r.Answer = "the indexer keeps no catalog graph revisions: nothing dropped"
|
||||
return r
|
||||
}
|
||||
if _, err := ix.Revisions.Get(ctx, name); err != nil {
|
||||
r.Answer = name + " is not kept (" + err.Error() + "): nothing dropped"
|
||||
return r
|
||||
}
|
||||
ix.keepMu.Lock()
|
||||
err := ix.Revisions.Delete(ctx, name)
|
||||
var removed []string
|
||||
if err == nil {
|
||||
removed, err = removeUnlisted(ctx, ix.Revisions)
|
||||
}
|
||||
ix.keepMu.Unlock()
|
||||
if err != nil {
|
||||
r.Answer = name + " could not be dropped whole: " + err.Error()
|
||||
ix.logf("%s", r.Answer)
|
||||
return r
|
||||
}
|
||||
r.Stored, r.Dropped = true, removed
|
||||
r.Answer = fmt.Sprintf("%s is dropped by %s (%s), with %d entries no other revision lists", name, by, why, len(removed))
|
||||
if roomed, err := ix.RoomAgain(ctx); err != nil {
|
||||
r.Answer += "; room is not evaluated: " + err.Error()
|
||||
} else if roomed {
|
||||
r.Answer += "; room said, what waited for it evaluated again"
|
||||
}
|
||||
ix.logf("%s", r.Answer)
|
||||
return r
|
||||
}
|
||||
|
||||
// RecomputeByHand computes the catalog graph again now from the view of the trunk, as a toolchain report or a merge
|
||||
// does, and compares it with the newest revision kept: the modules whose entry moved, and the revision kept for them
|
||||
// (none when the recompute equals the one kept).
|
||||
func (ix *Indexer) RecomputeByHand(ctx context.Context, by, why string) TakenIn {
|
||||
r := TakenIn{Module: "catalog graph"}
|
||||
if ix.Revisions == nil {
|
||||
r.Answer = "the indexer keeps no catalog graph revisions: nothing recomputed"
|
||||
return r
|
||||
}
|
||||
before, _, err := newestTrunk(ctx, ix.Revisions)
|
||||
if err != nil {
|
||||
r.Answer = "the newest catalog graph revision cannot be read: " + err.Error()
|
||||
return r
|
||||
}
|
||||
if err := ix.Recompute(ctx); err != nil {
|
||||
r.Answer = "the recompute failed: " + err.Error()
|
||||
return r
|
||||
}
|
||||
after, _, err := newestTrunk(ctx, ix.Revisions)
|
||||
if err != nil {
|
||||
r.Answer = "the newest catalog graph revision cannot be read after the recompute: " + err.Error()
|
||||
return r
|
||||
}
|
||||
r.Stored = true
|
||||
if after.Revision == before.Revision {
|
||||
r.Answer = fmt.Sprintf("recomputed by %s (%s): equal to the kept revision %d", by, why, before.Revision)
|
||||
} else {
|
||||
r.Handed = changedEntries(before.Entries, after.Entries)
|
||||
r.Answer = fmt.Sprintf("recomputed by %s (%s): revision %d differs from the kept %d in %s; revision %d is kept", by, why,
|
||||
after.Revision, before.Revision, strings.Join(r.Handed, ", "), after.Revision)
|
||||
}
|
||||
ix.logf("%s", r.Answer)
|
||||
return r
|
||||
}
|
||||
|
||||
// CatalogRevisionTools are the tools of the catalog graph revisions; ask writes an ask in takeins and waits for the
|
||||
// indexer's answer, record records a hand act through mesh-hand-acts before a drop is asked.
|
||||
func CatalogRevisionTools(o Objects, ask func(key string, value []byte) (TakeInAsk, error), now func() time.Time, record func(what, why string) error) []stdio.Tool {
|
||||
str := func(args map[string]any, k string) string { s, _ := args[k].(string); return strings.TrimSpace(s) }
|
||||
asked := func(key string, value []byte, err error) (any, error) {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
a, err := ask(key, value)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return a.Result, nil
|
||||
}
|
||||
return []stdio.Tool{
|
||||
{Name: CatalogRevisionTool,
|
||||
Description: "One catalog graph revision the indexer keeps in its objects catalog-revisions, whole: its list (kind, number, " +
|
||||
"source revision, trunk heads, a candidate's group and heads) and every entry it lists, each module as its manifest " +
|
||||
"declares it at that source revision (bases, packages, toolchains, provides, requires, invokes, serves, fingerprint, " +
|
||||
"acting rules, inputs). The trunk's newest with no input; revision a kept trunk revision; group and heads a candidate's. " +
|
||||
"An entry the store lost is an error naming it. indexer_catalog_revisions lists what is kept.",
|
||||
Input: map[string]any{"type": "object", "properties": map[string]any{
|
||||
"revision": map[string]any{"type": "integer", "description": "a kept trunk revision's number"},
|
||||
"group": map[string]any{"type": "string", "description": "a candidate's group, with heads"},
|
||||
"heads": map[string]any{"type": "string", "description": "a candidate's heads, with group"}}},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
ctx := context.Background()
|
||||
n, _ := args["revision"].(float64)
|
||||
name := ""
|
||||
switch {
|
||||
case str(args, "group") != "" || str(args, "heads") != "":
|
||||
name = RevisionObject(meshcatalog.CatalogRevision{Kind: "candidate", Group: str(args, "group"), Heads: str(args, "heads")})
|
||||
case n > 0:
|
||||
name = "trunk." + strconv.FormatInt(int64(n), 10)
|
||||
default:
|
||||
newest, ok, err := newestTrunk(ctx, o)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the catalog graph revisions cannot be read: %w", err)
|
||||
}
|
||||
if !ok {
|
||||
return nil, ErrNoRevision
|
||||
}
|
||||
name = RevisionObject(newest)
|
||||
}
|
||||
rev, entries, err := ReadRevision(ctx, o, name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return map[string]any{"name": name, "revision": rev, "entries": entries}, nil
|
||||
}},
|
||||
{Name: CatalogRevisionsTool,
|
||||
Description: "Every catalog graph revision the indexer keeps in its objects catalog-revisions, trunk and candidate: each " +
|
||||
"one's name, kind, number, source revision, when, how many entries it lists and whether it is the trunk's newest " +
|
||||
"(current); how many entries are stored once by digest, those no revision lists (unlisted), and the store's fill " +
|
||||
"in bytes against its cap (filling is said at 80 %, full refuses the next revision). An object it cannot read is " +
|
||||
"named in unread. indexer_catalog_revision reads one whole.",
|
||||
Input: map[string]any{"type": "object", "properties": map[string]any{}},
|
||||
Run: func(map[string]any) (any, error) {
|
||||
return ListRevisions(context.Background(), o, false)
|
||||
}},
|
||||
{Name: CandidateGraphsTool,
|
||||
Description: "Every candidate's catalog half the indexer keeps (a delivery group's or a pull request's heads merged onto " +
|
||||
"a trunk source revision): its group, its heads, its revision, the source revision it is merged onto, since when, " +
|
||||
"and how many entries it lists; and the store's fill. indexer_drop_candidate_graph drops one whose gate event never came.",
|
||||
Input: map[string]any{"type": "object", "properties": map[string]any{}},
|
||||
Run: func(map[string]any) (any, error) {
|
||||
return ListRevisions(context.Background(), o, true)
|
||||
}},
|
||||
{Name: RecomputeTool,
|
||||
Description: "Compute the catalog graph again now, as the indexer does on a toolchain report or a merge, from the view of " +
|
||||
"the trunk (each module's manifest, its bases, the agreed toolchains and the published packages), and compare it with " +
|
||||
"the newest revision kept: answers equal, or the modules whose entry differs and the revision kept for them.",
|
||||
Input: map[string]any{"type": "object", "properties": map[string]any{
|
||||
"why": map[string]any{"type": "string", "description": "why it is recomputed now, in plain words"}}},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
key, value, err := NewRecomputeAsk(str(args, "why"), stdio.CallerOf(args), now())
|
||||
return asked(key, value, err)
|
||||
}},
|
||||
{Name: DropCandidateGraphTool,
|
||||
Description: "Drop one candidate's catalog half by hand, for a candidate whose gate event never came: its revision and the " +
|
||||
"entries no other revision lists. A hand act, recorded through mesh-hand-acts before it is asked. When the drop takes " +
|
||||
"the store below its filling mark, room is said and a merge held for objects full is taken in. indexer_candidate_graphs " +
|
||||
"names the candidates.",
|
||||
Input: map[string]any{"type": "object", "required": []string{"group", "heads", "why"}, "properties": map[string]any{
|
||||
"group": map[string]any{"type": "string", "description": "the candidate's group"},
|
||||
"heads": map[string]any{"type": "string", "description": "the candidate's heads"},
|
||||
"why": map[string]any{"type": "string", "description": "why it is dropped, in plain words"}}},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
key, value, err := NewDropCandidateGraphAsk(str(args, "group"), str(args, "heads"), str(args, "why"), stdio.CallerOf(args), now())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := record("dropped the catalog half of the candidate "+str(args, "group")+" at "+str(args, "heads")+" by hand", str(args, "why")); err != nil {
|
||||
return nil, fmt.Errorf("the hand act was not recorded through mesh-hand-acts, so nothing was dropped: %w", err)
|
||||
}
|
||||
return asked(key, value, nil)
|
||||
}},
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,102 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/views"
|
||||
)
|
||||
|
||||
// RevisionInvoke is how a part declares the seat mesh-catalog's verb revision in its manifest's invokes, the one way a
|
||||
// part reads the catalog half (novox/hq ADR 0663 §2).
|
||||
const RevisionInvoke = "seat:" + meshcatalog.Seat + "." + meshcatalog.VerbRevision
|
||||
|
||||
// RevisionCallers are the parts the newest trunk revision's entries say declare the verb revision in their invokes,
|
||||
// sorted, the holder itself never among them (ADR 0663 §2: granted to exactly the parts that declare it).
|
||||
func RevisionCallers(entries []meshcatalog.CatalogEntry, holder string) stdio.Callers {
|
||||
var out stdio.Callers
|
||||
for _, e := range entries {
|
||||
if e.Module != holder && slices.Contains(e.Invokes, RevisionInvoke) {
|
||||
out = append(out, e.Module)
|
||||
}
|
||||
}
|
||||
slices.Sort(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// Revision answers the seat mesh-catalog's verb revision from the indexer's own objects (ADR 0663 §1, §3): one catalog
|
||||
// graph revision, the trunk's newest when the input names none, a kept trunk revision by its number, or a candidate's by
|
||||
// its group and heads, with one page of its entries whole, by module name. The caller is the runtime's proof: it is
|
||||
// answered only when the newest trunk revision's entry of its module declares the verb in invokes, and the holder is
|
||||
// refused too. With no trunk revision kept yet nobody is answered and the reader waits (ErrNoRevision), never an empty
|
||||
// catalog.
|
||||
func Revision(ctx context.Context, o Objects, holder, caller string, in meshcatalog.RevisionInput) (meshcatalog.RevisionAnswer, error) {
|
||||
var out meshcatalog.RevisionAnswer
|
||||
newest, ok, err := newestTrunk(ctx, o)
|
||||
if err != nil {
|
||||
return out, fmt.Errorf("the catalog graph revisions cannot be read: %w", err)
|
||||
}
|
||||
if !ok {
|
||||
return out, ErrNoRevision
|
||||
}
|
||||
_, trunkEntries, err := ReadRevision(ctx, o, RevisionObject(newest))
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
if err := views.Refusal(meshcatalog.Seat+"."+meshcatalog.VerbRevision, RevisionCallers(trunkEntries, holder), caller); err != nil {
|
||||
return out, fmt.Errorf("%w (novox/hq ADR 0663 §2: a part reads it by declaring %s in invokes)", err, RevisionInvoke)
|
||||
}
|
||||
name := RevisionObject(newest)
|
||||
switch {
|
||||
case in.Group != "" || in.Heads != "":
|
||||
if in.Group == "" || in.Heads == "" {
|
||||
return out, fmt.Errorf("a candidate's revision is named by its group and its heads, both")
|
||||
}
|
||||
name = RevisionObject(meshcatalog.CatalogRevision{Kind: "candidate", Group: in.Group, Heads: in.Heads})
|
||||
case in.Revision != 0 && in.Revision != newest.Revision:
|
||||
name = "trunk." + strconv.FormatInt(in.Revision, 10)
|
||||
}
|
||||
rev, entries := newest, trunkEntries
|
||||
if name != RevisionObject(newest) {
|
||||
if rev, entries, err = ReadRevision(ctx, o, name); err != nil {
|
||||
return out, err
|
||||
}
|
||||
if in.Revision != 0 && rev.Revision != in.Revision {
|
||||
return out, fmt.Errorf("the candidate %s.%s keeps its newest revision only, %d, not %d", in.Group, in.Heads, rev.Revision, in.Revision)
|
||||
}
|
||||
}
|
||||
slices.SortFunc(entries, func(a, b meshcatalog.CatalogEntry) int { return strings.Compare(a.Module, b.Module) })
|
||||
out.Revision, out.Entries = rev, []meshcatalog.CatalogEntry{}
|
||||
size := 0
|
||||
for _, e := range entries {
|
||||
if e.Module <= in.After {
|
||||
continue
|
||||
}
|
||||
data, _ := json.Marshal(e)
|
||||
if len(out.Entries) > 0 && size+len(data) > views.PageBytes {
|
||||
out.Next = out.Entries[len(out.Entries)-1].Module
|
||||
break
|
||||
}
|
||||
size += len(data)
|
||||
out.Entries = append(out.Entries, e)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// RevisionTool is the verb revision as the tool mesh-catalog.revision of the indexer's own runtime (ADR 0663 §1): the
|
||||
// tool runner describes it with the seat's definition, so it carries no description or input of its own.
|
||||
func RevisionTool(o Objects, holder string) stdio.Tool {
|
||||
return stdio.Tool{Name: meshcatalog.Seat + "." + meshcatalog.VerbRevision, Run: func(args map[string]any) (any, error) {
|
||||
var in meshcatalog.RevisionInput
|
||||
if raw, err := json.Marshal(args); err == nil {
|
||||
_ = json.Unmarshal(raw, &in)
|
||||
}
|
||||
return Revision(context.Background(), o, holder, stdio.CallerOf(args), in)
|
||||
}}
|
||||
}
|
||||
@@ -2,9 +2,14 @@ package index
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"maps"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
|
||||
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery"
|
||||
@@ -56,6 +61,9 @@ func (ix *Indexer) ApplySource(ctx context.Context, a meshdelivery.SourceAnswer)
|
||||
if ix.Modules == nil {
|
||||
return ix.RecomputeCandidates(ctx)
|
||||
}
|
||||
if err := ix.kickCatchUp(ctx, s.Heads); err != nil {
|
||||
return err
|
||||
}
|
||||
// A registered module the forge could not read is read again at each new source revision (issue 1945), in the
|
||||
// background: a forge that does not answer must not hold this revision. One taken in later is stamped when it is written
|
||||
// (applyManifests reads srcOf).
|
||||
@@ -100,6 +108,124 @@ func (ix *Indexer) ApplySource(ctx context.Context, a meshdelivery.SourceAnswer)
|
||||
return fmt.Errorf("modules.%s: %w", key, err)
|
||||
}
|
||||
}
|
||||
// The stamps are kept with the view (KeepView), so a restart reads them back.
|
||||
if err := ix.KeepView(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
// A trunk that moved makes the previews made onto the old one out of date.
|
||||
return ix.RecomputeCandidates(ctx)
|
||||
}
|
||||
|
||||
// kickCatchUp takes the trunk in at a first start, when the objects keep no trunk revision yet (the lead's ruling A on
|
||||
// milestone 47/G: the catch-up path): the head the source names for every repository, by the merged message the forge
|
||||
// bridge gives for it, each applied as its merge, and revision 1 kept from them all at its end. Until then the revision
|
||||
// the tools answer waits, never an empty catalog. The catch-up runs off the source handler, in its own goroutine, said
|
||||
// when it ends (lead-47's ruling R1 on the rebase): a bridge that does not answer holds the kept call (ADR 0668) to its
|
||||
// bound, and must not hold this source revision's stamps and the pulls' candidates with it. A repository whose message is
|
||||
// not read keeps no revision, said: the next source revision runs the catch-up again (no timer). Objects not on the bus
|
||||
// yet run none, said, until then.
|
||||
func (ix *Indexer) kickCatchUp(ctx context.Context, heads map[string]string) error {
|
||||
if ix.Revisions == nil || ix.Message == nil || len(heads) == 0 {
|
||||
return nil
|
||||
}
|
||||
if _, any, err := newestTrunk(ctx, ix.Revisions); errors.Is(err, process.ErrNotYet) {
|
||||
ix.logf("catalog graph: the trunk is not taken in yet: its objects %s are not on the bus", CatalogRevisionsObjects)
|
||||
return nil
|
||||
} else if err != nil || any {
|
||||
return err
|
||||
}
|
||||
if !ix.catchingUp.CompareAndSwap(false, true) {
|
||||
return nil
|
||||
}
|
||||
heads = maps.Clone(heads)
|
||||
ix.catchUpWG.Add(1)
|
||||
go func() {
|
||||
defer ix.catchUpWG.Done()
|
||||
unread := ix.takeInHeads(ctx, heads)
|
||||
ix.catchingUp.Store(false)
|
||||
if len(unread) > 0 {
|
||||
ix.logf("catalog graph: the first start's catch-up ended and no first revision is kept: %d repositories were not taken in (%s); taken in again at the next source revision",
|
||||
len(unread), strings.Join(unread, ", "))
|
||||
return
|
||||
}
|
||||
if err := ix.KeepView(ctx); err != nil {
|
||||
ix.logf("catalog graph: the first start's catch-up took in the trunk of %d repositories, but its revision could not be kept: %v; taken in again at the next source revision", len(heads), err)
|
||||
return
|
||||
}
|
||||
ix.logf("catalog graph: the first start's catch-up ended: the trunk of %d repositories taken in", len(heads))
|
||||
}()
|
||||
return nil
|
||||
}
|
||||
|
||||
// takeInHeads applies the forge bridge's merged message of each repository's head, as its merge, and answers the
|
||||
// repositories whose message was not read or not applied.
|
||||
func (ix *Indexer) takeInHeads(ctx context.Context, heads map[string]string) []string {
|
||||
var unread []string
|
||||
for _, repository := range slices.Sorted(maps.Keys(heads)) {
|
||||
head := heads[repository]
|
||||
if head == "" {
|
||||
continue
|
||||
}
|
||||
f, err := ix.mergedAt(ctx, repository, head)
|
||||
if err == nil {
|
||||
_, err = ix.ApplyMessage(ctx, f)
|
||||
}
|
||||
if err != nil {
|
||||
unread = append(unread, repository)
|
||||
ix.logf("catalog graph: the trunk of %s at %.12s is not taken in: %v", repository, head, err)
|
||||
}
|
||||
}
|
||||
return unread
|
||||
}
|
||||
|
||||
// CatchUp is the indexer's catch-up tool's work (indexer_take_in with no module and no repository, the catch-up its
|
||||
// manifest names beside the catalog graph revisions): after a restore point of them is restored, the node agent calls it
|
||||
// with no arguments, and the indexer takes in the head the gate's source names for every repository, as the first
|
||||
// start's catch-up does, whatever revision the objects keep, then keeps the view as the next revision when an entry
|
||||
// moved. Its answer names the repositories taken in and those whose merged message could not be read.
|
||||
func (ix *Indexer) CatchUp(ctx context.Context) TakenIn {
|
||||
out := TakenIn{Module: "catch-up"}
|
||||
ix.mu.Lock()
|
||||
heads := maps.Clone(ix.source.heads)
|
||||
ix.mu.Unlock()
|
||||
switch {
|
||||
case len(heads) == 0:
|
||||
out.Answer = "the gate's source names no head yet, so nothing is taken in; the next source revision is taken in as it comes"
|
||||
return out
|
||||
case ix.Modules == nil || ix.Message == nil:
|
||||
out.Answer = "the indexer holds no view of the trunk or reads no merged message, so nothing is taken in"
|
||||
return out
|
||||
case !ix.catchingUp.CompareAndSwap(false, true):
|
||||
out.Answer = "a catch-up runs already; its end keeps the revision"
|
||||
return out
|
||||
}
|
||||
unread := ix.takeInHeads(ctx, heads)
|
||||
ix.catchingUp.Store(false)
|
||||
if err := ix.KeepView(ctx); err != nil {
|
||||
out.Answer = "the repositories were taken in but the revision could not be kept: " + err.Error()
|
||||
return out
|
||||
}
|
||||
out.Stored = len(unread) == 0
|
||||
out.Unread = unread
|
||||
out.Answer = fmt.Sprintf("the heads the gate's source names for %d repositories taken in", len(heads)-len(unread))
|
||||
if len(unread) > 0 {
|
||||
out.Answer += fmt.Sprintf("; %d not (%s): their merged message was not read, and the call may be made again", len(unread), strings.Join(unread, ", "))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// mergedAt is the forge bridge's merged message of a repository's head.
|
||||
func (ix *Indexer) mergedAt(ctx context.Context, repository, head string) (forgemsg.Forge, error) {
|
||||
data, err := ix.Message(ctx, repository, head, 0)
|
||||
if err != nil {
|
||||
return forgemsg.Forge{}, fmt.Errorf("the forge bridge could not answer: %w", err)
|
||||
}
|
||||
f, err := forgemsg.DecodeForge(data)
|
||||
if err != nil {
|
||||
return f, fmt.Errorf("the forge bridge's answer is not a forge message: %w", err)
|
||||
}
|
||||
if f.Kind != forgemsg.ForgeMerged || f.Merge != head || !trunk.SameRepository(f.Repository, repository) {
|
||||
return f, fmt.Errorf("the forge bridge's answer is not the merge of %s at %s", repository, head)
|
||||
}
|
||||
return f, nil
|
||||
}
|
||||
|
||||
@@ -32,6 +32,9 @@ type TakeInAsk struct {
|
||||
// Repository asks the head the gate's source names for a repository to be taken in (TakeInRepository) instead of a module.
|
||||
Repository string `json:"repository,omitempty"`
|
||||
Why string `json:"why,omitempty"`
|
||||
// CatchUp asks the head the gate's source names for every repository to be taken in (CatchUp): the catch-up tool,
|
||||
// indexer_take_in called with no module and no repository, as the node agent calls it after a restore.
|
||||
CatchUp bool `json:"catch-up,omitempty"`
|
||||
// Judge asks the manifests to be judged again now (indexer_judge_again) instead of the module to be taken in.
|
||||
Judge bool `json:"judge,omitempty"`
|
||||
// Digest names, with Judge, the one manifest digest of the module to judge (indexer_judge with a digest).
|
||||
@@ -63,12 +66,17 @@ type TakeInAsk struct {
|
||||
// packages as they are read (indexer_fingerprints_again, fingerprints.go).
|
||||
Fingerprints bool `json:"fingerprints,omitempty"`
|
||||
// RecoverTries asks the open tries' candidates to be recovered now (indexer_recover_tries).
|
||||
RecoverTries bool `json:"recover-tries,omitempty"`
|
||||
Drop bool `json:"drop,omitempty"`
|
||||
By string `json:"by,omitempty"`
|
||||
At time.Time `json:"at"`
|
||||
Result *TakenIn `json:"result,omitempty"`
|
||||
Answered time.Time `json:"answered,omitzero"`
|
||||
RecoverTries bool `json:"recover-tries,omitempty"`
|
||||
// Recompute asks the catalog graph to be computed again now and compared with the newest kept revision (indexer_recompute).
|
||||
Recompute bool `json:"recompute,omitempty"`
|
||||
// CandidateGroup and CandidateHeads, with Drop, ask that candidate's catalog half to be dropped (indexer_drop_candidate_graph).
|
||||
CandidateGroup string `json:"candidate-group,omitempty"`
|
||||
CandidateHeads string `json:"candidate-heads,omitempty"`
|
||||
Drop bool `json:"drop,omitempty"`
|
||||
By string `json:"by,omitempty"`
|
||||
At time.Time `json:"at"`
|
||||
Result *TakenIn `json:"result,omitempty"`
|
||||
Answered time.Time `json:"answered,omitzero"`
|
||||
}
|
||||
|
||||
// Done says the ask is answered.
|
||||
@@ -83,7 +91,7 @@ func DecodeTakeIn(data []byte) (TakeInAsk, error) {
|
||||
if a.V != 1 {
|
||||
return a, fmt.Errorf("a takeins value of version %d, not 1", a.V)
|
||||
}
|
||||
if a.Module == "" && a.Repository == "" && a.Pending == "" && a.Merge == "" && a.TryCandidate == "" && !a.RecoverTries && !a.Fingerprints {
|
||||
if a.Module == "" && a.Repository == "" && a.Pending == "" && a.Merge == "" && a.TryCandidate == "" && !a.RecoverTries && !a.Fingerprints && !a.CatchUp && !a.Recompute && a.CandidateGroup == "" {
|
||||
return a, errors.New("a takeins value naming no module, repository, pending wait, merge or try")
|
||||
}
|
||||
return a, nil
|
||||
@@ -114,6 +122,20 @@ func NewTakeInRepository(repository, why, by string, at time.Time) (string, []by
|
||||
return "repository." + key, data, err
|
||||
}
|
||||
|
||||
// CatchUpKey is the takeins key of a catch-up: one at a time, a second written over an unanswered first.
|
||||
const CatchUpKey = "catch-up"
|
||||
|
||||
// NewCatchUp is the value indexer_take_in writes when it names no module and no repository: the catch-up of every
|
||||
// repository the gate's source names (CatchUp).
|
||||
func NewCatchUp(why, by string, at time.Time) (string, []byte, error) {
|
||||
why = strings.TrimSpace(why)
|
||||
if why == "" {
|
||||
why = "the catch-up after a restore"
|
||||
}
|
||||
data, err := json.Marshal(TakeInAsk{V: 1, CatchUp: true, Why: why, By: by, At: at.UTC()})
|
||||
return CatchUpKey, data, err
|
||||
}
|
||||
|
||||
// TakeInMergeTool takes one merge in by hand, its merged message asked of the forge bridge (ADR 0633 §5, issue 1585).
|
||||
const TakeInMergeTool = "indexer_take_in_merge"
|
||||
|
||||
|
||||
@@ -372,7 +372,10 @@ func (ix *Indexer) recomputeLocked(ctx context.Context, overlay map[string]meshc
|
||||
if err := ix.writeOverlay(ctx, overlay, finals, w); err != nil {
|
||||
return err
|
||||
}
|
||||
return ix.writeGraph(ctx, edges)
|
||||
if err := ix.writeGraph(ctx, edges); err != nil {
|
||||
return err
|
||||
}
|
||||
return ix.keepTrunkRevision(ctx, edges, wholeGraph(edges))
|
||||
}
|
||||
|
||||
// writeOverlay writes the records of a merge, each once and only if it differs from what the bucket holds; finals has
|
||||
|
||||
@@ -2,12 +2,15 @@ package index
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"maps"
|
||||
"os"
|
||||
"reflect"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -18,8 +21,10 @@ import (
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
|
||||
sdkstep "git.novox.be/novox/mesh-delivery/sdk/buildstep"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/keys"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
"git.novox.be/novox/mesh-sdk/go/views"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/sdk/graph"
|
||||
|
||||
@@ -285,16 +290,128 @@ func TestBasesAreComputedFirstAndNoFingerprintTakesTheBuilder(t *testing.T) { //
|
||||
if g := graphOf(t, ix, "other"); g.FP != "" || !strings.Contains(g.None, "ts-base") || !reflect.DeepEqual(g.Bases, []string{"ts-base"}) {
|
||||
t.Errorf("graph.other = %+v, want its base and its reason", g)
|
||||
}
|
||||
// Milestone 47/G W4.1, W4.2 (design 47, Two graphs): the catalog graph entry of a module is its graph node with where the
|
||||
// trunk holds it; a revision lists one entry per module by digest, an unchanged entry has the same digest in the next
|
||||
// revision, and a changed one only its own.
|
||||
entries := map[string]meshcatalog.CatalogEntry{}
|
||||
for name := range mods.vals {
|
||||
m, _ := moduleOf(t, mods, name)
|
||||
entries[name] = CatalogEntryOf(name, m, catalogue.Manifest{}, graphOf(t, ix, name))
|
||||
}
|
||||
// W3.1's main: the Go main package the module's process runs, read from its manifest (here the indexer's own); a
|
||||
// manifest that runs no process names none.
|
||||
own, err := os.ReadFile("../../module.json")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if m, err := catalogue.Decode(own); err != nil || MainOf(m) != "cmd/mesh-delivery-indexer" || MainOf(catalogue.Manifest{}) != "" {
|
||||
t.Errorf("the indexer's main = %q (%v), want cmd/mesh-delivery-indexer", MainOf(m), err)
|
||||
}
|
||||
if e := entries["avahi"]; e.Fp != want["avahi"] || e.Commit == "" || e.Manifest == "" || !reflect.DeepEqual(e.Bases, []string{"mesh-tools"}) ||
|
||||
e.Toolchains["go"] != (meshcatalog.BuildEdge{Module: "mesh-build-go", Version: "go1.27.1", Recipe: "1"}) {
|
||||
t.Errorf("the catalog entry of avahi = %+v", e)
|
||||
}
|
||||
// W3.1's inputs (the lead's ruling on finding 1): an entry carries what the next revision is computed from (the
|
||||
// fingerprint's inputs, the toolchains used, the package versions), so the module's value read back from it is the
|
||||
// value modules held, but for the old line's fields.
|
||||
for name := range mods.vals {
|
||||
m, _ := moduleOf(t, mods, name)
|
||||
m.Fp, m.None, m.Needs = "", "", nil
|
||||
if back := ModuleValueOf(entries[name]); !reflect.DeepEqual(back, m) {
|
||||
t.Errorf("%s read back from its entry = %+v, want %+v", name, back, m)
|
||||
}
|
||||
}
|
||||
if e := entries["avahi"]; e.Inputs == nil || e.Inputs.From == nil || e.Inputs.Toolchains["go"].Version != "go1.27.1" {
|
||||
t.Errorf("the catalog entry of avahi carries inputs %+v, want what its fingerprint is computed from", e.Inputs)
|
||||
}
|
||||
rev, objects, err := CatalogRevisionOf(meshcatalog.CatalogRevision{Revision: 1, Kind: "trunk", Source: 7}, entries)
|
||||
if err != nil || len(rev.Entries) != len(mods.vals) || len(objects) != len(mods.vals) || rev.V != CatalogEntryLayout {
|
||||
t.Fatalf("revision %+v with %d objects (%v), want one entry per module of %d", rev, len(objects), err, len(mods.vals))
|
||||
}
|
||||
moved := maps.Clone(entries)
|
||||
tools := moved["mesh-tools"]
|
||||
tools.Fp = "src2:" + strings.Repeat("b", 64)
|
||||
moved["mesh-tools"] = tools
|
||||
next, _, _ := CatalogRevisionOf(meshcatalog.CatalogRevision{Revision: 2, Kind: "trunk", Source: 8}, moved)
|
||||
for i, ref := range next.Entries {
|
||||
if same := ref.Digest == rev.Entries[i].Digest; same == (ref.Module == "mesh-tools") {
|
||||
t.Errorf("entry %s: digest %s against %s before; only mesh-tools' moves", ref.Module, ref.Digest, rev.Entries[i].Digest)
|
||||
}
|
||||
}
|
||||
// W4.2's store: both revisions kept in catalog-revisions hold one object per entry once, the moved entry twice, and
|
||||
// each list; a revision read back is its entries whole; an entry the bucket lost is an error, never left out.
|
||||
kept := newMemObjects(0)
|
||||
_, nextObjects, _ := CatalogRevisionOf(next, moved)
|
||||
if err := KeepRevision(context.Background(), kept, rev, objects); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := KeepRevision(context.Background(), kept, next, nextObjects); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if names, _ := kept.Names(context.Background()); len(names) != len(mods.vals)+1+2 || kept.puts != len(names) {
|
||||
t.Errorf("catalog-revisions holds %v after %d puts, want each entry once, the moved one twice, and two lists", names, kept.puts)
|
||||
}
|
||||
if back, got, err := ReadRevision(context.Background(), kept, "trunk.2"); err != nil || back.Source != 8 || len(got) != len(mods.vals) ||
|
||||
!slices.ContainsFunc(got, func(e meshcatalog.CatalogEntry) bool { return e.Module == "mesh-tools" && e.Fp == tools.Fp }) {
|
||||
t.Errorf("trunk.2 read back = %+v, %d entries, %v", back, len(got), err)
|
||||
}
|
||||
// W4.3's second part: the view of the trunk read at start is the newest trunk revision, each module's value as its
|
||||
// entry gives it back; a bucket that keeps none is an empty view at 0.
|
||||
view, nodes, at, err := TrunkView(context.Background(), kept)
|
||||
if err != nil || at != 2 {
|
||||
t.Fatalf("the view is read at %d (%v), want 2", at, err)
|
||||
}
|
||||
for name, e := range moved {
|
||||
key, _ := meshcatalogclient.ModulesKey(name)
|
||||
raw, _, found, _ := view.Get(context.Background(), key)
|
||||
got, err := meshcatalogclient.DecodeModule(raw)
|
||||
if want := ModuleValueOf(e); !found || err != nil || !reflect.DeepEqual(got, want) {
|
||||
t.Errorf("the view's %s = %+v (%v), want %+v", name, got, err, want)
|
||||
}
|
||||
// The node of the module graph the entry carries is read back with the view (NodeOf), so a revision kept before the
|
||||
// first recompute after a start loses no edge.
|
||||
gk, _ := graph.Key(name)
|
||||
rawNode, _, found, _ := nodes.Get(context.Background(), gk)
|
||||
if back, err := graph.Decode(rawNode); !found || err != nil || !reflect.DeepEqual(NodeOf(e), back) || back.FP != e.Fp ||
|
||||
!reflect.DeepEqual(back.Bases, e.Bases) || !reflect.DeepEqual(CatalogEntryOf(name, ModuleValueOf(e), catalogue.Manifest{}, back).Bases, e.Bases) {
|
||||
t.Errorf("the view's node of %s = %+v (%v), want %+v", name, back, err, NodeOf(e))
|
||||
}
|
||||
}
|
||||
if empty, _, at, err := TrunkView(context.Background(), newMemObjects(0)); err != nil || at != 0 || func() bool { k, _ := empty.Keys(context.Background()); return len(k) != 0 }() {
|
||||
t.Errorf("an empty bucket's view is read at %d (%v), want an empty view at 0", at, err)
|
||||
}
|
||||
_ = kept.Delete(context.Background(), EntryObject(next.Entries[0].Digest))
|
||||
if _, _, err := ReadRevision(context.Background(), kept, "trunk.2"); err == nil || !strings.Contains(err.Error(), next.Entries[0].Module) {
|
||||
t.Errorf("trunk.2 with a lost entry read without error naming it: %v", err)
|
||||
}
|
||||
// W4.3: a recompute keeps a trunk revision of the module graph and says catalog-revision; one that moves no entry keeps
|
||||
// none; a module leaving makes the next revision, naming it changed.
|
||||
ix.Revisions = newMemObjects(0)
|
||||
var revisions []meshcatalog.CatalogRevisionEvent
|
||||
ix.SayCatalogRevision = func(_ context.Context, e meshcatalog.CatalogRevisionEvent) error {
|
||||
revisions = append(revisions, e)
|
||||
return nil
|
||||
}
|
||||
if err := ix.Recompute(context.Background()); err != nil || len(revisions) != 1 || revisions[0].Revision != 1 || revisions[0].Kind != "trunk" ||
|
||||
revisions[0].Entries != int64(len(mods.vals)) || len(revisions[0].Changed) != len(mods.vals) {
|
||||
t.Fatalf("the first recompute said %+v (%v), want trunk revision 1 with every module", revisions, err)
|
||||
}
|
||||
graph := ix.Graph.(*memStore)
|
||||
written := graph.writes
|
||||
if err := ix.Recompute(context.Background()); err != nil || graph.writes != written {
|
||||
t.Errorf("a recompute that moves nothing wrote the graph %d times (%v)", graph.writes-written, err)
|
||||
if err := ix.Recompute(context.Background()); err != nil || graph.writes != written || len(revisions) != 1 {
|
||||
t.Errorf("a recompute that moves nothing wrote the graph %d times and said %d revisions (%v)", graph.writes-written, len(revisions), err)
|
||||
}
|
||||
// A module the modules state no longer holds (a merge dropped it, issue 1900) leaves the graph at the next recompute.
|
||||
delete(mods.vals, "other")
|
||||
if err := ix.Recompute(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(revisions) != 2 || revisions[1].Revision != 2 || !reflect.DeepEqual(revisions[1].Changed, []string{"other"}) {
|
||||
t.Errorf("after other left, said %+v, want revision 2 naming other changed", revisions)
|
||||
}
|
||||
if _, kept, err := ReadRevision(context.Background(), ix.Revisions, "trunk.2"); err != nil || len(kept) != len(mods.vals) {
|
||||
t.Errorf("trunk.2 holds %d entries (%v), want %d", len(kept), err, len(mods.vals))
|
||||
}
|
||||
if _, held := graph.vals["other"]; held {
|
||||
t.Error("graph.other is still held after its module went")
|
||||
}
|
||||
@@ -339,6 +456,137 @@ func TestBasesAreComputedFirstAndNoFingerprintTakesTheBuilder(t *testing.T) { //
|
||||
t.Errorf("%s moved with a build of build-agent: fp2 = %q, want %s", name, m.Fp2, fp)
|
||||
}
|
||||
}
|
||||
// The lead's ruling A on milestone 47/G: a first start whose objects keep no trunk revision takes in the head the source
|
||||
// names for every repository, by the forge bridge's merged message, and keeps revision 1 with every module; a message
|
||||
// the bridge does not give keeps none, and the next source revision takes the trunk in again.
|
||||
first, _, merged := newFormIndexerUnmerged(t, builderModules(t)...)
|
||||
first.Revisions = newMemObjects(0)
|
||||
var firstSaid []meshcatalog.CatalogRevisionEvent
|
||||
first.SayCatalogRevision = func(_ context.Context, e meshcatalog.CatalogRevisionEvent) error {
|
||||
firstSaid = append(firstSaid, e)
|
||||
return nil
|
||||
}
|
||||
// Lead-47's ruling R1: the catch-up runs off the source handler, so a bridge that has not answered yet holds no source
|
||||
// revision (the first message is answered only once ApplySource returned; held in the handler it is answered as failed).
|
||||
answers, applied := 0, make(chan struct{})
|
||||
first.Message = func(_ context.Context, repository, head string, _ int) ([]byte, error) {
|
||||
if answers++; answers == 1 {
|
||||
select {
|
||||
case <-applied:
|
||||
return nil, errors.New("the forge bridge does not answer")
|
||||
case <-time.After(10 * time.Second):
|
||||
return nil, errors.New("the catch-up held the source handler")
|
||||
}
|
||||
}
|
||||
if repository != catalogueRepo || head != merged.Merge {
|
||||
return nil, fmt.Errorf("no merge of %s at %s", repository, head)
|
||||
}
|
||||
return json.Marshal(merged)
|
||||
}
|
||||
heads := meshdelivery.SourceByMainValue{Heads: map[string]string{catalogueRepo: merged.Merge}}
|
||||
var firstLog []string
|
||||
var logMu sync.Mutex
|
||||
first.Logf = func(f string, a ...any) {
|
||||
logMu.Lock()
|
||||
firstLog = append(firstLog, fmt.Sprintf(f, a...))
|
||||
logMu.Unlock()
|
||||
}
|
||||
err = first.ApplySource(context.Background(), 5, heads)
|
||||
close(applied)
|
||||
first.catchUpWG.Wait()
|
||||
if err != nil || len(firstSaid) != 0 || !strings.Contains(strings.Join(firstLog, "\n"), "the forge bridge does not answer") {
|
||||
t.Fatalf("a catch-up whose message is not read said %+v (%v), log %q, want no revision and the bridge's silence said", firstSaid, err, firstLog)
|
||||
}
|
||||
err = first.ApplySource(context.Background(), 6, heads)
|
||||
first.catchUpWG.Wait()
|
||||
if err != nil || len(firstSaid) != 1 || firstSaid[0].Revision != 1 ||
|
||||
firstSaid[0].Entries != int64(len(builderModules(t))) || firstSaid[0].Source != 6 {
|
||||
t.Fatalf("the first start's catch-up said %+v (%v), want trunk revision 1 of source 6 with every module", firstSaid, err)
|
||||
}
|
||||
// KeepView: a write to the view outside a recompute (here a verdict carried into one module) is the next revision.
|
||||
viewKey, _ := meshcatalogclient.ModulesKey("avahi")
|
||||
_ = write(context.Background(), first.Modules, viewKey, func(value []byte, _ bool) ([]byte, bool, error) {
|
||||
m, _ := meshcatalogclient.DecodeModule(value)
|
||||
m.Verdict, m.Problems = meshcatalogclient.VerdictRefused, []string{"a field the judge refuses"}
|
||||
data, err := meshcatalogclient.EncodeModule(m)
|
||||
return data, false, err
|
||||
})
|
||||
if err := first.KeepView(context.Background()); err != nil || len(firstSaid) != 2 || !reflect.DeepEqual(firstSaid[1].Changed, []string{"avahi"}) {
|
||||
t.Errorf("KeepView after a write said %+v (%v), want revision 2 naming avahi", firstSaid, err)
|
||||
}
|
||||
if view, _, at, err := TrunkView(context.Background(), first.Revisions); err != nil || at != 2 {
|
||||
t.Errorf("the view at a restart is read at %d (%v), want 2", at, err)
|
||||
} else if raw, _, _, _ := view.Get(context.Background(), viewKey); !strings.Contains(string(raw), "a field the judge refuses") {
|
||||
t.Errorf("the view at a restart lost the verdict: %s", raw)
|
||||
}
|
||||
// W4.14: the catch-up tool (indexer_take_in with no module and no repository, as the node agent calls it after a
|
||||
// restore) takes in the head the source names for every repository again, whatever revision is kept, and keeps none
|
||||
// when nothing moved; a repository whose message is not read is named in unread.
|
||||
if key, value, err := NewCatchUp("", "node-engine@ace", time.Now()); err != nil || key != CatchUpKey {
|
||||
t.Fatalf("NewCatchUp = %q (%v), want key %q", key, err, CatchUpKey)
|
||||
} else if a, err := DecodeTakeIn(value); err != nil || !a.CatchUp {
|
||||
t.Fatalf("a catch-up ask decodes to %+v (%v), want a catch-up", a, err)
|
||||
}
|
||||
asked := answers
|
||||
if got := first.CatchUp(context.Background()); !got.Stored || answers != asked+1 || len(firstSaid) != 2 || len(got.Unread) != 0 {
|
||||
t.Errorf("the catch-up answered %+v after %d asks, said %d revisions, want the one repository taken in, no new revision", got, answers-asked, len(firstSaid))
|
||||
}
|
||||
first.Message = func(context.Context, string, string, int) ([]byte, error) {
|
||||
return nil, errors.New("the forge bridge does not answer")
|
||||
}
|
||||
if got := first.CatchUp(context.Background()); got.Stored || !reflect.DeepEqual(got.Unread, []string{catalogueRepo}) {
|
||||
t.Errorf("a catch-up whose message is not read answered %+v, want %s unread", got, catalogueRepo)
|
||||
}
|
||||
// The tools bundle answers modules and graph from the newest trunk revision, their marks at its number; with none kept
|
||||
// it says it waits (view-waits), never an empty catalog.
|
||||
trunkStates := TrunkStates{Objects: first.Revisions, Under: func(string) views.Source { return nil }}
|
||||
if keys, err := trunkStates.State("modules").Keys(); err != nil || len(keys) != len(builderModules(t)) {
|
||||
t.Errorf("the bundle's modules = %v (%v), want every module of revision 2", keys, err)
|
||||
}
|
||||
if e, ok, err := trunkStates.State(views.MarksState).Get("graph"); err != nil || !ok || !strings.Contains(string(e.Value), `"revision":2`) {
|
||||
t.Errorf("the bundle's mark of graph = %+v %v (%v), want revision 2", e, ok, err)
|
||||
}
|
||||
if _, err := (TrunkStates{Objects: newMemObjects(0)}).State("graph").Keys(); !errors.Is(err, ErrNoRevision) {
|
||||
t.Errorf("the bundle's graph with no revision kept answered %v, want it to wait", err)
|
||||
}
|
||||
// W4.4, ADR 0663: the seat mesh-catalog's verb revision answers, from the indexer's own objects, exactly the parts whose
|
||||
// entry in the newest trunk revision declares it, the holder refused too; a candidate's by its group and heads; none
|
||||
// kept, everyone waits.
|
||||
verbObjects := newMemObjects(0)
|
||||
keep := func(rev meshcatalog.CatalogRevision, declares map[string]bool) {
|
||||
entries := map[string][]byte{}
|
||||
for _, name := range []string{"avahi", Module, "mesh-grants", "node-tools"} {
|
||||
e := meshcatalog.CatalogEntry{Module: name, Repository: catalogueRepo}
|
||||
if declares[name] {
|
||||
e.Invokes = []string{RevisionInvoke}
|
||||
}
|
||||
data, _ := json.Marshal(e)
|
||||
digest := fmt.Sprintf("%s-%d-%s", rev.Kind, rev.Revision, name)
|
||||
entries[digest] = data
|
||||
rev.Entries = append(rev.Entries, meshcatalog.EntryRef{Module: name, Digest: digest})
|
||||
}
|
||||
if err := KeepRevision(context.Background(), verbObjects, rev, entries); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if _, err := Revision(context.Background(), verbObjects, Module, "n1.node-tools", meshcatalog.RevisionInput{}); !errors.Is(err, ErrNoRevision) {
|
||||
t.Errorf("revision with none kept answered %v, want it to wait", err)
|
||||
}
|
||||
keep(meshcatalog.CatalogRevision{Revision: 3, Kind: "trunk"}, map[string]bool{"node-tools": true, Module: true})
|
||||
keep(meshcatalog.CatalogRevision{Revision: 4, Kind: "candidate", Group: "feat/x", Heads: "0123456789ab"}, map[string]bool{"node-tools": true, "mesh-grants": true})
|
||||
for caller, answered := range map[string]bool{"n1.node-tools": true, "n1.mesh-grants": false, "n1." + Module: false, "person.someone": false, "unproven": false} {
|
||||
got, err := Revision(context.Background(), verbObjects, Module, caller, meshcatalog.RevisionInput{})
|
||||
if (err == nil) != answered || (answered && (got.Revision.Revision != 3 || len(got.Entries) != 4 || got.Next != "")) {
|
||||
t.Errorf("revision to %s answered %+v (%v), want answered %v with trunk revision 3 whole", caller, got, err, answered)
|
||||
}
|
||||
}
|
||||
if got, err := Revision(context.Background(), verbObjects, Module, "n1.node-tools", meshcatalog.RevisionInput{Group: "feat/x", Heads: "0123456789ab", After: "mesh-grants"}); err != nil ||
|
||||
got.Revision.Kind != "candidate" || len(got.Entries) != 1 || got.Entries[0].Module != "node-tools" {
|
||||
t.Errorf("a candidate's revision after mesh-grants answered %+v (%v), want its one entry node-tools", got, err)
|
||||
}
|
||||
if _, err := Revision(context.Background(), verbObjects, Module, "n1.node-tools", meshcatalog.RevisionInput{Revision: 9}); err == nil {
|
||||
t.Error("a trunk revision not kept was answered")
|
||||
}
|
||||
// ADR 0614 §2, §3: the graph names what each module invokes and serves, and leaves out, over the whole graph, the
|
||||
// verb edges of two modules calling each other, from either side; a third calling one of them keeps its edge.
|
||||
invoking := func(name, invokes string) mod {
|
||||
|
||||
@@ -252,6 +252,10 @@ func (ix *Indexer) judgeOverHeld(ctx context.Context, fresh map[string]bool, ove
|
||||
ix.logf("judge: modules.%s does not carry its verdict: %v", name, err)
|
||||
}
|
||||
}
|
||||
// The verdicts carried are kept with the view (KeepView), so a restart reads them back.
|
||||
if err := ix.KeepView(ctx); err != nil {
|
||||
ix.logf("judge: the verdicts carried are not kept in a catalog graph revision: %v", err)
|
||||
}
|
||||
if over != nil {
|
||||
return out, nil
|
||||
}
|
||||
|
||||
@@ -11,10 +11,11 @@ import (
|
||||
const MarksBucket = Module + "_" + views.MarksState
|
||||
|
||||
// FirstViewTools are the indexer's first-view tools (ADR 0621 §1, ADR 0622 §1, ADR 0631 §1): the tool, the state it answers, and
|
||||
// the words of what that state holds. Each answers only its callers list (CallersEnv), which the manifest writes.
|
||||
// the words of what that state holds. Each answers only its callers list (CallersEnv), which the manifest writes. The fourth
|
||||
// first view, of which modules exist, is the seat mesh-catalog's verb revision (ADR 0663 §1, RevisionTool), granted by
|
||||
// declaration, not by a list.
|
||||
var FirstViewTools = []ViewedTool{
|
||||
{Tool: "manifests", State: "manifests", Words: "every manifest the indexer keeps, by its digest."},
|
||||
{Tool: "modules", State: "modules", Words: "every module the indexer holds, its fingerprint, inputs and manifest digest."},
|
||||
{Tool: "verdicts", State: VerdictsState, Words: "the judge's verdict on each manifest: accepted, waiting or refused."},
|
||||
{Tool: "graph", State: graph.State, Words: "the module graph: each module's build, provision and verb edges, whose verb edges a want waits on."},
|
||||
}
|
||||
@@ -43,13 +44,11 @@ type ViewedTool struct {
|
||||
// ViewedStates are the states the indexer's views family says, each by its bucket: the ones its readers read.
|
||||
var ViewedStates = map[string]string{
|
||||
"manifests": ManifestsBucket,
|
||||
"modules": ModulesBucket,
|
||||
VerdictsState: VerdictsBucket,
|
||||
PredefinedState: PredefinedBucket,
|
||||
ConfigCandidatesState: ConfigCandidatesBucket,
|
||||
"candidates": CandidatesBucket,
|
||||
PackagesState: PackagesBucket,
|
||||
graph.State: GraphBucket,
|
||||
HeardState: HeardBucket,
|
||||
"taken": TakenBucket,
|
||||
mcc.FindingsState: mcc.FindingsBucket(Module),
|
||||
|
||||
@@ -16,7 +16,6 @@ import (
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
|
||||
mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
|
||||
"git.novox.be/novox/mesh-delivery/sdk/graph"
|
||||
"git.novox.be/novox/mesh-sdk/go/views"
|
||||
|
||||
"mesh-delivery-indexer/internal/index"
|
||||
@@ -57,7 +56,13 @@ type manifest struct {
|
||||
Slug string `json:"slug"`
|
||||
RunsAs string `json:"runs-as"`
|
||||
Streams []stream `json:"streams"`
|
||||
Claims []struct {
|
||||
Objects []struct {
|
||||
Name string `json:"name"`
|
||||
MaxMiB int `json:"max-mib"`
|
||||
Form int `json:"form"`
|
||||
CatchUp string `json:"catch-up"`
|
||||
} `json:"objects"`
|
||||
Claims []struct {
|
||||
Name string `json:"name"`
|
||||
Scope string `json:"scope"`
|
||||
Serves []string `json:"serves"`
|
||||
@@ -113,19 +118,26 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build
|
||||
t.Errorf("emits = %v, want the two take-in events (the verdict events are the seat mesh-catalog's)", m.Emits)
|
||||
}
|
||||
// 1126 note 76: heard keeps the last 64 FORGE messages the indexer handled, for indexer_heard.
|
||||
if want := []state{{Name: "candidates", History: 10}, {Name: "heard", History: 64}, {Name: "manifests", History: 1}, {Name: "modules", History: 20}, {Name: "predefined", History: 5}, {Name: "config-candidates", History: 5}, {Name: "taken", History: 1}, {Name: "takeins", History: 1}, {Name: "verdicts", History: 5}, {Name: "oldsections", History: 1}, {Name: "try-recovery", History: 1}, {Name: "own-build", History: 1}, {Name: index.PackagesState, History: index.PackagesHistory},
|
||||
{Name: graph.State, History: graph.History}, {Name: views.MarksState, History: 1}, {Name: mcc.FindingsState, History: 5}, {Name: views.StatusState, History: 1},
|
||||
if want := []state{{Name: "candidates", History: 10}, {Name: "heard", History: 64}, {Name: "manifests", History: 1}, {Name: "predefined", History: 5}, {Name: "config-candidates", History: 5}, {Name: "taken", History: 1}, {Name: "takeins", History: 1}, {Name: "verdicts", History: 5}, {Name: "oldsections", History: 1}, {Name: "try-recovery", History: 1}, {Name: "own-build", History: 1}, {Name: index.PackagesState, History: index.PackagesHistory},
|
||||
{Name: views.MarksState, History: 1}, {Name: mcc.FindingsState, History: 5}, {Name: views.StatusState, History: 1},
|
||||
{Name: index.RequirementsState, History: 5}, {Name: index.SeatDefinitionsState, History: 5},
|
||||
{Name: index.HoldsState, History: 5}, {Name: index.EngineReadsState, History: 1}, {Name: index.HoldBoundsState, History: 1, KeyTTL: true}, {Name: index.PendingState, History: 1}, {Name: index.UnjudgedState, History: 1}}; !reflect.DeepEqual(m.State, want) { // DLV-I7, DLV-I17, ADR 0632 §2
|
||||
t.Errorf("state = %+v, want %+v", m.State, want)
|
||||
}
|
||||
// Milestone 47/G W4.2 (design 47, Two graphs; ADR 0619 §2): the catalog graph revisions are objects in the old section,
|
||||
// capped at 512 MiB, under the local name the indexer opens. W4.14 (design 47, Reverted): their form is 1, and the
|
||||
// catch-up tool a restore of them is followed by is indexer_take_in, which the bundle serves.
|
||||
if len(m.Objects) != 1 || m.Objects[0].Name != index.CatalogRevisionsObjects || m.Objects[0].MaxMiB != 512 ||
|
||||
m.Objects[0].Form != 1 || m.Objects[0].CatchUp != index.TakeInTool || !slices.Contains(m.Tools, index.TakeInTool) {
|
||||
t.Errorf("objects = %+v, want %s capped at 512 MiB, form 1, catch-up %s (a tool of its own)", m.Objects, index.CatalogRevisionsObjects, index.TakeInTool)
|
||||
}
|
||||
// D-11 (ADR 0634 §1): the holders and buckets the indexer names (internal/index/names.go) are its manifest's: its own
|
||||
// name and states with their revisions, and the states it reads, each as `<holder>.<state>`.
|
||||
if index.Module != m.Module {
|
||||
t.Errorf("the indexer names itself %q, its manifest's module is %q", index.Module, m.Module)
|
||||
}
|
||||
for bucket, history := range map[string]int{index.CandidatesBucket: index.CandidatesHistory, index.ModulesBucket: index.ModulesHistory, index.ManifestsBucket: index.ManifestsHistory,
|
||||
index.VerdictsBucket: index.VerdictsHistory, index.PackagesBucket: index.PackagesHistory, index.TakenBucket: 1, index.PredefinedBucket: 5, index.ConfigCandidatesBucket: 5, index.GraphBucket: graph.History} {
|
||||
for bucket, history := range map[string]int{index.CandidatesBucket: index.CandidatesHistory, index.ManifestsBucket: index.ManifestsHistory,
|
||||
index.VerdictsBucket: index.VerdictsHistory, index.PackagesBucket: index.PackagesHistory, index.TakenBucket: 1, index.PredefinedBucket: 5, index.ConfigCandidatesBucket: 5} {
|
||||
if !slices.ContainsFunc(m.State, func(st state) bool { return m.Module+"_"+st.Name == bucket && st.History == history }) {
|
||||
t.Errorf("the indexer names its bucket %s with %d revisions, its manifest's states are %+v", bucket, history, m.State)
|
||||
}
|
||||
@@ -163,25 +175,30 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build
|
||||
// 1945: indexer_take_in, served by its tools bundle through the takeins state; the process serves none. The same bundle
|
||||
// reads the verdicts state (indexer_verdicts) and asks the indexer to judge again through the takeins state
|
||||
// (indexer_judge; issue 774, clause 3). 1966/AC11: indexer_old_sections reads the count of the old sections (state oldsections).
|
||||
// ADR 0621 §1, ADR 0622 §1, ADR 0631 §1: the first-view tools manifests, modules, verdicts and graph, and design 25 § 12.2's read
|
||||
// ADR 0621 §1, ADR 0622 §1, ADR 0631 §1, ADR 0663 §1: the first-view tools manifests, verdicts and graph (modules is the seat
|
||||
// verb revision's, served by the claim), and design 25 § 12.2's read
|
||||
// tools, all of the tools bundle, which runs in a runtime of its own on the indexer's account (runs-as).
|
||||
want := []string{"indexer_take_in", "indexer_old_sections", "indexer_verdicts", "indexer_judge"}
|
||||
for _, v := range append(append([]index.ViewedTool{}, index.FirstViewTools...), index.ReadTools...) {
|
||||
want = append(want, v.Tool)
|
||||
if _, ok := index.ViewedStates[v.State]; !ok {
|
||||
// Milestone 47/G: modules and graph are the newest catalog graph revision's, which catalog-revision says the changes of.
|
||||
if _, ok := index.ViewedStates[v.State]; !ok && v.State != index.ModulesState && v.State != "graph" {
|
||||
t.Errorf("%s answers the state %s, which the views family does not say", v.Tool, v.State)
|
||||
}
|
||||
}
|
||||
want = append(want, "indexer_first_views", "indexer_first_view_again", index.ConfigReadersTool, index.BusStateTool, index.RequirementsTool, index.PackageUsersTool, index.SayAcceptedTool, // DLV-F3r, DLV-I10, DLV-I11, DLV-I7, DLV-I8, DLV-I9c
|
||||
index.SeatDefinitionTool, index.SeatDefinitionsAgainTool, index.OutOfStepTool, // DLV-I19
|
||||
index.HoldsTool, index.EngineReadsTool, // ADR 0632 §2: the mesh-wide holds and the engines' reads they weigh
|
||||
index.PendingTool, index.TakeInPendingTool, index.DropPendingTool, index.TryCandidatesTool, index.DropTryCandidateTool, index.RecoverTriesTool, index.TakeInMergeTool, index.FingerprintsAgainTool) // issue 1966; 1585/T12, T13 (ADR 0633 §5)
|
||||
if !reflect.DeepEqual(m.Tools, want) || len(want) != 40 {
|
||||
index.PendingTool, index.TakeInPendingTool, index.DropPendingTool, index.TryCandidatesTool, index.DropTryCandidateTool, index.RecoverTriesTool, index.TakeInMergeTool, index.FingerprintsAgainTool, // issue 1966; 1585/T12, T13 (ADR 0633 §5)
|
||||
// milestone 47/G W4.12 (design 47, the indexer's tools): the catalog graph revisions read and steered.
|
||||
index.CatalogRevisionTool, index.CatalogRevisionsTool, index.CandidateGraphsTool, index.RecomputeTool, index.DropCandidateGraphTool)
|
||||
if !reflect.DeepEqual(m.Tools, want) || len(want) != 45 || slices.Contains(m.Tools, "modules") {
|
||||
t.Errorf("tools = %v, want %v", m.Tools, want)
|
||||
}
|
||||
// Every read tool of the indexer a reader of this repository invokes is one it serves (the checker's seventh first
|
||||
// view, indexer_read_seat_definitions, was invoked and not served).
|
||||
readers, _ := filepath.Glob("../*/module.json")
|
||||
var declaring []string
|
||||
for _, p := range readers {
|
||||
raw, err := os.ReadFile(p)
|
||||
if err != nil {
|
||||
@@ -194,24 +211,36 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build
|
||||
t.Fatalf("%s: %v", p, err)
|
||||
}
|
||||
for _, inv := range r.Invokes {
|
||||
// The grants part reads modules until its own subtask (W8) moves it to revision; the join removes this line.
|
||||
if p == "../mesh-grants/module.json" && inv == "mesh-delivery-indexer.modules" {
|
||||
continue
|
||||
}
|
||||
if tool, ok := strings.CutPrefix(inv, "mesh-delivery-indexer."); ok && !slices.Contains(m.Tools, tool) {
|
||||
t.Errorf("%s invokes %s, which the indexer does not serve", p, inv)
|
||||
}
|
||||
if inv == index.RevisionInvoke {
|
||||
declaring = append(declaring, filepath.Base(filepath.Dir(p)))
|
||||
}
|
||||
}
|
||||
}
|
||||
// ADR 0663 §2: the assigner reads which modules exist by declaring the seat's verb revision (the grants part too, from W8).
|
||||
if !slices.Contains(declaring, "mesh-delivery-assigner") {
|
||||
t.Errorf("the parts of this repository declaring %s are %v, want the assigner", index.RevisionInvoke, declaring)
|
||||
}
|
||||
if m.RunsAs != m.Module || m.SecretsOwner != m.RunsAs {
|
||||
t.Errorf("runs-as = %q, secrets-owner %q: the tools bundle runs on the indexer's own account (ADR 0621 §1)", m.RunsAs, m.SecretsOwner)
|
||||
}
|
||||
// ADR 0621 §2, ADR 0622 §1, ADR 0631 §1: each first-view tool answers exactly its readers; the indexer is on none, and
|
||||
// graph answers the assigner alone.
|
||||
callers := map[string]string{"manifests": "mesh-grants,node-tools,mesh-delivery-assigner", "modules": "mesh-grants,node-tools,mesh-delivery-assigner", "verdicts": "mesh-grants",
|
||||
// ADR 0663 §2: revision has no callers list; it answers the parts whose entry declares it.
|
||||
callers := map[string]string{"manifests": "mesh-grants,node-tools,mesh-delivery-assigner", "verdicts": "mesh-grants",
|
||||
"graph": "mesh-delivery-assigner"}
|
||||
for _, a := range m.Build.Artifacts {
|
||||
if a.Name != "tools" {
|
||||
continue
|
||||
}
|
||||
if len(a.Env) != len(callers) {
|
||||
t.Errorf("tools env = %v, want only the four callers lists", a.Env)
|
||||
t.Errorf("tools env = %v, want only the three callers lists", a.Env)
|
||||
}
|
||||
for tool, list := range callers {
|
||||
if got := a.Env[views.CallersEnv(tool)]; got != list {
|
||||
@@ -241,10 +270,10 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build
|
||||
}
|
||||
}
|
||||
// ADR 0604 §2, ADR 0636 §4: it holds the seat mesh-catalog, serving its four verbs and seat-definitions from its tools
|
||||
// bundle (which stays without a request subject).
|
||||
// bundle (which stays without a request subject); ADR 0663 §1: and revision, in the same runtime of its own.
|
||||
if len(m.Claims) != 1 || m.Claims[0].Name != index.CatalogSeat || m.Claims[0].Scope != catalogue.ScopeMesh ||
|
||||
!slices.Equal(m.Claims[0].Serves, []string{"modules", "module", "provides", "dependents", index.SeatDefinitionsVerb}) {
|
||||
t.Errorf("claims = %+v, want mesh-catalog with its five verbs", m.Claims)
|
||||
!slices.Equal(m.Claims[0].Serves, []string{"modules", "module", "provides", "dependents", index.SeatDefinitionsVerb, "revision"}) {
|
||||
t.Errorf("claims = %+v, want mesh-catalog with its six verbs", m.Claims)
|
||||
}
|
||||
for _, e := range []string{index.SeatDefinitionChangedEvent, index.SeatDefinitionRemovedEvent} {
|
||||
if slices.Contains(m.Emits, e) {
|
||||
|
||||
@@ -12,7 +12,8 @@
|
||||
"module",
|
||||
"provides",
|
||||
"dependents",
|
||||
"seat-definitions"
|
||||
"seat-definitions",
|
||||
"revision"
|
||||
]
|
||||
}
|
||||
],
|
||||
@@ -22,7 +23,6 @@
|
||||
"indexer_verdicts",
|
||||
"indexer_judge",
|
||||
"manifests",
|
||||
"modules",
|
||||
"verdicts",
|
||||
"graph",
|
||||
"indexer_read_manifests",
|
||||
@@ -56,7 +56,12 @@
|
||||
"indexer_drop_try_candidate",
|
||||
"indexer_recover_tries",
|
||||
"indexer_take_in_merge",
|
||||
"indexer_fingerprints_again"
|
||||
"indexer_fingerprints_again",
|
||||
"indexer_catalog_revision",
|
||||
"indexer_catalog_revisions",
|
||||
"indexer_candidate_graphs",
|
||||
"indexer_recompute",
|
||||
"indexer_drop_candidate_graph"
|
||||
],
|
||||
"consumes": [
|
||||
"go-package-registry.published",
|
||||
@@ -77,7 +82,6 @@
|
||||
{ "name": "candidates", "history": 10 },
|
||||
{ "name": "heard", "history": 64 },
|
||||
{ "name": "manifests", "history": 1 },
|
||||
{ "name": "modules", "history": 20 },
|
||||
{ "name": "predefined", "history": 5 },
|
||||
{ "name": "config-candidates", "history": 5 },
|
||||
{ "name": "taken", "history": 1 },
|
||||
@@ -87,7 +91,6 @@
|
||||
{ "name": "try-recovery", "history": 1 },
|
||||
{ "name": "own-build", "history": 1 },
|
||||
{ "name": "packages", "history": 5 },
|
||||
{ "name": "graph", "history": 5 },
|
||||
{ "name": "view-marks", "history": 1 },
|
||||
{ "name": "findings", "history": 5 },
|
||||
{ "name": "first-views", "history": 1 },
|
||||
@@ -99,6 +102,9 @@
|
||||
{ "name": "pending", "history": 1 },
|
||||
{ "name": "unjudged", "history": 1 }
|
||||
],
|
||||
"objects": [
|
||||
{ "name": "catalog-revisions", "max-mib": 512, "form": 1, "catch-up": "indexer_take_in" }
|
||||
],
|
||||
"emits": [
|
||||
"take-in.pending",
|
||||
"take-in.taken",
|
||||
@@ -194,7 +200,6 @@
|
||||
],
|
||||
"env": {
|
||||
"MESH_VIEW_CALLERS_MANIFESTS": "mesh-grants,node-tools,mesh-delivery-assigner",
|
||||
"MESH_VIEW_CALLERS_MODULES": "mesh-grants,node-tools,mesh-delivery-assigner",
|
||||
"MESH_VIEW_CALLERS_VERDICTS": "mesh-grants",
|
||||
"MESH_VIEW_CALLERS_GRAPH": "mesh-delivery-assigner"
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user