Merge origin/main into the builder reservation takeover work

This commit is contained in:
2026-11-19 16:59:19 +01:00
61 changed files with 1591 additions and 1817 deletions
@@ -93,6 +93,11 @@ type seatTools struct {
// seats is the seat list the bundle reads (seatlist.go): until it is read, propose judges no mesh seat the bootstrapped // seats is the seat list the bundle reads (seatlist.go): until it is read, propose judges no mesh seat the bootstrapped
// definitions do not name as held once (ADR 0636 §1). Nil reads nothing. // definitions do not name as held once (ADR 0636 §1). Nil reads nothing.
seatList *seatlist.Part seatList *seatlist.Part
// watched is the tools process's watched copy of the state seat-holders (tools.go wants.watched, which gate.Holders
// also reads: A-298), read by the verb assignments with seats instead of a bus get per key (issue 1982: under load
// that took about 14 s, where the watched copy answers at once). Nil, or a call it cannot answer, falls back to a
// direct listing.
watched *stdio.WatchedStates
} }
func newSeatTools(st seatStore, ask func(string, any) (json.RawMessage, error)) *seatTools { func newSeatTools(st seatStore, ask func(string, any) (json.RawMessage, error)) *seatTools {
@@ -554,25 +559,43 @@ func (s *seatTools) tools() []stdio.Tool {
return out return out
} }
// seats is the verb assignments with seats (ADR 0571 §4): who holds each seat and on which node, as the state seat-holders // seatHolders is every key of the state seat-holders and its raw value: from the watched copy (s.watched) when
// holds it; with node, the keys of seats held on that node. // there is one and it answers, else a direct listing, key by key, as the verb answered before (issue 1982).
func (s *seatTools) seats(node string) (any, error) { func (s *seatTools) seatHolders() (map[string][]byte, error) {
if s.watched != nil {
if vals, err := s.watched.Values(mma.StateSeatHolders); err == nil {
return vals, nil
}
}
keys, err := s.st.keys(mma.StateSeatHolders) keys, err := s.st.keys(mma.StateSeatHolders)
if err != nil { if err != nil {
return nil, fmt.Errorf("seat-holders cannot be listed: %w", err) return nil, fmt.Errorf("seat-holders cannot be listed: %w", err)
} }
sort.Strings(keys) out := make(map[string][]byte, len(keys))
out := []mma.SeatHolders{}
for _, key := range keys { for _, key := range keys {
raw, _, found, err := s.st.get(mma.StateSeatHolders, key) raw, _, found, err := s.st.get(mma.StateSeatHolders, key)
if err != nil { if err != nil {
return nil, fmt.Errorf("seat-holders %s cannot be read: %w", key, err) return nil, fmt.Errorf("seat-holders %s cannot be read: %w", key, err)
} }
if !found { if found {
continue out[key] = raw
} }
}
return out, nil
}
// seats is the verb assignments with seats (ADR 0571 §4): who holds each seat and on which node, as the state seat-holders
// holds it; with node, the keys of seats held on that node.
func (s *seatTools) seats(node string) (any, error) {
raws, err := s.seatHolders()
if err != nil {
return nil, err
}
keys := slices.Sorted(maps.Keys(raws))
out := []mma.SeatHolders{}
for _, key := range keys {
var sh mma.SeatHolders var sh mma.SeatHolders
if json.Unmarshal(raw, &sh) != nil { if json.Unmarshal(raws[key], &sh) != nil {
continue continue
} }
if node != "" && sh.Node != node && !slices.ContainsFunc(sh.Holders, func(h mma.SeatHoldersHoldersItem) bool { return h.Node == node }) { if node != "" && sh.Node != node && !slices.ContainsFunc(sh.Holders, func(h mma.SeatHoldersHoldersItem) bool { return h.Node == node }) {
@@ -352,6 +352,53 @@ func TestTheSeatsVerbs(t *testing.T) {
t.Errorf("seats on ace = %+v, %v", out, err) t.Errorf("seats on ace = %+v, %v", out, err)
} }
}) })
t.Run("issue 1982: assignments with seats reads the tools process's watched copy of seat-holders, the store only while it cannot", func(t *testing.T) {
seeded := map[string][]byte{}
put := func(seat, node string, holders ...mma.SeatHoldersHoldersItem) {
raw, err := json.Marshal(mma.SeatHolders{Seat: seat, Node: node, Holders: holders})
if err != nil {
t.Fatal(err)
}
seeded[seat] = raw
}
put("mesh-broker", "", mma.SeatHoldersHoldersItem{Module: "nats", Node: "ace"})
put("node-shell.g14", "g14")
st := &countingSeatStore{memStore: memStore{mma.StateSeatHolders: &memKV{values: seeded}}}
sv := newSeatTools(st, nil)
watches := 0
sv.watched = stdio.WatchedStatesOver(func(state string, handle func(stdio.StateChange) error) error {
watches++
for k, v := range seeded {
if err := handle(stdio.StateChange{Key: k, Op: stdio.OpPut, Value: v, Current: true}); err != nil {
return err
}
}
return nil
}, func(state, key string) ([]byte, bool, error) {
v, ok := seeded[key]
return v, ok, nil
})
out, err := sv.seats("")
if err != nil {
t.Fatal(err)
}
if got := out.(map[string]any)["seats"].([]mma.SeatHolders); len(got) != 2 || watches != 1 || st.keysCalls != 0 || st.getCalls != 0 {
t.Errorf("seats from the watched copy = %+v, watched %d time(s), the store's own keys %d get %d (want none)",
got, watches, st.keysCalls, st.getCalls)
}
// The lead's answer on issue 1982: the verb falls back to the store's own listing only while the watched copy
// cannot be read (here, its watch refused); the answer's form does not change.
sv.watched = stdio.WatchedStatesOver(func(string, func(stdio.StateChange) error) error {
return errors.New("refused")
}, func(state, key string) ([]byte, bool, error) { return nil, false, nil })
out, err = sv.seats("")
if err != nil {
t.Fatal(err)
}
if got := out.(map[string]any)["seats"].([]mma.SeatHolders); len(got) != 2 || st.keysCalls != 1 || st.getCalls != 2 {
t.Errorf("seats falling back to the store = %+v, the store's own keys %d get %d (want 1, 2)", got, st.keysCalls, st.getCalls)
}
})
t.Run("research 072 row 37: a node is written as its keys, and history reads the node key", func(t *testing.T) { t.Run("research 072 row 37: a node is written as its keys, and history reads the node key", func(t *testing.T) {
ctx := context.Background() ctx := context.Background()
kv := &revKV{values: map[string][]byte{}, revs: map[string]uint64{}} kv := &revKV{values: map[string][]byte{}, revs: map[string]uint64{}}
@@ -750,6 +797,23 @@ func readManifest() (catalogue.Manifest, error) {
return catalogue.Decode(raw) return catalogue.Decode(raw)
} }
// countingSeatStore is a memStore that counts its calls to keys and get, to prove a reader takes the state from the
// watched copy and the store only while that copy cannot be read (issue 1982).
type countingSeatStore struct {
memStore
keysCalls, getCalls int
}
func (c *countingSeatStore) keys(state string) ([]string, error) {
c.keysCalls++
return c.memStore.keys(state)
}
func (c *countingSeatStore) get(state, key string) ([]byte, uint64, bool, error) {
c.getCalls++
return c.memStore.get(state, key)
}
// countingKV is a memKV that counts its puts. // countingKV is a memKV that counts its puts.
type countingKV struct { type countingKV struct {
memKV memKV
@@ -419,6 +419,8 @@ func serveTools() error {
wants.watched = stdio.NewWatchedStates() // the large states read whole from one watched copy (ADR 0580 clause 3) wants.watched = stdio.NewWatchedStates() // the large states read whole from one watched copy (ADR 0580 clause 3)
// The callers' seat entries are resolved from the assigner's own seat-holders, its watched copy (A-298). // The callers' seat entries are resolved from the assigner's own seat-holders, its watched copy (A-298).
gate.Holders(seatlist.HoldersIn(func() (map[string][]byte, error) { return wants.watched.Values(mma.StateSeatHolders) })) gate.Holders(seatlist.HoldersIn(func() (map[string][]byte, error) { return wants.watched.Values(mma.StateSeatHolders) }))
// The verb assignments with seats answers from the same watched copy, so every caller of it is fast too (issue 1982).
seatVerbs.watched = wants.watched
wants.revisions = runtimeObjects{busstate.ObjectsOf(assigngraph.RevisionsObjects)} wants.revisions = runtimeObjects{busstate.ObjectsOf(assigngraph.RevisionsObjects)}
tools = append(tools, wants.tools()...) tools = append(tools, wants.tools()...)
keeper := &inputKeeper{vals: stateobjects.ForState(buckets.InputsState), now: time.Now, logf: func(string, ...any) {}} keeper := &inputKeeper{vals: stateobjects.ForState(buckets.InputsState), now: time.Now, logf: func(string, ...any) {}}
@@ -1,8 +1,6 @@
package main package main
import ( import (
"fmt"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
stdio "git.novox.be/novox/mesh-sdk/go" stdio "git.novox.be/novox/mesh-sdk/go"
@@ -15,59 +13,43 @@ import (
// catalogRead reads the indexer's view of the trunk (its modules and module graph, from the newest catalog graph // catalogRead reads the indexer's view of the trunk (its modules and module graph, from the newest catalog graph
// revision) and its manifests state for the seat's verbs. // revision) and its manifests state for the seat's verbs.
func catalogRead(modules views.Source, manifests stdio.KeptState, nodes views.Source, seatDefinitions, marks views.Source) index.CatalogRead { func catalogRead(modules views.Source, manifest func(digest string) (string, error), nodes views.Source, seatDefinitions, marks views.Source) index.CatalogRead {
return index.CatalogRead{ return index.CatalogRead{
SeatDefinitions: seatDefinitions, Marks: marks, SeatDefinitions: seatDefinitions, Marks: marks,
Modules: func() (map[string]meshcatalog.ModulesByModuleValue, error) { Modules: func() (map[string]meshcatalog.ModulesByModuleValue, error) {
keys, err := modules.Keys() values, err := index.SourceValues(modules)
if err != nil { if err != nil {
return nil, err return nil, err
} }
out := map[string]meshcatalog.ModulesByModuleValue{} out := map[string]meshcatalog.ModulesByModuleValue{}
for _, k := range keys { for k, v := range values {
e, ok, err := modules.Get(k) if m, err := meshcatalogclient.DecodeModule(v); err == nil {
if err != nil {
return nil, fmt.Errorf("modules.%s: %w", k, err)
}
if !ok {
continue
}
if m, err := meshcatalogclient.DecodeModule(e.Value); err == nil {
out[k] = m out[k] = m
} }
} }
return out, nil return out, nil
}, },
Manifest: func(digest string) (string, error) { Module: func(name string) (meshcatalog.ModulesByModuleValue, bool, error) {
key, err := meshcatalogclient.ManifestsKey(digest) key, err := meshcatalogclient.ModulesKey(name)
if err != nil { if err != nil {
return "", err return meshcatalog.ModulesByModuleValue{}, false, nil // a name no key holds is a module the trunk holds none of
} }
e, err := manifests.Get(key) e, ok, err := modules.Get(key)
if err != nil { if err != nil || !ok {
return "", err return meshcatalog.ModulesByModuleValue{}, false, err
} }
if e == nil { m, err := meshcatalogclient.DecodeModule(e.Value)
return "", fmt.Errorf("the manifests state holds no %s", digest) return m, err == nil, nil
}
kept, err := meshcatalogclient.DecodeManifest(e.Value)
return kept.Manifest, err
}, },
Manifest: manifest,
Graph: func() (graph.Graph, error) { Graph: func() (graph.Graph, error) {
keys, err := nodes.Keys() values, err := index.SourceValues(nodes)
if err != nil { if err != nil {
return nil, err return nil, err
} }
g := graph.Graph{} g := graph.Graph{}
for _, k := range keys { for k, v := range values {
e, ok, err := nodes.Get(k) if n, err := graph.Decode(v); err == nil {
if err != nil {
return nil, fmt.Errorf("graph.%s: %w", k, err)
}
if !ok {
continue
}
if n, err := graph.Decode(e.Value); err == nil {
g[k] = n g[k] = n
} }
} }
@@ -77,7 +59,27 @@ func catalogRead(modules views.Source, manifests stdio.KeptState, nodes views.So
} }
// catalogTools are the seat's verbs over the indexer's own states, served beside the indexer's own tools, since the indexer's manifest claims the seat (ADR 0604 §2). // catalogTools are the seat's verbs over the indexer's own states, served beside the indexer's own tools, since the indexer's manifest claims the seat (ADR 0604 §2).
func catalogTools(state func(string) views.Source) []stdio.Tool { // Every state a verb reads whole is answered from memory (the view of the trunk held, or a watched copy), so a call costs no
return index.CatalogTools(catalogRead(state("modules"), stdio.State("manifests"), state(graph.State), // round trip per key: four node-engines ask every module at once.
keptSource(index.SeatDefinitionsState), keptSource(views.MarksState))) func catalogTools(state func(string) views.Source, kept *index.KeptTrunk, watched *stdio.WatchedStates) []stdio.Tool {
return index.CatalogTools(catalogRead(state("modules"), kept.Manifest, state(graph.State),
watchedSource{watched, index.SeatDefinitionsState}, keptSource(views.MarksState)))
} }
// watchedSource is a state as its watched copy holds it (stdio.WatchedStates), as a Source; it offers every value in one read.
type watchedSource struct {
w *stdio.WatchedStates
state string
}
func (s watchedSource) Keys() ([]string, error) { return s.w.Keys(s.state) }
func (s watchedSource) Get(key string) (*views.Entry, bool, error) {
v, ok, err := s.w.Get(s.state, key)
if err != nil || !ok {
return nil, false, err
}
return &views.Entry{Value: v}, true, nil
}
func (s watchedSource) Values() (map[string][]byte, error) { return s.w.Values(s.state) }
@@ -17,9 +17,11 @@ import (
"time" "time"
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient" "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
stdio "git.novox.be/novox/mesh-sdk/go" stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/busstate" "git.novox.be/novox/mesh-sdk/go/busstate"
"git.novox.be/novox/mesh-sdk/go/process" "git.novox.be/novox/mesh-sdk/go/process"
"git.novox.be/novox/mesh-sdk/go/views"
"mesh-delivery-indexer/internal/index" "mesh-delivery-indexer/internal/index"
) )
@@ -286,13 +288,30 @@ func main() {
tools := Tools(asks, time.Now, stdio.State(index.VerdictsState), heldCounts(stdio.State(index.OldSectionsState)), acts) tools := Tools(asks, time.Now, stdio.State(index.VerdictsState), heldCounts(stdio.State(index.OldSectionsState)), acts)
// The modules and the module graph are the newest catalog graph revision's (milestone 47/G): revision.go. // The modules and the module graph are the newest catalog graph revision's (milestone 47/G): revision.go.
revisions := bundleObjects{busstate.ObjectsOf(index.CatalogRevisionsObjects)} revisions := bundleObjects{busstate.ObjectsOf(index.CatalogRevisionsObjects)}
trunk := index.TrunkStates{Objects: revisions, Under: keptSource, Kept: &index.KeptTrunk{Say: func(format string, a ...any) { // The view of the trunk is read from the indexer's own objects once when the runtime starts and again each time the seat's
// event catalog-revision is heard (design 47 G A-487b), through the bundle's view path.
say := func(format string, a ...any) {
fmt.Fprintf(os.Stderr, "[mesh-delivery-indexer-tools] "+format+"\n", a...) fmt.Fprintf(os.Stderr, "[mesh-delivery-indexer-tools] "+format+"\n", a...)
}}} }
tools = append(tools, catalogTools(trunk.State)...) kept := &index.KeptTrunk{Say: say}
trunk := index.TrunkStates{Objects: revisions, Under: keptSource, Kept: kept}
follow := &views.Reader{Module: os.Getenv(stdio.ServedAs), Logf: say, Acts: acts}
if err := follow.Adopt(trunkFollower(kept, revisions), meshcatalog.Seat, meshcatalog.EventCatalogRevision, func(ctx context.Context, data []byte) error {
var ev meshcatalog.CatalogRevisionEvent
if err := json.Unmarshal(data, &ev); err != nil {
return nil // not a revision event: offering it again changes nothing
}
return kept.Heard(ctx, revisions, ev)
}); err != nil {
fmt.Fprintf(os.Stderr, "[mesh-delivery-indexer-tools] %v\n", err)
os.Exit(1)
}
follow.Start(context.Background())
watched := stdio.NewWatchedStates()
tools = append(tools, catalogTools(trunk.State, kept, watched)...)
// The seat mesh-catalog's verb revision, a first view of the catalog half answered in this runtime of its own to exactly // The seat mesh-catalog's verb revision, a first view of the catalog half answered in this runtime of its own to exactly
// the parts whose entry declares it (novox/hq ADR 0663). // the parts whose entry declares it (novox/hq ADR 0663).
tools = append(tools, index.RevisionTool(revisions, os.Getenv(stdio.ServedAs)), index.EntryTool(revisions, keptRead{stdio.State(index.EntryIndexState)}), index.EntriesTool(revisions, keptRead{stdio.State(index.EntryIndexState)}, keptRead{stdio.State(index.NamedState)}), index.VersionsTool(keptRead{stdio.State(index.VersionsState)}), tools = append(tools, index.RevisionTool(revisions, kept, os.Getenv(stdio.ServedAs)), index.EntryTool(revisions, keptRead{stdio.State(index.EntryIndexState)}), index.EntriesTool(revisions, keptRead{stdio.State(index.EntryIndexState)}, keptRead{stdio.State(index.NamedState)}), index.VersionsTool(keptRead{stdio.State(index.VersionsState)}),
index.OutdatedTool(&index.Indexer{Versions: keptRead{stdio.State(index.VersionsState)}, Heard: keptRead{stdio.State(index.HeardState)}, Revisions: revisions}), index.OutdatedTool(&index.Indexer{Versions: keptRead{stdio.State(index.VersionsState)}, Heard: keptRead{stdio.State(index.HeardState)}, Revisions: revisions}),
index.InputsReadTool(keptRead{stdio.State(index.InputsState)}, stateobjects.ForState(index.InputsState).Objects)) index.InputsReadTool(keptRead{stdio.State(index.InputsState)}, stateobjects.ForState(index.InputsState).Objects))
tools = append(tools, index.CatalogRevisionTools(revisions, asks, time.Now, func(what, why string) error { tools = append(tools, index.CatalogRevisionTools(revisions, asks, time.Now, func(what, why string) error {
@@ -318,18 +337,26 @@ func main() {
})...) })...)
tools = append(tools, viewTools(os.Getenv, trunk.State, time.Now)...) tools = append(tools, viewTools(os.Getenv, trunk.State, time.Now)...)
tools = append(tools, index.PendingTools(keptRead{stdio.State(index.PendingState)}, keptRead{stdio.State(index.OwnBuildStateName)}, asks, time.Now)...) tools = append(tools, index.PendingTools(keptRead{stdio.State(index.PendingState)}, keptRead{stdio.State(index.OwnBuildStateName)}, asks, time.Now)...)
tools = append(tools, index.ValidationTools(asks, time.Now, func(what, why string) error {
_, err := acts.Record(meshhandactsclient.Request{What: what, Why: why, Cause: index.ValidateAgainTool, Verb: "validate-again"})
return err
})...)
tools = append(tools, index.FingerprintsTools(asks, time.Now, func(what, why string) error { tools = append(tools, index.FingerprintsTools(asks, time.Now, func(what, why string) error {
_, err := acts.Record(meshhandactsclient.Request{What: what, Why: why, Cause: index.FingerprintsAgainTool, Verb: "fingerprints-again"}) _, err := acts.Record(meshhandactsclient.Request{What: what, Why: why, Cause: index.FingerprintsAgainTool, Verb: "fingerprints-again"})
return err return err
}, 10*time.Second)...) }, 10*time.Second)...)
tools = append(tools, configReadersTool(stdio.NewWatchedStates(), trunk.State("modules")), busStateTool(trunk.State), requirementsTool(keptSource), tools = append(tools, configReadersTool(watched, trunk.State("modules")), busStateTool(trunk.State), requirementsTool(keptSource),
packageUsersTool(trunk.State), seatDefinitionTool(keptSource), outOfStepTool(os.Getenv(stdio.ServedAs), keptSource)) packageUsersTool(trunk.State), seatDefinitionTool(keptSource), outOfStepTool(os.Getenv(stdio.ServedAs), keptSource))
if err := stdio.Serve("", tools); err != nil { if err := stdio.Serve("", tools); err != nil {
fmt.Fprintf(os.Stderr, "[mesh-delivery-indexer-tools] %v\n", err) fmt.Fprintf(os.Stderr, "[mesh-delivery-indexer-tools] %v\n", err)
os.Exit(1) os.Exit(1)
} }
} }
// trunkFollower is the bundle's view of the trunk as a follower of the reader: its first view is the one read of the
// indexer's objects the start makes, and answers no value (the picture is KeptTrunk's, not the reader's).
func trunkFollower(kept *index.KeptTrunk, revisions index.Objects) *views.Follower {
return &views.Follower{Owner: meshcatalog.Seat, State: "catalog-half", Tool: "catalog-revisions",
Ask: func(ctx context.Context, _ string) (views.Answer, error) {
if err := kept.Reread(ctx, revisions); err != nil {
return views.Answer{}, err
}
return views.Answer{V: views.Version, Owner: meshcatalog.Seat, State: "catalog-half", Created: time.Unix(1, 0).UTC(), Values: map[string]json.RawMessage{}, At: time.Now()}, nil
}}
}
@@ -22,7 +22,6 @@ import (
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo"
sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep" sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/composer"
"github.com/nats-io/nats-server/v2/server" "github.com/nats-io/nats-server/v2/server"
"github.com/nats-io/nats.go" "github.com/nats-io/nats.go"
@@ -97,11 +96,7 @@ func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10; 1585/T14, T15
// The forge bridge's tool bridge_merged_message (issue 1585): answers the merged message published for a merge, by // The forge bridge's tool bridge_merged_message (issue 1585): answers the merged message published for a merge, by
// repository@merge, as the bridge does; a restarted indexer holds none in memory and asks it. // repository@merge, as the bridge does; a restarted indexer holds none in memory and asks it.
var bridgeMessages sync.Map var bridgeMessages sync.Map
// The build seat node-build-go's stand-in (design 47 G §712, §713): it hears the indexer's validation asks and ends each with js, ctx, candidates, _, _ := startIndexerRun(t, func(ctx context.Context, builds buildLog) {
// the manifest validator's verdict; refusing names, by the commit the job runs at, the manifest a revision's change refuses.
var validations *standIn
var refusing sync.Map
js, ctx, candidates, _, restart := startIndexerRun(t, func(ctx context.Context, builds buildLog) {
// The builder's log entries of two builds' ends (ADR 0571 clause 1), as the builder appends them. // The builder's log entries of two builds' ends (ADR 0571 clause 1), as the builder appends them.
builds.end(t, "novox/mesh-delivery", "a", "modules/a") builds.end(t, "novox/mesh-delivery", "a", "modules/a")
builds.end(t, "novox/mesh-catalog", "c", "modules/c") builds.end(t, "novox/mesh-catalog", "c", "modules/c")
@@ -128,9 +123,8 @@ func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10; 1585/T14, T15
if _, err := tc.Put(ctx, key, value); err != nil { if _, err := tc.Put(ctx, key, value); err != nil {
t.Fatal(err) t.Fatal(err)
} }
// The trunk holds the indexer's entry, taken in at an earlier commit with its own manifest; the validation names no Go module (A-268). // The trunk holds the indexer's entry, taken in at an earlier commit with its own manifest; the take-in asks no validation (A-476 a).
seedBuildModules(t, ctx, js, map[string]meshcatalog.CatalogEntry{testIndexer: trunkIndexerEntry(t, ctx, js)}) seedBuildModules(t, ctx, js, map[string]meshcatalog.CatalogEntry{testIndexer: trunkIndexerEntry(t, ctx, js)})
validations = standInBuildSeat(t, ctx, js, &refusing)
// The node-engine's composition on n1 (process.Conn.Call's subjects): the build of the indexer that runs there. // The node-engine's composition on n1 (process.Conn.Call's subjects): the build of the indexer that runs there.
answer := func(m *nats.Msg) { answer := func(m *nats.Msg) {
raw, _ := json.Marshal(map[string]any{"result": map[string]any{"runs": map[string]string{"fp": *running.Load()}}, "node": "n1"}) raw, _ := json.Marshal(map[string]any{"result": map[string]any{"runs": map[string]string{"fp": *running.Load()}}, "node": "n1"})
@@ -281,17 +275,13 @@ func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10; 1585/T14, T15
} }
return takein.TakeIn{}, false return takein.TakeIn{}, false
} }
held, ok := said(takein.TakeInPending, m1Key) // A-476 a, b: the revision changing the indexer's own tree is taken in at once, with the build this program runs and no
if !ok || held.What != takein.TakeInRevision || held.WaitsFor == "" || held.WaitsFor == first || len(held.Merges) != 1 { // validation ask: it is never said pending, and the build the revision names is not waited for.
t.Fatalf("take-in.pending.%s = %+v (said %v) within 20 s, want the revision waiting for another build than %s", m1Key, held, ok, first) if taken, ok := said(takein.TakeInTaken, m1Key); !ok || len(taken.Merges) != 1 || taken.Merges[0].Merge != m1.Merge {
t.Fatalf("take-in.taken.%s = %+v (said %v) within 20 s, want the revision taken in at once", m1Key, taken, ok)
} }
named := held.WaitsFor if held := saidAll(takein.TakeInPending, m1Key); len(held) != 0 {
// IX5 (§712): the revision was validated first, by one run ask whose job names the indexer entry's repository, path and t.Errorf("the revision was said pending: %+v", held)
// the revision's head, and no heads; accepted, it is never asked again while it waits for the named indexer.
if asks := validations.asked(); len(asks) != 1 || asks[0].Run == nil || asks[0].Kind != sdkstep.AskCheck ||
asks[0].Run.Program.Repository != m1.Repository || asks[0].Run.Program.Dir != "modules/"+ix || asks[0].Run.Program.Commit != m1.Merge ||
len(asks[0].Run.Heads) != 0 || asks[0].Run.Program.Package != "./cmd/mesh-judge" {
t.Fatalf("validation asks = %+v, want one run of the indexer's manifest validator at %s", asks, m1.Merge)
} }
get := func(bucket, key string) ([]byte, bool) { get := func(bucket, key string) ([]byte, bool) {
t.Helper() t.Helper()
@@ -312,11 +302,10 @@ func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10; 1585/T14, T15
} }
return e.Value(), true return e.Value(), true
} }
// Only what building and moving the named indexer needs is written.
raw, ok := get(index.ModulesState, ix) raw, ok := get(index.ModulesState, ix)
own, err := meshcatalogclient.DecodeModule(raw) own, err := meshcatalogclient.DecodeModule(raw)
if !ok || err != nil || own.Fp2 != named || own.Commit != m1.Merge { if !ok || err != nil || own.Commit != m1.Merge {
t.Fatalf("modules.%s = %+v (%v), want the named build %s at the merge", ix, own, err, named) t.Fatalf("modules.%s = %+v (%v), want the indexer at the merge", ix, own, err)
} }
if _, ok := get(index.Bucket(testIndexer, index.ManifestsState), own.Manifest); !ok { if _, ok := get(index.Bucket(testIndexer, index.ManifestsState), own.Manifest); !ok {
t.Errorf("manifests.%s is not written", own.Manifest) t.Errorf("manifests.%s is not written", own.Manifest)
@@ -325,16 +314,11 @@ func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10; 1585/T14, T15
t.Errorf("the verdict on the indexer's manifest %s is not written", own.Manifest) t.Errorf("the verdict on the indexer's manifest %s is not written", own.Manifest)
} }
repoKey, _ := trunk.RepositoryKey(m1.Repository) repoKey, _ := trunk.RepositoryKey(m1.Repository)
if _, ok := get(index.ModulesState, "a"); ok { if raw, ok := get(index.ModulesState, "a"); !ok || !strings.Contains(string(raw), m1.Merge) {
t.Errorf("modules.a is written while the revision is pending") t.Errorf("modules.a = %s, want it at the merge %s", raw, m1.Merge)
} }
if _, ok := get(index.Bucket(testIndexer, index.TakenState), repoKey); ok { if _, ok := get(index.Bucket(testIndexer, index.TakenState), repoKey); !ok {
t.Errorf("taken.%s is written while the revision is pending", repoKey) t.Errorf("taken.%s is not written", repoKey)
}
if e, err := candidates.Get(ctx, "feat/x"); err != nil {
t.Errorf("the candidate feat/x is gone while the merge of #12 is pending: %v", err)
} else if c, err := index.DecodeCandidate(e.Value()); err != nil || len(c.Pulls) != 2 {
t.Errorf("the candidate feat/x = %+v while the merge of #12 is pending, want its two pull requests", c)
} }
// A group's first member arrives meanwhile; its end is still to come (the knowledge base's remark: it stays across the restart). // A group's first member arrives meanwhile; its end is still to come (the knowledge base's remark: it stays across the restart).
@@ -365,34 +349,13 @@ func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10; 1585/T14, T15
} }
var w []takein.PendingWait var w []takein.PendingWait
for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) { for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
if w = waits(); len(w) == 2 && w[0].WaitsFor == named && w[1].Said != "" { if w = waits(); len(w) == 1 && w[0].Said != "" {
break break
} }
} }
if len(w) != 2 || w[0].Key != m1Key || w[0].WaitsFor != named || !slices.Contains(w[0].Holds, "a") || !slices.Contains(w[0].Holds, ix) || if len(w) != 1 || w[0].Key != groupKey || w[0].What != takein.TakeInGroup || len(w[0].Merges) != 1 || len(w[0].Keys) != 1 || w[0].Keys[0] != g1Key ||
w[1].Key != groupKey || w[1].What != takein.TakeInGroup || len(w[1].Merges) != 1 || w[1].Said != takein.TakeInPendingEvent { w[0].Said != takein.TakeInPendingEvent {
t.Fatalf("indexer_pending = %+v, want the revision %s waiting for %s holding a and the indexer, then the group gathered", w, m1Key, named) t.Fatalf("indexer_pending = %+v, want only the group gathered, its first member %s", w, g1Key)
}
// The named indexer starts (the node-engine now runs that build): it takes the revision in, and leaves the group.
running.Store(&named)
restart()
if taken, ok := said(takein.TakeInTaken, m1Key); !ok || len(taken.Merges) != 1 || taken.Merges[0].Merge != m1.Merge {
t.Fatalf("take-in.taken.%s = %+v (said %v) within 20 s of the start of %s", m1Key, taken, ok, named)
}
for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
if w = waits(); len(w) == 1 {
break
}
}
if raw, ok := get(index.ModulesState, "a"); !ok || !strings.Contains(string(raw), m1.Merge) {
t.Errorf("modules.a = %s after the take-in at the start, want it at the merge %s", raw, m1.Merge)
}
if _, ok := get(index.Bucket(testIndexer, index.TakenState), repoKey); !ok {
t.Errorf("taken.%s is not written by the take-in at the start", repoKey)
}
if len(w) != 1 || w[0].Key != groupKey || len(w[0].Keys) != 1 || w[0].Keys[0] != g1Key {
t.Fatalf("indexer_pending after the start = %+v, want only the group gathered, its first member %s", w, g1Key)
} }
if _, ok := get(index.ModulesState, "c"); ok { if _, ok := get(index.ModulesState, "c"); ok {
t.Errorf("modules.c is written while its group is gathered") t.Errorf("modules.c is written while its group is gathered")
@@ -425,23 +388,6 @@ func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10; 1585/T14, T15
t.Errorf("the group's first member %s was said pending as a revision of its own", g1Key) t.Errorf("the group's first member %s was said pending as a revision of its own", g1Key)
} }
// IX5 (§712, §713): a merge whose changed manifest gains a problem the trunk case lacks stays pending, refused naming it,
// said in take-in.pending; nothing of it is taken in.
m3 := forgemsg.Forge{Kind: forgemsg.ForgeMerged, Repository: "novox/mesh-delivery", Branch: "feat/y", Number: 13, Head: h1,
Merge: "3333333333333333333333333333333333333333", Root: "r0", Trees: map[string]string{"modules/a": "tree-a-3"},
Manifests: map[string]string{"modules/a": `{"module":"a","version":"2.0"}`}}
refusing.Store(m3.Merge, "a")
m3Key := takein.PendingKey(merged(m3))
if refused, ok := said(takein.TakeInPending, m3Key); !ok || !strings.Contains(refused.Refused, "a: the stand-in's problem") || refused.Failed != "" {
t.Fatalf("take-in.pending.%s = %+v (said %v) within 20 s, want the revision refused naming a's problem", m3Key, refused, ok)
}
if w = waits(); len(w) != 1 || w[0].Key != m3Key || w[0].Validation == nil || w[0].Validation.Accepted != 0 || len(validations.asked()) != 3 {
t.Fatalf("indexer_pending = %+v, asks %d: want the refused revision pending with its validation, asked once after m1 and the group", w, len(validations.asked()))
}
if raw, ok := get(index.ModulesState, "a"); !ok || strings.Contains(string(raw), m3.Merge) {
t.Errorf("modules.a = %s: the refused revision was taken in", raw)
}
} }
// trunkIndexerEntry keeps the indexer's own manifest by its digest, accepted, and answers its trunk entry at an earlier commit. // trunkIndexerEntry keeps the indexer's own manifest by its digest, accepted, and answers its trunk entry at an earlier commit.
@@ -475,63 +421,6 @@ func trunkIndexerEntry(t *testing.T, ctx context.Context, js jetstream.JetStream
return e return e
} }
// standIn is the build seat node-build-go as the indexer's validation meets it: the asks it heard, oldest first.
type standIn struct {
mu sync.Mutex
asks []sdkstep.StepAsk
}
func (s *standIn) asked() []sdkstep.StepAsk {
s.mu.Lock()
defer s.mu.Unlock()
return slices.Clone(s.asks)
}
// standInBuildSeat keeps the seat's check queue as a stream and hears the indexer's asks on it (accept.check.indexer): each is
// ended at once with step-ended on the seat's family, its answer inline (A-254), the program's last line: the manifest validator's verdict over both cases,
// the change case refusing the manifest refusing names for the job's commit.
func standInBuildSeat(t *testing.T, ctx context.Context, js jetstream.JetStream, refusing *sync.Map) *standIn {
t.Helper()
subject, err := sdkstep.AskSubjectBy(sdkstep.AskGo, sdkstep.AskCheck, index.ValidationAsker)
if err != nil || subject != "mesh.seat.node-build-go.accept.check.mesh-catalog" {
t.Fatalf("the validation's subject %q %v, want the indexer's seat name its asker (A-163, A-232)", subject, err)
}
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "SEAT_NODE_BUILD_GO_CHECK", Subjects: []string{"mesh.seat.node-build-go.accept.check.>"}}); err != nil {
t.Fatal(err)
}
// The indexer's state inputs and its objects, where the validation's request is kept for the build seat's holder (§354).
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: index.Bucket(testIndexer, index.InputsState), History: 1}); err != nil {
t.Fatal(err)
}
if _, err := js.CreateObjectStore(ctx, jetstream.ObjectStoreConfig{Bucket: catalogue.BusName(testIndexer, catalogue.StateObjectsName(index.InputsState))}); err != nil {
t.Fatal(err)
}
s := &standIn{}
if _, err := js.Conn().Subscribe(subject, func(m *nats.Msg) {
a, err := sdkstep.DecodeStepAsk(m.Data)
if err != nil || a.Run == nil {
t.Errorf("the stand-in build seat heard what is no run ask: %v", err)
return
}
s.mu.Lock()
s.asks = append(s.asks, a)
s.mu.Unlock()
change := composer.CaseVerdict{Name: "change", Manifests: map[string]composer.ManifestRead{}}
if module, ok := refusing.Load(a.Run.Program.Commit); ok {
change.Manifests[module.(string)] = composer.ManifestRead{Refused: "the stand-in's problem"}
}
verdict, _ := json.Marshal(composer.Verdict{V: 1, Cases: []composer.CaseVerdict{change, {Name: judge.TrunkCase, Manifests: map[string]composer.ManifestRead{}}}})
end, _ := json.Marshal(sdkstep.StepEnded{V: 1, ID: a.ID, Ask: a.ID, Asker: index.ValidationAsker, Node: "n1", Outcome: sdkstep.OutcomePassed,
Output: string(verdict), At: time.Now().UTC()})
if _, err := js.Publish(ctx, nodebuildgo.SubjectStepEnded("n1"), end); err != nil {
t.Errorf("the stand-in build seat could not say step-ended: %v", err)
}
}); err != nil {
t.Fatal(err)
}
return s
}
// jsSource is a bucket as a views.Source. // jsSource is a bucket as a views.Source.
type jsSource struct { type jsSource struct {
ctx context.Context ctx context.Context
@@ -1529,7 +1418,8 @@ func TestIndexerCountsOnlyThePlacedBuildNodesToolchains(t *testing.T) {
t.Fatal(err) t.Fatal(err)
} }
toolchains = kv toolchains = kv
for node, version := range map[string]string{"n1": "go1.27.3", "n2": "go1.27.2"} { // A-471: a toolchain is agreed by its language version (go1.27), not its patch; n2 reports another language version.
for node, version := range map[string]string{"n1": "go1.27", "n2": "go1.28.1"} {
key, _ := index.ToolchainKey(node, "go") key, _ := index.ToolchainKey(node, "go")
value, err := index.EncodeToolchain(nodebuildgo.Toolchain{Language: "go", Version: version, Recipe: "1", At: ptr(time.Now())}) value, err := index.EncodeToolchain(nodebuildgo.Toolchain{Language: "go", Version: version, Recipe: "1", At: ptr(time.Now())})
if err != nil { if err != nil {
@@ -1573,15 +1463,15 @@ func TestIndexerCountsOnlyThePlacedBuildNodesToolchains(t *testing.T) {
return got return got
} }
agreed := func(m meshcatalog.ModulesByModuleValue) bool { agreed := func(m meshcatalog.ModulesByModuleValue) bool {
return m.Fp2 != "" && m.Toolchains["go"].Version == "go1.27.3" return m.Fp2 != "" && m.Toolchains["go"].Version == "go1.27"
} }
held := read(agreed) held := read(agreed)
if !agreed(held) { if !agreed(held) {
t.Fatalf("within 20 s avahi has fp2 %q, none2 %q, toolchains %v: want a fingerprint of n1's go1.27.3 alone", held.Fp2, held.None2, held.Toolchains) t.Fatalf("within 20 s avahi has fp2 %q, none2 %q, toolchains %v: want a fingerprint of n1's go1.27 alone", held.Fp2, held.None2, held.Toolchains)
} }
place(ctx, t, "n2", "mesh-build-go") place(ctx, t, "n2", "mesh-build-go")
key, _ := index.ToolchainKey("n2", "go") key, _ := index.ToolchainKey("n2", "go")
value, err := index.EncodeToolchain(nodebuildgo.Toolchain{Language: "go", Version: "go1.27.2", Recipe: "1", At: ptr(time.Now().Add(time.Second))}) value, err := index.EncodeToolchain(nodebuildgo.Toolchain{Language: "go", Version: "go1.28.1", Recipe: "1", At: ptr(time.Now().Add(time.Second))})
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -1604,7 +1494,7 @@ func TestIndexerCountsOnlyThePlacedBuildNodesToolchains(t *testing.T) {
t.Fatal(err) t.Fatal(err)
} }
disagree := func(m meshcatalog.ModulesByModuleValue) bool { disagree := func(m meshcatalog.ModulesByModuleValue) bool {
return m.Fp2 == "" && strings.HasPrefix(m.None2, "builders disagree: go1.27.2 recipe 1 on n2") return m.Fp2 == "" && strings.HasPrefix(m.None2, "builders disagree: go1.27 recipe 1 on n1; go1.28 recipe 1 on n2")
} }
if m := read(disagree); !disagree(m) { if m := read(disagree); !disagree(m) {
t.Fatalf("within 20 s of a merge after n2's assignments avahi has fp2 %q, none2 %q: want the builders to disagree", m.Fp2, m.None2) t.Fatalf("within 20 s of a merge after n2's assignments avahi has fp2 %q, none2 %q: want the builders to disagree", m.Fp2, m.None2)
@@ -479,11 +479,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
// The take-in step (ADR 0633): its state pending and its stream take-in, both this delivery's, bound when they appear; // The take-in step (ADR 0633): its state pending and its stream take-in, both this delivery's, bound when they appear;
// until then every merge is taken in at once, as before. // until then every merge is taken in at once, as before.
Pending: awaitedStore{b: process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, index.PendingState), "no merge is held for the indexer its revision names: each is taken in at once", logger.Printf)}, Pending: awaitedStore{b: process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, index.PendingState), "no merge is held for the indexer its revision names: each is taken in at once", logger.Printf)},
SayTakeIn: sayTakeIn(js, cred), SayTakeIn: sayTakeIn(js, cred)}
// The validation's run ask (design 47 G §713, A-247, A-248), put under the indexer's own asker.
ValidationAsks: buildrun.Asker{Name: index.ValidationAsker, Publisher: buildrun.JetStreamAsks{JS: js}},
// The build seat's verb output, for a validation's answer kept as an object (A-254).
AskVerb: calls.CallFile}
// The registrations and the changes that wait for their warrant (design 47 G §2, §667): their states are bound when they // The registrations and the changes that wait for their warrant (design 47 G §2, §667): their states are bound when they
// appear; a warrant is heard on the router's decided event for this module, below. // appear; a warrant is heard on the router's decided event for this module, below.
registry := newRegistry(js, conn, cred, calls.Call, registry := newRegistry(js, conn, cred, calls.Call,
@@ -206,6 +206,7 @@ func TestAMeshSDKMergeMovesOnlyTheModulesImportingAChangedPackage(t *testing.T)
if pub.Commit != m2 || len(pub.Packages) != 3 { if pub.Commit != m2 || len(pub.Packages) != 3 {
t.Fatalf("packages.%s = %+v within 20 s, want the merge %s with a digest of go, announce and buildstep", key, pub, m2) t.Fatalf("packages.%s = %+v within 20 s, want the merge %s with a digest of go, announce and buildstep", key, pub, m2)
} }
// A-476a: mesh-sdk's merge moves importer, of another repository, which imports a package it moved.
moved := module("importer", func(m meshcatalog.ModulesByModuleValue) bool { return m.Fp2 != importer.Fp2 }) moved := module("importer", func(m meshcatalog.ModulesByModuleValue) bool { return m.Fp2 != importer.Fp2 })
if moved.Packages[sdkGo] != m2 { if moved.Packages[sdkGo] != m2 {
t.Errorf("importer moved to %+v, want it built against %s", moved.Packages, m2) t.Errorf("importer moved to %+v, want it built against %s", moved.Packages, m2)
@@ -28,7 +28,6 @@ type takerIn interface {
NamedDrop(ctx context.Context, fingerprint, member, by, why string) index.TakenIn NamedDrop(ctx context.Context, fingerprint, member, by, why string) index.TakenIn
Unpin(ctx context.Context, revision, record, by, why string) index.TakenIn Unpin(ctx context.Context, revision, record, by, why string) index.TakenIn
DropRunByHand(ctx context.Context, run, by, why string) index.TakenIn DropRunByHand(ctx context.Context, run, by, why string) index.TakenIn
ValidateAgainByHand(ctx context.Context, key, by, why string) index.TakenIn
RegistryAct(ctx context.Context, a index.TakeInAsk) index.TakenIn RegistryAct(ctx context.Context, a index.TakeInAsk) index.TakenIn
} }
@@ -75,9 +74,6 @@ func takenIn(ctx context.Context, ix takerIn, kv jetstream.KeyValue, e jetstream
} else if a.Merge != "" { } else if a.Merge != "" {
asked, r = "hand take-in of a merge", ix.TakeInMerge(ctx, a.MergeRepository, a.Merge, a.Sequence, a.By, a.Why) asked, r = "hand take-in of a merge", ix.TakeInMerge(ctx, a.MergeRepository, a.Merge, a.Sequence, a.By, a.Why)
a.Module = r.Module a.Module = r.Module
} else if a.Pending != "" && a.ValidateAgain {
asked, r = "validation asked again", ix.ValidateAgainByHand(ctx, a.Pending, a.By, a.Why)
a.Module = a.Pending
} else if a.Pending != "" && a.Drop { } else if a.Pending != "" && a.Drop {
asked, r = "drop of a pending merge", ix.DropPending(ctx, a.Pending, a.By, a.Why, a.Abandon) asked, r = "drop of a pending merge", ix.DropPending(ctx, a.Pending, a.By, a.Why, a.Abandon)
a.Module = a.Pending a.Module = a.Pending
@@ -7,8 +7,6 @@ import (
"sync/atomic" "sync/atomic"
"time" "time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
"github.com/nats-io/nats.go" "github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream" "github.com/nats-io/nats.go/jetstream"
@@ -155,15 +153,6 @@ func heardStepEnded(ctx context.Context, ix *index.Indexer, m jetstream.Msg, log
_ = m.Ack() _ = m.Ack()
return return
} }
// The end of a validation ask resumes its pending revision (design 47 G §713); its own fault is given again by the bus.
var end buildstep.StepEnded
if err := json.Unmarshal(m.Data(), &end); err == nil {
if err := ix.HearValidationEnded(ctx, end); err != nil {
logf("take-in: the end of the validation ask %s is not taken in, so the bus gives it again: %v", end.Ask, err)
leftForBus(m, "the end of the validation ask "+end.Ask, err, logf)
return
}
}
if err := ix.HearInputEnded(ctx, e); err != nil { if err := ix.HearInputEnded(ctx, e); err != nil {
logf("inputs: the input of the ask %s could not be deleted, so the bus gives it again: %v", e.Ask, err) logf("inputs: the input of the ask %s could not be deleted, so the bus gives it again: %v", e.Ask, err)
leftForBus(m, "the step-ended of the ask "+e.Ask, err, logf) leftForBus(m, "the step-ended of the ask "+e.Ask, err, logf)
@@ -91,7 +91,7 @@ func (ix *Indexer) TakeInBehind(ctx context.Context) []string {
} }
switch { switch {
case held[head]: case held[head]:
// A head the pending holds (a member waiting for its group's end or its build) is taken in by that path, whole with // A head the pending holds (a member waiting for its group's end) is taken in by that path, whole with
// its group: it is not behind (A-463a). // its group: it is not behind (A-463a).
ix.clearUntaken(ctx, key, "the head of "+repository+" waits in the indexer's pending") ix.clearUntaken(ctx, key, "the head of "+repository+" waits in the indexer's pending")
case merge == head: case merge == head:
@@ -6,7 +6,6 @@ import (
"context" "context"
"errors" "errors"
"fmt" "fmt"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
"git.novox.be/novox/mesh-sdk/go/stateobjects" "git.novox.be/novox/mesh-sdk/go/stateobjects"
"reflect" "reflect"
"slices" "slices"
@@ -234,12 +233,6 @@ type Indexer struct {
Pending Store Pending Store
// SayTakeIn publishes a take-in event (take-in.pending, take-in.taken; ADR 0633 §5); nil says none. // SayTakeIn publishes a take-in event (take-in.pending, take-in.taken; ADR 0633 §5); nil says none.
SayTakeIn func(ctx context.Context, t takein.TakeIn) error SayTakeIn func(ctx context.Context, t takein.TakeIn) error
// ValidationAsks puts the validation's run ask (design 47 G §713, A-247: buildrun.Asker, named as the indexer's asker,
// A-248); nil validates nothing, and a revision is taken in as before.
ValidationAsks ValidationAsks
// AskVerb asks a seat verb as the runtime calls it: the build seat's verb output, for a run's answer the seat keeps as an
// object (design 47 G A-254, buildstep.StepEnded.ReadOutput); nil reads none, and such an answer could not validate.
AskVerb buildstep.AskVerb
// Runs is the state runs: the runs on request the indexer holds a candidate for (runcandidates.go, A-174); nil keeps none. // Runs is the state runs: the runs on request the indexer holds a candidate for (runcandidates.go, A-174); nil keeps none.
Runs Store Runs Store
runMu sync.Mutex // one run event at a time (runcandidates.go) runMu sync.Mutex // one run event at a time (runcandidates.go)
@@ -25,6 +25,9 @@ import (
type CatalogRead struct { type CatalogRead struct {
// Modules is every value of the modules state, by module. // Modules is every value of the modules state, by module.
Modules func() (map[string]meshcatalog.ModulesByModuleValue, error) Modules func() (map[string]meshcatalog.ModulesByModuleValue, error)
// Module is the one value of the modules state of that module, and whether the state holds it: the verb module reads
// the one module it is asked, never every module (design 47 G A-487).
Module func(name string) (meshcatalog.ModulesByModuleValue, bool, error)
// Manifest is the manifest kept under a digest in the manifests state, as its JSON. // Manifest is the manifest kept under a digest in the manifests state, as its JSON.
Manifest func(digest string) (string, error) Manifest func(digest string) (string, error)
// Graph is the module graph, by module. // Graph is the module graph, by module.
@@ -51,17 +54,26 @@ func (c CatalogRead) CatalogModulesAnswer() (catalogue.CatalogModules, error) {
return catalogue.CatalogModules{Modules: out}, nil return catalogue.CatalogModules{Modules: out}, nil
} }
// module is one module's value: through Module when set, else out of every module's.
func (c CatalogRead) module(name string) (meshcatalog.ModulesByModuleValue, bool, error) {
if c.Module != nil {
return c.Module(name)
}
mods, err := c.Modules()
m, ok := mods[name]
return m, ok, err
}
// CatalogModuleAnswer is the verb module: one module whole, its manifest included. // CatalogModuleAnswer is the verb module: one module whole, its manifest included.
func (c CatalogRead) CatalogModuleAnswer(name string) (catalogue.CatalogOne, error) { func (c CatalogRead) CatalogModuleAnswer(name string) (catalogue.CatalogOne, error) {
name = strings.TrimSpace(name) name = strings.TrimSpace(name)
if name == "" { if name == "" {
return catalogue.CatalogOne{}, fmt.Errorf("name the module") return catalogue.CatalogOne{}, fmt.Errorf("name the module")
} }
mods, err := c.Modules() m, ok, err := c.module(name)
if err != nil { if err != nil {
return catalogue.CatalogOne{}, fmt.Errorf("the indexer's modules cannot be read: %w", err) return catalogue.CatalogOne{}, fmt.Errorf("the indexer's modules cannot be read: %w", err)
} }
m, ok := mods[name]
if !ok { if !ok {
return catalogue.CatalogOne{}, fmt.Errorf("the trunk holds no module %s", name) return catalogue.CatalogOne{}, fmt.Errorf("the trunk holds no module %s", name)
} }
@@ -45,7 +45,7 @@ func TestACatalogRevisionIsKeptPerWatchedBranch(t *testing.T) {
t.Errorf("said %+v, want main 1, then the escaped branch 1 and 2", said) t.Errorf("said %+v, want main 1, then the escaped branch 1 and 2", said)
} }
answer := func(branch string, revision int64) meshcatalog.RevisionAnswer { answer := func(branch string, revision int64) meshcatalog.RevisionAnswer {
a, err := Revision(ctx, ix.Revisions, "indexer", "reader", meshcatalog.RevisionInput{Branch: branch, Revision: revision}) a, err := Revision(ctx, ix.Revisions, nil, "indexer", "reader", meshcatalog.RevisionInput{Branch: branch, Revision: revision})
if err != nil { if err != nil {
t.Fatalf("revision of %q: %v", branch, err) t.Fatalf("revision of %q: %v", branch, err)
} }
@@ -66,7 +66,7 @@ func TestACatalogRevisionIsKeptPerWatchedBranch(t *testing.T) {
// A-205: `source` answers the branch's revision taken in from that source revision; a candidate's group alone answers its // A-205: `source` answers the branch's revision taken in from that source revision; a candidate's group alone answers its
// newest revision, whatever its heads. // newest revision, whatever its heads.
bySource := func(branch string, source int64) (meshcatalog.RevisionAnswer, error) { bySource := func(branch string, source int64) (meshcatalog.RevisionAnswer, error) {
return Revision(ctx, ix.Revisions, "indexer", "reader", meshcatalog.RevisionInput{Branch: branch, Source: source}) return Revision(ctx, ix.Revisions, nil, "indexer", "reader", meshcatalog.RevisionInput{Branch: branch, Source: source})
} }
if a, err := bySource("main", 5); err != nil || a.Revision.Revision != 1 || a.Entries[0].Commit != "c1" { if a, err := bySource("main", 5); err != nil || a.Revision.Revision != 1 || a.Entries[0].Commit != "c1" {
t.Errorf("main's source 5 answered %+v (%v)", a.Revision, err) t.Errorf("main's source 5 answered %+v (%v)", a.Revision, err)
@@ -89,10 +89,10 @@ func TestACatalogRevisionIsKeptPerWatchedBranch(t *testing.T) {
t.Fatal(err) t.Fatal(err)
} }
} }
if a, err := Revision(ctx, ix.Revisions, "indexer", "reader", meshcatalog.RevisionInput{Group: "run-ab"}); err != nil || a.Revision.Revision != 4 || a.Revision.Heads != "h2" || a.Entries[0].Commit != "c9" { if a, err := Revision(ctx, ix.Revisions, nil, "indexer", "reader", meshcatalog.RevisionInput{Group: "run-ab"}); err != nil || a.Revision.Revision != 4 || a.Revision.Heads != "h2" || a.Entries[0].Commit != "c9" {
t.Errorf("a group alone answered %+v (%v), want its newest revision 4", a.Revision, err) t.Errorf("a group alone answered %+v (%v), want its newest revision 4", a.Revision, err)
} }
if _, err := Revision(ctx, ix.Revisions, "indexer", "reader", meshcatalog.RevisionInput{Group: "run-none"}); err == nil { if _, err := Revision(ctx, ix.Revisions, nil, "indexer", "reader", meshcatalog.RevisionInput{Group: "run-none"}); err == nil {
t.Error("a candidate never kept was answered") t.Error("a candidate never kept was answered")
} }
} }
@@ -123,7 +123,7 @@ func TestTheNodeEngineReadsTheRevisionAsTheModuleHoldingItsSeat(t *testing.T) {
if err := ix.KeepBranchEntries(ctx, "main", 5, map[string]meshcatalog.CatalogEntry{"mesh-host": engine}); err != nil { if err := ix.KeepBranchEntries(ctx, "main", 5, map[string]meshcatalog.CatalogEntry{"mesh-host": engine}); err != nil {
t.Fatal(err) t.Fatal(err)
} }
_, err := Revision(ctx, ix.Revisions, "indexer", c.caller, meshcatalog.RevisionInput{Branch: "main"}) _, err := Revision(ctx, ix.Revisions, nil, "indexer", c.caller, meshcatalog.RevisionInput{Branch: "main"})
if (err == nil) != c.want { if (err == nil) != c.want {
t.Errorf("caller %q: err %v, want answered %v", c.caller, err, c.want) t.Errorf("caller %q: err %v, want answered %v", c.caller, err, c.want)
} }
@@ -0,0 +1,72 @@
package index
import (
"context"
"encoding/json"
"fmt"
"strings"
"sync/atomic"
"testing"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-sdk/go/views"
)
// listingObjects counts the listings of the bucket: each costs the whole bucket (56 ms at 11 000 objects, more through the runtime).
type listingObjects struct {
*memObjects
listings atomic.Int64
}
func (l *listingObjects) Names(ctx context.Context) ([]string, error) {
l.listings.Add(1)
return l.memObjects.Names(ctx)
}
// A seat verb that answers from the whole view of the trunk (modules, provides, dependents, module) reads the catalog-revisions
// bucket's listing once per call, never once per module: the node-engines ask `module` for every module at once, and 33 asks
// in 5 minutes passed the tool runner's 30 s (2026-11-17).
func TestTrunkViewIsReadOnceForAWholeAnswer(t *testing.T) {
objects := &listingObjects{memObjects: newMemObjects(0)}
const modules = 40
var refs []meshcatalog.EntryRef
for m := 0; m < modules; m++ {
name := fmt.Sprintf("mod%02d", m)
digest := fmt.Sprintf("%064x", m)
e := meshcatalog.CatalogEntry{Module: name, Repository: "o/r", Path: "m/" + name, Commit: "c", Fp: "src2:" + strings.Repeat("a", 64),
Manifest: map[string]any{"name": name}, ManifestDigest: "sha256:" + digest}
b, _ := json.Marshal(e)
if err := objects.Put(context.Background(), EntryObject(digest), b); err != nil {
t.Fatal(err)
}
refs = append(refs, meshcatalog.EntryRef{Module: name, Digest: digest})
}
b, _ := json.Marshal(meshcatalog.CatalogRevision{Kind: "trunk", Revision: 1, Entries: refs})
if err := objects.Put(context.Background(), "trunk.1", b); err != nil {
t.Fatal(err)
}
ts := TrunkStates{Objects: objects, Under: func(string) views.Source { return nil }, Kept: &KeptTrunk{}}
for _, state := range []string{"modules", "graph"} {
objects.listings.Store(0)
values, err := SourceValues(ts.State(state))
if err != nil || len(values) != modules {
t.Fatalf("the %s of the view = %d values (%v), want %d", state, len(values), err, modules)
}
if n := objects.listings.Load(); n > 2 {
t.Errorf("a whole read of the %s listed the bucket %d times, want at most 2 (once per call, not once per module)", state, n)
}
}
if _, err := meshcatalogclient.DecodeModule(mustValue(t, ts)); err != nil {
t.Errorf("a value of the view does not decode: %v", err)
}
}
func mustValue(t *testing.T, ts TrunkStates) []byte {
t.Helper()
v, err := SourceValues(ts.State("modules"))
if err != nil {
t.Fatal(err)
}
return v["mod00"]
}
@@ -182,7 +182,7 @@ func newestOf(ctx context.Context, o Objects, branch string) (meshcatalog.Catalo
name := RevisionObject(meshcatalog.CatalogRevision{Branch: branch, Revision: newest}) name := RevisionObject(meshcatalog.CatalogRevision{Branch: branch, Revision: newest})
data, err := o.Get(ctx, name) data, err := o.Get(ctx, name)
if err != nil { if err != nil {
return meshcatalog.CatalogRevision{}, false, err return meshcatalog.CatalogRevision{Branch: branch, Revision: newest}, false, err // the number that failed, for readNewestOf
} }
var rev meshcatalog.CatalogRevision var rev meshcatalog.CatalogRevision
if err := json.Unmarshal(data, &rev); err != nil { if err := json.Unmarshal(data, &rev); err != nil {
@@ -191,6 +191,31 @@ func newestOf(ctx context.Context, o Objects, branch string) (meshcatalog.Catalo
return rev, true, nil return rev, true, nil
} }
// readNewestOf is the newest revision of a branch and its entries whole (design 47 G A-487d). The pin judge removes a revision
// as soon as a newer one is kept, so a revision whose list or entry is removed under the read is read again as the newest,
// while the newest is newer than the one that failed; the same revision failing twice is the read's failure. Progress bounds
// the loop. No revision kept is false.
func readNewestOf(ctx context.Context, o Objects, branch string) (meshcatalog.CatalogRevision, []meshcatalog.CatalogEntry, bool, error) {
var failed int64
for {
newest, any, err := newestOf(ctx, o, branch)
if err == nil && !any {
return newest, nil, false, nil
}
var entries []meshcatalog.CatalogEntry
if err == nil {
_, entries, err = ReadRevision(ctx, o, RevisionObject(newest))
if err == nil {
return newest, entries, true, nil
}
}
if !errors.Is(err, ErrNoObject) || newest.Revision == 0 || newest.Revision <= failed {
return newest, nil, false, err
}
failed = newest.Revision
}
}
// changedEntries are the modules whose entry differs between two revisions' lists, one gone or added included, by name. // changedEntries are the modules whose entry differs between two revisions' lists, one gone or added included, by name.
func changedEntries(before, after []meshcatalog.EntryRef) []string { func changedEntries(before, after []meshcatalog.EntryRef) []string {
was := map[string]string{} was := map[string]string{}
@@ -438,15 +463,17 @@ func TrunkView(ctx context.Context, o Objects) (modules, nodes Store, at int64,
// BranchView is TrunkView of a watched branch: the view read from the newest revision the bucket keeps of that branch // BranchView is TrunkView of a watched branch: the view read from the newest revision the bucket keeps of that branch
// (design 47 G §291, A-93), empty and at 0 when it keeps none. // (design 47 G §291, A-93), empty and at 0 when it keeps none.
func BranchView(ctx context.Context, o Objects, branch string) (modules, nodes Store, at int64, err error) { func BranchView(ctx context.Context, o Objects, branch string) (modules, nodes Store, at int64, err error) {
view, held := newOverlay(nil), newOverlay(nil) newest, entries, any, err := readNewestOf(ctx, o, branch)
newest, any, err := newestOf(ctx, o, branch)
if err != nil || !any { if err != nil || !any {
return view, held, 0, err return newOverlay(nil), newOverlay(nil), 0, err
}
_, entries, err := ReadRevision(ctx, o, RevisionObject(newest))
if err != nil {
return nil, nil, 0, err
} }
return viewOfEntries(newest.Revision, entries)
}
// viewOfEntries is the view of the entries of one trunk revision, as BranchView reads them.
func viewOfEntries(revision int64, entries []meshcatalog.CatalogEntry) (modules, nodes Store, at int64, err error) {
view, held := newOverlay(nil), newOverlay(nil)
ctx, newest := context.Background(), meshcatalog.CatalogRevision{Revision: revision}
for _, e := range entries { for _, e := range entries {
key, err := meshcatalogclient.ModulesKey(e.Module) key, err := meshcatalogclient.ModulesKey(e.Module)
if err != nil { if err != nil {
@@ -496,14 +523,106 @@ type TrunkStates struct {
Kept *KeptTrunk Kept *KeptTrunk
} }
// KeptTrunk is the view of the trunk as last read, at its revision, shared by every read of a TrunkStates; Say, when set, // KeptTrunk is the view of the newest trunk revision, held in memory (design 47 G A-487, A-487b): it reads the newest
// says each read of a revision and how long it took. // revision from the indexer's own objects catalog-revisions once when the runtime starts (Reread) and again each time it
// hears the seat's event catalog-revision of a newer trunk revision (Heard); a read answers from the view held and never
// lists the bucket. Say, when set, says each read of a revision with its number, its module count and how long it took.
type KeptTrunk struct { type KeptTrunk struct {
Say func(format string, a ...any) Say func(format string, a ...any)
mu sync.Mutex
at int64 read sync.Mutex // one re-read at a time; a read of the view never holds it
mods Store mu sync.Mutex
held Store started bool
at int64
mods Store
held Store
rev meshcatalog.CatalogRevision
entries []meshcatalog.CatalogEntry
// manifests are the manifests the revision's entries carry whole, as JSON, by their digest as a module's value names it
// (A-487c: the verbs module and provides answer them from here, never from the manifests state).
manifests map[string]string
}
// Reread reads the newest trunk revision of the bucket and holds its view; a bucket keeping none holds an empty view at 0.
func (k *KeptTrunk) Reread(ctx context.Context, o Objects) error {
k.read.Lock()
defer k.read.Unlock()
return k.reread(ctx, o)
}
func (k *KeptTrunk) reread(ctx context.Context, o Objects) error {
began := time.Now()
fail := func(err error) error {
if k.Say != nil {
k.Say("the view of the trunk is BEHIND, its re-read failed after %s (the view held stays, the next heard catalog-revision reads again): %v", time.Since(began).Round(time.Millisecond), err)
}
return err
}
newest, entries, any, err := readNewestOf(ctx, o, TrunkBranch)
if err != nil {
return fail(err)
}
mods, held, at := Store(newOverlay(nil)), Store(newOverlay(nil)), int64(0)
if any {
if mods, held, at, err = viewOfEntries(newest.Revision, entries); err != nil {
return fail(err)
}
}
manifests := map[string]string{}
for _, e := range entries {
if d := meshcatalog.BareManifestDigest(e.ManifestDigest); d != "" && e.Manifest != nil {
if raw, err := json.Marshal(e.Manifest); err == nil {
manifests[d] = string(raw)
}
}
}
k.mu.Lock()
was := k.at
k.mods, k.held, k.at, k.started, k.rev, k.entries, k.manifests = mods, held, at, true, newest, entries, manifests
k.mu.Unlock()
if k.Say != nil {
n, _ := mods.Keys(ctx)
k.Say("the view of the trunk: read revision %d (was %d), %d modules, in %s", at, was, len(n), time.Since(began).Round(time.Millisecond))
}
return nil
}
// Newest is the newest trunk revision held, with a copy of its entries; false when the bucket kept none at the last read.
func (k *KeptTrunk) Newest(ctx context.Context, o Objects) (meshcatalog.CatalogRevision, []meshcatalog.CatalogEntry, bool, error) {
if _, _, _, err := k.current(ctx, o); err != nil {
return meshcatalog.CatalogRevision{}, nil, false, err
}
k.mu.Lock()
defer k.mu.Unlock()
return k.rev, slices.Clone(k.entries), k.at != 0, nil
}
// Manifest is the manifest of that digest (as a module's value names it) the held revision's entries carry.
func (k *KeptTrunk) Manifest(digest string) (string, error) {
k.mu.Lock()
defer k.mu.Unlock()
raw, ok := k.manifests[digest]
if !ok {
return "", fmt.Errorf("the newest trunk revision's entries carry no manifest %s", digest)
}
return raw, nil
}
// Heard is the seat's event catalog-revision: one of a trunk revision newer than the one held wakes one re-read of the
// objects; any other event is no change. A failed re-read is an error, so the event is offered again.
func (k *KeptTrunk) Heard(ctx context.Context, o Objects, e meshcatalog.CatalogRevisionEvent) error {
if e.Kind != "trunk" || !isTrunkBranch(e.Branch) {
return nil
}
k.read.Lock()
defer k.read.Unlock()
k.mu.Lock()
held := k.started && e.Revision <= k.at
k.mu.Unlock()
if held {
return nil
}
return k.reread(ctx, o)
} }
// read is the view's modules and nodes, and the revision they were read at; none kept yet is ErrNoRevision. // read is the view's modules and nodes, and the revision they were read at; none kept yet is ErrNoRevision.
@@ -518,36 +637,37 @@ func (t TrunkStates) read(nodes bool) (Store, int64, error) {
return mods, at, err return mods, at, err
} }
// view is the view of the newest trunk revision: the one kept when it is still the newest, else read whole and kept. // view is the view of the trunk: the one held while the follower's half is still at its revision, else made from the
// half and held.
func (t TrunkStates) view() (modules, nodes Store, at int64, err error) { func (t TrunkStates) view() (modules, nodes Store, at int64, err error) {
ctx := context.Background()
if t.Kept == nil { if t.Kept == nil {
return TrunkView(ctx, t.Objects) return TrunkView(context.Background(), t.Objects)
} }
k := t.Kept return t.Kept.current(context.Background(), t.Objects)
newest, any, err := newestOf(ctx, t.Objects, TrunkBranch) }
if err != nil || !any {
return newOverlay(nil), newOverlay(nil), 0, err // current is the view held; before the start's read it reads the newest revision once.
func (k *KeptTrunk) current(ctx context.Context, o Objects) (Store, Store, int64, error) {
k.mu.Lock()
started := k.started
k.mu.Unlock()
if !started {
k.read.Lock()
k.mu.Lock()
started = k.started
k.mu.Unlock()
var err error
if !started {
err = k.reread(ctx, o)
}
k.read.Unlock()
if err != nil {
return nil, nil, 0, err
}
} }
k.mu.Lock() k.mu.Lock()
defer k.mu.Unlock() defer k.mu.Unlock()
if k.mods != nil && k.at == newest.Revision { return k.mods, k.held, k.at, nil
return k.mods, k.held, k.at, nil
}
started := time.Now()
mods, held, at, err := TrunkView(ctx, t.Objects)
if err != nil {
if k.Say != nil {
k.Say("the view of the trunk: revision %d cannot be read after %s: %v", newest.Revision, time.Since(started).Round(time.Millisecond), err)
}
return nil, nil, 0, err
}
if k.Say != nil {
n, _ := mods.Keys(ctx)
k.Say("the view of the trunk: read revision %d (was %d), %d modules, in %s", at, k.at, len(n), time.Since(started).Round(time.Millisecond))
}
k.mods, k.held, k.at = mods, held, at
return mods, held, at, nil
} }
// state is the source of a state by its name: modules and graph from the revision, their marks at its number, every // state is the source of a state by its name: modules and graph from the revision, their marks at its number, every
@@ -588,6 +708,57 @@ func (r revisionSource) Get(key string) (*views.Entry, bool, error) {
return &views.Entry{Value: value, Revision: uint64(at)}, true, nil return &views.Entry{Value: value, Revision: uint64(at)}, true, nil
} }
// Values is every value of the modules (or the module graph) of the newest trunk revision from one read of it: a seat verb
// that answers from the whole view asks which revision is newest once, never once per key (each ask lists the whole
// catalog-revisions bucket: 6 s for the 110 modules of a bucket of 11 000 objects, and 35 s with 12 callers at once).
func (r revisionSource) Values() (map[string][]byte, error) {
s, _, err := r.t.read(r.nodes)
if err != nil {
return nil, err
}
ctx := context.Background()
keys, err := s.Keys(ctx)
if err != nil {
return nil, err
}
out := make(map[string][]byte, len(keys))
for _, k := range keys {
value, _, found, err := s.Get(ctx, k)
if err != nil {
return nil, err
}
if found {
out[k] = value
}
}
return out, nil
}
// SourceValues is every key's value of a source: in one read where the source offers it (the view of the trunk, a
// watched copy of a state), else a get per key.
func SourceValues(src views.Source) (map[string][]byte, error) {
if all, ok := src.(interface {
Values() (map[string][]byte, error)
}); ok {
return all.Values()
}
keys, err := src.Keys()
if err != nil {
return nil, err
}
out := make(map[string][]byte, len(keys))
for _, k := range keys {
e, ok, err := src.Get(k)
if err != nil {
return nil, fmt.Errorf("%s: %w", k, err)
}
if ok && e != nil {
out[k] = e.Value
}
}
return out, nil
}
// Page is the values in key order after a key from one read of the revision, as many as fit maxBytes and at least one // Page is the values in key order after a key from one read of the revision, as many as fit maxBytes and at least one
// (views.Pager): a first view's page reads the revision once, never once per key. // (views.Pager): a first view's page reads the revision once, never once per key.
func (r revisionSource) Page(after string, maxBytes int, _ time.Duration) ([]views.KeyedEntry, string, error) { func (r revisionSource) Page(after string, maxBytes int, _ time.Duration) ([]views.KeyedEntry, string, error) {
@@ -60,7 +60,7 @@ func (ix *Indexer) FingerprintsAgain(ctx context.Context, module string) TakenIn
ix.freshMu.Unlock() ix.freshMu.Unlock()
// The path of a merge's recompute: the modules, then the candidates their fingerprints reach (recomputeWith), then the // The path of a merge's recompute: the modules, then the candidates their fingerprints reach (recomputeWith), then the
// pending merges a changed fingerprint settles (PendingAgain, as after a toolchain report). // pending merges a changed fingerprint settles (PendingAgain, as after a toolchain report).
err = ix.recomputeWith(ctx, nil, nil) err = ix.recomputeWith(ctx, nil, nil, "")
ix.freshMu.Lock() ix.freshMu.Lock()
ix.fresh = nil ix.fresh = nil
ix.freshMu.Unlock() ix.freshMu.Unlock()
@@ -0,0 +1,325 @@
package index
import (
"context"
"encoding/json"
"errors"
"fmt"
"strings"
"sync/atomic"
"testing"
"time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-sdk/go/views"
)
// listedObjects counts the listings of its bucket.
type listedObjects struct {
*memObjects
n atomic.Int64
fail atomic.Bool
}
func (l *listedObjects) Names(ctx context.Context) ([]string, error) {
l.n.Add(1)
if l.fail.Load() {
return nil, errors.New("the bucket does not answer")
}
return l.memObjects.Names(ctx)
}
func keptDigest(name string) string {
return "sha256:" + fmt.Sprintf("%064x", len(name))
}
// A-487, A-487b, A-487c: the view of the trunk is read from the bucket once at the start and once for each heard event of a newer
// trunk revision; 100 reads between events list the bucket zero times.
func TestTheKeptTrunkIsReadAtTheStartAndOncePerHeardRevision(t *testing.T) {
objects := &listedObjects{memObjects: newMemObjects(0)}
keep := func(revision int64, modules ...string) {
t.Helper()
rev, entries := meshcatalog.CatalogRevision{Revision: revision, Kind: "trunk"}, map[string][]byte{}
for _, name := range modules {
data, _ := json.Marshal(meshcatalog.CatalogEntry{Module: name, Repository: catalogueRepo, None: "not built", ManifestDigest: keptDigest(name), Manifest: map[string]any{"module": name}})
digest := fmt.Sprintf("%d-%s", revision, name)
entries[digest] = data
rev.Entries = append(rev.Entries, meshcatalog.EntryRef{Module: name, Digest: digest})
}
if err := KeepRevision(context.Background(), objects, rev, entries); err != nil {
t.Fatal(err)
}
}
said := []string{}
kept := &KeptTrunk{Say: func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) }}
modules := TrunkStates{Objects: objects, Under: func(string) views.Source { return nil }, Kept: kept}.State("modules")
keep(1, "avahi")
if err := kept.Reread(context.Background(), objects); err != nil { // the start's read
t.Fatal(err)
}
listed := objects.n.Load()
for i := 0; i < 50; i++ {
if _, _, err := modules.Get("avahi"); err != nil {
t.Fatal(err)
}
if _, err := modules.Keys(); err != nil {
t.Fatal(err)
}
}
if got := objects.n.Load() - listed; got != 0 {
t.Errorf("100 reads listed the bucket %d times, want none", got)
}
// a newer revision kept is no reason to read: only the event says it
keep(2, "avahi", "ca-trust")
listed = objects.n.Load()
if keys, _ := modules.Keys(); len(keys) != 1 {
t.Errorf("the view holds %v before the event, want revision 1's one module", keys)
}
for _, e := range []meshcatalog.CatalogRevisionEvent{{Kind: "candidate", Revision: 9}, {Kind: "trunk", Branch: "other", Revision: 9}, {Kind: "trunk", Revision: 1}} {
if err := kept.Heard(context.Background(), objects, e); err != nil {
t.Errorf("%+v: %v", e, err)
}
}
if got := objects.n.Load() - listed; got != 0 {
t.Errorf("events of a candidate, another branch and the revision held listed the bucket %d times, want none", got)
}
if err := kept.Heard(context.Background(), objects, meshcatalog.CatalogRevisionEvent{Kind: "trunk", Branch: "main", Revision: 2}); err != nil {
t.Fatal(err)
}
if got := objects.n.Load() - listed; got != 1 {
t.Errorf("a heard catalog-revision listed the bucket %d times, want exactly one re-read", got)
}
if e, ok, err := modules.Get("ca-trust"); err != nil || !ok || e.Revision != 2 {
t.Errorf("after the event ca-trust = %+v %v (%v), want it at revision 2", e, ok, err)
}
// A-487c: the manifests by digest and the verb revision asked of the trunk's newest answer from the view held.
listed = objects.n.Load()
if raw, err := kept.Manifest(meshcatalog.BareManifestDigest(keptDigest("ca-trust"))); err != nil || raw != `{"module":"ca-trust"}` {
t.Errorf("the held manifest = %q (%v), want the entry's manifest", raw, err)
}
if a, err := Revision(context.Background(), objects, kept, "avahi", "avahi", meshcatalog.RevisionInput{}); err != nil || a.Revision.Revision != 2 || len(a.Entries) != 2 {
t.Errorf("revision of the newest answered %+v (%v), want revision 2 whole", a, err)
}
if got := objects.n.Load() - listed; got != 0 {
t.Errorf("the manifest and the verb revision listed the bucket %d times, want none", got)
}
if len(said) != 2 || !strings.Contains(said[1], "read revision 2 (was 1), 2 modules, in ") {
t.Errorf("said %q, want each read with its revision, module count and time", said)
}
// A-487b: a re-read that fails is said loudly, the view held stays, and the next heard event reads again.
keep(3, "avahi", "ca-trust", "dbus")
objects.fail.Store(true)
if err := kept.Heard(context.Background(), objects, meshcatalog.CatalogRevisionEvent{Kind: "trunk", Revision: 3}); err == nil {
t.Error("a re-read that failed was taken")
}
if last := said[len(said)-1]; !strings.Contains(last, "BEHIND") {
t.Errorf("a failed re-read said %q, want it said loudly", last)
}
if keys, _ := modules.Keys(); len(keys) != 2 {
t.Errorf("the view held after a failed re-read has %v, want revision 2's two modules", keys)
}
objects.fail.Store(false)
if err := kept.Heard(context.Background(), objects, meshcatalog.CatalogRevisionEvent{Kind: "trunk", Revision: 3}); err != nil {
t.Fatal(err)
}
if keys, _ := modules.Keys(); len(keys) != 3 {
t.Errorf("the next heard event left the view at %v, want revision 3's three modules", keys)
}
}
// A-487: the verb module reads the one module it is asked.
func TestTheVerbModuleReadsOneModule(t *testing.T) {
c := CatalogRead{
Modules: func() (map[string]meshcatalog.ModulesByModuleValue, error) {
return nil, errors.New("every module was read")
},
Module: func(name string) (meshcatalog.ModulesByModuleValue, bool, error) {
return meshcatalog.ModulesByModuleValue{Manifest: "d1"}, name == "avahi", nil
},
Manifest: func(string) (string, error) { return `{"module":"avahi"}`, nil },
}
if one, err := c.CatalogModuleAnswer("avahi"); err != nil || one.Module.Module != "avahi" {
t.Errorf("module avahi = %+v (%v)", one, err)
}
if _, err := c.CatalogModuleAnswer("nope"); err == nil || !strings.Contains(err.Error(), "holds no module nope") {
t.Errorf("module nope = %v, want the trunk holds none", err)
}
}
// goneUnderRead is a bucket whose pin judge runs while the first entry of the revision named newest is read: a newer revision
// is kept and the one named is removed, its list and its entries, as the indexer's judge does.
type goneUnderRead struct {
*memObjects
once atomic.Bool
prune func()
on string // when set, only the read of this entry runs the pruning
}
func (g *goneUnderRead) Get(ctx context.Context, name string) ([]byte, error) {
if strings.HasPrefix(name, "entry.") && (g.on == "" || name == g.on) && g.once.CompareAndSwap(false, true) {
g.prune()
}
return g.memObjects.Get(ctx, name)
}
// A revision removed while it is read (take-ins keep one every few seconds, each removing the last) is read again as the newest
// one, never failed: on novox the first view of the catalog half never came while revisions 487 to 493 were kept.
func TestTheKeptTrunkReadsTheNewerRevisionWhenTheOneNamedIsRemovedUnderIt(t *testing.T) {
mem := newMemObjects(0)
keep := func(revision int64) {
data, _ := json.Marshal(meshcatalog.CatalogEntry{Module: "avahi", Repository: catalogueRepo, None: "not built"})
digest := fmt.Sprintf("d%d", revision)
rev := meshcatalog.CatalogRevision{Revision: revision, Kind: "trunk", Entries: []meshcatalog.EntryRef{{Module: "avahi", Digest: digest}}}
if err := KeepRevision(context.Background(), mem, rev, map[string][]byte{digest: data}); err != nil {
t.Fatal(err)
}
}
keep(1)
objects := &goneUnderRead{memObjects: mem}
objects.prune = func() {
keep(2)
_ = mem.Delete(context.Background(), RevisionObject(meshcatalog.CatalogRevision{Revision: 1, Kind: "trunk"}))
_ = mem.Delete(context.Background(), EntryObject("d1"))
}
kept := &KeptTrunk{}
if err := kept.Reread(context.Background(), objects); err != nil {
t.Fatalf("the read failed when the revision named was removed under it: %v", err)
}
if rev, _, ok, err := kept.Newest(context.Background(), objects); err != nil || !ok || rev.Revision != 2 {
t.Errorf("the view holds revision %d (%v, %v), want 2", rev.Revision, ok, err)
}
}
// A-487d: the same revision failing twice is the read's failure, said as it is; a read by number never reads another.
func TestTheKeptTrunkFailsWhenTheSameRevisionIsGoneTwice(t *testing.T) {
mem := newMemObjects(0)
data, _ := json.Marshal(meshcatalog.CatalogEntry{Module: "avahi", Repository: catalogueRepo, None: "not built"})
rev := meshcatalog.CatalogRevision{Revision: 1, Kind: "trunk", Entries: []meshcatalog.EntryRef{{Module: "avahi", Digest: "d1"}}}
if err := KeepRevision(context.Background(), mem, rev, map[string][]byte{"d1": data}); err != nil {
t.Fatal(err)
}
_ = mem.Delete(context.Background(), EntryObject("d1")) // revision 1 stays listed, its entry lost, no newer revision
for name, read := range map[string]func() error{
"kept trunk": func() error { return (&KeptTrunk{}).Reread(context.Background(), mem) },
"TrunkView": func() error { _, _, _, err := TrunkView(context.Background(), mem); return err },
"Outdated": func() error {
_, err := outdatedOfAvahi(mem).Outdated(context.Background(), meshcatalog.OutdatedInput{})
return err
},
} {
if err := read(); !errors.Is(err, ErrNoObject) {
t.Errorf("%s: %v, want the lost entry said (no such object)", name, err)
}
}
}
// A-487d: BranchView and TrunkView name the newest again when the one named is removed under the read.
func TestTrunkViewReadsTheNewerRevisionWhenTheOneNamedIsRemovedUnderIt(t *testing.T) {
mem := newMemObjects(0)
keep := func(revision int64) {
data, _ := json.Marshal(meshcatalog.CatalogEntry{Module: "avahi", Repository: catalogueRepo, None: "not built"})
digest := fmt.Sprintf("d%d", revision)
rev := meshcatalog.CatalogRevision{Revision: revision, Kind: "trunk", Entries: []meshcatalog.EntryRef{{Module: "avahi", Digest: digest}}}
if err := KeepRevision(context.Background(), mem, rev, map[string][]byte{digest: data}); err != nil {
t.Fatal(err)
}
}
keep(1)
objects := &goneUnderRead{memObjects: mem}
objects.prune = func() {
keep(2)
_ = mem.Delete(context.Background(), RevisionObject(meshcatalog.CatalogRevision{Revision: 1, Kind: "trunk"}))
_ = mem.Delete(context.Background(), EntryObject("d1"))
}
if _, _, at, err := TrunkView(context.Background(), objects); err != nil || at != 2 {
t.Errorf("TrunkView read revision %d (%v), want 2", at, err)
}
}
// A-487d: every reader of the newest revision in the indexer program reads the newest again when the revision it named is
// removed under it (ModuleInNewestView, SeedEntryHeld, entryOf, SeedModulesFrom, FillVersions, the tool indexer_catalog_revision
// and the verb revision, trunk's and a watched branch's).
func TestTheNewestRevisionReadersReadAgainWhenTheRevisionIsRemovedUnderThem(t *testing.T) {
ctx := context.Background()
cases := map[string]struct {
branch string
read func(ix *Indexer) (bool, error)
}{
"ModuleInNewestView": {"", func(ix *Indexer) (bool, error) { return ix.ModuleInNewestView(ctx, "avahi") }},
"SeedEntryHeld": {"", func(ix *Indexer) (bool, error) { return ix.SeedEntryHeld(ctx, "avahi", "fp2") }},
"entryOf": {"", func(ix *Indexer) (bool, error) {
e, err := ix.entryOf(ctx, "avahi")
return e.Fp == "fp2", err
}},
"SeedModulesFrom": {"", func(ix *Indexer) (bool, error) { _, err := ix.SeedModulesFrom(ctx, nil, nil); return err == nil, err }},
"FillVersions": {"", func(ix *Indexer) (bool, error) {
ix.Versions, ix.Heard = MemoryStore(), MemoryStore()
err := ix.FillVersions(ctx, func(context.Context, string, string) (BuilderVersionsAnswer, error) {
return BuilderVersionsAnswer{}, nil
})
return err == nil, err
}},
"indexer_catalog_revision": {"", func(ix *Indexer) (bool, error) {
var tool map[string]any
for _, tl := range CatalogRevisionTools(ix.Revisions, nil, time.Now, nil) {
if tl.Name == CatalogRevisionTool {
got, err := tl.Run(map[string]any{})
if err != nil {
return false, err
}
tool = got.(map[string]any)
}
}
return tool != nil && tool["name"] == "trunk.2", nil
}},
"Outdated": {"", func(ix *Indexer) (bool, error) {
a, err := outdatedOfAvahi(ix.Revisions).Outdated(ctx, meshcatalog.OutdatedInput{})
return len(a.Behind) == 1 && a.Behind[0].Kind == 5 && strings.Contains(a.Behind[0].Newest, "fp2"), err
}},
"Revision": {"", func(ix *Indexer) (bool, error) {
a, err := Revision(ctx, ix.Revisions, nil, "avahi", "avahi", meshcatalog.RevisionInput{})
return a.Revision != nil && a.Revision.Revision == 2, err
}},
"Revision of a branch": {"dev", func(ix *Indexer) (bool, error) {
a, err := Revision(ctx, ix.Revisions, nil, "avahi", "avahi", meshcatalog.RevisionInput{Branch: "dev"})
return a.Revision != nil && a.Revision.Revision == 2 && len(a.Entries) == 1 && a.Entries[0].Fp == "fp2", err
}},
}
for name, c := range cases {
t.Run(name, func(t *testing.T) {
mem := newMemObjects(0)
keep := func(branch string, revision int64) {
data, _ := json.Marshal(meshcatalog.CatalogEntry{Module: "avahi", Repository: catalogueRepo, None: "not built", Fp: fmt.Sprintf("fp%d", revision)})
digest := fmt.Sprintf("%s-d%d", branch, revision)
rev := meshcatalog.CatalogRevision{Branch: branch, Revision: revision, Kind: "trunk", Entries: []meshcatalog.EntryRef{{Module: "avahi", Digest: digest}}}
if err := KeepRevision(ctx, mem, rev, map[string][]byte{digest: data}); err != nil {
t.Fatal(err)
}
}
keep("", 1)
if c.branch != "" { // the trunk stays at revision 1; the branch's revision 1 is removed under the read
keep("", 2)
keep(c.branch, 1)
}
objects := &goneUnderRead{memObjects: mem}
pruned := c.branch
objects.on = EntryObject(fmt.Sprintf("%s-d1", pruned))
objects.prune = func() {
keep(pruned, 2)
_ = mem.Delete(ctx, RevisionObject(meshcatalog.CatalogRevision{Branch: pruned, Revision: 1, Kind: "trunk"}))
_ = mem.Delete(ctx, objects.on)
}
if ok, err := c.read(&Indexer{Revisions: objects}); err != nil || !ok {
t.Errorf("answered %v (%v), want revision 2's entry", ok, err)
}
})
}
}
// outdatedOfAvahi is an indexer whose state versions holds avahi at a kind-5 edge, so Outdated reads the trunk's entry of avahi.
func outdatedOfAvahi(o Objects) *Indexer {
ix := &Indexer{Versions: MemoryStore(), Heard: MemoryStore(), Revisions: o}
data, _ := json.Marshal(meshcatalog.VersionEntry{V: 1, Module: "avahi", Version: "0.1.1", Edges: []meshcatalog.VersionEdge{{Kind: 5, Part: "actor:program"}}})
_ = ix.Versions.Create(context.Background(), meshcatalog.VersionsByModuleVersionKey("avahi", "0.1.1"), data)
return ix
}
@@ -144,7 +144,7 @@ func (ix *Indexer) applyMoving(ctx context.Context, f forgemsg.Forge, drop bool,
ix.logf("%s at %s: its packages are not published: %v", f.Repository, f.Merge, err) ix.logf("%s at %s: its packages are not published: %v", f.Repository, f.Merge, err)
} }
// What this merge moved reaches the modules standing on it (a base's tree, build-agent's fingerprint), in the graph's order. // What this merge moved reaches the modules standing on it (a base's tree, build-agent's fingerprint), in the graph's order.
return w, ix.recomputeWith(ctx, overlay, w) return w, ix.recomputeWith(ctx, overlay, w, f.Repository)
} }
// baseFP answers the new-form fingerprint of a base module as modules.<base> holds it, or why it has none. // baseFP answers the new-form fingerprint of a base module as modules.<base> holds it, or why it has none.
@@ -3,7 +3,6 @@ package index
import ( import (
"context" "context"
"errors" "errors"
"fmt"
"slices" "slices"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
@@ -51,14 +50,10 @@ func (ix *Indexer) SeedEntryHeld(ctx context.Context, module, fp string) (bool,
if ctx.Err() != nil || ix.Revisions == nil { if ctx.Err() != nil || ix.Revisions == nil {
return false, ctx.Err() return false, ctx.Err()
} }
rev, found, err := newestTrunk(ctx, ix.Revisions) _, entries, found, err := readNewestOf(ctx, ix.Revisions, TrunkBranch)
if err != nil || !found { if err != nil || !found {
return false, err return false, err
} }
_, entries, err := ReadRevision(ctx, ix.Revisions, fmt.Sprintf("trunk.%d", rev.Revision))
if err != nil {
return false, err
}
return slices.ContainsFunc(entries, func(e meshcatalog.CatalogEntry) bool { return e.Module == module && e.Fp == fp }), nil return slices.ContainsFunc(entries, func(e meshcatalog.CatalogEntry) bool { return e.Module == module && e.Fp == fp }), nil
} }
@@ -67,14 +62,10 @@ func (ix *Indexer) ModuleInNewestView(ctx context.Context, module string) (bool,
if ctx.Err() != nil || ix.Revisions == nil { if ctx.Err() != nil || ix.Revisions == nil {
return false, ctx.Err() return false, ctx.Err()
} }
rev, found, err := newestTrunk(ctx, ix.Revisions) _, entries, found, err := readNewestOf(ctx, ix.Revisions, TrunkBranch)
if err != nil || !found { if err != nil || !found {
return false, err return false, err
} }
_, entries, err := ReadRevision(ctx, ix.Revisions, fmt.Sprintf("trunk.%d", rev.Revision))
if err != nil {
return false, err
}
return slices.ContainsFunc(entries, func(e meshcatalog.CatalogEntry) bool { return e.Module == module }), nil return slices.ContainsFunc(entries, func(e meshcatalog.CatalogEntry) bool { return e.Module == module }), nil
} }
@@ -13,7 +13,6 @@ import (
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein"
@@ -23,13 +22,10 @@ import (
"git.novox.be/novox/mesh-sdk/go/process" "git.novox.be/novox/mesh-sdk/go/process"
"mesh-delivery-indexer/judge" "mesh-delivery-indexer/judge"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/graph"
) )
// The take-in step (novox/hq ADR 0633): a source revision is taken in only by the indexer it names. A merged message is // The take-in step (novox/hq ADR 0633, A-476): a merged message is kept in the indexer's own state pending, one key per
// kept in the indexer's own state pending, one key per merge, until the take-in that ends it; a revision naming another // merge, until the take-in that ends it; a revision is taken in by the running indexer whatever build of the indexer it names.
// build of the indexer writes only what building and moving that indexer needs, and the rest stays pending for it.
const ( const (
// PendingState is the indexer's state of held merges: the bucket mesh-delivery-indexer_pending, keyed // PendingState is the indexer's state of held merges: the bucket mesh-delivery-indexer_pending, keyed
// takein.PendingKey, so the key order is the merge order. An old `state` entry (the indexer is foundation, ADR // takein.PendingKey, so the key order is the merge order. An old `state` entry (the indexer is foundation, ADR
@@ -48,10 +44,6 @@ var errClaimLost = errors.New("another take-in claimed the merges first, so this
// runs such a build, so the revision waits until a later merge gives it one or a hand take-in (indexer_take_in_pending). // runs such a build, so the revision waits until a later merge gives it one or a hand take-in (indexer_take_in_pending).
const NoBuild = "no build: " const NoBuild = "no build: "
// ErrOwnBuildUnknown is what a merged message answers while the indexer cannot learn the build it runs (ADR 0633 §1):
// the message is left unacknowledged, so the bus delivers it again, and nothing is written.
var ErrOwnBuildUnknown = errors.New("the build of the indexer this program runs is not known")
// PendingRef is what a pending value keeps of its merged message instead of a copy (a merge of a repository with many // PendingRef is what a pending value keeps of its merged message instead of a copy (a merge of a repository with many
// modules is about 0.5 MB, over the state's 256 KiB value cap; issue 1585): the small fields the waits are worked out // modules is about 0.5 MB, over the state's 256 KiB value cap; issue 1585): the small fields the waits are worked out
// from. The message itself is read from the forge bridge's stream by Pending.Seq when a take-in needs it. // from. The message itself is read from the forge bridge's stream by Pending.Seq when a take-in needs it.
@@ -98,8 +90,6 @@ type Pending struct {
// Held is why a ready merge is not taken in although its revision names this build: objects full (HeldObjectsFull), // Held is why a ready merge is not taken in although its revision names this build: objects full (HeldObjectsFull),
// until room is said (RoomAgain). // until room is said (RoomAgain).
Held string `json:"held,omitempty"` Held string `json:"held,omitempty"`
// Validation is the revision's validation (design 47 G §713, A-247), on the revision's newest value alone.
Validation *takein.Validation `json:"validation,omitempty"`
} }
// PendingEntry is one key of the pending state as read: its key, revision, value and the merged message decoded. For a // PendingEntry is one key of the pending state as read: its key, revision, value and the merged message decoded. For a
@@ -389,26 +379,13 @@ func (ix *Indexer) pendingOnBus(ctx context.Context) bool {
} }
// ownBuildNow is the build this program runs, as FixOwnBuild fixed it at the start: empty while unknown, and never asked again. // ownBuildNow is the build this program runs, as FixOwnBuild fixed it at the start: empty while unknown, and never asked again.
// It names only the writer of what is written (LastWriter); nothing waits for it (A-476 b).
func (ix *Indexer) ownBuildNow(context.Context) string { func (ix *Indexer) ownBuildNow(context.Context) string {
ix.ownMu.Lock() ix.ownMu.Lock()
defer ix.ownMu.Unlock() defer ix.ownMu.Unlock()
return ix.ownFP return ix.ownFP
} }
// sayOwnBuildUnknown says, for a merge this process cannot take in because it does not know its own build (ADR 0633 §1), a
// take-in.pending that names why and since when, so the watchman's take-in-waits bound covers it. The merge is not written to
// pending; the stream delivers it again to the process that knows its build.
func (ix *Indexer) sayOwnBuildUnknown(ctx context.Context, seq uint64, f forgemsg.Forge) {
ix.ownMu.Lock()
since, why := ix.ownSince, ix.ownWhy
ix.ownMu.Unlock()
if since.IsZero() { // never fixed: the start did not run (no engine wired)
since, why = ix.now(), "the build this program runs was never asked"
}
ix.sayTakeIn(ctx, takein.TakeIn{Kind: takein.TakeInPending, Key: takein.PendingKey(seq), What: takein.TakeInRevision,
Merges: []takein.Merge{{Repository: f.Repository, Number: f.Number, Merge: f.Merge}}, Since: since, Unknown: why})
}
// sayTakeIn publishes a take-in event and keeps what it said on the entries, so indexer_pending reads the last event of // sayTakeIn publishes a take-in event and keeps what it said on the entries, so indexer_pending reads the last event of
// each wait from the state. A publish that fails is said in the log: the next evaluation says it again. // each wait from the state. A publish that fails is said in the log: the next evaluation says it again.
func (ix *Indexer) sayTakeIn(ctx context.Context, t takein.TakeIn) bool { func (ix *Indexer) sayTakeIn(ctx context.Context, t takein.TakeIn) bool {
@@ -456,10 +433,6 @@ func (ix *Indexer) Arrive(ctx context.Context, seq uint64, f forgemsg.Forge) (Ap
return a, "", err return a, "", err
} }
own := ix.ownBuildNow(ctx) own := ix.ownBuildNow(ctx)
if own == "" {
ix.sayOwnBuildUnknown(ctx, seq, f)
return Applied{}, "", ErrOwnBuildUnknown
}
ix.pendingMu.Lock() ix.pendingMu.Lock()
defer ix.pendingMu.Unlock() defer ix.pendingMu.Unlock()
key := takein.PendingKey(seq) key := takein.PendingKey(seq)
@@ -502,9 +475,6 @@ func (ix *Indexer) EndGroup(ctx context.Context, end forgemsg.ForgeGroupEnd) (Ap
return Applied{}, "no pending state: the group's merges were taken in one by one", nil return Applied{}, "no pending state: the group's merges were taken in one by one", nil
} }
own := ix.ownBuildNow(ctx) own := ix.ownBuildNow(ctx)
if own == "" {
return Applied{}, "", ErrOwnBuildUnknown
}
ix.pendingMu.Lock() ix.pendingMu.Lock()
defer ix.pendingMu.Unlock() defer ix.pendingMu.Unlock()
entries, err := ReadPending(ctx, ix.Pending, ix.logf) entries, err := ReadPending(ctx, ix.Pending, ix.logf)
@@ -677,21 +647,15 @@ func (ix *Indexer) takenAlready(ctx context.Context, m forgemsg.GroupMerge) (boo
} }
// TakeInAtStart is the take-in at the start (ADR 0633 §2), before the consumer on the forge bridge's stream starts: // TakeInAtStart is the take-in at the start (ADR 0633 §2), before the consumer on the forge bridge's stream starts:
// it evaluates pending once with the build this program runs, taking in the whole revisions that name this build; a // it evaluates pending once, taking in the whole revisions whatever build of the indexer they name (A-476 b); a group
// group still being gathered stays, and a revision naming yet another build is held for that one. A claim left by an // still being gathered stays. A claim left by an indexer that ran before is void here. The build this program runs, known
// indexer that ran before is void here. When the build is not known (the node-engine did not answer), it waits for // or not, only names the writer. No timer.
// nothing: it says so and the consumer starts all the same; a merged message is then left unacknowledged until the
// build is known (clause 1), and its redelivery, or a toolchain report (ToolchainReported), asks again. No timer.
func (ix *Indexer) TakeInAtStart(ctx context.Context) (Applied, string, error) { func (ix *Indexer) TakeInAtStart(ctx context.Context) (Applied, string, error) {
if ix.Pending == nil || !ix.pendingOnBus(ctx) { if ix.Pending == nil || !ix.pendingOnBus(ctx) {
// Nothing can wait in a state not on the bus: the start asks no build and does not wait for one. // Nothing can wait in a state not on the bus: the start asks no build and does not wait for one.
return Applied{}, "the pending state is not on the bus yet: nothing waits", nil return Applied{}, "the pending state is not on the bus yet: nothing waits", nil
} }
own := ix.ownBuildNow(ctx) own := ix.ownBuildNow(ctx)
if own == "" {
ix.logf("take-in: the build this program runs is not known at its start, so nothing pending is taken in now; the forge consumer starts, and a merged message left unacknowledged asks it again")
return Applied{}, "the build this program runs is not known yet: nothing pending is taken in at the start", nil
}
ix.pendingMu.Lock() ix.pendingMu.Lock()
defer ix.pendingMu.Unlock() defer ix.pendingMu.Unlock()
a, words, err := ix.evaluate(ctx, own, true) a, words, err := ix.evaluate(ctx, own, true)
@@ -699,8 +663,8 @@ func (ix *Indexer) TakeInAtStart(ctx context.Context) (Applied, string, error) {
return a, "the pending state is not on the bus yet: nothing waits", nil return a, "the pending state is not on the bus yet: nothing waits", nil
} }
if err == nil && len(a.Written) == 0 { if err == nil && len(a.Written) == 0 {
// A start that takes nothing in says so, with the build it learnt (the wait's own build is in the line holdFor says). // A start that takes nothing in says so.
ix.logf("take-in: at the start, this program runs build %s and took nothing in: %s", own, words) ix.logf("take-in: at the start, this program (build %q) took nothing in: %s", own, words)
} }
return a, words, err return a, words, err
} }
@@ -731,16 +695,12 @@ func (ix *Indexer) UnreadAgain(ctx context.Context) error {
} }
// PendingAgain evaluates pending again after something a fingerprint is computed from changed, a toolchain report // PendingAgain evaluates pending again after something a fingerprint is computed from changed, a toolchain report
// (ADR 0633 §1, §3), or the verdicts state bound (issue 1966): a revision that now names the running build is taken in, and one that names another build is held // (ADR 0633 §3), or the verdicts state bound (issue 1966): what waits is taken in. It does nothing while the pending state is not on the bus.
// for that one. It does nothing while the pending state is not on the bus or the build this program runs is not known.
func (ix *Indexer) PendingAgain(ctx context.Context) error { func (ix *Indexer) PendingAgain(ctx context.Context) error {
if ix.Pending == nil || !ix.pendingOnBus(ctx) { if ix.Pending == nil || !ix.pendingOnBus(ctx) {
return nil return nil
} }
own := ix.ownBuildNow(ctx) own := ix.ownBuildNow(ctx)
if own == "" {
return nil
}
ix.pendingMu.Lock() ix.pendingMu.Lock()
defer ix.pendingMu.Unlock() defer ix.pendingMu.Unlock()
_, words, err := ix.evaluate(ctx, own, false) _, words, err := ix.evaluate(ctx, own, false)
@@ -786,31 +746,37 @@ func (ix *Indexer) evaluate(ctx context.Context, own string, start bool) (Applie
if len(lost) > 0 { if len(lost) > 0 {
return Applied{}, ix.waitingLine(w), ix.holdLost(ctx, w.Ready, lost, reasons, unread) return Applied{}, ix.waitingLine(w), ix.holdLost(ctx, w.Ready, lost, reasons, unread)
} }
if wait, words, err := ix.validationGate(ctx, ready, own); wait || err != nil { dry, err := ix.computeOver(ctx, ready)
return Applied{}, words, err
}
names, dry, err := ix.computeOver(ctx, ready)
if err != nil { if err != nil {
return Applied{}, "", err return Applied{}, "", err
} }
if words, refused, err := ix.refuseCycle(ctx, ready, dry); refused || err != nil { if words, refused, err := ix.refuseCycle(ctx, ready, dry); refused || err != nil {
return Applied{}, words, err return Applied{}, words, err
} }
if names == own { // The indexer takes in with the build it runs, whatever build of the indexer the revision names (A-476 b).
a, err := ix.takeInWhole(ctx, ready, own, "") a, err := ix.takeInWhole(ctx, ready, own, "")
if errors.Is(err, errClaimLost) { if errors.Is(err, errClaimLost) {
return a, "claimed by another take-in", nil return a, "claimed by another take-in", nil
}
if errors.Is(err, errVerdictsNotYet) {
return a, "the verdicts state is not on the bus yet: the revision waits for it, taken in when it binds", nil
}
if errors.Is(err, ErrObjectsFull) {
ix.holdFull(ctx, ready)
return a, ix.waitingLine(w) + "; held: " + HeldObjectsFull, nil
}
return a, ix.waitingLine(w), err
} }
return Applied{}, ix.waitingLine(w), ix.holdFor(ctx, ready, names, own, dry) if errors.Is(err, errVerdictsNotYet) {
return a, "the verdicts state is not on the bus yet: the revision waits for it, taken in when it binds", nil
}
if errors.Is(err, ErrObjectsFull) {
ix.holdFull(ctx, ready)
return a, ix.waitingLine(w) + "; held: " + HeldObjectsFull, nil
}
return a, ix.waitingLine(w), err
}
// newestOfRevision is the index in ready of the revision's newest merge.
func newestOfRevision(ready []PendingEntry) int {
n := 0
for i, e := range ready {
if e.Value.Seq > ready[n].Value.Seq {
n = i
}
}
return n
} }
// refuseCycle refuses the revision whose catalog graph would hold a cycle with a build edge the take-in refuses (design 47 G // refuseCycle refuses the revision whose catalog graph would hold a cycle with a build edge the take-in refuses (design 47 G
@@ -951,7 +917,6 @@ func (ix *Indexer) dry() *dryRun {
x := &Indexer{Module: ix.Module, Manifests: ix.Manifests, Packages: ix.Packages, JudgeID: ix.JudgeID, Files: ix.Files, GoSum: ix.GoSum, x := &Indexer{Module: ix.Module, Manifests: ix.Manifests, Packages: ix.Packages, JudgeID: ix.JudgeID, Files: ix.Files, GoSum: ix.GoSum,
GoMod: ix.GoMod, Now: ix.Now, Builds: ix.Builds, Named: ix.Named, Trunk: ix.Trunk, BranchHead: ix.BranchHead, Read: ix.Read, Contains: ix.Contains, GoMod: ix.GoMod, Now: ix.Now, Builds: ix.Builds, Named: ix.Named, Trunk: ix.Trunk, BranchHead: ix.BranchHead, Read: ix.Read, Contains: ix.Contains,
ReadOpen: nil, OwnBuild: nil, Message: ix.Message, SayRefused: nil, SayAccepted: nil, SayTakeIn: nil, Pending: nil, ReadOpen: nil, OwnBuild: nil, Message: ix.Message, SayRefused: nil, SayAccepted: nil, SayTakeIn: nil, Pending: nil,
ValidationAsks: nil, AskVerb: nil, // a dry run asks no validation (§713)
// The registry milestone's fields: what a take-in would write on the bus stays out of a dry run, so the states of the // The registry milestone's fields: what a take-in would write on the bus stays out of a dry run, so the states of the
// requirements, findings, seat definitions, holds and their bounds are not overlaid, nothing is said, and no hold is // requirements, findings, seat definitions, holds and their bounds are not overlaid, nothing is said, and no hold is
// weighed (Holds and Counted nil hold nothing); ReadFile only reads. // weighed (Holds and Counted nil hold nothing); ReadFile only reads.
@@ -991,16 +956,15 @@ func (ix *Indexer) dry() *dryRun {
} }
// computeOver applies the ready merges, in merge order, to a dry indexer, as one overlay: each later merge's trees // computeOver applies the ready merges, in merge order, to a dry indexer, as one overlay: each later merge's trees
// govern (ADR 0633 §3). It answers the fingerprint the indexer's own module has once they are applied (none is "", // govern (ADR 0633 §3). It answers the dry run, whose overlays hold what the take-in would write. A merge that can never be read
// with its reason), and the dry run, whose overlays hold what the take-in would write. A merge that can never be read
// is left out, as its take-in will drop it. // is left out, as its take-in will drop it.
func (ix *Indexer) computeOver(ctx context.Context, ready []PendingEntry) (string, *dryRun, error) { func (ix *Indexer) computeOver(ctx context.Context, ready []PendingEntry) (*dryRun, error) {
d := ix.dry() d := ix.dry()
// The manifests are judged against the seat definitions the revision carries (seatsof.go, ADR 0633 §1), so the indexer // The manifests are judged against the seat definitions the revision carries (seatsof.go, ADR 0633 §1), so the indexer
// it names is not refused for a verb or an event the built-in list lacks. // it names is not refused for a verb or an event the built-in list lacks.
seats, err := ix.revisionSeats(ctx, ready) seats, err := ix.revisionSeats(ctx, ready)
if err != nil { if err != nil {
return "", d, fmt.Errorf("reading the seat definitions of %s: %w", ready[0].Key, err) return d, fmt.Errorf("reading the seat definitions of %s: %w", ready[0].Key, err)
} }
var applyErr error var applyErr error
judge.WithSeats(seats, func() { judge.WithSeats(seats, func() {
@@ -1018,24 +982,9 @@ func (ix *Indexer) computeOver(ctx context.Context, ready []PendingEntry) (strin
} }
}) })
if applyErr != nil { if applyErr != nil {
return "", d, applyErr return d, applyErr
} }
key, _ := meshcatalogclient.ModulesKey(ix.Module) return d, nil
value, _, found, err := d.ix.Modules.Get(ctx, key)
if err != nil {
return "", d, err
}
if !found {
return NoBuild + "the indexer holds no record of itself", d, nil
}
m, err := meshcatalogclient.DecodeModule(value)
if err != nil {
return "", d, nil
}
if m.Fp2 == "" {
return NoBuild + m.None2, d, nil
}
return m.Fp2, d, nil
} }
// claim marks the entries as taken in by this take-in, by compare-and-set on the revisions read. A lost one (another // claim marks the entries as taken in by this take-in, by compare-and-set on the revisions read. A lost one (another
@@ -1238,8 +1187,7 @@ func (ix *Indexer) nextSource(branch string) int64 {
// takenWaits are the taken events of a take-in: every wait the entries were said under (saidWaits), every wait // takenWaits are the taken events of a take-in: every wait the entries were said under (saidWaits), every wait
// indexer_pending names the claimed entries by now (WaitsOf: a revision by its oldest merge's key, a group by its g. key) and // indexer_pending names the claimed entries by now (WaitsOf: a revision by its oldest merge's key, a group by its g. key) and
// the key of every claimed entry, since indexer_pending named it by that key earlier when it waited behind a group. So a wait // the key of every claimed entry, since indexer_pending named it by that key earlier when it waited behind a group. So a wait
// the watchman learned of without the indexer marking it (seeded from indexer_pending, or said by the own-build-unknown // the watchman learned of without the indexer marking it (seeded from indexer_pending) ends too. No key is said twice.
// path) ends too (design 47, Taken in by the indexer it names). No key is said twice.
func takenWaits(entries []PendingEntry) []takein.TakeIn { func takenWaits(entries []PendingEntry) []takein.TakeIn {
out := saidWaits(entries) out := saidWaits(entries)
said := map[string]bool{} said := map[string]bool{}
@@ -1298,83 +1246,6 @@ func saidWaits(entries []PendingEntry) []takein.TakeIn {
return out return out
} }
// holdFor keeps the ready merges pending for the indexer build they name and writes only what building and moving
// that indexer needs (ADR 0633 §1): the indexer's own module entry, its key of the module graph and the verdict on its
// manifest, from the dry run; the manifests and the packages are written by the dry run itself. The revision is said
// pending under its oldest merge's key, and a wait it was said under before by another key is said taken.
func (ix *Indexer) holdFor(ctx context.Context, ready []PendingEntry, names, own string, d *dryRun) error {
key, _ := meshcatalogclient.ModulesKey(ix.Module)
if err := copyKey(ctx, d.modules, ix.Modules, key); err != nil {
return fmt.Errorf("the indexer's own module entry: %w", err)
}
if gk, err := graph.Key(ix.Module); err == nil {
if err := copyKey(ctx, d.graph, ix.Graph, gk); err != nil && !errors.Is(err, process.ErrNotYet) {
return fmt.Errorf("the indexer's key of the module graph: %w", err)
}
}
// Why the manifest of the indexer the revision names was refused, when it was (judged with the seat definitions the
// revision carries): said with the wait, as a merge the forge bridge could not give is (holdLost).
refused := ""
if value, _, found, err := d.ix.Modules.Get(ctx, key); err == nil && found {
if m, err := meshcatalogclient.DecodeModule(value); err == nil && m.Manifest != "" {
if vk, err := meshcatalogclient.VerdictsKey(m.Manifest); err == nil {
if err := copyKey(ctx, d.verdicts, ix.Verdicts, vk); err != nil && !errors.Is(err, process.ErrNotYet) {
return fmt.Errorf("the verdict on the indexer's manifest: %w", err)
}
if vv, _, ok, err := d.ix.Verdicts.Get(ctx, vk); err == nil && ok {
if v, err := meshcatalogclient.DecodeVerdict(vv); err == nil && meshcatalogclient.VerdictRefuses(v) {
refused = strings.Join(v.Problems, "; ")
}
}
}
}
}
// The indexer's own entry the held revision names is kept as the trunk revision after the newest, every other entry as
// it was (milestone 47/G W4.7: a merged revision naming a newer indexer replaces only the indexer's entry).
if err := ix.KeepView(ctx); err != nil {
return fmt.Errorf("the revision with the indexer's own entry: %w", err)
}
t := revisionWait(takein.TakeInPending, ready, names)
t.Refused = refused
var ended, unmarked []PendingEntry
said, changed := false, false
for _, e := range ready {
switch {
case e.Value.Said == t.Key && e.Value.SaidKind == takein.TakeInPending:
said = true
changed = changed || e.Value.WaitsFor != names || e.Value.Refused != refused
if e.Value.WaitsFor != names || e.Value.Refused != refused {
unmarked = append(unmarked, e)
}
case e.Value.Said != "" && e.Value.SaidKind == takein.TakeInPending:
ended = append(ended, e)
unmarked = append(unmarked, e)
default:
unmarked = append(unmarked, e)
}
}
for _, old := range saidWaits(ended) {
ix.sayTakeIn(ctx, old)
}
if said && !changed {
// Nothing to say on the bus, but the start decided nothing and a reader of the journal sees why: the build learnt, the build the wait names.
ix.logf("take-in: %d merges, oldest %s, still name the indexer build %s, this program runs %s: they wait, as already said", len(ready), t.Key, names, own)
}
if !said || changed {
ix.logf("take-in: %d merges, oldest %s, name the indexer build %s, not %s, which runs: they wait for it", len(ready), t.Key, names, own)
if refused != "" {
ix.logf("take-in: the manifest of the indexer they name is refused, judged with the seat definitions they carry, so no such indexer runs: %s", refused)
}
if !ix.sayTakeIn(ctx, t) {
ix.markSaid(ctx, unmarked, nil, func(p *Pending) { p.WaitsFor, p.LastWriter, p.Refused = names, own, refused })
return nil
}
unmarked = ready
}
ix.markSaid(ctx, unmarked, &t, func(p *Pending) { p.WaitsFor, p.LastWriter, p.Refused = names, own, refused })
return nil
}
// copyKey writes what the overlay holds of key to the store, when it wrote or deleted it. // copyKey writes what the overlay holds of key to the store, when it wrote or deleted it.
func copyKey(ctx context.Context, from *overlayStore, to Store, key string) error { func copyKey(ctx context.Context, from *overlayStore, to Store, key string) error {
if from == nil || to == nil { if from == nil || to == nil {
@@ -1439,9 +1310,6 @@ func waitOf(entries []PendingEntry, what, behind string) takein.PendingWait {
if e.Value.Refused != "" { if e.Value.Refused != "" {
pw.Refused = e.Value.Refused pw.Refused = e.Value.Refused
} }
if e.Value.Validation != nil {
pw.Validation = e.Value.Validation
}
if e.Value.Said != "" && e.Value.SaidAt.After(pw.SaidAt) { if e.Value.Said != "" && e.Value.SaidAt.After(pw.SaidAt) {
pw.Said, pw.SaidAt = "take-in."+e.Value.SaidKind, e.Value.SaidAt pw.Said, pw.SaidAt = "take-in."+e.Value.SaidKind, e.Value.SaidAt
} }
@@ -1450,9 +1318,9 @@ func waitOf(entries []PendingEntry, what, behind string) takein.PendingWait {
return pw return pw
} }
// TakeInPendingByHand is the hand take-in (indexer_take_in_pending, ADR 0633 §5): the wait named by its key, the // TakeInPendingByHand is the hand take-in (indexer_take_in_pending, ADR 0633 §5, A-476, principle 4): the pending revision (its
// ready revision (its m. key) or a group gathered so far (its g. key), is taken in with this indexer and its judge, // m. key) or the group gathered so far (its g. key) is taken in now with this indexer and its judge,
// whatever build it names, for when the named indexer can never run or the group's end never comes. by and why are // for a revision whose message is unread or lost or held for room, or a group whose end never comes. by and why are
// said in the journal and on the take-in.taken event. What waited behind it is evaluated again after. // said in the journal and on the take-in.taken event. What waited behind it is evaluated again after.
func (ix *Indexer) TakeInPendingByHand(ctx context.Context, key, by, why string) TakenIn { func (ix *Indexer) TakeInPendingByHand(ctx context.Context, key, by, why string) TakenIn {
r := TakenIn{Module: key} r := TakenIn{Module: key}
@@ -106,6 +106,16 @@ func (b *takeInBed) merge(number int, c, avahi, indexer string) forgemsg.Forge {
return f return f
} }
// arriveHeld arrives f while the verdicts state is not on the bus, so the take-in holds it (ADR 0668 §4): the way a merge is
// left pending now that no merge waits for the indexer build it names (A-476 b).
func (b *takeInBed) arriveHeld(t *testing.T, seq uint64, f forgemsg.Forge) {
t.Helper()
bound := b.ix.Verdicts
b.ix.Verdicts = &memStore{unbound: process.ErrNotYet}
defer func() { b.ix.Verdicts = bound }()
b.arrive(t, seq, f)
}
func (b *takeInBed) arrive(t *testing.T, seq uint64, f forgemsg.Forge) { func (b *takeInBed) arrive(t *testing.T, seq uint64, f forgemsg.Forge) {
t.Helper() t.Helper()
b.publish(t, f) b.publish(t, f)
@@ -176,63 +186,21 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633)
}) })
} }
}) })
t.Run("T6 a merge while the own build is unknown writes nothing", func(t *testing.T) { t.Run("T6 a merge while the own build is unknown is taken in all the same (A-476 b)", func(t *testing.T) {
b := newTakeInBed(t) b := newTakeInBed(t)
b.ix.ownFP, b.ix.ownFixed = "", false // this process is started again and asks the engine, which does not answer b.ix.ownFP, b.ix.ownFixed = "", false // this process is started again and asks the engine, which does not answer
b.ix.OwnBuild = func(context.Context) (string, error) { return "", errors.New("the engine does not answer") } b.ix.OwnBuild = func(context.Context) (string, error) { return "", errors.New("the engine does not answer") }
b.ix.FixOwnBuild(ctx) b.ix.FixOwnBuild(ctx)
b.said = nil b.said = nil
_, _, err := b.ix.Arrive(ctx, 1, b.merge(20, sha('b'), "tree-avahi-2", "")) if _, _, err := b.ix.Arrive(ctx, 1, b.merge(20, sha('b'), "tree-avahi-2", "tree-indexer-2")); err != nil || len(b.keys()) != 0 || b.commit(t, "avahi") != sha('b') {
if !errors.Is(err, ErrOwnBuildUnknown) || len(b.keys()) != 0 || b.commit(t, "avahi") == sha('b') { t.Errorf("err %v, pending %v, avahi at %s: want the merge taken in at once", err, b.keys(), b.commit(t, "avahi"))
t.Errorf("err %v, pending %v, avahi at %s: want ErrOwnBuildUnknown and nothing written", err, b.keys(), b.commit(t, "avahi"))
} }
// The merge is said pending with why and since when (so the watchman's bound covers it), and indexer_pending says the if len(b.said) != 1 || b.said[0].Kind != takein.TakeInTaken {
// build is unknown, from the own-build state. t.Errorf("said %+v, want one take-in.taken and no wait said", b.said)
if len(b.said) != 1 || b.said[0].Kind != takein.TakeInPending || b.said[0].Key != takein.PendingKey(1) || !strings.Contains(b.said[0].Unknown, "does not answer") ||
b.said[0].Since.IsZero() || b.said[0].WaitsFor != "" {
t.Errorf("said %+v, want one take-in.pending of m.1 saying why the own build is unknown", b.said)
} else if _, err := takein.EncodeTakeIn(b.said[0]); err != nil {
t.Errorf("the unknown-build event is refused by its own contract: %v", err)
} }
answer, err := PendingTools(b.pending, b.ix.OwnBuildState, nil, nil)[0].Run(map[string]any{}) answer, err := PendingTools(b.pending, b.ix.OwnBuildState, nil, nil)[0].Run(map[string]any{})
if a, _ := answer.(takein.PendingAnswer); err != nil || !strings.Contains(a.OwnBuildUnknown, "does not answer") || a.OwnBuildSince.IsZero() { if a, _ := answer.(takein.PendingAnswer); err != nil || a.OwnBuild != "" || a.OwnBuildSince.IsZero() || len(a.Waits) != 0 {
t.Errorf("indexer_pending = %+v %v, want the build unknown with why and since", answer, err) t.Errorf("indexer_pending = %+v %v, want no wait and no known build", answer, err)
}
// Before the pending state is on the bus nothing waits: the start asks no build, and the merge is taken in at once,
// as before ADR 0633, though the build is unknown (TestIndexerWhole found the start waiting on it for ever).
b.ix.Pending = &memStore{unbound: process.ErrNotYet}
if _, _, err := b.ix.TakeInAtStart(ctx); err != nil {
t.Fatal(err)
}
if _, _, err := b.ix.Arrive(ctx, 1, b.merge(20, sha('b'), "tree-avahi-2", "")); err != nil || b.commit(t, "avahi") != sha('b') {
t.Errorf("no pending state: err %v, avahi at %s: want the merge taken in at once", err, b.commit(t, "avahi"))
}
})
t.Run("1966/AC195-2 AC195-3 a merge said pending with its build unknown and taken in on redelivery ends that wait under its key; a revision wait is said once", func(t *testing.T) {
b := newTakeInBed(t)
ownBuild := b.ix.OwnBuild
b.ix.ownFP, b.ix.ownFixed = "", false
b.ix.OwnBuild = func(context.Context) (string, error) { return "", errors.New("the engine does not answer") }
b.ix.FixOwnBuild(ctx)
b.said = nil
if _, _, err := b.ix.Arrive(ctx, 7, b.merge(30, sha('b'), "tree-avahi-2", "")); !errors.Is(err, ErrOwnBuildUnknown) {
t.Fatal(err)
}
b.ix.OwnBuild, b.ix.ownFP, b.ix.ownFixed = ownBuild, "", false
b.ix.FixOwnBuild(ctx)
b.arrive(t, 7, b.merge(30, sha('b'), "tree-avahi-2", "")) // the stream delivers it again
if want := []string{"pending " + takein.PendingKey(7), "taken " + takein.PendingKey(7)}; !slices.Equal(b.events(), want) || len(b.keys()) != 0 {
t.Fatalf("said %v, pending %v: want %v and nothing pending", b.events(), b.keys(), want)
}
// Two merges said pending under the oldest's key (waiting for a named indexer), then taken in together: one taken, under
// that key, and no entry's key said twice.
b.said = nil
b.arrive(t, 8, b.merge(31, sha('c'), "tree-avahi-3", "tree-indexer-2"))
b.arrive(t, 9, b.merge(32, sha('d'), "tree-avahi-4", "tree-indexer-2"))
b.said = nil
b.arrive(t, 10, b.merge(33, sha('e'), "tree-avahi-5", b.base.Trees["modules/"+testSelf]))
if want := []string{"taken " + takein.PendingKey(8), "taken " + takein.PendingKey(9), "taken " + takein.PendingKey(10)}; !slices.Equal(b.events(), want) || len(b.keys()) != 0 {
t.Errorf("said %v, pending %v: want %v, the wait's key (m.8, said pending) once and each other entry's own", b.events(), b.keys(), want)
} }
}) })
t.Run("T7 and T3 a merge alone, an indexer change, a merge behind it and a revert", func(t *testing.T) { t.Run("T7 and T3 a merge alone, an indexer change, a merge behind it and a revert", func(t *testing.T) {
@@ -259,42 +227,28 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633)
t.Fatalf("alone: avahi %s (was %s), pending %v, said %v: want taken in, said taken under its key (1966/AC195-1)", avahi1, avahi0, b.keys(), b.events()) t.Fatalf("alone: avahi %s (was %s), pending %v, said %v: want taken in, said taken under its key (1966/AC195-1)", avahi1, avahi0, b.keys(), b.events())
} }
b.said = nil b.said = nil
avahi1, verdicts := b.fp(t, "avahi"), len(b.ix.Verdicts.(*memStore).vals) avahi1 := b.fp(t, "avahi")
// T4: a change to the indexer writes only its entry (with its graph key and verdict); the rest is pending. // A-476 b: a merge naming another indexer build is taken in at once, with the build this program runs: the indexer's entry
// and avahi are written, nothing is held and said pending.
b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2")) b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2"))
named := b.fp(t, testSelf) named := b.fp(t, testSelf)
if named == own || b.fp(t, "avahi") != avahi1 || b.commit(t, "avahi") != sha('b') { if named == own || b.fp(t, "avahi") == avahi1 || b.commit(t, "avahi") != sha('c') || len(b.keys()) != 0 {
t.Fatalf("indexer change: indexer %s (own %s), avahi %s at %s: want only the indexer written", named, own, b.fp(t, "avahi"), b.commit(t, "avahi")) t.Fatalf("indexer change: indexer %s (own %s), avahi %s at %s, pending %v: want the whole revision taken in at once", named, own, b.fp(t, "avahi"), b.commit(t, "avahi"), b.keys())
} }
if g := graphOf(t, b.ix, testSelf); g.FP != named { if g := graphOf(t, b.ix, testSelf); g.FP != named {
t.Errorf("graph of the indexer names %s, want %s", g.FP, named) t.Errorf("graph of the indexer names %s, want %s", g.FP, named)
} }
if n := len(b.ix.Verdicts.(*memStore).vals); n > verdicts+1 { if want := []string{"taken " + takein.PendingKey(2)}; !slices.Equal(b.events(), want) {
t.Errorf("verdicts %d, want at most %d: only the new indexer manifest's", n, verdicts+1) t.Errorf("said %v, want %v: nothing said pending", b.events(), want)
} }
if want := []string{takein.PendingKey(2)}; !reflect.DeepEqual(b.keys(), want) || !slices.Equal(b.events(), []string{"pending " + takein.PendingKey(2)}) || b.said[0].WaitsFor != named { // A later merge is taken in at once too, and a revert to the first indexer.
t.Fatalf("pending %v, said %+v: want m.2 pending for %s", b.keys(), b.said, named)
}
// T5: the merge delivered again writes no second key and says nothing again.
b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2"))
if len(b.keys()) != 1 || len(b.said) != 1 {
t.Errorf("redelivered: pending %v, said %v", b.keys(), b.events())
}
// A later merge joins: the newest governs and still names the new indexer.
b.arrive(t, 3, b.merge(22, sha('d'), "tree-avahi-4", "tree-indexer-2")) b.arrive(t, 3, b.merge(22, sha('d'), "tree-avahi-4", "tree-indexer-2"))
if len(b.keys()) != 2 || b.commit(t, "avahi") != sha('b') || len(b.said) != 1 {
t.Fatalf("behind: pending %v, avahi at %s, said %v: want both pending and nothing said again", b.keys(), b.commit(t, "avahi"), b.events())
}
// A revert brings the indexer back to the running build: everything is taken in, oldest first.
b.arrive(t, 4, b.merge(23, sha('e'), "tree-avahi-5", b.base.Trees["modules/"+testSelf])) b.arrive(t, 4, b.merge(23, sha('e'), "tree-avahi-5", b.base.Trees["modules/"+testSelf]))
if len(b.keys()) != 0 || b.fp(t, testSelf) != own || b.commit(t, "avahi") != sha('e') { if len(b.keys()) != 0 || b.fp(t, testSelf) != own || b.commit(t, "avahi") != sha('e') {
t.Fatalf("revert: pending %v, indexer %s, avahi at %s: want all taken in", b.keys(), b.fp(t, testSelf), b.commit(t, "avahi")) t.Fatalf("revert: pending %v, indexer %s, avahi at %s: want all taken in", b.keys(), b.fp(t, testSelf), b.commit(t, "avahi"))
} }
if want := []string{"pending " + takein.PendingKey(2), "taken " + takein.PendingKey(2), "taken " + takein.PendingKey(3), "taken " + takein.PendingKey(4)}; !slices.Equal(b.events(), want) {
t.Errorf("said %v, want %v", b.events(), want)
}
}) })
t.Run("IX22 a merged revision naming a newer indexer is the previous catalog graph revision with only the indexer's entry replaced", func(t *testing.T) { // design 47 G §273, §237 t.Run("IX22 a merged revision naming a newer indexer is a catalog graph revision with its entry and the merge's others replaced", func(t *testing.T) { // design 47 G §273, §237
b := newTakeInBed(t) b := newTakeInBed(t)
store := newMemObjects(0) store := newMemObjects(0)
b.ix.Revisions = store b.ix.Revisions = store
@@ -305,14 +259,14 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633)
if err != nil || before.Revision != 1 { if err != nil || before.Revision != 1 {
t.Fatalf("the view's revision %+v (%v), want revision 1", before, err) t.Fatalf("the view's revision %+v (%v), want revision 1", before, err)
} }
// The merge changes avahi and the indexer: only the indexer is written, the revision is made at once. // The merge changes avahi and the indexer: both are taken in at once (A-476 b), in a revision after the first.
b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2")) b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2"))
after, _, err := newestTrunk(ctx, store) after, _, err := newestTrunk(ctx, store)
if err != nil || after.Revision != 2 { if err != nil || after.Revision <= before.Revision {
t.Fatalf("after the merge naming a newer indexer the newest revision is %+v (%v), want revision 2", after, err) t.Fatalf("after the merge naming a newer indexer the newest revision is %+v (%v), want one after revision %d", after, err, before.Revision)
} }
if changed := changedEntries(before.Entries, after.Entries); !slices.Equal(changed, []string{testSelf}) { if changed := changedEntries(before.Entries, after.Entries); !slices.Equal(changed, []string{"avahi", testSelf}) && !slices.Equal(changed, []string{testSelf, "avahi"}) {
t.Errorf("the revision differs from the previous in %v, want the indexer's entry alone", changed) t.Errorf("the revision differs from the previous in %v, want avahi and the indexer", changed)
} }
}) })
t.Run("P1 a revision that adds a Go module its own indexer uses is computed against that module's packages at its merge", func(t *testing.T) { t.Run("P1 a revision that adds a Go module its own indexer uses is computed against that module's packages at its merge", func(t *testing.T) {
@@ -353,71 +307,13 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633)
t.Fatal(err) t.Fatal(err)
} }
b.arrive(t, 2, f) b.arrive(t, 2, f)
if len(b.said) != 1 || !strings.HasPrefix(b.said[0].WaitsFor, "src2:") { if len(b.said) != 1 || b.said[0].Kind != takein.TakeInTaken || b.said[0].WaitsFor != "" || len(b.keys()) != 0 {
t.Errorf("said %+v, want m.2 waiting for the indexer build its merge's packages give, not for a version no take-in has published", b.said) t.Errorf("said %+v, pending %v, want m.2 taken in at once", b.said, b.keys())
} }
if _, ok := b.ix.publishedAll(ctx)[sibling]; !ok { if _, ok := b.ix.publishedAll(ctx)[sibling]; !ok {
t.Errorf("the packages the revision publishes at its merge lack %s", sibling) t.Errorf("the packages the revision publishes at its merge lack %s", sibling)
} }
}) })
t.Run("K1 the indexer a revision names is judged with the seat definitions the revision carries", func(t *testing.T) {
// A successor's manifest invokes a verb of the node-engine's seat that the seat list lacks and the revision's
// definitions add (ADR 0633 §1, §6): refused without them, with its reason said on the wait and in indexer_pending;
// accepted with them, and the revision waits for the build it names.
const verb, contracts = "revision-verb", "modules/seat-contracts"
// The third row's definer is the Delivery domain's contracts module defining a Core seat: not taken (ADR 0636 §5).
for _, row := range []struct {
name string
defined bool
refused bool
module string
}{{"definitions the revision does not carry", false, true, ""}, {"definitions the revision carries", true, false, "seat-contracts"},
{"definitions of a seat outside the carrier's domain", true, true, "mesh-delivery-contracts"}} {
b := newTakeInBed(t)
f := b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2")
f.Manifests = map[string]string{}
for dir, raw := range b.base.Manifests {
f.Manifests[dir] = raw
}
dir := "modules/" + testSelf
f.Manifests[dir] = strings.Replace(f.Manifests[dir], `"module": "`+testSelf+`"`,
`"module": "`+testSelf+`", "invokes": ["seat:node-engine.`+verb+`"]`, 1)
if row.defined {
f.Trees[contracts] = "tree-seat-contracts"
f.Locks = map[string]map[string]string{}
for d, l := range b.base.Locks {
f.Locks[d] = l
}
f.Locks[contracts] = map[string]string{}
f.Manifests[contracts] = `{"module":"` + row.module + `","version":"1.0","build":{"artifacts":[{"name":"go-seats","kind":"package","language":"go","paths":["go"],"series":"0.1","generated-from":"."}]}}`
b.ix.Files = func(_ context.Context, repository, commit string, dirs []string) (map[string]map[string]string, error) {
if repository != f.Repository || commit != f.Merge || !slices.Equal(dirs, []string{contracts + "/seats"}) {
t.Errorf("listed %v of %s at %s", dirs, repository, commit)
}
return map[string]map[string]string{dirs[0]: {"node-engine/seat.json": "1", "node-engine/fixtures/a.json": "2"}}, nil
}
b.ix.ReadFile = func(_ context.Context, _, _, p string) ([]byte, error) {
switch p {
case contracts + "/seats/node-engine/seat.json":
return []byte(`{"format":1,"seat":"node-engine","scope":"node","grain":"seat-list","decision":"d","domain":"Core","verbs":[{"name":"` + verb + `"}]}`), nil
}
return nil, nil
}
}
b.arrive(t, 2, f)
waits := PendingWaits(readPending(t, b))
refused := ""
if len(b.said) > 0 {
refused = b.said[0].Refused
}
if got := strings.Contains(refused, `serves no verb "`+verb+`"`); got != row.refused || (len(waits) == 1 && waits[0].Refused != refused) {
t.Errorf("%s: said %q, indexer_pending %+v: want a refusal naming the verb %v", row.name, refused, waits, row.refused)
}
if _, ok := catalogue.SeatNamed("node-engine"); !ok || judgeSeatsHave("node-engine", verb) {
t.Errorf("%s: the seats are not put back after the dry run", row.name)
}
}
})
t.Run("A-326 a revision whose catalog graph holds a cycle with a build edge no build seat breaks is refused, naming it", func(t *testing.T) { t.Run("A-326 a revision whose catalog graph holds a cycle with a build edge no build seat breaks is refused, naming it", func(t *testing.T) {
// avahi's artifact needs the indexer's (kind 2), and the indexer is built on avahi (kind 3): avahi holds no build seat, so // avahi's artifact needs the indexer's (kind 2), and the indexer is built on avahi (kind 3): avahi holds no build seat, so
// no running version breaks the cycle, and the revision is refused at take-in naming every artifact on it (design 47 G §49). // no running version breaks the cycle, and the revision is refused at take-in naming every artifact on it (design 47 G §49).
@@ -443,15 +339,15 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633)
t.Errorf("indexer_pending %+v, want the refusal naming the cycle", waits) t.Errorf("indexer_pending %+v, want the refusal naming the cycle", waits)
} }
}) })
t.Run("T10 the named indexer takes the revision in at its start; a gathered group stays", func(t *testing.T) { t.Run("T10 a start takes what waits in whatever its build; a gathered group stays", func(t *testing.T) {
b := newTakeInBed(t) b := newTakeInBed(t)
b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2")) b.arriveHeld(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2"))
g := b.merge(30, sha('f'), "", "") g := b.merge(30, sha('f'), "", "")
g.Repository, g.Trees, g.Manifests, g.Locks, g.Group = "novox/mesh-sdk", map[string]string{}, nil, nil, "feat/x" g.Repository, g.Trees, g.Manifests, g.Locks, g.Group = "novox/mesh-sdk", map[string]string{}, nil, nil, "feat/x"
g.Members = []meshdeliveryclient.Head{{Repository: "novox/mesh-sdk", Number: 30, Head: g.Head}, {Repository: "novox/mesh-host", Number: 4, Head: sha('9')}} g.Members = []meshdeliveryclient.Head{{Repository: "novox/mesh-sdk", Number: 30, Head: g.Head}, {Repository: "novox/mesh-host", Number: 4, Head: sha('9')}}
b.arrive(t, 5, g) b.arriveHeld(t, 5, g)
named := b.fp(t, testSelf) own, _ := b.ix.OwnBuild(ctx)
// The new indexer starts; its build is not known (the engine does not answer at its start): a hand ask is answered // The indexer starts; its build is not known (the engine does not answer at its start): a hand ask is answered
// meanwhile (R5), and the start waits for nothing. It is never asked again, so it stays unknown until it is started again. // meanwhile (R5), and the start waits for nothing. It is never asked again, so it stays unknown until it is started again.
mk := func(ask func(context.Context) (string, error)) *Indexer { mk := func(ask func(context.Context) (string, error)) *Indexer {
n := &Indexer{Module: testSelf, Store: b.ix.Store, Modules: b.ix.Modules, Manifests: b.ix.Manifests, Graph: b.ix.Graph, Verdicts: b.ix.Verdicts, n := &Indexer{Module: testSelf, Store: b.ix.Store, Modules: b.ix.Modules, Manifests: b.ix.Manifests, Graph: b.ix.Graph, Verdicts: b.ix.Verdicts,
@@ -462,39 +358,21 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633)
n.FixOwnBuild(ctx) n.FixOwnBuild(ctx)
return n return n
} }
// A start that decides nothing says so (issue 1921): the build it learnt, the build the held wait names, also when the // The build this program runs is not known (the engine does not answer at its start): nothing waits for it, so the start
// wait was said before; and indexer_pending names the known build and since when. // takes the revision in all the same (A-476 b), and a hand ask names what waits.
oldBuild, _ := b.ix.OwnBuild(ctx) next := mk(func(context.Context) (string, error) { return "", errors.New("not yet") })
oldStart := mk(func(context.Context) (string, error) { return oldBuild, nil })
b.logs.Reset()
if _, words, err := oldStart.TakeInAtStart(ctx); err != nil || len(b.keys()) != 2 || !strings.Contains(b.logs.String(), "still name the indexer build") ||
!strings.Contains(b.logs.String(), "at the start, this program runs build") {
t.Fatalf("a start that holds: %q %v, journal %q: want the build learnt and the build the wait names said", words, err, b.logs.String())
}
if answer, err := PendingTools(b.pending, b.ix.OwnBuildState, nil, nil)[0].Run(map[string]any{}); err != nil || answer.(takein.PendingAnswer).OwnBuild != oldBuild || answer.(takein.PendingAnswer).OwnBuildSince.IsZero() {
t.Errorf("indexer_pending = %+v %v, want the known build and since", answer, err)
}
asked := 0
next := mk(func(context.Context) (string, error) { asked++; return "", errors.New("not yet") })
if r := next.TakeInPendingByHand(ctx, "m.nothing", "lead@shanks", "a test"); r.Stored || !strings.Contains(r.Answer, "nothing waits") { if r := next.TakeInPendingByHand(ctx, "m.nothing", "lead@shanks", "a test"); r.Stored || !strings.Contains(r.Answer, "nothing waits") {
t.Errorf("a hand ask while the own build is unknown: %+v", r) t.Errorf("a hand ask for what does not wait: %+v", r)
} }
if _, words, err := next.TakeInAtStart(ctx); err != nil || len(b.keys()) != 2 || !strings.Contains(words, "not known") {
t.Fatalf("a start with its build unknown: %q %v, pending %v: want it to answer at once and take nothing in", words, err, b.keys())
}
// The engine answers now, and the same process does not ask it again: it learnt nothing it did not know at its start.
next.OwnBuild = func(context.Context) (string, error) { asked++; return named, nil }
if _, words, err := next.TakeInAtStart(ctx); err != nil || asked != 1 || len(b.keys()) != 2 || !strings.Contains(words, "not known") {
t.Fatalf("the window of a replacement: %q %v, asked %d, pending %v: want the build fixed at the start, never learnt later", words, err, asked, b.keys())
}
// The module started again fixes the build at its start and takes the revision in.
next = mk(func(context.Context) (string, error) { return named, nil })
if _, _, err := next.TakeInAtStart(ctx); err != nil { if _, _, err := next.TakeInAtStart(ctx); err != nil {
t.Fatal(err) t.Fatal(err)
} }
if !reflect.DeepEqual(b.keys(), []string{takein.PendingKey(5)}) || b.commit(t, "avahi") != sha('c') { if !reflect.DeepEqual(b.keys(), []string{takein.PendingKey(5)}) || b.commit(t, "avahi") != sha('c') {
t.Errorf("start: pending %v, avahi at %s: want the revision taken in, the group left", b.keys(), b.commit(t, "avahi")) t.Errorf("start: pending %v, avahi at %s: want the revision taken in, the group left", b.keys(), b.commit(t, "avahi"))
} }
if answer, err := PendingTools(b.pending, b.ix.OwnBuildState, nil, nil)[0].Run(map[string]any{}); err != nil || answer.(takein.PendingAnswer).OwnBuild != own {
t.Errorf("indexer_pending = %+v %v, want the build the original program knows", answer, err)
}
}) })
t.Run("T8 T9 T11 a group gathered until its end, a merge behind it, one alone, an end again", func(t *testing.T) { t.Run("T8 T9 T11 a group gathered until its end, a merge behind it, one alone, an end again", func(t *testing.T) {
b := newTakeInBed(t) b := newTakeInBed(t)
@@ -544,7 +422,7 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633)
if _, words, err := b.ix.EndGroup(ctx, end); err != nil || !strings.Contains(words, "taken in already") { if _, words, err := b.ix.EndGroup(ctx, end); err != nil || !strings.Contains(words, "taken in already") {
t.Errorf("an end again: %q %v", words, err) t.Errorf("an end again: %q %v", words, err)
} }
// T11 (the lead's row): a group whose revision changes the indexer: the group's wait is taken, then the revision pending. // T11: a group whose revision changes the indexer: the group's wait is taken and the revision taken in at once (A-476 b).
b.said = nil b.said = nil
g := member(catalogueRepo, 40, sha('c')) g := member(catalogueRepo, 40, sha('c'))
g.Group, g.Trees = "feat/two", b.merge(0, "", "", "tree-indexer-7").Trees g.Group, g.Trees = "feat/two", b.merge(0, "", "", "tree-indexer-7").Trees
@@ -553,7 +431,7 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633)
if _, _, err := b.ix.EndGroup(ctx, forgemsg.ForgeGroupEnd{Group: "feat/two", Members: []forgemsg.GroupMerge{{Repository: catalogueRepo, Number: 40, Merge: sha('c')}}}); err != nil { if _, _, err := b.ix.EndGroup(ctx, forgemsg.ForgeGroupEnd{Group: "feat/two", Members: []forgemsg.GroupMerge{{Repository: catalogueRepo, Number: 40, Merge: sha('c')}}}); err != nil {
t.Fatal(err) t.Fatal(err)
} }
if want := []string{"pending g.feat/two", "taken g.feat/two", "pending " + takein.PendingKey(9)}; !slices.Equal(b.events(), want) { if want := []string{"pending g.feat/two", "taken g.feat/two", "taken " + takein.PendingKey(9)}; !slices.Equal(b.events(), want) {
t.Errorf("a group naming another indexer: said %v, want %v", b.events(), want) t.Errorf("a group naming another indexer: said %v, want %v", b.events(), want)
} }
}) })
@@ -593,11 +471,8 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633)
} }
held := func() *takeInBed { held := func() *takeInBed {
b := newTakeInBed(t) b := newTakeInBed(t)
b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2")) b.arriveHeld(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2"))
b.arrive(t, 3, b.merge(22, sha('d'), "tree-avahi-4", "tree-indexer-2")) b.arriveHeld(t, 3, b.merge(22, sha('d'), "tree-avahi-4", "tree-indexer-2"))
b.ix.ownMu.Lock()
b.ix.ownFP = b.fp(t, testSelf)
b.ix.ownMu.Unlock()
return b return b
} }
// A probe that fails keeps the group's wait and stops nothing else: an unrelated merge is still taken in. // A probe that fails keeps the group's wait and stops nothing else: an unrelated merge is still taken in.
@@ -719,7 +594,7 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633)
}) })
t.Run("T5 two take-ins of one pending revision: the one that loses the claim writes nothing", func(t *testing.T) { t.Run("T5 two take-ins of one pending revision: the one that loses the claim writes nothing", func(t *testing.T) {
b := newTakeInBed(t) b := newTakeInBed(t)
b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2")) b.arriveHeld(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2"))
named := b.fp(t, testSelf) named := b.fp(t, testSelf)
b.ix.ownMu.Lock() b.ix.ownMu.Lock()
b.ix.ownFP = named b.ix.ownFP = named
@@ -774,8 +649,8 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633)
{"a take-in that fails", func(b *takeInBed) { b.mods.failWrite = once(func(string) bool { return true }) }}, {"a take-in that fails", func(b *takeInBed) { b.mods.failWrite = once(func(string) bool { return true }) }},
} { } {
b := newTakeInBed(t) b := newTakeInBed(t)
b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2")) b.arriveHeld(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2"))
b.arrive(t, 3, b.merge(22, sha('d'), "tree-avahi-4", "tree-indexer-2")) b.arriveHeld(t, 3, b.merge(22, sha('d'), "tree-avahi-4", "tree-indexer-2"))
named := b.fp(t, testSelf) named := b.fp(t, testSelf)
b.ix.ownMu.Lock() b.ix.ownMu.Lock()
b.ix.ownFP = named b.ix.ownFP = named
@@ -794,7 +669,7 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633)
}) })
t.Run("W4.6 a full store holds the merge with the reason objects full, says full once, and room takes it in", func(t *testing.T) { t.Run("W4.6 a full store holds the merge with the reason objects full, says full once, and room takes it in", func(t *testing.T) {
b := newTakeInBed(t) b := newTakeInBed(t)
b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2")) b.arriveHeld(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2"))
named := b.fp(t, testSelf) named := b.fp(t, testSelf)
b.ix.ownMu.Lock() b.ix.ownMu.Lock()
b.ix.ownFP = named b.ix.ownFP = named
@@ -826,7 +701,7 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633)
// The same take-in into a store it fills to 85 % of its cap says filling once, and is taken in. // The same take-in into a store it fills to 85 % of its cap says filling once, and is taken in.
kept := store.used() kept := store.used()
b2 := newTakeInBed(t) b2 := newTakeInBed(t)
b2.arrive(t, 2, b2.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2")) b2.arriveHeld(t, 2, b2.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2"))
b2.ix.ownMu.Lock() b2.ix.ownMu.Lock()
b2.ix.ownFP = b2.fp(t, testSelf) b2.ix.ownFP = b2.fp(t, testSelf)
b2.ix.ownMu.Unlock() b2.ix.ownMu.Unlock()
@@ -845,7 +720,7 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633)
t.Run("W4.12 a candidate's half dropped by hand frees its own entries, says room and takes the held merge in; the read tools list it; a recompute that moves nothing is equal", func(t *testing.T) { t.Run("W4.12 a candidate's half dropped by hand frees its own entries, says room and takes the held merge in; the read tools list it; a recompute that moves nothing is equal", func(t *testing.T) {
bed := func(store *memObjects) *takeInBed { bed := func(store *memObjects) *takeInBed {
b := newTakeInBed(t) b := newTakeInBed(t)
b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2")) b.arriveHeld(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2"))
b.ix.ownMu.Lock() b.ix.ownMu.Lock()
b.ix.ownFP = b.fp(t, testSelf) b.ix.ownFP = b.fp(t, testSelf)
b.ix.ownMu.Unlock() b.ix.ownMu.Unlock()
@@ -930,40 +805,10 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633)
t.Errorf("indexer_recompute = %+v (%v): want equal to the kept revision 2", got, err) t.Errorf("indexer_recompute = %+v (%v): want equal to the kept revision 2", got, err)
} }
}) })
t.Run("T7 a toolchain report evaluates pending again: the revision then names the running build and is taken in", func(t *testing.T) {
b := newTakeInBed(t)
b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2"))
entries, err := ReadPending(ctx, b.pending, nil)
if err != nil {
t.Fatal(err)
}
// The build the revision names under the next toolchain, worked out on a dry indexer; this one runs it.
x := b.ix.dry().ix
for _, node := range []string{"novox", "ace"} {
key, _ := ToolchainKey(node, "go")
x.PutToolchain("mesh-build-go", key, toolchainValue(t, "go", "go1.28.0", "1"), false)
}
ready, _, _, _ := b.ix.loadNewest(ctx, WaitsOf(entries).Ready)
next, _, err := x.computeOver(ctx, ready)
if err != nil || next == "" || next == b.said[0].WaitsFor {
t.Fatalf("the build under the next toolchain: %q %v (the revision names %s)", next, err, b.said[0].WaitsFor)
}
b.ix.ownMu.Lock()
b.ix.ownFP = next
b.ix.ownMu.Unlock()
reportToolchain(t, b.ix, "mesh-build-go", "novox", "go", "go1.28.0")
if len(b.keys()) != 1 {
t.Fatalf("one node's report: pending %v, want the revision still held (the nodes disagree)", b.keys())
}
reportToolchain(t, b.ix, "mesh-build-go", "ace", "go", "go1.28.0")
if len(b.keys()) != 0 || b.commit(t, "avahi") != sha('c') {
t.Errorf("after the toolchain report: pending %v, avahi at %s: want the revision taken in", b.keys(), b.commit(t, "avahi"))
}
})
t.Run("T12 T13 indexer_pending names the wait; a hand take-in and a drop say what they ended", func(t *testing.T) { t.Run("T12 T13 indexer_pending names the wait; a hand take-in and a drop say what they ended", func(t *testing.T) {
b := newTakeInBed(t) b := newTakeInBed(t)
b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2")) b.arriveHeld(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2"))
b.arrive(t, 3, b.merge(22, sha('d'), "tree-avahi-4", "tree-indexer-2")) b.arriveHeld(t, 3, b.merge(22, sha('d'), "tree-avahi-4", "tree-indexer-2"))
var answer takein.PendingAnswer var answer takein.PendingAnswer
for _, tool := range PendingTools(b.pending, nil, nil, nil) { for _, tool := range PendingTools(b.pending, nil, nil, nil) {
if tool.Name == PendingTool { if tool.Name == PendingTool {
@@ -975,7 +820,7 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633)
} }
} }
if len(answer.Waits) != 1 || answer.Waits[0].Key != takein.PendingKey(2) || !slices.Equal(answer.Waits[0].Holds, []string{testSelf, "avahi"}) || if len(answer.Waits) != 1 || answer.Waits[0].Key != takein.PendingKey(2) || !slices.Equal(answer.Waits[0].Holds, []string{testSelf, "avahi"}) ||
answer.Waits[0].Said != takein.TakeInPendingEvent || len(answer.Waits[0].Keys) != 2 { len(answer.Waits[0].Keys) != 2 {
t.Fatalf("indexer_pending = %+v", answer) t.Fatalf("indexer_pending = %+v", answer)
} }
// ADR 0668 §6: a later merge covers a pending one only when its commit has the pending merge's as an ancestor. // ADR 0668 §6: a later merge covers a pending one only when its commit has the pending merge's as an ancestor.
@@ -990,32 +835,47 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633)
} }
history = "cd" history = "cd"
b.said = nil b.said = nil
bound := b.ix.Verdicts
b.ix.Verdicts = &memStore{unbound: process.ErrNotYet} // m.3 stays held while m.2 is dropped
if r := b.ix.DropPending(ctx, takein.PendingKey(2), "lead@shanks", "replaced by m.3", false); !r.Stored { if r := b.ix.DropPending(ctx, takein.PendingKey(2), "lead@shanks", "replaced by m.3", false); !r.Stored {
t.Fatalf("drop: %+v", r) t.Fatalf("drop: %+v", r)
} }
if len(b.said) < 2 || !b.said[0].Dropped || b.said[0].Key != takein.PendingKey(2) || b.said[0].By != "lead@shanks" || b.said[1].Key != takein.PendingKey(3) { b.ix.Verdicts = bound
t.Fatalf("drop said %+v: want m.2 taken (dropped, by) and m.3 pending", b.said) if len(b.keys()) != 1 || b.keys()[0] != takein.PendingKey(3) {
t.Fatalf("pending %v: want m.2 dropped and m.3 pending", b.keys())
} }
// Issue 1966 (#650), ADR 0633 §5: the newest pending merge of its repository, which no taken merge covers, is not // Issue 1966 (#650), ADR 0633 §5: the newest pending merge of its repository, which no taken merge covers, is not
// dropped; the answer names the read again. abandon drops it all the same (below). // dropped; the answer names abandon, which drops it all the same (below).
if r := b.ix.DropPending(ctx, takein.PendingKey(3), "lead@shanks", "lost message", false); r.Stored || if r := b.ix.DropPending(ctx, takein.PendingKey(3), "lead@shanks", "lost message", false); r.Stored ||
!strings.Contains(r.Answer, TakeInPendingTool) || len(b.keys()) != 1 { !strings.Contains(r.Answer, "abandon") || len(b.keys()) != 1 {
t.Fatalf("a drop nothing covers: %+v, pending %v", r, b.keys()) t.Fatalf("a drop nothing covers: %+v, pending %v", r, b.keys())
} }
// A pending revision is taken in by the tool, by its key (A-476, principle 4).
if _, _, err := NewPendingAsk(takein.PendingKey(3), "x", "lead@shanks", false, false, time.Unix(0, 0)); err != nil {
t.Fatalf("the ask to take a pending revision in: %v", err)
}
if r := b.ix.TakeInPendingByHand(ctx, takein.PendingKey(3), "lead@shanks", "its message was unread"); !r.Stored || len(b.keys()) != 0 || b.commit(t, "avahi") != sha('d') {
t.Fatalf("hand take-in of a revision: %+v, pending %v, avahi at %s", r, b.keys(), b.commit(t, "avahi"))
}
// A delivery group gathered is taken in by hand (A-476 b: only a group is).
b.ix.Pins = newMem() b.ix.Pins = newMem()
b.ix.Revisions = newMemObjects(0) // A-448: the taken event names the catalog graph revision the take-in made b.ix.Revisions = newMemObjects(0) // A-448: the taken event names the catalog graph revision the take-in made
gm := b.merge(24, sha('f'), "tree-avahi-6", "")
gm.Group, gm.Head = "feat/h", sha('1')
gm.Members = []meshdeliveryclient.Head{{Repository: catalogueRepo, Number: 24, Head: sha('1')}, {Repository: "novox/mesh-sdk", Number: 25, Head: sha('2')}}
b.arrive(t, 5, gm)
b.said = nil b.said = nil
if r := b.ix.TakeInPendingByHand(ctx, takein.PendingKey(3), "lead@shanks", "the named indexer can never run"); !r.Stored { if r := b.ix.TakeInPendingByHand(ctx, takein.GroupTakeInKey("feat/h"), "lead@shanks", "the group's end never comes"); !r.Stored {
t.Fatalf("hand take-in: %+v", r) t.Fatalf("hand take-in: %+v", r)
} }
if len(b.keys()) != 0 || b.commit(t, "avahi") != sha('d') || len(b.said) != 1 || b.said[0].Kind != takein.TakeInTaken || b.said[0].By != "lead@shanks" { if len(b.keys()) != 0 || b.commit(t, "avahi") != sha('f') || len(b.said) < 1 || b.said[0].Kind != takein.TakeInTaken || b.said[0].By != "lead@shanks" {
t.Errorf("hand take-in: pending %v, avahi at %s, said %+v", b.keys(), b.commit(t, "avahi"), b.said) t.Errorf("hand take-in: pending %v, avahi at %s, said %+v", b.keys(), b.commit(t, "avahi"), b.said)
} }
// §476, §493: a hand take-in's taken event carries the branch, the source revision it makes and its entries. // §476, §493: a hand take-in's taken event carries the branch, the source revision it makes and its entries.
if len(b.said) == 1 { if len(b.said) >= 1 {
got := b.said[0] got := b.said[0]
if got.Branch != TrunkBranch || got.Source <= 0 || len(got.Entries) != 1 || got.Entries[0].Module != "avahi" || if got.Branch != TrunkBranch || got.Source <= 0 || len(got.Entries) != 2 || got.Entries[1].Module != "avahi" ||
got.Entries[0].Commit != sha('d') || got.Entries[0].Fingerprint == "" { got.Entries[1].Commit != sha('f') || got.Entries[1].Fingerprint == "" {
t.Errorf("hand take-in said branch %q source %d entries %+v: want main, a source revision and avahi at its commit", got.Branch, got.Source, got.Entries) t.Errorf("hand take-in said branch %q source %d entries %+v: want main, a source revision and avahi at its commit", got.Branch, got.Source, got.Entries)
} else if newest, any, _ := newestTrunk(ctx, b.ix.Revisions); !any || got.Revision != newest.Revision { } else if newest, any, _ := newestTrunk(ctx, b.ix.Revisions); !any || got.Revision != newest.Revision {
t.Errorf("hand take-in said revision %d, want the revision it made, %d (A-448)", got.Revision, newest.Revision) t.Errorf("hand take-in said revision %d, want the revision it made, %d (A-448)", got.Revision, newest.Revision)
@@ -1025,7 +885,7 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633)
t.Errorf("the taken event is refused by its own contract: %v", err) t.Errorf("the taken event is refused by its own contract: %v", err)
} }
} }
b.arrive(t, 4, b.merge(23, sha('e'), "tree-avahi-5", "tree-indexer-2")) b.arriveHeld(t, 4, b.merge(23, sha('e'), "tree-avahi-5", "tree-indexer-2"))
if r := b.ix.DropPending(ctx, takein.PendingKey(4), "lead@shanks", "abandoned on the operator's word", true); !r.Stored || len(b.keys()) != 0 { if r := b.ix.DropPending(ctx, takein.PendingKey(4), "lead@shanks", "abandoned on the operator's word", true); !r.Stored || len(b.keys()) != 0 {
t.Errorf("an abandoned drop: %+v, pending %v", r, b.keys()) t.Errorf("an abandoned drop: %+v, pending %v", r, b.keys())
} }
@@ -1062,8 +922,8 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633)
if _, _, found, _ := b.ix.Store.Get(ctx, key); !found { if _, _, found, _ := b.ix.Store.Get(ctx, key); !found {
t.Fatal("the group has no candidate") t.Fatal("the group has no candidate")
} }
b.arrive(t, 2, big(b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2"))) b.arriveHeld(t, 2, big(b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2")))
b.arrive(t, 3, big(b.merge(22, sha('d'), "tree-avahi-4", "tree-indexer-2"))) b.arriveHeld(t, 3, big(b.merge(22, sha('d'), "tree-avahi-4", "tree-indexer-2")))
raw, _, _, _ := b.pending.Get(ctx, takein.PendingKey(2)) raw, _, _, _ := b.pending.Get(ctx, takein.PendingKey(2))
if len(b.keys()) != 2 || len(raw) > 4096 || b.commit(t, "avahi") != sha('b') { if len(b.keys()) != 2 || len(raw) > 4096 || b.commit(t, "avahi") != sha('b') {
t.Fatalf("a big merge held: pending %v, value %d bytes, avahi at %s: want two small values", b.keys(), len(raw), b.commit(t, "avahi")) t.Fatalf("a big merge held: pending %v, value %d bytes, avahi at %s: want two small values", b.keys(), len(raw), b.commit(t, "avahi"))
@@ -1139,7 +999,7 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633)
b := newTakeInBed(t) b := newTakeInBed(t)
f := b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2") f := b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2")
f.Words, f.Branch = strings.Repeat("x", 300*1024), "feat/g" f.Words, f.Branch = strings.Repeat("x", 300*1024), "feat/g"
b.arrive(t, 2, f) // held by reference, for an indexer build that is not this one b.arriveHeld(t, 2, f) // held by reference, with the verdicts state not on the bus
b.ix.Taken = newMem() b.ix.Taken = newMem()
b.ix.Contains = func(context.Context, string, string, string) (bool, error) { return c.behind, nil } b.ix.Contains = func(context.Context, string, string, string) (bool, error) { return c.behind, nil }
data, _ := takein.EncodeTaken(takein.Taken{Repository: trunk.RepositoryName(catalogueRepo), Merge: sha('d')}) data, _ := takein.EncodeTaken(takein.Taken{Repository: trunk.RepositoryName(catalogueRepo), Merge: sha('d')})
@@ -4,7 +4,6 @@ import (
"context" "context"
"encoding/json" "encoding/json"
"fmt" "fmt"
"strings"
"time" "time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein"
@@ -44,15 +43,13 @@ func PendingTools(pending, ownBuild Store, ask func(key string, value []byte) (T
return []stdio.Tool{ return []stdio.Tool{
{Name: PendingTool, {Name: PendingTool,
Description: "What waits for a take-in (ADR 0633): merges the indexer holds in its pending state instead of taking them in. " + Description: "What waits for a take-in (ADR 0633): merges the indexer holds in its pending state instead of taking them in. " +
"A revision (what revision, its key its oldest merge's m.<sequence>) waits for the build of the indexer it names " + "A revision (what revision, its key its oldest merge's m.<sequence>) waits only while its message is unread or lost or the " +
"(waits-for): the running indexer wrote only what building and moving that indexer needs, and the new indexer takes " + "objects are full. A group (what group, key g.<group>) is a delivery group whose members are gathered until the " +
"it in at its start. A group (what group, key g.<group>) is a delivery group whose members are gathered until the " +
"forge bridge says its end; a merge in one of its members' repositories waits behind it (behind). Each names its " + "forge bridge says its end; a merge in one of its members' repositories waits behind it (behind). Each names its " +
"merges, since when it waits, the modules it holds back (holds), the merges whose message the forge bridge's stream no " + "merges, since when it waits, the modules it holds back (holds), the merges whose message the forge bridge's stream no " +
"longer holds (lost: they are not taken in; indexer_drop_pending drops them) and the last take-in event said of it. Empty waits: " + "longer holds (lost: they are not taken in; indexer_drop_pending drops them) and the last take-in event said of it. Empty waits: " +
"every merge is taken in. The watchman raises take-in-waits past 60 minutes for a revision, 30 for a group; " + "every merge is taken in. The watchman raises take-in-waits past 60 minutes for a revision, 30 for a group; " +
"indexer_take_in_pending and indexer_drop_pending steer a wait. own-build says which build the indexer knows it runs and own-build-since since when; own-build-unknown, when set, says the indexer cannot learn the build it " + "indexer_take_in_pending and indexer_drop_pending steer a wait. own-build says which build the indexer knows it runs and own-build-since since when; it only names the writer, and no merge waits for it.",
"runs (since when): it fixes it once at its start, so every merge waits (a take-in.pending that says so) until the module is started again.",
Input: map[string]any{"type": "object", "properties": map[string]any{}}, Input: map[string]any{"type": "object", "properties": map[string]any{}},
Run: func(map[string]any) (any, error) { Run: func(map[string]any) (any, error) {
var unread []string var unread []string
@@ -63,7 +60,7 @@ func PendingTools(pending, ownBuild Store, ask func(key string, value []byte) (T
return nil, err return nil, err
} }
answer := takein.PendingAnswer{Waits: PendingWaits(entries), Unread: unread} answer := takein.PendingAnswer{Waits: PendingWaits(entries), Unread: unread}
// Whether this indexer knows the build it runs: when it does not, every merge waits, and this says since when and why. // The build this indexer knows it runs, and since when.
if ownBuild != nil { if ownBuild != nil {
if v, _, found, err := ownBuild.Get(context.Background(), OwnBuildKey); err == nil && found { if v, _, found, err := ownBuild.Get(context.Background(), OwnBuildKey); err == nil && found {
var o OwnBuild var o OwnBuild
@@ -71,8 +68,6 @@ func PendingTools(pending, ownBuild Store, ask func(key string, value []byte) (T
answer.OwnBuildSince = o.Since answer.OwnBuildSince = o.Since
if o.Known { if o.Known {
answer.OwnBuild = o.FP answer.OwnBuild = o.FP
} else {
answer.OwnBuildUnknown = o.Why
} }
} }
} }
@@ -80,10 +75,10 @@ func PendingTools(pending, ownBuild Store, ask func(key string, value []byte) (T
return answer, nil return answer, nil
}}, }},
{Name: TakeInPendingTool, {Name: TakeInPendingTool,
Description: "Take a wait in by hand with the running indexer and its judge (ADR 0633 §5): the revision under its m. key, " + Description: "Take in now a pending revision (its m.<sequence> key), for a merge whose message is unread or lost or that is held " +
"whatever build of the indexer it names, or a delivery group under its g. key with the members gathered so far. For " + "for room, or a delivery group (its g.<group> key) with the members gathered so far, for when its end never comes (ADR 0633 §5). " +
"when the named indexer can never run, or a group's end never comes. A hand act: who and why are said in the " + "A hand act: who and why are said in the indexer's journal and on the take-in.taken event, which clears take-in-waits. " +
"indexer's journal and on the take-in.taken event, which clears take-in-waits. indexer_pending names the keys.", "indexer_pending names the keys.",
Input: map[string]any{"type": "object", "required": []string{"key", "why"}, "properties": map[string]any{ Input: map[string]any{"type": "object", "required": []string{"key", "why"}, "properties": map[string]any{
"key": map[string]any{"type": "string", "description": "the wait's key, m.<sequence> or g.<group>, as indexer_pending names it"}, "key": map[string]any{"type": "string", "description": "the wait's key, m.<sequence> or g.<group>, as indexer_pending names it"},
"why": map[string]any{"type": "string", "description": "why it is taken in by hand, in plain words"}}}, "why": map[string]any{"type": "string", "description": "why it is taken in by hand, in plain words"}}},
@@ -112,33 +107,3 @@ func PendingTools(pending, ownBuild Store, ask func(key string, value []byte) (T
Run: merge}, Run: merge},
} }
} }
// ValidationTools is indexer_validate_again (class act, design 47 G §713): the tools bundle writes an ask in takeins and the
// indexer asks the revision's validation again; record says the hand act in the hand-act log before it is asked.
func ValidationTools(ask func(key string, value []byte) (TakeInAsk, error), now func() time.Time, record func(what, why string) error) []stdio.Tool {
return []stdio.Tool{{Name: ValidateAgainTool,
Description: "Ask again the validation of a pending revision that could not validate (design 47 G §712, §713): the manifest " +
"validator's run on the build seat node-build-go ended could not run, exited other than 0, or answered not both cases. " +
"The revision is asked under its next try; the answer says the validation's state (the ask, its try, accepted, refused, " +
"or why it could not validate). A revision accepted, refused or whose ask is out is asked nothing: a refusal is asked " +
"again only by a later merge of its branch. A hand act, recorded through mesh-hand-acts before it is asked. " +
"indexer_pending names the keys and shows the validation whole.",
Input: map[string]any{"type": "object", "required": []string{"key", "why"}, "properties": map[string]any{
"key": map[string]any{"type": "string", "description": "the revision's wait key, m.<sequence>, as indexer_pending names it"},
"why": map[string]any{"type": "string", "description": "why it is asked again, in plain words"}}},
Run: func(args map[string]any) (any, error) {
str := func(k string) string { s, _ := args[k].(string); return s }
key, value, err := NewValidateAgainAsk(str("key"), str("why"), stdio.CallerOf(args), now())
if err != nil {
return nil, err
}
if err := record("asked the validation of the pending revision "+strings.TrimSpace(str("key"))+" again by hand", strings.TrimSpace(str("why"))); err != nil {
return nil, fmt.Errorf("the hand act was not recorded through mesh-hand-acts, so nothing was asked: %w", err)
}
a, err := ask(key, value)
if err != nil {
return nil, err
}
return a.Result, nil
}}}
}
@@ -324,24 +324,28 @@ func CatalogRevisionTools(o Objects, ask func(key string, value []byte) (TakeInA
ctx := context.Background() ctx := context.Background()
n, _ := args["revision"].(float64) n, _ := args["revision"].(float64)
name := "" name := ""
var rev meshcatalog.CatalogRevision
var entries []meshcatalog.CatalogEntry
switch { switch {
case str(args, "group") != "" || str(args, "heads") != "": case str(args, "group") != "" || str(args, "heads") != "":
name = RevisionObject(meshcatalog.CatalogRevision{Kind: "candidate", Group: str(args, "group"), Heads: str(args, "heads")}) name = RevisionObject(meshcatalog.CatalogRevision{Kind: "candidate", Group: str(args, "group"), Heads: str(args, "heads")})
case n > 0: case n > 0:
name = "trunk." + strconv.FormatInt(int64(n), 10) name = "trunk." + strconv.FormatInt(int64(n), 10)
default: default:
newest, ok, err := newestTrunk(ctx, o) newest, newestEntries, ok, err := readNewestOf(ctx, o, TrunkBranch)
if err != nil { if err != nil {
return nil, fmt.Errorf("the catalog graph revisions cannot be read: %w", err) return nil, fmt.Errorf("the catalog graph revisions cannot be read: %w", err)
} }
if !ok { if !ok {
return nil, ErrNoRevision return nil, ErrNoRevision
} }
name = RevisionObject(newest) name, rev, entries = RevisionObject(newest), newest, newestEntries
} }
rev, entries, err := ReadRevision(ctx, o, name) if entries == nil {
if err != nil { var err error
return nil, err if rev, entries, err = ReadRevision(ctx, o, name); err != nil {
return nil, err
}
} }
return map[string]any{"name": name, "revision": rev, "entries": entries}, nil return map[string]any{"name": name, "revision": rev, "entries": entries}, nil
}}, }},
@@ -65,9 +65,20 @@ const ToolRunner = "node-tools"
// answered only when the newest trunk revision's entry of its module declares the verb in invokes, and the holder is // answered only when the newest trunk revision's entry of its module declares the verb in invokes, and the holder is
// refused too. With no trunk revision kept yet the answer is `none-yet` alone (design 47 G §530), never an empty // refused too. With no trunk revision kept yet the answer is `none-yet` alone (design 47 G §530), never an empty
// catalog and never an error. // catalog and never an error.
func Revision(ctx context.Context, o Objects, holder, caller string, in meshcatalog.RevisionInput) (meshcatalog.RevisionAnswer, error) { //
// kept, when not nil, is the trunk's newest revision held in memory: the trunk's newest answers from it with no read of
// the bucket (A-487c); a candidate, a branch and an older number read the bucket.
func Revision(ctx context.Context, o Objects, kept *KeptTrunk, holder, caller string, in meshcatalog.RevisionInput) (meshcatalog.RevisionAnswer, error) {
var out meshcatalog.RevisionAnswer var out meshcatalog.RevisionAnswer
newest, ok, err := newestTrunk(ctx, o) var newest meshcatalog.CatalogRevision
var ok bool
var trunkEntries, branchEntries []meshcatalog.CatalogEntry
var err error
if kept != nil {
newest, trunkEntries, ok, err = kept.Newest(ctx, o)
} else {
newest, trunkEntries, ok, err = readNewestOf(ctx, o, TrunkBranch)
}
if err != nil { if err != nil {
return out, fmt.Errorf("the catalog graph revisions cannot be read: %w", err) return out, fmt.Errorf("the catalog graph revisions cannot be read: %w", err)
} }
@@ -75,10 +86,6 @@ func Revision(ctx context.Context, o Objects, holder, caller string, in meshcata
// An answer is told by its fields (design 47 G §530): no trunk revision kept yet is `none-yet`, no error. // An answer is told by its fields (design 47 G §530): no trunk revision kept yet is `none-yet`, no error.
return meshcatalog.RevisionAnswer{NoneYet: true}, nil return meshcatalog.RevisionAnswer{NoneYet: true}, nil
} }
_, trunkEntries, err := ReadRevision(ctx, o, RevisionObject(newest))
if err != nil {
return out, err
}
caller = CallerModule(trunkEntries, caller) caller = CallerModule(trunkEntries, caller)
if err := views.Refusal(meshcatalog.Seat+"."+meshcatalog.VerbRevision, RevisionCallers(trunkEntries, holder), caller); err != nil { if err := views.Refusal(meshcatalog.Seat+"."+meshcatalog.VerbRevision, RevisionCallers(trunkEntries, holder), caller); err != nil {
return out, fmt.Errorf("%w (novox/hq ADR 0663 §2: a part reads it by declaring %s in invokes)", err, RevisionInvoke) return out, fmt.Errorf("%w (novox/hq ADR 0663 §2: a part reads it by declaring %s in invokes)", err, RevisionInvoke)
@@ -90,14 +97,14 @@ func Revision(ctx context.Context, o Objects, holder, caller string, in meshcata
return out, fmt.Errorf("the branch %q is no escaped name: %w", in.Branch, err) return out, fmt.Errorf("the branch %q is no escaped name: %w", in.Branch, err)
} }
if !isTrunkBranch(branch) { if !isTrunkBranch(branch) {
other, found, err := newestOf(ctx, o, branch) other, entries, found, err := readNewestOf(ctx, o, branch)
if err != nil { if err != nil {
return out, fmt.Errorf("the catalog graph revisions of %s cannot be read: %w", branch, err) return out, fmt.Errorf("the catalog graph revisions of %s cannot be read: %w", branch, err)
} }
if !found { if !found {
return meshcatalog.RevisionAnswer{NoneYet: true}, nil return meshcatalog.RevisionAnswer{NoneYet: true}, nil
} }
newest = other newest, branchEntries = other, entries
} }
name := RevisionObject(newest) name := RevisionObject(newest)
switch { switch {
@@ -121,7 +128,9 @@ func Revision(ctx context.Context, o Objects, holder, caller string, in meshcata
} }
rev, entries := newest, trunkEntries rev, entries := newest, trunkEntries
if !isTrunkBranch(branch) && in.Group == "" && in.Heads == "" { if !isTrunkBranch(branch) && in.Group == "" && in.Heads == "" {
if rev, entries, err = ReadRevision(ctx, o, name); err != nil { if name == RevisionObject(newest) {
entries = branchEntries // the newest of the branch, read by readNewestOf (A-487d)
} else if rev, entries, err = ReadRevision(ctx, o, name); err != nil {
return out, err return out, err
} }
} else if name != RevisionObject(newest) { } else if name != RevisionObject(newest) {
@@ -152,13 +161,13 @@ func Revision(ctx context.Context, o Objects, holder, caller string, in meshcata
// RevisionTool is the verb revision as the tool mesh-catalog.revision of the indexer's own runtime (ADR 0663 §1): the // RevisionTool is the verb revision as the tool mesh-catalog.revision of the indexer's own runtime (ADR 0663 §1): the
// tool runner describes it with the seat's definition, so it carries no description or input of its own. // tool runner describes it with the seat's definition, so it carries no description or input of its own.
func RevisionTool(o Objects, holder string) stdio.Tool { func RevisionTool(o Objects, kept *KeptTrunk, holder string) stdio.Tool {
return stdio.Tool{Name: meshcatalog.Seat + "." + meshcatalog.VerbRevision, Run: func(args map[string]any) (any, error) { return stdio.Tool{Name: meshcatalog.Seat + "." + meshcatalog.VerbRevision, Run: func(args map[string]any) (any, error) {
var in meshcatalog.RevisionInput var in meshcatalog.RevisionInput
if raw, err := json.Marshal(args); err == nil { if raw, err := json.Marshal(args); err == nil {
_ = json.Unmarshal(raw, &in) _ = json.Unmarshal(raw, &in)
} }
return Revision(context.Background(), o, holder, stdio.CallerOf(args), in) return Revision(context.Background(), o, kept, holder, stdio.CallerOf(args), in)
}} }}
} }
@@ -63,17 +63,13 @@ func (ix *Indexer) SeedModulesFrom(ctx context.Context, defs map[string]seatdef.
if ix.Revisions == nil { if ix.Revisions == nil {
return nil, fmt.Errorf("the catalog graph is not kept here: no trunk revision to seed from") return nil, fmt.Errorf("the catalog graph is not kept here: no trunk revision to seed from")
} }
rev, found, err := newestTrunk(ctx, ix.Revisions) _, entries, found, err := readNewestOf(ctx, ix.Revisions, TrunkBranch)
if err != nil { if err != nil {
return nil, err return nil, err
} }
if !found { if !found {
return nil, fmt.Errorf("the indexer keeps no trunk revision yet: the seed waits for its first") return nil, fmt.Errorf("the indexer keeps no trunk revision yet: the seed waits for its first")
} }
_, entries, err := ReadRevision(ctx, ix.Revisions, fmt.Sprintf("trunk.%d", rev.Revision))
if err != nil {
return nil, err
}
seatOf := map[string]string{} // artifact reference -> its build seat, all found before any call is made seatOf := map[string]string{} // artifact reference -> its build seat, all found before any call is made
unbuilt := map[string][]string{} // module -> each artifact of it no build seat builds, with its language or kind unbuilt := map[string][]string{} // module -> each artifact of it no build seat builds, with its language or kind
for _, e := range entries { for _, e := range entries {
@@ -51,9 +51,6 @@ type TakeInAsk struct {
// Abandon, with Drop, drops a pending merge no later merge covers (ADR 0633 §5 asks a later merge; issue 1966, ADR // Abandon, with Drop, drops a pending merge no later merge covers (ADR 0633 §5 asks a later merge; issue 1966, ADR
// 0668 §5): the hand act says why it is abandoned all the same. // 0668 §5): the hand act says why it is abandoned all the same.
Abandon bool `json:"abandon,omitempty"` Abandon bool `json:"abandon,omitempty"`
// ValidateAgain, with Pending, asks the validation of the revision under that key again (indexer_validate_again, design 47
// G §713): one that could not validate.
ValidateAgain bool `json:"validate-again,omitempty"`
// Merge asks the merge of MergeRepository at this commit to be taken in by hand (indexer_take_in_merge): its merged message // Merge asks the merge of MergeRepository at this commit to be taken in by hand (indexer_take_in_merge): its merged message
// is asked of the forge bridge and run through the take-in as a delivered one, under its stream sequence Sequence. // is asked of the forge bridge and run through the take-in as a delivered one, under its stream sequence Sequence.
Merge string `json:"merge,omitempty"` Merge string `json:"merge,omitempty"`
@@ -167,20 +164,6 @@ func NewMergeAsk(repository, merge string, sequence uint64, why, by string, at t
return "merge." + merge, data, err return "merge." + merge, data, err
} }
// NewValidateAgainAsk is the value indexer_validate_again writes: its key in takeins, `pending.validate-again.<key>`, and the
// value. key is a revision's wait key, m.<sequence>, as indexer_pending names it.
func NewValidateAgainAsk(key, why, by string, at time.Time) (string, []byte, error) {
key = strings.TrimSpace(key)
if !strings.HasPrefix(key, "m.") {
return "", nil, errors.New("name the revision whose validation is asked again by its key, m.<sequence>, as indexer_pending names it")
}
if strings.TrimSpace(why) == "" {
return "", nil, errors.New("say why (why): a hand act is recorded through mesh-hand-acts")
}
data, err := json.Marshal(TakeInAsk{V: 1, Pending: key, ValidateAgain: true, Why: strings.TrimSpace(why), By: by, At: at.UTC()})
return "pending.validate-again." + key, data, err
}
// NewPendingAsk is the value indexer_take_in_pending (drop false) or indexer_drop_pending (drop true) writes: its key // NewPendingAsk is the value indexer_take_in_pending (drop false) or indexer_drop_pending (drop true) writes: its key
// in takeins, `pending.<take-in|drop>.<key>`, and the value. key is a wait's key as indexer_pending names it, for a // in takeins, `pending.<take-in|drop>.<key>`, and the value. key is a wait's key as indexer_pending names it, for a
// drop one pending merge's; abandon drops one no later merge covers. // drop one pending merge's; abandon drops one no later merge covers.
@@ -13,6 +13,7 @@ import (
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep" sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/trunk"
"git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/catalogue"
"mesh-delivery-indexer/judge" "mesh-delivery-indexer/judge"
@@ -127,24 +128,26 @@ type known struct {
// the order the bucket lists them in; modules that stand on each other have a cycle as their reason. A module is written // the order the bucket lists them in; modules that stand on each other have a cycle as their reason. A module is written
// only when its fingerprint or its reason changes, and only if its manifest is still the one computed from. A module with // only when its fingerprint or its reason changes, and only if its manifest is still the one computed from. A module with
// no kept inputs (a message that carried no manifest) is left as it is. // no kept inputs (a message that carried no manifest) is left as it is.
func (ix *Indexer) Recompute(ctx context.Context) error { return ix.recomputeWith(ctx, nil, nil) } func (ix *Indexer) Recompute(ctx context.Context) error { return ix.recomputeWith(ctx, nil, nil, "") }
// recomputeWith is Recompute for a merge as well: the records the merge gives (overlay, by module key) stand in place // recomputeWith is Recompute for a merge as well: the records the merge gives (overlay, by module key) stand in place
// of what the bucket holds, are computed with the others, and are written once, with their final fingerprint or reason // of what the bucket holds, are computed with the others, and are written once, with their final fingerprint or reason
// (issue 1444: a merge writes each module once). A record of the overlay that has no kept inputs is written as it is. w is // (issue 1444: a merge writes each module once). A record of the overlay that has no kept inputs is written as it is. w is
// the merge's write (order.go), nil for none; it is told each module written. // the merge's write (order.go), nil for none; it is told each module written. A merge's recompute names its repository
func (ix *Indexer) recomputeWith(ctx context.Context, overlay map[string]meshcatalog.ModulesByModuleValue, w *mergeWrite) error { // (A-476 c, A-476a): the modules registered for it are moved, and those of another repository only when they import a package it moved; "" moves every module (a toolchain
// report, a hand recompute).
func (ix *Indexer) recomputeWith(ctx context.Context, overlay map[string]meshcatalog.ModulesByModuleValue, w *mergeWrite, repository string) error {
if err := ix.fillPaths(ctx); err != nil { if err := ix.fillPaths(ctx); err != nil {
return err return err
} }
if err := ix.recomputeModules(ctx, overlay, w); err != nil { if err := ix.recomputeModules(ctx, overlay, w, repository); err != nil {
return err return err
} }
// What the modules moved to reaches the candidates' fingerprints too (a base, the builder, a toolchain). // What the modules moved to reaches the candidates' fingerprints too (a base, the builder, a toolchain).
return ix.RecomputeCandidates(ctx) return ix.RecomputeCandidates(ctx)
} }
func (ix *Indexer) recomputeModules(ctx context.Context, overlay map[string]meshcatalog.ModulesByModuleValue, w *mergeWrite) error { func (ix *Indexer) recomputeModules(ctx context.Context, overlay map[string]meshcatalog.ModulesByModuleValue, w *mergeWrite, repository string) error {
if ix.Modules == nil { if ix.Modules == nil {
return nil return nil
} }
@@ -153,11 +156,11 @@ func (ix *Indexer) recomputeModules(ctx context.Context, overlay map[string]mesh
} }
ix.recompute.Lock() ix.recompute.Lock()
defer ix.recompute.Unlock() defer ix.recompute.Unlock()
return ix.recomputeLocked(ctx, overlay, w) return ix.recomputeLocked(ctx, overlay, w, repository)
} }
// recomputeLocked is recomputeModules with ix.recompute held by the caller (a hold that moves, holds.go). // recomputeLocked is recomputeModules with ix.recompute held by the caller (a hold that moves, holds.go).
func (ix *Indexer) recomputeLocked(ctx context.Context, overlay map[string]meshcatalog.ModulesByModuleValue, w *mergeWrite) error { func (ix *Indexer) recomputeLocked(ctx context.Context, overlay map[string]meshcatalog.ModulesByModuleValue, w *mergeWrite, repository string) error {
if ix.Manifests == nil { if ix.Manifests == nil {
return ix.writeOverlay(ctx, overlay, nil, w) return ix.writeOverlay(ctx, overlay, nil, w)
} }
@@ -298,6 +301,14 @@ func (ix *Indexer) recomputeLocked(ctx context.Context, overlay map[string]meshc
r.tcs, r.pkgs = toolchainsUsed(toolchains), f.versions r.tcs, r.pkgs = toolchainsUsed(toolchains), f.versions
} }
} }
if _, merged := overlay[module]; !merged && repository != "" && !trunk.SameRepository(k.cur.Repository, repository) &&
maps.Equal(k.held.Packages, map[string]string(r.pkgs)) {
// A module registered for another repository is moved by this repository's commit only when it imports a package the
// commit moved (A-476a, ADR 0593 §3); any other stands as held (A-476 c), and so it is for a base of this repository's.
r = result{fp: k.held.Fp2, none: k.held.None2, tcs: k.held.Toolchains, pkgs: sdkstep.PackageVersions(k.held.Packages)}
done[module] = r
return r
}
done[module] = r done[module] = r
k.cur.Inputs2 = &kept k.cur.Inputs2 = &kept
return r return r
@@ -645,22 +645,22 @@ func TestBasesAreComputedFirstAndNoFingerprintTakesTheBuilder(t *testing.T) { //
t.Fatal(err) t.Fatal(err)
} }
} }
if got, err := Revision(context.Background(), verbObjects, testSelf, "n1.node-tools", meshcatalog.RevisionInput{}); err != nil || !got.NoneYet || got.Revision != nil || len(got.Entries) != 0 { if got, err := Revision(context.Background(), verbObjects, nil, testSelf, "n1.node-tools", meshcatalog.RevisionInput{}); err != nil || !got.NoneYet || got.Revision != nil || len(got.Entries) != 0 {
t.Errorf("revision with none kept answered %+v, %v, want none-yet alone with no error (design 47 G §530)", got, err) t.Errorf("revision with none kept answered %+v, %v, want none-yet alone with no error (design 47 G §530)", got, err)
} }
keep(meshcatalog.CatalogRevision{Revision: 3, Kind: "trunk"}, map[string]bool{"node-tools": true, testSelf: true}) keep(meshcatalog.CatalogRevision{Revision: 3, Kind: "trunk"}, map[string]bool{"node-tools": true, testSelf: true})
keep(meshcatalog.CatalogRevision{Revision: 4, Kind: "candidate", Group: "feat/x", Heads: "0123456789ab"}, map[string]bool{"node-tools": true, "mesh-grants": true}) keep(meshcatalog.CatalogRevision{Revision: 4, Kind: "candidate", Group: "feat/x", Heads: "0123456789ab"}, map[string]bool{"node-tools": true, "mesh-grants": true})
for caller, answered := range map[string]bool{"n1.node-tools": true, "n1.avahi": true, "n1.mesh-grants": true, "n1." + testSelf: true, "person.someone": false, "unproven": false} { for caller, answered := range map[string]bool{"n1.node-tools": true, "n1.avahi": true, "n1.mesh-grants": true, "n1." + testSelf: true, "person.someone": false, "unproven": false} {
got, err := Revision(context.Background(), verbObjects, testSelf, caller, meshcatalog.RevisionInput{}) got, err := Revision(context.Background(), verbObjects, nil, testSelf, caller, meshcatalog.RevisionInput{})
if (err == nil) != answered || (answered && (got.Revision.Revision != 3 || len(got.Entries) != 4 || got.Next != "")) { if (err == nil) != answered || (answered && (got.Revision.Revision != 3 || len(got.Entries) != 4 || got.Next != "")) {
t.Errorf("revision to %s answered %+v (%v), want answered %v with trunk revision 3 whole", caller, got, err, answered) t.Errorf("revision to %s answered %+v (%v), want answered %v with trunk revision 3 whole", caller, got, err, answered)
} }
} }
if got, err := Revision(context.Background(), verbObjects, testSelf, "n1.node-tools", meshcatalog.RevisionInput{Group: "feat/x", Heads: "0123456789ab", After: "mesh-grants"}); err != nil || if got, err := Revision(context.Background(), verbObjects, nil, testSelf, "n1.node-tools", meshcatalog.RevisionInput{Group: "feat/x", Heads: "0123456789ab", After: "mesh-grants"}); err != nil ||
got.Revision.Kind != "candidate" || len(got.Entries) != 1 || got.Entries[0].Module != "node-tools" { got.Revision.Kind != "candidate" || len(got.Entries) != 1 || got.Entries[0].Module != "node-tools" {
t.Errorf("a candidate's revision after mesh-grants answered %+v (%v), want its one entry node-tools", got, err) t.Errorf("a candidate's revision after mesh-grants answered %+v (%v), want its one entry node-tools", got, err)
} }
if _, err := Revision(context.Background(), verbObjects, testSelf, "n1.node-tools", meshcatalog.RevisionInput{Revision: 9}); err == nil { if _, err := Revision(context.Background(), verbObjects, nil, testSelf, "n1.node-tools", meshcatalog.RevisionInput{Revision: 9}); err == nil {
t.Error("a trunk revision not kept was answered") t.Error("a trunk revision not kept was answered")
} }
// ADR 0614 §2, §3: the graph names what each module invokes and serves, and leaves out, over the whole graph, the // ADR 0614 §2, §3: the graph names what each module invokes and serves, and leaves out, over the whole graph, the
@@ -832,6 +832,8 @@ func TestAModuleKeepsTheToolchainsItsFingerprintWasComputedWith(t *testing.T) {
if err := ix.Apply(context.Background(), f); err != nil { if err := ix.Apply(context.Background(), f); err != nil {
t.Fatal(err) t.Fatal(err)
} }
// A-476a: a commit of mesh-sdk moves avahi, of another repository, which imports its package, and leaves mesh-tools,
// which imports nothing it moved.
a, _ := moduleOf(t, mods, "avahi") a, _ := moduleOf(t, mods, "avahi")
if a.Fp2 == "" || !reflect.DeepEqual(a.Packages, map[string]string{sdk: commit}) { if a.Fp2 == "" || !reflect.DeepEqual(a.Packages, map[string]string{sdk: commit}) {
t.Fatalf("after mesh-sdk's merge %.8s: fp2 %q none2 %q packages %v", commit, a.Fp2, a.None2, a.Packages) t.Fatalf("after mesh-sdk's merge %.8s: fp2 %q none2 %q packages %v", commit, a.Fp2, a.None2, a.Packages)
@@ -841,12 +843,24 @@ func TestAModuleKeepsTheToolchainsItsFingerprintWasComputedWith(t *testing.T) {
} }
return a return a
} }
// A-476 c: mesh-tools, of another repository than mesh-sdk, holds a fingerprint a recompute would move (here a stale one); a
// commit of mesh-sdk, which moved nothing it imports, leaves it as held, and only a full recompute moves it.
stale := tools
stale.Fp2 = "src2:" + strings.Repeat("5", 64)
mods.put("mesh-tools", mustEncode(t, stale))
tools = stale
first := sdkMerge(strings.Repeat("1", 40), "tree-sdk-1") first := sdkMerge(strings.Repeat("1", 40), "tree-sdk-1")
second := sdkMerge(strings.Repeat("2", 40), "tree-sdk-2") second := sdkMerge(strings.Repeat("2", 40), "tree-sdk-2")
if second.Fp2 == first.Fp2 { if second.Fp2 == first.Fp2 {
t.Errorf("a mesh-sdk trunk merge kept avahi's fingerprint %q", first.Fp2) t.Errorf("a mesh-sdk trunk merge kept avahi's fingerprint %q", first.Fp2)
} }
n = second if err := ix.Recompute(context.Background()); err != nil {
t.Fatal(err)
}
if o, _ := moduleOf(t, mods, "mesh-tools"); o.Fp2 == stale.Fp2 {
t.Errorf("a full recompute left mesh-tools at %q: the check above proves nothing", o.Fp2)
}
n, _ = moduleOf(t, mods, "avahi") // moved by the recompute too: its base mesh-tools moved
// Nodes that disagree keep the fingerprint already computed, with its toolchains (issue 1940, 2026-10-25 20:56: a // Nodes that disagree keep the fingerprint already computed, with its toolchains (issue 1940, 2026-10-25 20:56: a
// stale report blanked every module and the controller's take-in refused modules already built); only a new // stale report blanked every module and the controller's take-in refused modules already built); only a new
// fingerprint needs the agreement: a merge that moves the module's tree gives it none, naming the disagreement. // fingerprint needs the agreement: a merge that moves the module's tree gives it none, naming the disagreement.
@@ -1,264 +0,0 @@
package index
import (
"context"
"encoding/json"
"errors"
"fmt"
"maps"
"slices"
"strings"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/composer"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/trunk"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"mesh-delivery-indexer/judge"
)
// The validation before take-in (design 47 G §712, A-213, A-232): for a pending revision of a watched branch the running
// indexer asks the seat node-build-go a `run` of its own manifest validator over two cases, change and trunk, and refuses only
// what the revision causes (ADR 0579 §2). The ask rides the check queue under the indexer's name, its kind `run` in the body.
const (
// validationProgram is the manifest validator's program, by its directory under the indexer entry's Go module: the indexer's
// own code, so no repository or module name is fixed here (ADR 0666 §5).
validationProgram = "cmd/mesh-judge"
// ValidationAsker is the asker the ask is put under (buildrun.Asker.Name, A-248): the indexer's seat name, mesh-catalog
// (design 47 G A-163, A-232).
ValidationAsker = meshcatalog.Seat
// validationCatalogSeat is the seat the indexer entry claims.
validationCatalogSeat = "mesh-catalog"
// The cases of the request, as the manifest validator names them.
validationChange = "change"
validationTrunk = judge.TrunkCase
)
// validationInputVerb is where the build seat's holder asks the input: the seat mesh-catalog's verb input (§354).
var validationInputVerb = "seat:" + meshcatalog.Seat + "." + meshcatalog.VerbInput
// validationRequest is the manifest validator's request for the ready merges of branch over the branch's current revision
// (§712): case change, every module of the revision by its entry's manifest, each one the merges carry replaced by the merged
// message's bytes (the newest merge governs) and one added for a new directory; case trunk, every module as it stands; the
// seat definitions the merges' trees carry; the packages and the predefined list as the take-in's own validator has them;
// no machines. It answers the indexer entry of the change case too, the one that claims the seat mesh-catalog.
func (ix *Indexer) validationRequest(ctx context.Context, branch string, ready []PendingEntry) (composer.Request, meshcatalog.CatalogEntry, error) {
var indexer meshcatalog.CatalogEntry
if ix.Revisions == nil {
return composer.Request{}, indexer, errors.New("the catalog graph is not kept here: no revision to validate over")
}
rev, found, err := newestOf(ctx, ix.Revisions, branch)
if err != nil {
return composer.Request{}, indexer, err
}
var entries []meshcatalog.CatalogEntry
if found {
if _, entries, err = ReadRevision(ctx, ix.Revisions, RevisionObject(rev)); err != nil {
return composer.Request{}, indexer, err
}
}
trunkCase := composer.Case{Name: validationTrunk, Manifests: map[string]composer.Manifest{}}
byPlace := map[string]string{} // <repository> <path> -> module
of := map[string]meshcatalog.CatalogEntry{}
for _, e := range entries {
raw := ix.manifestBytes(ctx, e.ManifestDigest)
if raw == "" {
b, err := json.Marshal(e.Manifest)
if err != nil {
return composer.Request{}, indexer, fmt.Errorf("the manifest of %s: %w", e.Module, err)
}
raw = string(b)
}
trunkCase.Manifests[e.Module] = composer.Manifest{Raw: json.RawMessage(raw)}
byPlace[placeKey(e.Repository, e.Path)] = e.Module
of[e.Module] = e
}
change := composer.Case{Name: validationChange, Manifests: maps.Clone(trunkCase.Manifests)}
for _, e := range ready {
if e.Covered {
continue
}
for _, dir := range slices.Sorted(maps.Keys(e.Forge.Manifests)) {
raw := e.Forge.Manifests[dir]
module, known := byPlace[placeKey(e.Forge.Repository, dir)]
if !known {
m, err := judge.ForInputs([]byte(raw))
if err != nil || m.Module == "" {
module = e.Forge.Repository + "/" + dir // a new directory whose manifest names no module: judged under its place
} else {
module = m.Module
}
byPlace[placeKey(e.Forge.Repository, dir)] = module
of[module] = meshcatalog.CatalogEntry{Module: module, Repository: e.Forge.Repository, Path: dir}
}
change.Manifests[module] = composer.Manifest{Raw: json.RawMessage(raw)}
}
}
for _, module := range slices.Sorted(maps.Keys(change.Manifests)) {
m, err := judge.ForInputs(change.Manifests[module].Raw)
if err != nil || !slices.ContainsFunc(m.Claims, func(c catalogue.Claim) bool { return c.Name == validationCatalogSeat }) {
continue
}
indexer = of[module]
if indexer.Module == "" {
indexer.Module = module
}
break
}
seats, err := ix.revisionDefinitions(ctx, ready)
if err != nil {
return composer.Request{}, indexer, err
}
req := composer.Request{V: composer.Version, Input: json.RawMessage("{}"), Cases: []composer.Case{change, trunkCase}, Seats: seats,
Predefined: ix.predefinedList(ctx)}
if known := ix.knownPackages(ctx); known != nil {
req.Packages = slices.Sorted(maps.Keys(known))
}
return req, indexer, nil
}
func placeKey(repository, path string) string {
return strings.ToLower(trunk.RepositoryName(repository)) + " " + path
}
// revisionDefinitions are the validation's seat definitions, raw (§712 `seats`, A-241): the branch's revision's seat
// definitions with the definitions the merges' trees carry laid over them, so a revision that touches no definition is still
// judged against the whole seat list. A revision's own definitions are not kept yet (A-234), so the revision's are the
// trunk's, read from the state seat-definitions as the verb seat-definitions answers them.
func (ix *Indexer) revisionDefinitions(ctx context.Context, ready []PendingEntry) ([]json.RawMessage, error) {
defs, err := ix.trunkDefinitions(ctx)
if err != nil {
return nil, err
}
merged, err := ix.revisionSeatDefinitions(ctx, ready)
if err != nil {
return nil, err
}
for seat, d := range merged {
raw, err := json.Marshal(d)
if err != nil {
return nil, fmt.Errorf("the definition of %s: %w", seat, err)
}
defs[seat] = raw
}
var out []json.RawMessage
for _, seat := range slices.Sorted(maps.Keys(defs)) {
out = append(out, defs[seat])
}
return out, nil
}
// trunkDefinitions are the trunk's seat definitions of the state seat-definitions, raw, by seat: every `<seat>.trunk` entry
// that holds a definition; none where the indexer keeps no such state.
func (ix *Indexer) trunkDefinitions(ctx context.Context) (map[string]json.RawMessage, error) {
defs := map[string]json.RawMessage{}
if ix.SeatDefinitions == nil {
return defs, nil
}
keys, err := ix.SeatDefinitions.Keys(ctx)
if err != nil {
return nil, fmt.Errorf("the %s state cannot be listed: %w", SeatDefinitionsState, err)
}
for _, key := range keys {
seat, ok := strings.CutSuffix(key, "."+SourceTrunk)
if !ok || strings.HasPrefix(key, outOfStepPrefix) {
continue
}
v, _, found, err := ix.SeatDefinitions.Get(ctx, key)
if err != nil {
return nil, fmt.Errorf("%s.%s cannot be read: %w", SeatDefinitionsState, key, err)
}
var e SeatDefinitionEntry
if !found || json.Unmarshal(v, &e) != nil || e.Digest == "" || len(e.Definition) == 0 {
continue // an entry that holds no definition is no seat of the list
}
defs[seat] = e.Definition
}
return defs, nil
}
// validationJob is the run of the manifest validator for the request (§712): the program cmd/mesh-judge, its directory under
// the Go module the go.mod at the entry's path names (read by the build, A-268), at the entry's path, built at head, the revision's head of the entry's repository; no
// heads and no arguments; the input the request's bytes, served by the seat mesh-catalog's verb input.
func validationJob(indexer meshcatalog.CatalogEntry, head, branch string, seq uint64, input []byte) (buildstep.RunJob, error) {
switch {
case indexer.Repository == "":
return buildstep.RunJob{}, fmt.Errorf("no entry of the revision claims the seat %s", validationCatalogSeat)
case head == "":
return buildstep.RunJob{}, fmt.Errorf("the revision names no head of %s, the repository of the indexer entry %s", indexer.Repository, indexer.Module)
}
in, err := buildstep.RunInputOf(validationInputVerb, input)
if err != nil {
return buildstep.RunJob{}, err
}
dir := indexer.Path
if dir == "" {
dir = "."
}
return buildstep.RunJob{
Program: buildstep.RunProgram{Repository: indexer.Repository, Commit: head, Dir: dir, Package: "./" + validationProgram},
Input: in,
Reader: fmt.Sprintf("manifest validator of %s at %d", branchName(branch), seq),
}, nil
}
func branchName(branch string) string {
if isTrunkBranch(branch) {
return TrunkBranch
}
return branch
}
// ValidationRefusal is what a validation's verdict refuses (ADR 0579 §2, §712): of each manifest the change case refuses that
// the revision adds or changes (its bytes differ from the trunk case's), each problem the trunk case lacks; and each problem
// between manifests the trunk case lacks. A problem only in unchanged manifests refuses nothing. It is empty when the revision
// is accepted. An answer that holds not both cases is no verdict: it answers an error, so the run could not validate.
func ValidationRefusal(req composer.Request, v composer.Verdict) (map[string][]string, error) {
cases := map[string]composer.CaseVerdict{}
for _, c := range v.Cases {
cases[c.Name] = c
}
change, okChange := cases[validationChange]
trunkV, okTrunk := cases[validationTrunk]
if !okChange || !okTrunk {
return nil, fmt.Errorf("the manifest validator's answer holds not both cases %s and %s", validationChange, validationTrunk)
}
raws := map[string]map[string]string{}
for _, c := range req.Cases {
raws[c.Name] = map[string]string{}
for name, m := range c.Manifests {
raws[c.Name][name] = InputDigest(m.Raw)
}
}
refused := map[string][]string{}
for _, module := range slices.Sorted(maps.Keys(change.Manifests)) {
r := change.Manifests[module]
if r.Accepted || r.Refused == "" {
continue
}
before, had := raws[validationTrunk][module]
if module != judge.Unnamed && had && before == raws[validationChange][module] {
continue // unchanged: its problems refuse nothing, and its entry keeps its verdict (ADR 0579 §1)
}
old := map[string]bool{}
for _, p := range strings.Split(trunkV.Manifests[module].Refused, "\n") {
old[p] = true
}
for _, p := range strings.Split(r.Refused, "\n") {
if p != "" && !old[p] {
refused[module] = append(refused[module], p)
}
}
}
return refused, nil
}
// RefusedWords is a refusal as the pending value's `refused` says it: each manifest and its problems, one manifest a line.
func RefusedWords(refused map[string][]string) string {
var lines []string
for _, module := range slices.Sorted(maps.Keys(refused)) {
lines = append(lines, module+": "+strings.Join(refused[module], "; "))
}
return strings.Join(lines, "\n")
}
@@ -1,130 +0,0 @@
package index
import (
"context"
"encoding/json"
"reflect"
"strings"
"testing"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/composer"
"mesh-delivery-indexer/judge"
)
// The validation refuses only what the revision causes (design 47 G §712, ADR 0579 §2), and its run names the indexer entry's
// repository, path and the revision's head, and no heads.
func TestValidationRefusesWhatTheRevisionCauses(t *testing.T) {
manifests := func(m map[string]string) map[string]composer.Manifest {
out := map[string]composer.Manifest{}
for k, v := range m {
out[k] = composer.Manifest{Raw: json.RawMessage(v)}
}
return out
}
req := composer.Request{Cases: []composer.Case{
{Name: "change", Manifests: manifests(map[string]string{"a": `{"module":"a","v":2}`, "b": `{"module":"b"}`})},
{Name: "trunk", Manifests: manifests(map[string]string{"a": `{"module":"a"}`, "b": `{"module":"b"}`})},
}}
verdict := func(change, trunk map[string]string) composer.Verdict {
read := func(m map[string]string) map[string]composer.ManifestRead {
out := map[string]composer.ManifestRead{"a": {Accepted: true}, "b": {Accepted: true}}
for k, v := range m {
out[k] = composer.ManifestRead{Refused: v}
}
return out
}
return composer.Verdict{Cases: []composer.CaseVerdict{{Name: "change", Manifests: read(change)}, {Name: "trunk", Manifests: read(trunk)}}}
}
for _, c := range []struct {
name string
v composer.Verdict
want, wantErr string
}{
{"a changed manifest gains a problem", verdict(map[string]string{"a": "new\nold"}, map[string]string{"a": "old"}), "a: new", ""},
{"an unchanged manifest's problem refuses nothing", verdict(map[string]string{"b": "bad"}, nil), "", ""},
{"a problem between manifests the trunk lacks", verdict(map[string]string{judge.Unnamed: "x\ny"}, map[string]string{judge.Unnamed: "y"}), judge.Unnamed + ": x", ""},
{"an answer with one case is no verdict", composer.Verdict{Cases: verdict(nil, nil).Cases[:1]}, "", "not both cases"},
} {
got, err := ValidationRefusal(req, c.v)
if c.wantErr != "" {
if err == nil || !strings.Contains(err.Error(), c.wantErr) {
t.Errorf("%s: %v, want %q", c.name, err, c.wantErr)
}
continue
}
if err != nil || RefusedWords(got) != c.want {
t.Errorf("%s: %q, %v, want %q", c.name, RefusedWords(got), err, c.want)
}
}
head := strings.Repeat("a", 40)
entry := meshcatalog.CatalogEntry{Module: "mesh-delivery-indexer", Repository: "novox/mesh-delivery", Path: "modules/mesh-delivery-indexer",
Artifacts: []meshcatalog.Artifact{{Language: "go", Kind: "bundle"}}}
job, err := validationJob(entry, head, "main", 7, []byte(`{"v":1}`))
if err != nil {
t.Fatal(err)
}
want := buildstep.RunProgram{Repository: "novox/mesh-delivery", Commit: head, Dir: "modules/mesh-delivery-indexer", Package: "./cmd/mesh-judge"}
if job.Program != want || len(job.Heads) != 0 || len(job.Args) != 0 || job.Input.Verb != "seat:mesh-catalog.input" || job.Reader != "manifest validator of main at 7" {
t.Errorf("the job %+v", job)
}
if _, err := validationJob(meshcatalog.CatalogEntry{}, head, "main", 7, nil); err == nil || !strings.Contains(err.Error(), "claims the seat mesh-catalog") {
t.Errorf("no indexer entry: %v", err)
}
}
// The validation's seats are the branch's revision's seat definitions with the merges' changed ones laid over them (A-241): a
// revision that touches no definition is judged against the whole seat list, and a merge's definition replaces its seat's.
func TestValidationSeatsAreTheRevisionsWithTheMergesOver(t *testing.T) {
ctx := context.Background()
defs := newMem()
for seat, raw := range map[string]string{"node-engine": `{"seat":"node-engine","verbs":[{"name":"old"}]}`, "git": `{"seat":"git"}`} {
v, _ := json.Marshal(SeatDefinitionEntry{Seat: seat, Source: SourceTrunk, Digest: "d-" + seat, Definition: json.RawMessage(raw)})
defs.put(SeatDefinitionKey(seat, SourceTrunk, "", 0), v)
}
v, _ := json.Marshal(SeatDefinitionEntry{Seat: "pulled", Source: SourcePull, Pull: 3, Digest: "d", Definition: json.RawMessage(`{"seat":"pulled"}`)})
defs.put(SeatDefinitionKey("pulled", SourcePull, "r", 3), v)
ix := &Indexer{Module: testSelf, SeatDefinitions: defs}
seatsOf := func(raws []json.RawMessage) map[string]string {
out := map[string]string{}
for _, raw := range raws {
var d struct {
Seat string `json:"seat"`
Verbs []struct {
Name string `json:"name"`
} `json:"verbs"`
}
if err := json.Unmarshal(raw, &d); err != nil {
t.Fatal(err)
}
out[d.Seat] = ""
if len(d.Verbs) > 0 {
out[d.Seat] = d.Verbs[0].Name
}
}
return out
}
got, err := ix.revisionDefinitions(ctx, nil)
if err != nil || !reflect.DeepEqual(seatsOf(got), map[string]string{"node-engine": "old", "git": ""}) {
t.Errorf("a revision touching no definition: %v (%v), want the trunk's whole list and no pull request's", seatsOf(got), err)
}
const contracts = "modules/seat-contracts"
f := forgemsg.Forge{Repository: "r", Merge: strings.Repeat("c", 40), Manifests: map[string]string{
contracts: `{"module":"seat-contracts","version":"1.0","build":{"artifacts":[{"name":"go-seats","kind":"package","language":"go","paths":["go"],"series":"0.1","generated-from":"."}]}}`}}
ix.Files = func(_ context.Context, _, _ string, dirs []string) (map[string]map[string]string, error) {
return map[string]map[string]string{dirs[0]: {"node-engine/seat.json": "1"}}, nil
}
ix.ReadFile = func(_ context.Context, _, _, p string) ([]byte, error) {
if p == contracts+"/seats/node-engine/seat.json" {
return []byte(`{"format":1,"seat":"node-engine","scope":"node","grain":"seat-list","decision":"d","domain":"Core","verbs":[{"name":"new"}]}`), nil
}
return nil, nil
}
got, err = ix.revisionDefinitions(ctx, []PendingEntry{{Forge: f}})
if err != nil || !reflect.DeepEqual(seatsOf(got), map[string]string{"node-engine": "new", "git": ""}) {
t.Errorf("a merge changing a definition: %v (%v), want its definition over the trunk's and the rest kept", seatsOf(got), err)
}
}
@@ -1,329 +0,0 @@
package index
import (
"context"
"encoding/json"
"errors"
"fmt"
"strings"
"time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/composer"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein"
)
// The pending revision's wait for its validation (design 47 G §713, A-232, A-247): the take-in puts the run ask of §712, writes
// `validation` on the revision's newest pending value and ends there; the build seat's step-ended of that ask, heard through the
// indexer's own durable consumer, resumes it: accepted takes the revision in as before, refused keeps it pending with `refused`,
// and an end that could not validate keeps it pending with `failed`, asked again by a later merge of the branch or by hand.
// ValidationAsks puts a run ask on the build seat (buildrun.Asker).
type ValidationAsks interface {
Put(ctx context.Context, a buildstep.StepAsk) error
}
// newestOfRevision is the index in ready of the revision's newest merge, the value that carries `validation` (A-247).
func newestOfRevision(ready []PendingEntry) int {
n := 0
for i, e := range ready {
if e.Value.Seq > ready[n].Value.Seq {
n = i
}
}
return n
}
// branchOfRevision is the watched branch the revision's merges landed on, its newest merged message's base (design 47 G §714);
// a message that names none is the trunk's. A pending value's own branch is its pull request's, never the branch validated.
func branchOfRevision(ready []PendingEntry) string {
if base := ready[newestOfRevision(ready)].Forge.Base; base != "" {
return base
}
return TrunkBranch
}
// validationGate says whether the ready revision waits for its validation (§713): it goes on to the take-in once its newest
// value's validation was accepted at that value's sequence; it waits while its ask is out, while it is refused, and while it
// could not validate; otherwise it puts a new ask, the next try, and waits. Without an asker nothing is validated.
func (ix *Indexer) validationGate(ctx context.Context, ready []PendingEntry, own string) (bool, string, error) {
if ix.ValidationAsks == nil {
return false, "", nil
}
newest := ready[newestOfRevision(ready)]
seq := newest.Value.Seq
if v := newest.Value.Validation; v != nil && v.Seq == seq {
switch {
case v.Accepted == seq:
return false, "", nil
case v.Failed != "":
return true, fmt.Sprintf("the validation of the revision at %d could not validate: %s", seq, v.Failed), nil
case newest.Value.Refused != "":
return true, fmt.Sprintf("the revision at %d is refused: %s", seq, newest.Value.Refused), nil
case v.Ask != "":
return true, fmt.Sprintf("the revision at %d waits for its validation, the ask %s", seq, v.Ask), nil
}
}
return true, "", ix.askValidation(ctx, ready, own)
}
// askValidation puts the revision's validation ask (§712) and writes it on its newest value, the try after every earlier try
// of the revision; an older value's validation goes (A-247: on the newest alone). A request or job that cannot be made could not
// validate: it is written as `failed`, said, and no ask is put.
func (ix *Indexer) askValidation(ctx context.Context, ready []PendingEntry, own string) error {
n := newestOfRevision(ready)
newest := ready[n]
try := 0
for _, e := range ready {
if v := e.Value.Validation; v != nil && v.Try > try {
try = v.Try
}
}
v := &takein.Validation{Seq: newest.Value.Seq, Try: try + 1, Asked: ix.now()}
branch := branchOfRevision(ready)
ask := buildstep.StepAsk{ID: buildstep.NewID(), Kind: buildstep.AskCheck, Language: buildstep.AskGo, Asker: ValidationAsker, At: ix.now().UTC()}
req, entry, err := ix.validationRequest(ctx, branch, ready)
var raw []byte
if err == nil {
raw, err = json.Marshal(req)
}
var job buildstep.RunJob
if err == nil {
job, err = validationJob(entry, revisionHead(ready, entry.Repository, entry.Commit), branch, v.Seq, raw)
}
if err == nil {
v.Input, err = ix.KeepInput(ctx, ask.ID, raw)
}
if err != nil {
v.Failed = "the validation could not be asked: " + err.Error()
} else {
v.Ask, ask.Run = ask.ID, &job
}
if err := ix.writeValidation(ctx, ready, n, v, own); err != nil {
return err
}
if v.Failed != "" {
ix.sayValidation(ctx, ready, v)
return nil
}
if err := ix.ValidationAsks.Put(ctx, ask); err != nil {
return fmt.Errorf("the validation ask %s of the revision at %d could not be put: %w", ask.ID, v.Seq, err)
}
ix.logf("take-in: the revision at %d is validated by the ask %s (try %d); it waits for its end", v.Seq, ask.ID, v.Try)
return nil
}
// revisionHead is the revision's head of repository: the newest merge of it the revision covers, else the commit its entry stands at.
func revisionHead(ready []PendingEntry, repository, commit string) string {
var seq uint64
head := commit
for _, e := range ready {
if e.Forge.Merge != "" && strings.EqualFold(e.Forge.Repository, repository) && e.Value.Seq >= seq {
seq, head = e.Value.Seq, e.Forge.Merge
}
}
return head
}
// writeValidation writes v on ready[n] and takes it from every other value, each by compare-and-set; a new validation clears the
// value's refusal. A value lost under it is an error: the evaluation is made again.
func (ix *Indexer) writeValidation(ctx context.Context, ready []PendingEntry, n int, v *takein.Validation, own string) error {
for i, e := range ready {
if i != n && e.Value.Validation == nil {
continue
}
p := e.Value
p.Validation, p.LastWriter = nil, own
if i == n {
p.Validation, p.Refused = v, ""
}
data, err := encodePending(p)
if err != nil {
return err
}
if err := ix.Pending.Update(ctx, e.Key, data, e.Revision); err != nil {
return fmt.Errorf("%s.%s: %w", PendingState, e.Key, err)
}
}
return nil
}
// sayValidation says the revision's wait in take-in.pending with its refusal or why it could not validate.
func (ix *Indexer) sayValidation(ctx context.Context, ready []PendingEntry, v *takein.Validation) {
t := revisionWait(takein.TakeInPending, ready, "")
t.Failed = v.Failed
t.Refused = ready[newestOfRevision(ready)].Value.Refused
ix.sayTakeIn(ctx, t)
}
// errNoValidationAnswer is an end whose answer is no verdict of both cases.
var errNoValidationAnswer = errors.New("no answer")
// HearValidationEnded takes in the end of a validation ask (§713), heard on the build seat's step-ended through the indexer's own
// durable consumer: only an end of the indexer's asks outside a pipeline; an end whose ask no pending value names now is dropped,
// written in the journal. The answer is read from the event's own output or output object (A-247).
func (ix *Indexer) HearValidationEnded(ctx context.Context, e buildstep.StepEnded) error {
if e.Asker != ValidationAsker || e.Pipeline != "" || e.Ask == "" || ix.Pending == nil || !ix.pendingOnBus(ctx) {
return nil
}
own := ix.ownBuildNow(ctx)
ix.pendingMu.Lock()
defer ix.pendingMu.Unlock()
entries, err := ReadPending(ctx, ix.Pending, ix.logf)
if err != nil {
return err
}
ready := WaitsOf(entries).Ready
n := -1
for i, p := range ready {
if p.Value.Validation != nil && p.Value.Validation.Ask == e.Ask {
n = i
}
}
if n < 0 {
ix.logf("take-in: the end of the validation ask %s (%s on %s) is dropped: no pending revision names that ask now", e.Ask, e.Outcome, e.Node)
return nil
}
ready, lost, _, _ := ix.loadNewest(ctx, ready)
if len(lost) > 0 {
return fmt.Errorf("the revision the validation ask %s ended for holds a merge that cannot be read", e.Ask)
}
n = newestOfRevision(ready)
v := *ready[n].Value.Validation
refused, failed := ix.validationAnswer(ctx, ready, e, v)
p := ready[n].Value
switch {
case failed != "":
v.Failed = failed
case refused != "":
p.Refused = refused
default:
v.Accepted = v.Seq
}
p.Validation, p.LastWriter = &v, own
data, err := encodePending(p)
if err != nil {
return err
}
if err := ix.Pending.Update(ctx, ready[n].Key, data, ready[n].Revision); err != nil {
return fmt.Errorf("%s.%s: %w", PendingState, ready[n].Key, err)
}
if v.Accepted == 0 {
ready[n].Value = p
ix.sayValidation(ctx, ready, &v)
ix.logf("take-in: the validation ask %s of the revision at %d ended: refused %q, could not validate %q", e.Ask, v.Seq, refused, failed)
return nil
}
ix.logf("take-in: the validation ask %s accepted the revision at %d", e.Ask, v.Seq)
if own == "" {
return nil // the take-in waits for the build this program runs, as every take-in does
}
_, _, err = ix.evaluate(ctx, own, false)
return err
}
// validationAnswer reads the end's answer: the refusal of the revision, or why it could not validate (the step did not pass, the
// program exited other than 0, its answer is no verdict of both cases, or its request is no longer the one asked).
func (ix *Indexer) validationAnswer(ctx context.Context, ready []PendingEntry, e buildstep.StepEnded, v takein.Validation) (string, string) {
if e.Outcome != buildstep.OutcomePassed {
return "", strings.TrimSpace(e.Outcome + ": " + e.Said)
}
// The answer is the program's last line of standard output, inline or kept by the seat as an object (A-254); a program that
// exited other than 0 ended the step failed, read above.
out, err := e.ReadOutput(buildstep.AskGo, ix.AskVerb)
if err != nil {
return "", "the run's answer could not be read: " + err.Error()
}
var verdict composer.Verdict
if err := json.Unmarshal(out, &verdict); err != nil {
return "", fmt.Sprintf("%v: the manifest validator's output is no verdict: %v", errNoValidationAnswer, err)
}
req, _, err := ix.validationRequest(ctx, branchOfRevision(ready), ready)
var raw []byte
if err == nil {
raw, err = json.Marshal(req)
}
if err != nil {
return "", "the request cannot be made again to read the verdict: " + err.Error()
}
if runDigest(raw) != v.Input {
return "", "the revision's request is no longer the one the ask " + v.Ask + " was asked with"
}
refused, err := ValidationRefusal(req, verdict)
if err != nil {
return "", err.Error()
}
return RefusedWords(refused), ""
}
// ValidateAgainTool asks the validation of a revision that could not validate again by hand (§713).
const ValidateAgainTool = "indexer_validate_again"
// validationWords is the validation's state as the hand ask answers it.
func validationWords(v *takein.Validation, refused string) string {
switch {
case v == nil:
return "no validation is asked yet"
case v.Accepted != 0:
return fmt.Sprintf("accepted at %d (try %d, the ask %s)", v.Accepted, v.Try, v.Ask)
case v.Failed != "":
return fmt.Sprintf("could not validate at %d (try %d): %s", v.Seq, v.Try, v.Failed)
case refused != "":
return fmt.Sprintf("refused at %d (try %d): %s", v.Seq, v.Try, refused)
default:
return fmt.Sprintf("the ask %s is out since %s (try %d at %d)", v.Ask, v.Asked.UTC().Format(time.RFC3339), v.Try, v.Seq)
}
}
// ValidateAgainByHand is indexer_validate_again answered (§713): the ready revision under key (its oldest merge's m. key, as
// indexer_pending names it) whose validation could not validate is asked again, the next try; its answer says the ask's state.
// A revision accepted, refused or whose ask is out is asked nothing: a refusal is asked again only by a later merge of its branch.
func (ix *Indexer) ValidateAgainByHand(ctx context.Context, key, by, why string) TakenIn {
r := TakenIn{Module: key}
if ix.Pending == nil || ix.ValidationAsks == nil {
r.Answer = "this indexer asks no validation: nothing asked"
return r
}
own := ix.ownBuildNow(ctx)
ix.pendingMu.Lock()
defer ix.pendingMu.Unlock()
entries, err := ReadPending(ctx, ix.Pending, ix.logf)
if err != nil {
r.Answer = err.Error()
return r
}
ready := WaitsOf(entries).Ready
if len(ready) == 0 || ready[0].Key != key {
r.Answer = fmt.Sprintf("no revision waits under %s: indexer_pending names the waits", key)
return r
}
ready, lost, reasons, _ := ix.loadNewest(ctx, ready)
if len(lost) > 0 {
r.Answer = fmt.Sprintf("nothing asked: the message of %s is not read (%s)", lost[0].Key, reasons[lost[0].Key])
return r
}
newest := ready[newestOfRevision(ready)]
v := newest.Value.Validation
if v == nil || v.Seq != newest.Value.Seq || v.Failed == "" {
r.Answer = "nothing asked: the validation is " + validationWords(v, newest.Value.Refused) + "; only one that could not validate is asked again by hand"
return r
}
ix.logf("take-in: the validation of %s is asked again by hand by %s: %s", key, by, why)
if err := ix.askValidation(ctx, ready, own); err != nil {
r.Answer = "the validation was not asked again: " + err.Error()
return r
}
after, err := ReadPending(ctx, ix.Pending, ix.logf)
if err == nil {
ready = WaitsOf(after).Ready
}
if err != nil || len(ready) == 0 {
r.Answer = "asked again; its state is not read back"
return r
}
n := ready[newestOfRevision(ready)]
r.Stored = n.Value.Validation != nil && n.Value.Validation.Failed == ""
r.Answer = fmt.Sprintf("asked again by %s (%s): the validation is %s", by, why, validationWords(n.Value.Validation, n.Value.Refused))
return r
}
@@ -1,113 +0,0 @@
package index
import (
"context"
"encoding/json"
"strings"
"testing"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/composer"
)
type askRecorder struct{ asks []buildstep.StepAsk }
func (r *askRecorder) Put(_ context.Context, a buildstep.StepAsk) error {
r.asks = append(r.asks, a)
return nil
}
// Design 47 G §713, A-247: a pending revision names its ask on its newest value and resumes on that ask's end; an end of a
// replaced ask is dropped; a run that could not validate is no refusal and never passed; the start asks nothing again while
// the ask is out.
func TestAPendingRevisionWaitsForItsValidation(t *testing.T) {
ctx := context.Background()
b := newTakeInBed(t)
asks := &askRecorder{}
b.ix.ValidationAsks, b.ix.Inputs, b.ix.Revisions = asks, newMem(), newMemObjects(0)
// No entry of the revision claims the seat mesh-catalog: the validation could not be asked, so it is failed, said, and
// nothing is taken in.
b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", ""))
entries := readPending(t, b)
if len(entries) != 1 || entries[0].Value.Validation == nil || !strings.Contains(entries[0].Value.Validation.Failed, "claims the seat mesh-catalog") || len(asks.asks) != 0 {
t.Fatalf("pending %+v, asks %d: want m.2 pending with failed and no ask", entries, len(asks.asks))
}
if last := b.said[len(b.said)-1]; last.Kind != takein.TakeInPending || last.Failed == "" || last.Refused != "" {
t.Errorf("said %+v, want take-in.pending naming failed, never a refusal", last)
}
if w := PendingWaits(entries); len(w) != 1 || w[0].Validation == nil || w[0].Validation.Try != 1 {
t.Errorf("indexer_pending %+v, want the validation whole", w)
}
// An ask out: the newest value names it; the start asks nothing new while it is out.
loaded, _, _, _ := b.ix.loadNewest(ctx, entries)
req, _, err := b.ix.validationRequest(ctx, TrunkBranch, loaded)
if err != nil {
t.Fatal(err)
}
raw, _ := json.Marshal(req)
setValidation := func(v takein.Validation) {
e := readPending(t, b)[0]
p := e.Value
p.Validation = &v
data, _ := encodePending(p)
if err := b.pending.Update(ctx, e.Key, data, e.Revision); err != nil {
t.Fatal(err)
}
}
setValidation(takein.Validation{Ask: "mvl0frv18d1957", Input: runDigest(raw), Seq: 2, Try: 2})
if _, _, err := b.ix.TakeInAtStart(ctx); err != nil || len(asks.asks) != 0 || len(b.keys()) != 1 {
t.Fatalf("the start: %v, asks %d, pending %v: want it waiting for its ask", err, len(asks.asks), b.keys())
}
end := func(ask, outcome, said string, answer any) buildstep.StepEnded {
out, _ := json.Marshal(answer)
return buildstep.StepEnded{Ask: ask, ID: ask, Asker: ValidationAsker, Node: "ace", Outcome: outcome, Said: said, Output: string(out)}
}
// The answer is the manifest validator's verdict itself, the program's last line (A-254).
accepted := composer.Verdict{Cases: []composer.CaseVerdict{{Name: "change"}, {Name: "trunk"}}}
// The end of a replaced ask is dropped, written in the journal.
if err := b.ix.HearValidationEnded(ctx, end("mvl0frv18d1950", buildstep.OutcomePassed, "", accepted)); err != nil || len(b.keys()) != 1 || !strings.Contains(b.logs.String(), "is dropped") {
t.Fatalf("a replaced ask's end: %v, pending %v", err, b.keys())
}
// could not run: pending with failed, never passed.
if err := b.ix.HearValidationEnded(ctx, end("mvl0frv18d1957", buildstep.OutcomeCouldNotRun, "no build node", nil)); err != nil {
t.Fatal(err)
}
if v := readPending(t, b)[0].Value.Validation; v == nil || v.Failed != "could not run: no build node" || v.Accepted != 0 {
t.Fatalf("after could not run: %+v", v)
}
// indexer_validate_again (§713): its hand act recorded first, then the next try asked; the answer says the ask's state.
var recorded []string
tools := ValidationTools(func(key string, value []byte) (TakeInAsk, error) {
a, err := DecodeTakeIn(value)
if err != nil || key != "pending.validate-again."+takein.PendingKey(2) || !a.ValidateAgain {
t.Fatalf("asked %s %+v %v", key, a, err)
}
r := b.ix.ValidateAgainByHand(ctx, a.Pending, a.By, a.Why)
a.Result = &r
return a, nil
}, b.ix.now, func(what, why string) error { recorded = append(recorded, what); return nil })
if len(tools) != 1 || tools[0].Name != ValidateAgainTool {
t.Fatalf("tools = %v", tools)
}
if _, err := tools[0].Run(map[string]any{"key": takein.PendingKey(2)}); err == nil || len(recorded) != 0 {
t.Fatalf("asked again with no why: %v, %v", err, recorded)
}
got, err := tools[0].Run(map[string]any{"key": takein.PendingKey(2), "why": "the build node is back"})
if v := readPending(t, b)[0].Value.Validation; err != nil || len(recorded) != 1 || v == nil || v.Try != 3 || v.Failed == "could not run: no build node" ||
!strings.Contains(got.(*TakenIn).Answer, "try 3") {
t.Fatalf("asked again: %v, recorded %v, validation %+v, answer %+v: want the try after 2 asked", err, recorded, v, got)
}
setValidation(takein.Validation{Ask: "mvl0frv18d1959", Input: runDigest(raw), Seq: 2, Try: 3})
if r := b.ix.ValidateAgainByHand(ctx, takein.PendingKey(2), "lead", "again"); r.Stored || !strings.Contains(r.Answer, "is out") || readPending(t, b)[0].Value.Validation.Try != 3 {
t.Fatalf("an ask out was asked again: %+v", r)
}
// Asked again (by hand or by a later merge) and accepted: the revision is taken in, and its keys go.
setValidation(takein.Validation{Ask: "mvl0frv18d1958", Input: runDigest(raw), Seq: 2, Try: 3})
if err := b.ix.HearValidationEnded(ctx, end("mvl0frv18d1958", buildstep.OutcomePassed, "", accepted)); err != nil {
t.Fatal(err)
}
if len(b.keys()) != 0 || b.commit(t, "avahi") != sha('c') {
t.Fatalf("accepted: pending %v %+v, avahi at %s: want it taken in\n%s", b.keys(), readPending(t, b)[0].Value, b.commit(t, "avahi"), b.logs.String())
}
}
@@ -506,8 +506,8 @@ func (ix *Indexer) writeVerdict(ctx context.Context, digest string, v meshcatalo
} }
// OwnBuildState is the indexer's state of the build it runs (the bucket mesh-delivery-indexer_own-build, one key, OwnBuildKey): // OwnBuildState is the indexer's state of the build it runs (the bucket mesh-delivery-indexer_own-build, one key, OwnBuildKey):
// written once, at the process's start by FixOwnBuild, so indexer_pending says whether this process knows its build and, when it // written once, at the process's start by FixOwnBuild, so indexer_pending says which build this process knows it runs, and since
// does not, since when and why. // when. The build only names the writer of what is written (A-476 b).
const ( const (
OwnBuildStateName = "own-build" OwnBuildStateName = "own-build"
OwnBuildKey = "state" OwnBuildKey = "state"
@@ -526,8 +526,7 @@ type OwnBuild struct {
// seat (OwnBuild; never the trunk's value of the indexer, which names a build that may be merged and not delivered), before // seat (OwnBuild; never the trunk's value of the indexer, which names a build that may be merged and not delivered), before
// this process hears a merge. It is never asked again: while the node-engine replaces the indexer the seat already names the // this process hears a merge. It is never asked again: while the node-engine replaces the indexer the seat already names the
// successor, and an old process that learnt it later would take revisions in with its old judge. Where the start's ask is not // successor, and an old process that learnt it later would take revisions in with its old judge. Where the start's ask is not
// answered the build stays unknown for the life of this process: every merge waits, loudly (a take-in.pending that says so, the // answered the build stays unknown for the life of this process, said in the own-build state and the journal; nothing waits for it.
// own-build state, indexer_pending), and the module is started again to learn it.
func (ix *Indexer) FixOwnBuild(ctx context.Context) { func (ix *Indexer) FixOwnBuild(ctx context.Context) {
ix.ownMu.Lock() ix.ownMu.Lock()
if ix.ownFixed || ix.OwnBuild == nil { if ix.ownFixed || ix.OwnBuild == nil {
@@ -543,7 +542,7 @@ func (ix *Indexer) FixOwnBuild(ctx context.Context) {
st := OwnBuild{V: 1, Since: ix.now().UTC()} st := OwnBuild{V: 1, Since: ix.now().UTC()}
if err != nil { if err != nil {
st.Why = err.Error() st.Why = err.Error()
ix.logf("judge: the build this program runs is not known (%v), so every merge waits and a manifest with a field it does not know waits and is not refused; it is learnt only at the start, so the indexer is started again", err) ix.logf("judge: the build this program runs is not known (%v); merges are taken in all the same, and it is learnt only at the start", err)
} else { } else {
st.Known, st.FP = true, fp st.Known, st.FP = true, fp
ix.logf("judge: this program runs the indexer build %s, fixed here at its start", fp) ix.logf("judge: this program runs the indexer build %s, fixed here at its start", fp)
@@ -177,17 +177,13 @@ func (ix *Indexer) entryOf(ctx context.Context, module string) (meshcatalog.Cata
if ix.Revisions == nil { if ix.Revisions == nil {
return meshcatalog.CatalogEntry{}, errors.New("the catalog graph is not kept here: no entry names the module's artifacts") return meshcatalog.CatalogEntry{}, errors.New("the catalog graph is not kept here: no entry names the module's artifacts")
} }
rev, found, err := newestTrunk(ctx, ix.Revisions) _, entries, found, err := readNewestOf(ctx, ix.Revisions, TrunkBranch)
if err != nil { if err != nil {
return meshcatalog.CatalogEntry{}, err return meshcatalog.CatalogEntry{}, err
} }
if !found { if !found {
return meshcatalog.CatalogEntry{}, errors.New("the indexer keeps no trunk revision yet") return meshcatalog.CatalogEntry{}, errors.New("the indexer keeps no trunk revision yet")
} }
_, entries, err := ReadRevision(ctx, ix.Revisions, fmt.Sprintf("trunk.%d", rev.Revision))
if err != nil {
return meshcatalog.CatalogEntry{}, err
}
for _, e := range entries { for _, e := range entries {
if e.Module == module { if e.Module == module {
return e, nil return e, nil
@@ -194,11 +194,13 @@ func (ix *Indexer) Outdated(ctx context.Context, in meshcatalog.OutdatedInput) (
} }
trunk := map[string]meshcatalog.CatalogEntry{} // the newest trunk revision's entries, read where a kind needs them trunk := map[string]meshcatalog.CatalogEntry{} // the newest trunk revision's entries, read where a kind needs them
if ix.Revisions != nil { if ix.Revisions != nil {
if rev, found, err := newestTrunk(ctx, ix.Revisions); err == nil && found { _, entries, found, err := readNewestOf(ctx, ix.Revisions, TrunkBranch)
if _, entries, err := ReadRevision(ctx, ix.Revisions, fmt.Sprintf("trunk.%d", rev.Revision)); err == nil { if err != nil {
for _, e := range entries { return out, fmt.Errorf("the trunk's entries, which the kinds 3 and 5 are told by, cannot be read: %w", err)
trunk[e.Module] = e }
} if found {
for _, e := range entries {
trunk[e.Module] = e
} }
} }
} }
@@ -79,17 +79,13 @@ func (ix *Indexer) FillVersions(ctx context.Context, ask VersionsAsker) error {
if ix.Revisions == nil { if ix.Revisions == nil {
return errors.New("the catalog graph is not kept here: no entry names the modules' artifacts") return errors.New("the catalog graph is not kept here: no entry names the modules' artifacts")
} }
rev, found, err := newestTrunk(ctx, ix.Revisions) _, entries, found, err := readNewestOf(ctx, ix.Revisions, TrunkBranch)
if err != nil { if err != nil {
return err return err
} }
if !found { if !found {
return errors.New("the indexer keeps no trunk revision yet: the first fill of the versions waits for its first") return errors.New("the indexer keeps no trunk revision yet: the first fill of the versions waits for its first")
} }
_, entries, err := ReadRevision(ctx, ix.Revisions, fmt.Sprintf("trunk.%d", rev.Revision))
if err != nil {
return err
}
type found1 struct { type found1 struct {
v BuilderVersion v BuilderVersion
name string name string
+87 -33
View File
@@ -2476,24 +2476,14 @@ func invokeProblems(m catalogue.Manifest) []string {
return problems return problems
} }
// ownSecretOwnerProblems is what is wrong with an own secret's declared owner (novox/hq ADR 0436 §1): naming // ownSecretOwnerProblems is what is wrong with an own secret's declared owner (novox/hq ADR 0680 §1, §2, which
// an account the module makes — a `user` resource of that name — never root, which a secret file owned by it // ends ADR 0436 §1's refusal of root and the operator account here): an account the module makes — a `user`
// would make readable by anything, and never the operator account's placeholder (`${machine:account}`), which // resource of that name — the operator account's placeholder (`${machine:account}`) or root. Any other account
// the mesh fills per machine and a manifest cannot say safely (novox/hq ADR 0436, settled by the lead). // the module does not make is still refused.
func ownSecretOwnerProblems(m catalogue.Manifest, name, owner string) []string { func ownSecretOwnerProblems(m catalogue.Manifest, name, owner string) []string {
if owner == "" { if owner == "" || owner == "root" || owner == "${machine:account}" {
return nil return nil
} }
if owner == "root" {
return []string{fmt.Sprintf(
"%s names %q as the owner of its secret %q; an own secret's owner is an account the module "+
"makes, never root (novox/hq ADR 0436 §1)", m.Module, owner, name)}
}
if owner == "${machine:account}" {
return []string{fmt.Sprintf(
"%s names %q as the owner of its secret %q; an own secret's owner is an account the module "+
"makes, never the operator account (novox/hq ADR 0436 §1)", m.Module, owner, name)}
}
for _, r := range m.Resources { for _, r := range m.Resources {
if fmt.Sprint(r["type"]) == "user" && fmt.Sprint(r["name"]) == owner { if fmt.Sprint(r["type"]) == "user" && fmt.Sprint(r["name"]) == owner {
return nil return nil
@@ -2501,7 +2491,7 @@ func ownSecretOwnerProblems(m catalogue.Manifest, name, owner string) []string {
} }
return []string{fmt.Sprintf( return []string{fmt.Sprintf(
"%s names %q as the owner of its secret %q, which it does not make: a user resource named %q "+ "%s names %q as the owner of its secret %q, which it does not make: a user resource named %q "+
"(novox/hq ADR 0436 §1)", m.Module, owner, name, owner)} "(novox/hq ADR 0680 §2)", m.Module, owner, name, owner)}
} }
// ownSecretPromptProblems is what is wrong with an own secret's declared prompt, in the module check's words: // ownSecretPromptProblems is what is wrong with an own secret's declared prompt, in the module check's words:
@@ -3218,6 +3208,9 @@ func parseManifest(raw []byte, predefined catalogue.PredefinedList) (catalogue.M
problems = append(problems, ownSecretPromptProblems(m, name, own.Prompt, catalogue.IsFamily(name))...) problems = append(problems, ownSecretPromptProblems(m, name, own.Prompt, catalogue.IsFamily(name))...)
problems = append(problems, ownSecretOwnerProblems(m, name, own.Owner)...) problems = append(problems, ownSecretOwnerProblems(m, name, own.Owner)...)
} }
// The module's secrets-owner is held to the same accounts as an own secret's owner (novox/hq ADR 0680 §2).
problems = append(problems, ownSecretOwnerProblems(m, "(its secrets-owner)", m.SecretsOwner)...)
problems = append(problems, ownSecretReadProblems(m)...)
localOf := map[string]string{} localOf := map[string]string{}
for _, to := range m.SecretRequirements() { for _, to := range m.SecretRequirements() {
if _, plain := m.Secrets[to]; plain { if _, plain := m.Secrets[to]; plain {
@@ -4204,25 +4197,19 @@ func passableDir(m catalogue.Manifest, id string, depth int) bool {
return false return false
} }
// processAccountProblems is what is wrong with one process's bus account (novox/hq ADR 0436 §1, §2): an // processAccountProblems is what is wrong with one process's bus account (novox/hq ADR 0680 §1, §2, which ends ADR
// account the module makes — a `user` resource of that name — never root and never the operator account, // 0436 §1's refusal of root and the operator account here): an account the module makes — a `user` resource of
// which a manifest cannot say safely (a node with no account has nothing to resolve it to, novox/hq ADR // that name — the operator account's placeholder (`${machine:account}`) or root. A process that declares bus
// 0436, settled by the lead). // and names no account, or one the module does not make, is refused.
func processAccountProblems(m catalogue.Manifest, processID, user string) []string { func processAccountProblems(m catalogue.Manifest, processID, user string) []string {
switch { switch user {
case user == "": case "":
return []string{fmt.Sprintf( return []string{fmt.Sprintf(
"%s's process %s declares bus and names no account; a process account is a user resource the "+ "%s's process %s declares bus and names no account; a process account is a user resource the "+
"module makes, never root and never the operator account (novox/hq ADR 0436 §1, §2)", "module makes, the operator account or root (novox/hq ADR 0680 §2)",
m.Module, processID)} m.Module, processID)}
case user == "root": case "root", "${machine:account}":
return []string{fmt.Sprintf( return nil
"%s's process %s runs as root; a process account is the module's own, never root (novox/hq ADR "+
"0436 §1, §2)", m.Module, processID)}
case user == "${machine:account}":
return []string{fmt.Sprintf(
"%s's process %s runs as the operator account; a process account is the module's own, never the "+
"operator's (novox/hq ADR 0436 §1, §2)", m.Module, processID)}
} }
for _, r := range m.Resources { for _, r := range m.Resources {
if fmt.Sprint(r["type"]) == "user" && fmt.Sprint(r["name"]) == user { if fmt.Sprint(r["type"]) == "user" && fmt.Sprint(r["name"]) == user {
@@ -4230,8 +4217,75 @@ func processAccountProblems(m catalogue.Manifest, processID, user string) []stri
} }
} }
return []string{fmt.Sprintf( return []string{fmt.Sprintf(
"%s's process %s runs as %s, which it does not make: a user resource named %s (novox/hq ADR 0436 §1, "+ "%s's process %s runs as %s, which it does not make: a user resource named %s (novox/hq ADR 0680 §2)",
"§2)", m.Module, processID, user, user)} m.Module, processID, user, user)}
}
// ownSecretReadProblems is what is wrong with a process that names an own secret it cannot read (novox/hq ADR
// 0680 §3): a process resource whose `run` or `env` names an own secret's path must run as that secret's
// resolved owner (its `owner`, else the module's `secrets-owner`, else root) or as root. A module with a
// container is excepted: a container reads its secrets as its own account. A process with no `user` runs as
// root, as the service manager runs it (lead answer A-485 a, b: a container module is any module with a
// `container` resource). A word names a secret only when it is the secret's path (a family's: begins with the path
// up to its `*`).
func ownSecretReadProblems(m catalogue.Manifest) []string {
for _, r := range m.Resources {
if fmt.Sprint(r["type"]) == "container" {
return nil
}
}
var problems []string
for _, r := range m.Resources {
if fmt.Sprint(r["type"]) != "process" {
continue
}
user, _ := r["user"].(string)
if user == "" || user == "root" {
continue
}
var words []string
if run, ok := r["run"].([]any); ok {
for _, w := range run {
words = append(words, fmt.Sprint(w))
}
}
if env, ok := r["env"].(map[string]any); ok {
for _, v := range env {
words = append(words, fmt.Sprint(v))
}
}
for name, own := range m.OwnSecrets {
path, _, _ := strings.Cut(own.Path, "*")
if path == "" {
continue
}
named := false
for _, w := range words {
if w == own.Path || (strings.Contains(own.Path, "*") && strings.HasPrefix(w, path)) {
named = true
}
}
owner := own.Owner
if owner == "" {
owner = m.SecretsOwner
}
if named && owner != user {
problems = append(problems, fmt.Sprintf(
"%s's process %v runs as %s and names its secret %q, which belongs to %s; a process runs as "+
"the owner of an own secret it names, or as root (novox/hq ADR 0680 §3)",
m.Module, r["id"], user, name, ownerWord(owner)))
}
}
}
sort.Strings(problems)
return problems
}
func ownerWord(owner string) string {
if owner == "" {
return "root"
}
return owner
} }
// busKeeps says the process resource r's bus declares `keeps` (novox/hq ADR 0626 §1): read from the resource as // busKeeps says the process resource r's bus declares `keeps` (novox/hq ADR 0626 §1): read from the resource as
@@ -2113,19 +2113,79 @@ func caseAnOwnSecretSaysWhenAnOutsidePartyIssuesIt(t *testing.T) {
} }
} }
// 1170/AC2: an owner naming an account the module does not make, root, or the operator account's placeholder is // ADR 0680 §2, §4: an own secret's owner of root or the operator account's placeholder is accepted; an owner naming an
// refused, naming the secret and the account. // account the module does not make is refused, naming the secret and the account.
func caseAnOwnSecretsOwnerIsRefusedWhenItIsNotAnAccountTheModuleMakes(t *testing.T) { func caseAnOwnSecretsOwnerIsRefusedWhenItIsNotAnAccountTheModuleMakes(t *testing.T) {
for name, c := range map[string]struct{ owner, says string }{ for name, c := range map[string]struct {
"root": {"root", "root"}, owner string
"the operator account": {"${machine:account}", "operator account"}, ok bool
"an account not declared": {"not-made", "not-made"}, }{
"root": {"root", true},
"the operator account": {"${machine:account}", true},
"an account the module makes": {"mesh-mail-send", true},
"an account not declared": {"not-made", false},
} { } {
_, err := ParseManifest([]byte(`{"module":"mesh-mail","version":"1.0","own-secrets":{` + _, err := ParseManifest([]byte(`{"module":"mesh-mail","version":"1.0","own-secrets":{` +
`"mail-credential-*":{"path":"/var/lib/mesh-mail/mail-credential-*.secret","issued-by":"outside","owner":"` + `"mail-credential-*":{"path":"/var/lib/mesh-mail/mail-credential-*.secret","issued-by":"outside","owner":"` +
c.owner + `"}},"resources":[{"id":"account","type":"user","name":"mesh-mail-send"}]}`)) c.owner + `"}},"resources":[{"id":"account","type":"user","name":"mesh-mail-send"}]}`))
if err == nil || !strings.Contains(err.Error(), "mail-credential-*") || !strings.Contains(err.Error(), c.says) { switch {
t.Errorf("%s: %v; want a refusal naming the secret and %q", name, err, c.says) case c.ok && err != nil:
t.Errorf("%s: refused: %v", name, err)
case !c.ok && (err == nil || !strings.Contains(err.Error(), "mail-credential-*") || !strings.Contains(err.Error(), c.owner)):
t.Errorf("%s: %v; want a refusal naming the secret and %q", name, err, c.owner)
}
}
}
// ADR 0680 §2, §4: the module's secrets-owner is held to the same accounts as an own secret's owner.
func caseASecretsOwnerIsRefusedWhenItIsNotAnAccountTheModuleMakes(t *testing.T) {
for name, c := range map[string]struct {
owner string
ok bool
}{
"root": {"root", true},
"the operator account": {"${machine:account}", true},
"an account the module makes": {"mesh-mail-send", true},
"an account not declared": {"not-made", false},
} {
_, err := ParseManifest([]byte(`{"module":"mesh-mail","version":"1.0","secrets-owner":"` + c.owner +
`","resources":[{"id":"account","type":"user","name":"mesh-mail-send"}]}`))
switch {
case c.ok && err != nil:
t.Errorf("%s: refused: %v", name, err)
case !c.ok && (err == nil || !strings.Contains(err.Error(), c.owner)):
t.Errorf("%s: %v; want a refusal naming %q", name, err, c.owner)
}
}
}
// ADR 0680 §3, §4: a process naming an own secret runs as its resolved owner (its owner, else the module's
// secrets-owner) or as root; any other account is refused; a module with a container is excepted.
func caseAProcessNamingAnOwnSecretMustBeAbleToReadIt(t *testing.T) {
const users = `{"id":"u1","type":"user","name":"alice"},{"id":"u2","type":"user","name":"bob"}`
for name, c := range map[string]struct {
secret, extra, proc string
ok bool
}{
"the resolved owner": {`"path":"/var/lib/x/k","owner":"alice"`, ``, `"user":"alice","run":["./x","--key","/var/lib/x/k"]`, true},
"the owner by secrets-owner": {`"path":"/var/lib/x/k"`, `"secrets-owner":"alice",`, `"user":"alice","env":{"KEY":"/var/lib/x/k"}`, true},
"root": {`"path":"/var/lib/x/k","owner":"alice"`, ``, `"user":"root","run":["./x","/var/lib/x/k"]`, true},
"the operator account": {`"path":"/var/lib/x/k","owner":"${machine:account}"`, ``, `"user":"${machine:account}","run":["./x","/var/lib/x/k"]`, true},
"a secret it does not name": {`"path":"/var/lib/x/k","owner":"alice"`, ``, `"user":"bob","run":["./x"]`, true},
"another account": {`"path":"/var/lib/x/k","owner":"alice"`, ``, `"user":"bob","run":["./x","/var/lib/x/k"]`, false},
"another account by secrets-owner": {`"path":"/var/lib/x/k"`, `"secrets-owner":"alice",`, `"user":"bob","env":{"KEY":"/var/lib/x/k"}`, false},
"a secret owned by root": {`"path":"/var/lib/x/k"`, ``, `"user":"bob","run":["./x","/var/lib/x/k"]`, false},
"a longer word holding the path": {`"path":"/var/lib/x/k","owner":"alice"`, ``, `"user":"bob","env":{"KEY":"/var/lib/x/keyring"}`, true},
"a container module": {`"path":"/var/lib/x/k","owner":"alice"`, ``, `"user":"bob","run":["./x","/var/lib/x/k"]},{"id":"c","type":"container","image":"x"`, true},
} {
raw := `{"module":"x","version":"1.0",` + c.extra + `"own-secrets":{"k":{` + c.secret + `}},"resources":[` + users +
`,{"id":"p","type":"process","artifact":"x",` + c.proc + `}]}`
_, err := ParseManifest([]byte(raw))
switch {
case c.ok && err != nil:
t.Errorf("%s: refused: %v", name, err)
case !c.ok && (err == nil || !strings.Contains(err.Error(), "ADR 0680 §3")):
t.Errorf("%s: %v; want the ADR 0680 §3 refusal", name, err)
} }
} }
} }
@@ -2419,13 +2479,18 @@ func caseAPortWithNoSourceIsRefused(t *testing.T) {
} }
} }
// 1179/AC2: a bus on a process with no user, user root, or the operator account is refused naming the // ADR 0680 §2, §4: a bus on a process with no user, or a user the module does not make, is refused naming the
// process and the account rule. // process; root and the operator account are accepted.
func caseAProcessBusAccountMustBeTheModulesOwn(t *testing.T) { func caseAProcessBusAccountMustBeTheModulesOwn(t *testing.T) {
for name, c := range map[string]struct{ user, says string }{ for name, c := range map[string]struct {
"no user": {"", "send"}, user string
"user root": {"root", "root"}, ok bool
"the operator account": {"${machine:account}", "operator"}, }{
"no user": {"", false},
"an account not declared": {"not-made", false},
"user root": {"root", true},
"the operator account": {"${machine:account}", true},
"the module's own": {"mesh-mail-send", true},
} { } {
raw := `{"module":"mesh-mail","version":"1.0","emits":["mail.send.sent"],"state":["sends"],` + raw := `{"module":"mesh-mail","version":"1.0","emits":["mail.send.sent"],"state":["sends"],` +
`"streams":[{"name":"mail-sends","events":["mail.send.approved"],"retention":"work-queue",` + `"streams":[{"name":"mail-sends","events":["mail.send.approved"],"retention":"work-queue",` +
@@ -2434,10 +2499,13 @@ func caseAProcessBusAccountMustBeTheModulesOwn(t *testing.T) {
if c.user != "" { if c.user != "" {
raw += `,"user":"` + c.user + `"` raw += `,"user":"` + c.user + `"`
} }
raw += `,"bus":{"emits":["mail.send.sent"]}}]}` raw += `,"bus":{"emits":["mail.send.sent"],"queues":["mail-sends"]}}]}`
_, err := ParseManifest([]byte(raw)) _, err := ParseManifest([]byte(raw))
if err == nil || !strings.Contains(err.Error(), "send") || !strings.Contains(err.Error(), c.says) { switch {
t.Errorf("%s: %v; want a refusal naming the process and %q", name, err, c.says) case c.ok && err != nil:
t.Errorf("%s: refused: %v", name, err)
case !c.ok && (err == nil || !strings.Contains(err.Error(), "send")):
t.Errorf("%s: %v; want a refusal naming the process", name, err)
} }
} }
} }
@@ -6414,6 +6482,8 @@ func TestRulesSecrets(t *testing.T) {
t.Run("AnOffersIdentityIsParsedStrictly", caseAnOffersIdentityIsParsedStrictly) t.Run("AnOffersIdentityIsParsedStrictly", caseAnOffersIdentityIsParsedStrictly)
t.Run("AnOwnSecretSaysWhenAnOutsidePartyIssuesIt", caseAnOwnSecretSaysWhenAnOutsidePartyIssuesIt) t.Run("AnOwnSecretSaysWhenAnOutsidePartyIssuesIt", caseAnOwnSecretSaysWhenAnOutsidePartyIssuesIt)
t.Run("AnOwnSecretsOwnerIsRefusedWhenItIsNotAnAccountTheModuleMakes", caseAnOwnSecretsOwnerIsRefusedWhenItIsNotAnAccountTheModuleMakes) t.Run("AnOwnSecretsOwnerIsRefusedWhenItIsNotAnAccountTheModuleMakes", caseAnOwnSecretsOwnerIsRefusedWhenItIsNotAnAccountTheModuleMakes)
t.Run("ASecretsOwnerIsRefusedWhenItIsNotAnAccountTheModuleMakes", caseASecretsOwnerIsRefusedWhenItIsNotAnAccountTheModuleMakes)
t.Run("AProcessNamingAnOwnSecretMustBeAbleToReadIt", caseAProcessNamingAnOwnSecretMustBeAbleToReadIt)
t.Run("AnOwnSecretTakenSomeOtherWayIsRefused", caseAnOwnSecretTakenSomeOtherWayIsRefused) t.Run("AnOwnSecretTakenSomeOtherWayIsRefused", caseAnOwnSecretTakenSomeOtherWayIsRefused)
t.Run("AnUnknownAccessReferenceIsRefusedAtParse", caseAnUnknownAccessReferenceIsRefusedAtParse) t.Run("AnUnknownAccessReferenceIsRefusedAtParse", caseAnUnknownAccessReferenceIsRefusedAtParse)
t.Run("AnUnknownPlaceholderKeyIsRefused", caseAnUnknownPlaceholderKeyIsRefused) t.Run("AnUnknownPlaceholderKeyIsRefused", caseAnUnknownPlaceholderKeyIsRefused)
@@ -165,7 +165,7 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build
// "Reads FORGE" is a consumes of the indexer's stream: the broker grants only buckets by reads (mesh-control // "Reads FORGE" is a consumes of the indexer's stream: the broker grants only buckets by reads (mesh-control
// internal/broker/state.go:62-93). // internal/broker/state.go:62-93).
// The registry seats' published fill each (§413: no consume of version.numbered) package's series (DLV-I5, design 51 *The versions*). // The registry seats' published fill each (§413: no consume of version.numbered) package's series (DLV-I5, design 51 *The versions*).
if want := []string{"go-package-registry.published", "mesh-build-go.toolchain.reported", "mesh-build-node.toolchain.reported", "mesh-builds.package-published", "mesh-builds.toolchain-agreed", "mesh-conditions.findings-asked", "mesh-delivery-checker.verdict", "mesh-delivery-forge-bridge.forge.**", if want := []string{"go-package-registry.published", "mesh-build-go.toolchain.reported", "mesh-build-node.toolchain.reported", "mesh-builds.package-published", "mesh-builds.toolchain-agreed", "mesh-catalog.catalog-revision", "mesh-conditions.findings-asked", "mesh-delivery-checker.verdict", "mesh-delivery-forge-bridge.forge.**",
"mesh-delivery.freed", "mesh-delivery.heads-replaced", "mesh-delivery.pinned", "mesh-delivery.pins-stated", "mesh-delivery.source-moved", "mesh-pipeline-runner.pipeline-ended", "mesh-pipeline-runner.run-started", "mesh-wants.version-delivered", "mesh-wants.wait-ended", "mesh-wants.wanted", "node-build-go.step-ended", "node-engine.running", "npm-package-registry.published", "operator-channel.decided", "seat:mesh-module-assigner.views.assignments", "seat:mesh-module-assigner.views.seat-holders"}; !reflect.DeepEqual(sorted(m.Consumes), want) { "mesh-delivery.freed", "mesh-delivery.heads-replaced", "mesh-delivery.pinned", "mesh-delivery.pins-stated", "mesh-delivery.source-moved", "mesh-pipeline-runner.pipeline-ended", "mesh-pipeline-runner.run-started", "mesh-wants.version-delivered", "mesh-wants.wait-ended", "mesh-wants.wanted", "node-build-go.step-ended", "node-engine.running", "npm-package-registry.published", "operator-channel.decided", "seat:mesh-module-assigner.views.assignments", "seat:mesh-module-assigner.views.seat-holders"}; !reflect.DeepEqual(sorted(m.Consumes), want) {
t.Errorf("consumes = %v, want %v", m.Consumes, want) t.Errorf("consumes = %v, want %v", m.Consumes, want)
} }
@@ -179,8 +179,6 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build
// ADR 0499 clauses 2 and 3, ADR 0495 clause 1; and the module directories of the trunk, issue 1900) and whether a // ADR 0499 clauses 2 and 3, ADR 0495 clause 1; and the module directories of the trunk, issue 1900) and whether a
// commit is in another (issue 1900). // commit is in another (issue 1900).
if want := []string{"seat:git.trees", "seat:git.file", "seat:git.contains", "seat:git.open-pulls", "seat:git.branch", "mesh-delivery-forge-bridge.bridge_merged_message", "seat:node-engine.composition", "seat:mesh-delivery.source", "seat:mesh-module-assigner.assignments", "seat:mesh-module-assigner.placement", "seat:mesh-hand-acts.hand-act", "seat:mesh-watchdogs.presence", "seat:node-build-go.steps", if want := []string{"seat:git.trees", "seat:git.file", "seat:git.contains", "seat:git.open-pulls", "seat:git.branch", "mesh-delivery-forge-bridge.bridge_merged_message", "seat:node-engine.composition", "seat:mesh-delivery.source", "seat:mesh-module-assigner.assignments", "seat:mesh-module-assigner.placement", "seat:mesh-hand-acts.hand-act", "seat:mesh-watchdogs.presence", "seat:node-build-go.steps",
"seat:node-build-go.accept.check", // design 47 G A-231: the validation's run ask (§712) on the check queue
"seat:node-build-go.output", // design 47 G A-254: a validation's answer kept as an object
"seat:node-build-node.steps", "seat:node-build-image.steps", "mesh-delivery-builder.builder_versions"}; !reflect.DeepEqual(m.Invokes, want) { "seat:node-build-node.steps", "seat:node-build-image.steps", "mesh-delivery-builder.builder_versions"}; !reflect.DeepEqual(m.Invokes, want) {
t.Errorf("invokes = %v, want %v", m.Invokes, want) t.Errorf("invokes = %v, want %v", m.Invokes, want)
} }
@@ -204,9 +202,8 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build
// milestone 47/G W4.12 (design 47, the indexer's tools): the catalog graph revisions read and steered. // milestone 47/G W4.12 (design 47, the indexer's tools): the catalog graph revisions read and steered.
index.CatalogRevisionTool, index.CatalogRevisionsTool, index.CandidateGraphsTool, index.RecomputeTool, index.DropCandidateGraphTool, index.CatalogRevisionTool, index.CatalogRevisionsTool, index.CandidateGraphsTool, index.RecomputeTool, index.DropCandidateGraphTool,
index.EntryIndexTool, index.NamedDropTool, // design 47 G §675 index.EntryIndexTool, index.NamedDropTool, // design 47 G §675
index.ProposalsTool, index.ProposalDropTool, index.SeedAgainTool, index.InputsTool, index.PinsTool, index.UnpinTool, index.ConsumerTool, index.RunsTool, index.RunDropTool, // design 47 G §667, §8, §579, §64 (A-194) index.ProposalsTool, index.ProposalDropTool, index.SeedAgainTool, index.InputsTool, index.PinsTool, index.UnpinTool, index.ConsumerTool, index.RunsTool, index.RunDropTool) // design 47 G §667, §8, §579, §64 (A-194)
index.ValidateAgainTool) // design 47 G §713 if !reflect.DeepEqual(m.Tools, want) || len(want) != 51 || !slices.Contains(m.Tools, "modules") {
if !reflect.DeepEqual(m.Tools, want) || len(want) != 52 || !slices.Contains(m.Tools, "modules") {
t.Errorf("tools = %v, want %v", m.Tools, want) t.Errorf("tools = %v, want %v", m.Tools, want)
} }
// Design 47 G §2, §9: every version of every registration is kept with no age limit, 64 MiB, on the old streams section. // Design 47 G §2, §9: every version of every registration is kept with no age limit, 64 MiB, on the old streams section.
+2 -4
View File
@@ -77,8 +77,7 @@
"indexer_unpin", "indexer_unpin",
"indexer_consumer", "indexer_consumer",
"indexer_runs", "indexer_runs",
"indexer_run_drop", "indexer_run_drop"
"indexer_validate_again"
], ],
"consumes": [ "consumes": [
"go-package-registry.published", "go-package-registry.published",
@@ -94,6 +93,7 @@
"mesh-wants.wait-ended", "mesh-wants.wait-ended",
"mesh-delivery-checker.verdict", "mesh-delivery-checker.verdict",
"mesh-delivery-forge-bridge.forge.**", "mesh-delivery-forge-bridge.forge.**",
"mesh-catalog.catalog-revision",
"mesh-delivery.source-moved", "mesh-delivery.source-moved",
"mesh-delivery.pinned", "mesh-delivery.pinned",
"mesh-delivery.freed", "mesh-delivery.freed",
@@ -179,8 +179,6 @@
"seat:mesh-hand-acts.hand-act", "seat:mesh-hand-acts.hand-act",
"seat:mesh-watchdogs.presence", "seat:mesh-watchdogs.presence",
"seat:node-build-go.steps", "seat:node-build-go.steps",
"seat:node-build-go.accept.check",
"seat:node-build-go.output",
"seat:node-build-node.steps", "seat:node-build-node.steps",
"seat:node-build-image.steps", "seat:node-build-image.steps",
"mesh-delivery-builder.builder_versions" "mesh-delivery-builder.builder_versions"
+11 -35
View File
@@ -1,5 +1,5 @@
// Package takein is the indexer's take-in contract (novox/hq ADR 0633 §1, §5): the events it publishes on its own stream // Package takein is the indexer's take-in contract (novox/hq ADR 0633 §1, §5): the events it publishes on its own stream
// `take-in` while a source revision waits for the indexer it names or a delivery group is gathered, their keys, and the // `take-in` while a source revision waits (its message unread, lost or held for room) or a delivery group is gathered, their keys, and the
// answer of its tool indexer_pending. The indexer writes them; the watchman and the view read them. It is the indexer's // answer of its tool indexer_pending. The indexer writes them; the watchman and the view read them. It is the indexer's
// own package: one module owns these types, so they live with it and not in mesh-sdk (the operator, 2026-11-04). // own package: one module owns these types, so they live with it and not in mesh-sdk (the operator, 2026-11-04).
package takein package takein
@@ -118,7 +118,7 @@ func checkTakeInKey(key string) error {
} }
// TakeIn is the body of a take-in event. Kind is the event's kind and Key its key. What says what waits: a revision // TakeIn is the body of a take-in event. Kind is the event's kind and Key its key. What says what waits: a revision
// (Key its oldest merge's PendingKey, WaitsFor the indexer build it names) or a group being gathered (Key its // (Key its oldest merge's PendingKey) or a group being gathered (Key its
// GroupTakeInKey, Members every approved member with its head, Merges the members held so far). Merges are the merges // GroupTakeInKey, Members every approved member with its head, Merges the members held so far). Merges are the merges
// it holds, oldest first; Since is when the wait began. A `taken` event names the same once the wait ended; By and Why // it holds, oldest first; Since is when the wait began. A `taken` event names the same once the wait ended; By and Why
// name a hand act (a hand take-in, or a drop, Dropped), and are carried by nothing else. // name a hand act (a hand take-in, or a drop, Dropped), and are carried by nothing else.
@@ -138,16 +138,9 @@ type TakeIn struct {
// Lost are the keys of merges whose message the forge bridge's stream no longer holds: they are not taken in, and the // Lost are the keys of merges whose message the forge bridge's stream no longer holds: they are not taken in, and the
// revision waits until indexer_drop_pending drops them (issue 1585). // revision waits until indexer_drop_pending drops them (issue 1585).
Lost []string `json:"lost,omitempty"` Lost []string `json:"lost,omitempty"`
// Refused is why the manifest of the indexer the revision names was refused when the revision was worked out: with // Refused is why the revision is refused: the cycle with a build edge its catalog graph would hold (design 47 G §49). The
// the seat definitions the revision carries (ADR 0633 §1), the manifest validator's problems. The revision waits for // revision waits, and says why here, as it says what it lost.
// an indexer whose manifest is accepted, and says why here, as it says what it lost.
Refused string `json:"refused,omitempty"` Refused string `json:"refused,omitempty"`
// Failed is why the revision's validation could not validate (design 47 G §713): the build seat's words, never a refusal
// and never passed; it is asked again by a later merge of the branch and by indexer_validate_again.
Failed string `json:"failed,omitempty"`
// Unknown is why the indexer holds a merge it does not take in although no indexer build is named: it cannot learn the build
// it runs itself (ADR 0633 §1), so Since is when it began not to know. A pending revision names WaitsFor or Unknown.
Unknown string `json:"own-build-unknown,omitempty"`
// Branch, Source and Entries are what a take-in took in (design 47 G §476, §493, §648): the watched branch, the source // Branch, Source and Entries are what a take-in took in (design 47 G §476, §493, §648): the watched branch, the source
// revision it makes of that branch, and one entry per module it wrote. Every taken event of one take-in carries the same; // revision it makes of that branch, and one entry per module it wrote. Every taken event of one take-in carries the same;
// a taken event that ends a wait without a take-in (a group's end, a drop) carries none and starts no pipeline. // a taken event that ends a wait without a take-in (a group's end, a drop) carries none and starts no pipeline.
@@ -184,8 +177,8 @@ func (t TakeIn) check() error {
if !strings.HasPrefix(t.Key, "m.") { if !strings.HasPrefix(t.Key, "m.") {
return fmt.Errorf("delivery: a revision's take-in key is its oldest merge's m.<sequence>, not %q", t.Key) return fmt.Errorf("delivery: a revision's take-in key is its oldest merge's m.<sequence>, not %q", t.Key)
} }
if t.Kind == TakeInPending && t.WaitsFor == "" && t.Unknown == "" && len(t.Lost) == 0 && t.Refused == "" && t.Failed == "" { if t.Kind == TakeInPending && len(t.Lost) == 0 && t.Refused == "" {
return fmt.Errorf("delivery: a pending revision %s names the indexer build it waits for, why the indexer does not know its own, the merges it lost, its refusal or why it could not validate", t.Key) return fmt.Errorf("delivery: a pending revision %s names the merges it lost or its refusal", t.Key)
} }
case TakeInGroup: case TakeInGroup:
if t.Group == "" { if t.Group == "" {
@@ -266,32 +259,15 @@ type PendingWait struct {
Unread []string `json:"unread,omitempty"` Unread []string `json:"unread,omitempty"`
// Refused is why the manifest of the indexer it names was refused (TakeIn.Refused). // Refused is why the manifest of the indexer it names was refused (TakeIn.Refused).
Refused string `json:"refused,omitempty"` Refused string `json:"refused,omitempty"`
// Validation is the revision's validation whole, as its newest pending value keeps it (design 47 G §713, A-247).
Validation *Validation `json:"validation,omitempty"`
}
// Validation is a pending revision's validation (design 47 G §712, §713, A-232, A-247), kept on the revision's newest pending
// value alone: the run ask's id, when it was asked, its input's digest, the sequence of the newest merge it covers and the try
// number for it, why it could not validate, and the sequence it was accepted at (never asked again once set).
type Validation struct {
Ask string `json:"ask,omitempty"`
Asked time.Time `json:"asked,omitzero"`
Input string `json:"input,omitempty"`
Seq uint64 `json:"seq"`
Try int `json:"try"`
Failed string `json:"failed,omitempty"`
Accepted uint64 `json:"accepted,omitempty"`
} }
// PendingAnswer is indexer_pending's answer: every wait, oldest first, and the pending keys that do not read. // PendingAnswer is indexer_pending's answer: every wait, oldest first, and the pending keys that do not read.
type PendingAnswer struct { type PendingAnswer struct {
Waits []PendingWait `json:"waits"` Waits []PendingWait `json:"waits"`
Unread []string `json:"unread,omitempty"` Unread []string `json:"unread,omitempty"`
// OwnBuildUnknown is why the running indexer does not know its own build, and since when: while it is set, every merge waits OwnBuildSince time.Time `json:"own-build-since,omitzero"`
// (a take-in.pending with the same words) and the indexer is started again to learn it. Empty when the build is known. // OwnBuild is the build the running indexer knows it runs, fixed at its start; OwnBuildSince is since when. Empty when unknown:
OwnBuildUnknown string `json:"own-build-unknown,omitempty"` // it names only the writer of what the indexer writes, and nothing waits for it.
OwnBuildSince time.Time `json:"own-build-since,omitzero"`
// OwnBuild is the build the running indexer knows it runs, fixed at its start; OwnBuildSince is since when. Empty when unknown.
OwnBuild string `json:"own-build,omitempty"` OwnBuild string `json:"own-build,omitempty"`
} }
@@ -16,15 +16,12 @@ import (
func sampleTakeIns(since time.Time) []TakeIn { func sampleTakeIns(since time.Time) []TakeIn {
m := []Merge{{Repository: "novox/mesh-delivery", Number: 9, Merge: "eeee"}} m := []Merge{{Repository: "novox/mesh-delivery", Number: 9, Merge: "eeee"}}
return []TakeIn{ return []TakeIn{
{Kind: TakeInPending, Key: PendingKey(12), What: TakeInRevision, Merges: m, WaitsFor: "src2:abc", Since: since}, {Kind: TakeInPending, Key: PendingKey(12), What: TakeInRevision, Merges: m, Lost: []string{"m.00000000000000000011"}, Since: since},
{Kind: TakeInPending, Key: GroupTakeInKey("feat/take-in"), What: TakeInGroup, Merges: m, Group: "feat/take-in", {Kind: TakeInPending, Key: GroupTakeInKey("feat/take-in"), What: TakeInGroup, Merges: m, Group: "feat/take-in",
Members: []meshdeliveryclient.Head{{Repository: "novox/mesh-delivery", Number: 9, Head: "ffff"}, {Repository: "novox/mesh-sdk", Number: 37, Head: "dddd"}}, Since: since}, Members: []meshdeliveryclient.Head{{Repository: "novox/mesh-delivery", Number: 9, Head: "ffff"}, {Repository: "novox/mesh-sdk", Number: 37, Head: "dddd"}}, Since: since},
{Kind: TakeInTaken, Key: PendingKey(12), What: TakeInRevision, Merges: m, WaitsFor: "src2:abc", Since: since, {Kind: TakeInTaken, Key: PendingKey(12), What: TakeInRevision, Merges: m, WaitsFor: "src2:abc", Since: since,
By: "lead-47@shanks", Why: "the named indexer can never run"}, By: "lead-47@shanks", Why: "the named indexer can never run"},
{Kind: TakeInPending, Key: PendingKey(13), What: TakeInRevision, Merges: m, Unknown: "the engine does not answer", Since: since}, // the indexer cannot learn its own build {Kind: TakeInPending, Key: PendingKey(14), What: TakeInRevision, Merges: m, Refused: "a cycle with a build edge", Since: since},
// design 47 G §713: a revision refused by its validation, and one that could not validate.
{Kind: TakeInPending, Key: PendingKey(14), What: TakeInRevision, Merges: m, Refused: "a: names no version", Since: since},
{Kind: TakeInPending, Key: PendingKey(15), What: TakeInRevision, Merges: m, Failed: "could not run: no build node", Since: since},
} }
} }
@@ -73,7 +70,7 @@ func TestTakeInEvents(t *testing.T) {
}{ }{
{"group take-in without its group", "names its group", 1, func(t *TakeIn) { t.Group = "" }}, {"group take-in without its group", "names its group", 1, func(t *TakeIn) { t.Group = "" }},
{"group take-in under another group's key", "g.other", 1, func(t *TakeIn) { t.Key = GroupTakeInKey("other") }}, {"group take-in under another group's key", "g.other", 1, func(t *TakeIn) { t.Key = GroupTakeInKey("other") }},
{"pending revision waiting for no build", "lost", 0, func(t *TakeIn) { t.WaitsFor = "" }}, {"pending revision naming neither a lost merge nor a refusal", "lost", 0, func(t *TakeIn) { t.Lost = nil }},
{"pending with a hand act", "only a taken event", 0, func(t *TakeIn) { t.By = "someone@node" }}, {"pending with a hand act", "only a taken event", 0, func(t *TakeIn) { t.By = "someone@node" }},
{"hand act without its by", "by", 2, func(t *TakeIn) { t.By = "" }}, {"hand act without its by", "by", 2, func(t *TakeIn) { t.By = "" }},
{"take-in without merges", "no merge", 0, func(t *TakeIn) { t.Merges = nil }}, {"take-in without merges", "no merge", 0, func(t *TakeIn) { t.Merges = nil }},
@@ -15,6 +15,8 @@ mesh-catalog b5dbfbb784507e211372a7c9640d91f7d6dadd84 modules/*/module.json, m
mesh-host 233bb23bbcb1a829ce2a6164a31a5c5cc4f3d950 internal/apply/placement_guard.go mesh-host 233bb23bbcb1a829ce2a6164a31a5c5cc4f3d950 internal/apply/placement_guard.go
mesh-catalog 2d5358b8 (milestone 47/G's AM-K1) modules/zsh/module.json alone: the login shell uses operator-channel for mesh-catalog 2d5358b8 (milestone 47/G's AM-K1) modules/zsh/module.json alone: the login shell uses operator-channel for
its seat's marked execute (design 46 approval mark §21), which the validator requires (§4) its seat's marked execute (design 46 approval mark §21), which the validator requires (§4)
mesh-catalog cd3b8fe8 (restic's secrets-owner, ADR 0680) modules/restic/module.json and modules/restic/files/** alone: its
processes run as the operator account, so the validator (ADR 0680 §3) needs the secrets-owner it names
To move them, from mesh-delivery's modules/mesh-delivery-indexer directory, with the two repositories checked out To move them, from mesh-delivery's modules/mesh-delivery-indexer directory, with the two repositories checked out
beside mesh-delivery: beside mesh-delivery:
@@ -0,0 +1 @@
${config:put-off-bound}
@@ -1,6 +1,9 @@
{ {
"module": "restic", "module": "restic",
"version": "1", "version": "0.1",
"packages": {
"git.novox.be/novox/mesh-catalog/modules/disk-load/diskreadings": "0.1"
},
"moved": true, "moved": true,
"claims": [ "claims": [
{ {
@@ -13,13 +16,27 @@
] ]
} }
], ],
"invokes": [
"seat:mesh-module-assigner.assignments",
"seat:mesh-broker.show"
],
"consumes": [
"mesh-recovery.restore-started",
"mesh-recovery.restored",
"mesh-conditions.findings-asked"
],
"emits": [
"backed-up"
],
"tools": [ "tools": [
"backup_health", "backup_health",
"backup_delete_retired", "backup_delete_retired",
"backup_deleted" "backup_deleted"
], ],
"secrets-owner": "${machine:account}",
"own-secrets": { "own-secrets": {
"repository": "${dir:state}/repository.secret" "repository": "${dir:state}/repository.secret",
"broker": "${dir:state}/broker"
}, },
"data": { "data": {
"own": [ "own": [
@@ -38,40 +55,48 @@
} }
] ]
}, },
"config": { "config": {
"keep-daily": { "keep-daily": {
"type": "number", "type": "number",
"description": "How many nightly restore points are kept, a count from 1 to 9999: two weeks undo a mistake noticed within a fortnight. Read by the night, which thins nothing while it does not read (novox/hq issue 416); lowering it deletes the restore points beyond the new count at the next night, and that cannot be undone.", "description": "How many nightly restore points are kept, a count from 1 to 9999: two weeks undo a mistake noticed within a fortnight. Read by the night, which thins nothing while it does not read (novox/hq issue 416); lowering it deletes the restore points beyond the new count at the next night, and that cannot be undone.",
"minimum": 1, "minimum": 1,
"maximum": 9999, "maximum": 9999,
"trusted": true, "trusted": true,
"default": 14 "default": 14
},
"keep-weekly": {
"type": "number",
"description": "How many weekly restore points are kept behind the dailies, a count from 1 to 9999, refused by the night like keep-daily; lowering it deletes the restore points beyond the new count at the next night, and that cannot be undone.",
"minimum": 1,
"maximum": 9999,
"trusted": true,
"default": 8
},
"keep-monthly": {
"type": "number",
"description": "How many monthly restore points are kept behind the weeklies, a count from 1 to 9999, refused by the night like keep-daily; lowering it deletes the restore points beyond the new count at the next night, and that cannot be undone.",
"minimum": 1,
"maximum": 9999,
"trusted": true,
"default": 6
},
"keep-hand-taken-days": {
"type": "number",
"description": "How many days a restore point taken by hand (node-backup.now, or the mesh's before a module moves) is kept at least, a count from 1 to 9999; it is kept until a later good night of its module exists, and this many days in any case, so the next thinning never takes the one taken against a risky change (novox/hq issue 453). Lowering it lets go of the older ones taken by hand at the next thinning, and that cannot be undone.",
"minimum": 1,
"maximum": 9999,
"trusted": true,
"default": 7
}
}, },
"keep-weekly": {
"type": "number",
"description": "How many weekly restore points are kept behind the dailies, a count from 1 to 9999, refused by the night like keep-daily; lowering it deletes the restore points beyond the new count at the next night, and that cannot be undone.",
"minimum": 1,
"maximum": 9999,
"trusted": true,
"default": 8
},
"keep-monthly": {
"type": "number",
"description": "How many monthly restore points are kept behind the weeklies, a count from 1 to 9999, refused by the night like keep-daily; lowering it deletes the restore points beyond the new count at the next night, and that cannot be undone.",
"minimum": 1,
"maximum": 9999,
"trusted": true,
"default": 6
},
"keep-hand-taken-days": {
"type": "number",
"description": "How many days a restore point taken by hand (node-backup.now, or the mesh's before a module moves) is kept at least, a count from 1 to 9999; it is kept until a later good night of its module exists, and this many days in any case, so the next thinning never takes the one taken against a risky change (novox/hq issue 453). Lowering it lets go of the older ones taken by hand at the next thinning, and that cannot be undone.",
"minimum": 1,
"maximum": 9999,
"trusted": true,
"default": 7
},
"put-off-bound": {
"type": "number",
"description": "minutes an item put off for a restore of the recovery module is waited for before the finding backup-waits is written and the wait ends, so the night never holds its lock without a bound; timed by a bound key of the state put-off (design 47 G sections 561, 557, A-155).",
"minimum": 1,
"maximum": 10080,
"trusted": false,
"default": 60
}
},
"resources": [ "resources": [
{ {
"id": "state", "id": "state",
@@ -108,6 +133,13 @@
"trusted": true, "trusted": true,
"template": "files/retention.conf" "template": "files/retention.conf"
}, },
{
"id": "put-off-bound",
"type": "file",
"path": "${dir:state}/put-off-bound",
"mode": "0600",
"template": "files/put-off-bound"
},
{ {
"id": "tool", "id": "tool",
"type": "package", "type": "package",
@@ -133,7 +165,32 @@
"MESH_BACKUP_REPOSITORY": "${dir:repository}", "MESH_BACKUP_REPOSITORY": "${dir:repository}",
"MESH_BACKUP_PASSWORD_FILE": "${dir:state}/repository.secret", "MESH_BACKUP_PASSWORD_FILE": "${dir:state}/repository.secret",
"MESH_BACKUP_STATE": "${dir:state}", "MESH_BACKUP_STATE": "${dir:state}",
"MESH_BACKUP_DATA": "${dir:state}/data.conf" "MESH_BACKUP_DATA": "${dir:state}/data.conf",
"MESH_BROKER_FILE": "${dir:state}/broker",
"MESH_BACKUP_PUT_OFF_BOUND_FILE": "${dir:state}/put-off-bound"
},
"health": {
"kind": "unit"
}
},
{
"id": "awake",
"type": "process",
"name": "restic-awake",
"artifact": "tools",
"run": [
"./restic-backups",
"awake"
],
"user": "${machine:account}",
"env": {
"MESH_BACKUP_DECLARED": "${dir:state}/backups.conf",
"MESH_BACKUP_REPOSITORY": "${dir:repository}",
"MESH_BACKUP_PASSWORD_FILE": "${dir:state}/repository.secret",
"MESH_BACKUP_STATE": "${dir:state}",
"MESH_BACKUP_DATA": "${dir:state}/data.conf",
"MESH_BROKER_FILE": "${dir:state}/broker",
"MESH_BACKUP_PUT_OFF_BOUND_FILE": "${dir:state}/put-off-bound"
}, },
"health": { "health": {
"kind": "tool", "kind": "tool",
@@ -143,27 +200,6 @@
"looks": 2, "looks": 2,
"grace": "60s" "grace": "60s"
} }
},
{
"id": "measure",
"type": "process",
"name": "restic-measure",
"artifact": "tools",
"run": [
"./restic-backups",
"measure"
],
"user": "${machine:account}",
"env": {
"MESH_BACKUP_DECLARED": "${dir:state}/backups.conf",
"MESH_BACKUP_REPOSITORY": "${dir:repository}",
"MESH_BACKUP_PASSWORD_FILE": "${dir:state}/repository.secret",
"MESH_BACKUP_STATE": "${dir:state}",
"MESH_BACKUP_DATA": "${dir:state}/data.conf"
},
"health": {
"kind": "unit"
}
} }
], ],
"build": { "build": {
@@ -183,7 +219,8 @@
"MESH_BACKUP_REPOSITORY": "${dir:repository}", "MESH_BACKUP_REPOSITORY": "${dir:repository}",
"MESH_BACKUP_PASSWORD_FILE": "${dir:state}/repository.secret", "MESH_BACKUP_PASSWORD_FILE": "${dir:state}/repository.secret",
"MESH_BACKUP_STATE": "${dir:state}", "MESH_BACKUP_STATE": "${dir:state}",
"MESH_BACKUP_DATA": "${dir:state}/data.conf" "MESH_BACKUP_DATA": "${dir:state}/data.conf",
"MESH_BACKUP_PUT_OFF_BOUND_FILE": "${dir:state}/put-off-bound"
} }
}, },
{ {
@@ -192,5 +229,40 @@
"from": "files" "from": "files"
} }
] ]
} },
"reads": [
"disk-load.readings"
],
"state": [
"good"
],
"requires": [
{
"name": "put-off",
"storage": "key-value",
"template": "state",
"options": {
"key-ttl": true
}
},
{
"name": "findings",
"storage": "key-value",
"template": "state"
},
{
"name": "finding-events",
"storage": "stream",
"template": "events",
"options": {
"events": [
"finding-written.*",
"finding-cleared.*",
"findings-stated.*"
],
"max-age-days": 7,
"max-mib": 16
}
}
]
} }
@@ -152,6 +152,7 @@ func main() {
tools = append(tools, UnloggedTools(stdio.State(seat.StatePipelines), stdio.Log(seat.StateStepLog), meshhandactsclient.Recorder{Ask: stdio.Ask}, tools = append(tools, UnloggedTools(stdio.State(seat.StatePipelines), stdio.Log(seat.StateStepLog), meshhandactsclient.Recorder{Ask: stdio.Ask},
func(f string, a ...any) { fmt.Fprintf(os.Stderr, "[mesh-delivery-pipeline-tools] "+f+"\n", a...) })...) func(f string, a ...any) { fmt.Fprintf(os.Stderr, "[mesh-delivery-pipeline-tools] "+f+"\n", a...) })...)
tools = append(tools, CancelTool(r.List, stdio.Ask), RerunTool(stdio.Ask)) tools = append(tools, CancelTool(r.List, stdio.Ask), RerunTool(stdio.Ask))
tools = append(tools, VerbTools(stdio.Ask)...)
tools = append(tools, TestRunsTools(stdio.Ask)...) tools = append(tools, TestRunsTools(stdio.Ask)...)
tools = append(tools, HealthTool(r.Pipelines, kept{stdio.State(pipeline.StateConsumers)}, time.Now)) tools = append(tools, HealthTool(r.Pipelines, kept{stdio.State(pipeline.StateConsumers)}, time.Now))
if err := stdio.Serve("", tools); err != nil { if err := stdio.Serve("", tools); err != nil {
@@ -32,13 +32,16 @@ func TestTools(t *testing.T) {
names = append(names, tl.Name) names = append(names, tl.Name)
} }
names = append(names, CancelTool(nil, nil).Name, RerunTool(nil).Name) names = append(names, CancelTool(nil, nil).Name, RerunTool(nil).Name)
for _, tl := range VerbTools(nil) {
names = append(names, tl.Name)
}
// RN25 (§468, §76; A-164): the health check is the bundle's runner_health. // RN25 (§468, §76; A-164): the health check is the bundle's runner_health.
names = append(names, HealthTool(none{}, none{}, nil).Name) names = append(names, HealthTool(none{}, none{}, nil).Name)
// §606, §549: the unlogged entries are listed and appended by hand. // §606, §549: the unlogged entries are listed and appended by hand.
for _, tl := range UnloggedTools(stdio.State("pipelines"), stdio.Log("step-log"), nil, nil) { for _, tl := range UnloggedTools(stdio.State("pipelines"), stdio.Log("step-log"), nil, nil) {
names = append(names, tl.Name) names = append(names, tl.Name)
} }
want := "mesh-pipeline-runner.show mesh-pipeline-runner.list mesh-pipeline-runner.stands mesh-pipeline-runner.why-carried mesh-pipeline-runner.kept runner_first_views runner_first_view_again runner_cancel runner_rerun runner_health runner_unlogged runner_unlogged_append" want := "mesh-pipeline-runner.show mesh-pipeline-runner.list mesh-pipeline-runner.stands mesh-pipeline-runner.why-carried mesh-pipeline-runner.kept runner_first_views runner_first_view_again runner_cancel runner_rerun runner_start runner_run runner_accept runner_ask_again runner_health runner_unlogged runner_unlogged_append"
if got := strings.Join(names, " "); got != want { if got := strings.Join(names, " "); got != want {
t.Errorf("tools %q, want %q", got, want) t.Errorf("tools %q, want %q", got, want)
} }
@@ -0,0 +1,81 @@
package main
import (
"encoding/json"
"errors"
"strings"
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
stdio "git.novox.be/novox/mesh-sdk/go"
)
// VerbTools are runner_start, runner_run, runner_accept and runner_ask_again: the runner's seat verbs start, run, accept and
// ask-again, reachable as tools (design 47 G §83, §482), as runner_rerun reaches rerun. Each asks its verb over the bus with
// the arguments it was given and answers the verb's answer; the verb records the hand act and refuses what it refuses.
func VerbTools(ask func(key string, body any) (json.RawMessage, error)) []stdio.Tool {
str := map[string]any{"type": "string"}
obj := func(required []string, props map[string]any) map[string]any {
return map[string]any{"type": "object", "properties": props, "required": required}
}
return []stdio.Tool{
verbTool[seat.StartInput](ask, "start", seat.VerbStart,
"Start by hand the delivering pipeline of the head of a watched branch for a module, by the seat verb start: the branch, the module and why. A head whose pipeline is written already is refused by the verb, naming it.",
obj([]string{"branch", "module", "why"}, map[string]any{"branch": str, "module": str, "why": str}),
func(in seat.StartInput) string {
if in.Branch == "" || in.Module == "" {
return "start names the watched branch and the module"
}
return whyOf(in.Why)
}),
verbTool[seat.RunInput](ask, "run", seat.VerbRun,
"Start a run on request by the seat verb run: the members (module, branch, commit as the seat's Member) and optionally why.",
obj([]string{"members"}, map[string]any{"members": map[string]any{"type": "array", "items": map[string]any{"type": "object"}}, "why": str}),
func(in seat.RunInput) string {
if len(in.Members) == 0 {
return "run names its members"
}
return ""
}),
verbTool[seat.AcceptInput](ask, "accept", seat.VerbAccept,
"Accept one step of a pipeline by hand by the seat verb accept: the pipeline's key, the step, the artifact where the step is per artifact, and why.",
obj([]string{"pipeline", "step", "why"}, map[string]any{"pipeline": str, "step": str, "artifact": str, "why": str}),
func(in seat.AcceptInput) string {
if in.Pipeline == "" || in.Step == "" {
return "accept names the pipeline and the step"
}
return whyOf(in.Why)
}),
verbTool[seat.AskAgainInput](ask, "ask_again", seat.VerbAskAgain,
"Ask a declined approval step of a pipeline once more by the seat verb ask-again: the pipeline's key and why.",
obj([]string{"pipeline", "why"}, map[string]any{"pipeline": str, "why": str}),
func(in seat.AskAgainInput) string {
if in.Pipeline == "" {
return "ask-again names the pipeline"
}
return whyOf(in.Why)
}),
}
}
func whyOf(why string) string {
if strings.TrimSpace(why) == "" {
return "a hand act names why"
}
return ""
}
// verbTool is one tool over one seat verb: it decodes the arguments into the verb's input, refuses what the check refuses
// before asking, and answers the verb's answer.
func verbTool[T any](ask func(string, any) (json.RawMessage, error), name, verb, description string, input map[string]any, check func(T) string) stdio.Tool {
return stdio.Tool{Name: Prefix + "_" + name, Description: description, Input: input,
Run: func(args map[string]any) (any, error) {
in, err := as[T](args)
if err != nil {
return nil, err
}
if msg := check(in); msg != "" {
return nil, errors.New(msg)
}
return ask("seat:"+seat.Seat+"."+verb, in)
}}
}
@@ -0,0 +1,59 @@
package main
import (
"encoding/json"
"strings"
"testing"
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
)
// Each tool asks its seat verb with the arguments it was given and refuses, before asking, a call the verb would refuse.
func TestVerbTools(t *testing.T) {
for _, c := range []struct {
name, tool, key, refusal string
args map[string]any
body any
}{
{"start", "runner_start", "seat:mesh-pipeline-runner.start", "", map[string]any{"branch": "main", "module": "i3", "why": "cancelled"},
seat.StartInput{Branch: "main", Module: "i3", Why: "cancelled"}},
{"start no why", "runner_start", "", "names why", map[string]any{"branch": "main", "module": "i3"}, nil},
{"start no module", "runner_start", "", "branch and the module", map[string]any{"branch": "main", "why": "x"}, nil},
{"run", "runner_run", "seat:mesh-pipeline-runner.run", "", map[string]any{"members": []any{map[string]any{}}, "why": "x"},
seat.RunInput{Members: []seat.Member{{}}, Why: "x"}},
{"run no members", "runner_run", "", "members", map[string]any{}, nil},
{"accept", "runner_accept", "seat:mesh-pipeline-runner.accept", "", map[string]any{"pipeline": "p", "step": "lint", "artifact": "a", "why": "x"},
seat.AcceptInput{Pipeline: "p", Step: "lint", Artifact: "a", Why: "x"}},
{"accept no step", "runner_accept", "", "pipeline and the step", map[string]any{"pipeline": "p", "why": "x"}, nil},
{"ask-again", "runner_ask_again", "seat:mesh-pipeline-runner.ask-again", "", map[string]any{"pipeline": "p", "why": "x"},
seat.AskAgainInput{Pipeline: "p", Why: "x"}},
{"ask-again no why", "runner_ask_again", "", "names why", map[string]any{"pipeline": "p"}, nil},
} {
t.Run(c.name, func(t *testing.T) {
var key string
var body any
ask := func(k string, b any) (json.RawMessage, error) {
key, body = k, b
return json.RawMessage(`{"ok":1}`), nil
}
var run func(map[string]any) (any, error)
for _, tl := range VerbTools(ask) {
if tl.Name == c.tool {
run = tl.Run
}
}
out, err := run(c.args)
if c.refusal != "" {
if err == nil || !strings.Contains(err.Error(), c.refusal) || key != "" {
t.Fatalf("want refusal %q and no ask, got %v (asked %q)", c.refusal, err, key)
}
return
}
want, _ := json.Marshal(c.body)
got, _ := json.Marshal(body)
if err != nil || key != c.key || string(got) != string(want) || out == nil {
t.Fatalf("asked %q with %s, want %s, err %v", key, got, want, err)
}
})
}
}
@@ -127,10 +127,17 @@ func place(p seat.Pipeline, reg catalog.Registration, entries map[string]catalog
steps = append(steps, s) steps = append(steps, s)
continue continue
} }
// The ask names the language of the seat it is asked of, never the artifact's word (design 47 G A-488).
language, err := sdkstep.AskLanguageOfSeat(bs)
if err != nil {
s.Outcome, s.Started, s.Ended, s.Why = seat.StepResultOutcomeCouldNotRun, &now, &now, err.Error()
steps = append(steps, s)
continue
}
s.Seat, s.Outcome, s.Started = bs, seat.StepResultOutcomeRunning, &now s.Seat, s.Outcome, s.Started = bs, seat.StepResultOutcomeRunning, &now
s.Ask = AskID(startOf(p, now), p.Pipeline, a.Ref, step) s.Ask = AskID(startOf(p, now), p.Pipeline, a.Ref, step)
body := stepAsk(p, entry, a, step, s.Ask, asker, now) body := stepAsk(p, entry, a, language, step, s.Ask, asker, now)
ask := Asked{Seat: bs, ID: s.Ask, Ask: body} ask := Asked{Seat: bs, ID: s.Ask, Ask: body, Toolchain: a.Language}
if share && BoundSeat(bs) { if share && BoundSeat(bs) {
digest := ShareDigest(p.Fingerprint, step, status, body.Step.Fetch) digest := ShareDigest(p.Fingerprint, step, status, body.Step.Fetch)
run := TestRunKey(candidate, step, digest) run := TestRunKey(candidate, step, digest)
@@ -189,17 +196,20 @@ type Asked struct {
// Run is the test run the ask asks, its key in test-runs, with its level, its share digest and the steps of the pipeline // Run is the test run the ask asks, its key in test-runs, with its level, its share digest and the steps of the pipeline
// reading it, the asking step first (§724, §725); empty for an ask of no test run. // reading it, the asking step first (§724, §725); empty for an ask of no test run.
Run, Level, Digest string Run, Level, Digest string
Readers []seat.TestRunReader // Toolchain is the language of the artifact's toolchain as the catalog keys it (go, typescript; design 47 G A-104, A-488b),
// never the ask's language, which names the seat; empty for an artifact with none (an image).
Toolchain string
Readers []seat.TestRunReader
} }
func stepAsk(p seat.Pipeline, e catalog.CatalogEntry, a catalog.Artifact, step, id, asker string, now time.Time) nodebuildgo.StepAsk { func stepAsk(p seat.Pipeline, e catalog.CatalogEntry, a catalog.Artifact, language, step, id, asker string, now time.Time) nodebuildgo.StepAsk {
// A step is for its module alone (principle 10, A-447): the fetch always names the module's directory, and the // A step is for its module alone (principle 10, A-447): the fetch always names the module's directory, and the
// artifact's named paths in it when the entry has them. // artifact's named paths in it when the entry has them.
fetch := nodebuildgo.Fetch{Repository: p.Repository, Commit: p.Commit, Dir: e.Path} fetch := nodebuildgo.Fetch{Repository: p.Repository, Commit: p.Commit, Dir: e.Path}
if paths := artifactPaths(e, a.Ref); len(paths) > 0 { if paths := artifactPaths(e, a.Ref); len(paths) > 0 {
fetch.Paths = paths fetch.Paths = paths
} }
return nodebuildgo.StepAsk{V: 1, ID: id, Asker: asker, Kind: "step", Language: a.Language, At: &now, return nodebuildgo.StepAsk{V: 1, ID: id, Asker: asker, Kind: "step", Language: language, At: &now,
Step: &nodebuildgo.StepRun{Artifact: a.Ref, Branch: p.Branch, Commit: p.Commit, Fetch: fetch, Fingerprint: p.Fingerprint, Step: &nodebuildgo.StepRun{Artifact: a.Ref, Branch: p.Branch, Commit: p.Commit, Fetch: fetch, Fingerprint: p.Fingerprint,
Module: p.Module, Pipeline: p.Pipeline, Revision: p.Source, Step: step}} Module: p.Module, Pipeline: p.Pipeline, Revision: p.Source, Step: step}}
} }
@@ -242,7 +242,9 @@ func TestAStepAskCarriesTheKeptResultsAndTheToolchain(t *testing.T) {
reg, entries := placeFixture() reg, entries := placeFixture()
e := entries["app"] e := entries["app"]
e.Artifacts[0].Package = &catalog.ArtifactPackage{Ecosystem: "go", Name: "git.example/o/app"} e.Artifacts[0].Package = &catalog.ArtifactPackage{Ecosystem: "go", Name: "git.example/o/app"}
e.Artifacts = append(e.Artifacts, catalog.Artifact{Ref: "app:lib", Language: "typescript"})
entries["app"] = e entries["app"] = e
entries["mesh-build-node"] = catalog.CatalogEntry{Module: "mesh-build-node", Acts: []catalog.ActingRule{{Seat: "node-build-node", Step: "build", On: "typescript"}}}
store, asks := created{}, &stepAsks{} store, asks := created{}, &stepAsks{}
reads := 0 reads := 0
s := &Starter{Pipelines: store, Say: func(string, any) error { return nil }, Steps: asks, Name: "the-runner", s := &Starter{Pipelines: store, Say: func(string, any) error { return nil }, Steps: asks, Name: "the-runner",
@@ -260,7 +262,7 @@ func TestAStepAskCarriesTheKeptResultsAndTheToolchain(t *testing.T) {
if _, err := s.Start(Begin{Branch: "main", Source: 4, Module: "app", Commit: "c1", Fp: "src2:aa", Repository: "o/app", Entries: entries}); err != nil { if _, err := s.Start(Begin{Branch: "main", Source: 4, Module: "app", Commit: "c1", Fp: "src2:aa", Repository: "o/app", Entries: entries}); err != nil {
t.Fatal(err) t.Fatal(err)
} }
var program, tools *nodebuildgo.StepRun var program, tools, lib *nodebuildgo.StepRun
for _, a := range asks.asks { for _, a := range asks.asks {
if ask, ok := a.(nodebuildgo.StepAsk); ok { if ask, ok := a.(nodebuildgo.StepAsk); ok {
switch { switch {
@@ -268,6 +270,8 @@ func TestAStepAskCarriesTheKeptResultsAndTheToolchain(t *testing.T) {
program = ask.Step program = ask.Step
case ask.Step.Artifact == "app:tools" && ask.Step.Step == "unit": case ask.Step.Artifact == "app:tools" && ask.Step.Step == "unit":
tools = ask.Step tools = ask.Step
case ask.Step.Artifact == "app:lib" && ask.Step.Step == "unit":
lib = ask.Step
} }
} }
} }
@@ -277,6 +281,10 @@ func TestAStepAskCarriesTheKeptResultsAndTheToolchain(t *testing.T) {
if tools == nil || len(tools.Kept) != 0 || tools.Toolchain == nil { if tools == nil || len(tools.Kept) != 0 || tools.Toolchain == nil {
t.Errorf("an artifact with no package carries no kept result and its language's toolchain: %+v", tools) t.Errorf("an artifact with no package carries no kept result and its language's toolchain: %+v", tools)
} }
// Design 47 G A-488b: the toolchain is read by the artifact's toolchain language (typescript), never by the ask's (node).
if lib == nil || lib.Toolchain == nil || lib.Toolchain.Language != "typescript" {
t.Errorf("the TypeScript artifact's ask carries the toolchain %+v, want the one of typescript", lib)
}
for pkg, want := range map[string]bool{"a/b": true, "a/b/c": true, "a/bc": false, "a": false} { for pkg, want := range map[string]bool{"a/b": true, "a/b/c": true, "a/bc": false, "a": false} {
if InPackage(pkg, "a/b") != want { if InPackage(pkg, "a/b") != want {
t.Errorf("InPackage(%s, a/b) = %v, want %v", pkg, !want, want) t.Errorf("InPackage(%s, a/b) = %v, want %v", pkg, !want, want)
@@ -295,8 +303,10 @@ func TestAnArtifactWithoutALanguageIsPlacedByItsKind(t *testing.T) {
reg := catalog.Registration{Module: "app", Policy: catalog.Policy{Steps: catalog.StepStatuses{Lint: "required", Unit: "required", Integration: "required"}}} reg := catalog.Registration{Module: "app", Policy: catalog.Policy{Steps: catalog.StepStatuses{Lint: "required", Unit: "required", Integration: "required"}}}
entries := map[string]catalog.CatalogEntry{ entries := map[string]catalog.CatalogEntry{
"app": {Module: "app", Artifacts: []catalog.Artifact{{Ref: "app:server", Kind: "image"}, {Ref: "app:files", Kind: "archive"}, "app": {Module: "app", Artifacts: []catalog.Artifact{{Ref: "app:server", Kind: "image"}, {Ref: "app:files", Kind: "archive"},
{Ref: "app:mirror", Kind: "upstream"}, {Ref: "app:web", Language: "rust"}}}, {Ref: "app:mirror", Kind: "upstream"}, {Ref: "app:web", Language: "rust"}, {Ref: "app:lib", Language: "typescript"}, {Ref: "app:cli", Language: "go"}}},
"mesh-build-go": {Module: "mesh-build-go", Acts: []catalog.ActingRule{{Seat: "node-build-go", Step: "build", On: "go"}}},
"mesh-build-image": {Module: "mesh-build-image", Acts: []catalog.ActingRule{{Seat: "node-build-image", Step: "build", On: seatdef.OnKindImage}}}, "mesh-build-image": {Module: "mesh-build-image", Acts: []catalog.ActingRule{{Seat: "node-build-image", Step: "build", On: seatdef.OnKindImage}}},
"mesh-build-node": {Module: "mesh-build-node", Acts: []catalog.ActingRule{{Seat: "node-build-node", Step: "build", On: "typescript"}}},
} }
steps, asks := Place(seat.Pipeline{Pipeline: "main.4.app", Module: "app"}, reg, entries, time.Unix(0, 0), "the-runner") steps, asks := Place(seat.Pipeline{Pipeline: "main.4.app", Module: "app"}, reg, entries, time.Unix(0, 0), "the-runner")
want := map[string]struct{ outcome, seat, status string }{ want := map[string]struct{ outcome, seat, status string }{
@@ -304,6 +314,8 @@ func TestAnArtifactWithoutALanguageIsPlacedByItsKind(t *testing.T) {
"app:files": {seat.StepResultOutcomeBypassed, "", seat.StepResultStatusBypass}, "app:files": {seat.StepResultOutcomeBypassed, "", seat.StepResultStatusBypass},
"app:mirror": {seat.StepResultOutcomeBypassed, "", seat.StepResultStatusBypass}, "app:mirror": {seat.StepResultOutcomeBypassed, "", seat.StepResultStatusBypass},
"app:web": {seat.StepResultOutcomeCouldNotRun, "", "required"}, "app:web": {seat.StepResultOutcomeCouldNotRun, "", "required"},
"app:lib": {seat.StepResultOutcomeRunning, "node-build-node", "required"},
"app:cli": {seat.StepResultOutcomeRunning, "node-build-go", "required"},
} }
n := 0 n := 0
for _, s := range steps { for _, s := range steps {
@@ -316,16 +328,21 @@ func TestAnArtifactWithoutALanguageIsPlacedByItsKind(t *testing.T) {
t.Errorf("%s %s is %s/%s at %q (%s), want %s/%s at %q", s.Artifact, s.Step, s.Status, s.Outcome, s.Seat, s.Why, w.status, w.outcome, w.seat) t.Errorf("%s %s is %s/%s at %q (%s), want %s/%s at %q", s.Artifact, s.Step, s.Status, s.Outcome, s.Seat, s.Why, w.status, w.outcome, w.seat)
} }
} }
if n != 12 { if n != 18 {
t.Errorf("placed %d build steps of the four artifacts, want 12", n) t.Errorf("placed %d build steps of the six artifacts, want 18", n)
} }
for _, a := range asks { for _, a := range asks {
if a.Seat != builds.Seat && a.Seat != wants.Seat && a.Seat != "node-build-image" { if a.Seat != builds.Seat && a.Seat != wants.Seat && a.Seat != "node-build-image" && a.Seat != "node-build-node" && a.Seat != "node-build-go" {
t.Errorf("asked %s, want only the build, the composition and the image's build seat", a.Seat) t.Errorf("asked %s, want only the build, the composition and the image's and the node's build seats", a.Seat)
}
// Design 47 G A-488: a step's ask names the language of the seat it is asked of (node-build-node node, node-build-image
// image), never the artifact's word (typescript).
if ask, ok := a.Ask.(nodebuildgo.StepAsk); ok && ask.Language != strings.TrimPrefix(a.Seat, "node-build-") {
t.Errorf("the ask %s on %s names the language %q, want %q", a.ID, a.Seat, ask.Language, strings.TrimPrefix(a.Seat, "node-build-"))
} }
} }
if len(asks) != 5 { if len(asks) != 11 {
t.Errorf("asked %d, want the build, the image's three steps and the composition", len(asks)) t.Errorf("asked %d, want the build, the image's and the node's three steps each and the composition", len(asks))
} }
} }
@@ -483,16 +483,16 @@ func (s *Starter) carry(key string, asks []Asked) {
if a.Package != "" { if a.Package != "" {
ask.Step.Kept = kept[a.Package] ask.Step.Kept = kept[a.Package]
} }
if s.Toolchain == nil || ask.Language == "" { if s.Toolchain == nil || a.Toolchain == "" {
continue continue
} }
t, done := chains[ask.Language] t, done := chains[a.Toolchain]
if !done { if !done {
var err error var err error
if t, err = s.Toolchain(ask.Language); err != nil { if t, err = s.Toolchain(a.Toolchain); err != nil {
s.logf("the toolchain of %s for the steps of %s is not read: %v", ask.Language, key, err) s.logf("the toolchain of %s for the steps of %s is not read: %v", a.Toolchain, key, err)
} }
chains[ask.Language] = t chains[a.Toolchain] = t
} }
ask.Step.Toolchain = t ask.Step.Toolchain = t
} }
@@ -114,6 +114,7 @@ func TestManifest(t *testing.T) {
{"uses operator-channel", equalSets(m.Uses, []string{"operator-channel"})}, {"uses operator-channel", equalSets(m.Uses, []string{"operator-channel"})},
// RN25 (§539): the runner's tools carry the prefix runner_. // RN25 (§539): the runner's tools carry the prefix runner_.
{"first-view tools", equalSets(m.Tools, []string{"runner_first_views", "runner_first_view_again", "runner_health", "runner_unlogged", "runner_unlogged_append", {"first-view tools", equalSets(m.Tools, []string{"runner_first_views", "runner_first_view_again", "runner_health", "runner_unlogged", "runner_unlogged_append",
"runner_start", "runner_run", "runner_accept", "runner_ask_again",
"runner_test_runs", "runner_test_runs_drop"})}, "runner_test_runs", "runner_test_runs_drop"})},
// A-305 (ADR 0227 rule 8, ADR 0240 rule 2): the process is judged by its unit, as every process of mesh-delivery, // A-305 (ADR 0227 rule 8, ADR 0240 rule 2): the process is judged by its unit, as every process of mesh-delivery,
// never by a tool of its own; runner_health stays a tool. // never by a tool of its own; runner_health stays a tool.
@@ -30,6 +30,10 @@
"runner_health", "runner_health",
"runner_unlogged", "runner_unlogged",
"runner_unlogged_append", "runner_unlogged_append",
"runner_start",
"runner_run",
"runner_accept",
"runner_ask_again",
"runner_test_runs", "runner_test_runs",
"runner_test_runs_drop" "runner_test_runs_drop"
], ],
@@ -141,6 +141,19 @@ func BuildSeat(language string) (string, error) {
return "node-build-" + language, nil return "node-build-" + language, nil
} }
// AskLanguageOfSeat is the language of the asks a build seat takes: node-build-go go, node-build-node node, node-build-image
// image (BuildSeat's inverse). A pipeline step's ask names it, never the artifact's language word (design 47 G A-488).
func AskLanguageOfSeat(seat string) (string, error) {
language, ok := strings.CutPrefix(seat, "node-build-")
if !ok {
return "", fmt.Errorf("buildstep: %q is no build seat", seat)
}
if err := checkAskLanguage(language); err != nil {
return "", err
}
return language, nil
}
// AskSubject is the subject a build seat accepts a queue's asks on: the seat's accept subject of main's form, which names no // AskSubject is the subject a build seat accepts a queue's asks on: the seat's accept subject of main's form, which names no
// asker (design 47 G §600, §483). // asker (design 47 G §600, §483).
func AskSubject(language, queue string) (string, error) { func AskSubject(language, queue string) (string, error) {
@@ -108,7 +108,7 @@ func tiersToolSaysEachModulesWait(t *testing.T) {
t.Fatalf("view_tiers with waits = %+v, %v", out, err) t.Fatalf("view_tiers with waits = %+v, %v", out, err)
} }
web = held.Tiers[1][0] web = held.Tiers[1][0]
if last := web.Waits[len(web.Waits)-1]; last.Step != view.TakeInStep || !last.Since.Equal(heldSince) || !strings.Contains(last.Words, "src2:abc") { if last := web.Waits[len(web.Waits)-1]; last.Step != view.TakeInStep || !last.Since.Equal(heldSince) || !strings.Contains(last.Words, "unread or lost") {
t.Errorf("web = %+v, want its last wait on the edge take-in since %s", web, heldSince) t.Errorf("web = %+v, want its last wait on the edge take-in since %s", web, heldSince)
} }
if lib := held.Tiers[0][1]; lib.Module != "lib" || lib.Waits[len(lib.Waits)-1].Step != view.TakeInStep || !strings.Contains(lib.Waits[len(lib.Waits)-1].Words, "group feat") { if lib := held.Tiers[0][1]; lib.Module != "lib" || lib.Waits[len(lib.Waits)-1].Step != view.TakeInStep || !strings.Contains(lib.Waits[len(lib.Waits)-1].Words, "group feat") {
@@ -219,7 +219,7 @@ func (r Reader) heldBack(out *Tiers) (map[string]TierWait, []string) {
return held, nil return held, nil
} }
for _, w := range a.Waits { for _, w := range a.Waits {
words := "the indexer holds back " + w.Key + ", which waits for the indexer build " + w.WaitsFor words := "the indexer holds back " + w.Key + " (its message is unread or lost, or it is held for room)"
if w.What == takein.TakeInGroup { if w.What == takein.TakeInGroup {
words = "the indexer gathers the group " + w.Group + " and its end has not come" words = "the indexer gathers the group " + w.Group + " and its end has not come"
} else if w.Behind != "" { } else if w.Behind != "" {
@@ -132,7 +132,7 @@ func TestRunNeedsItsCredential(t *testing.T) { // 1135/AC6
} }
out.findings[old.Key()] = true out.findings[old.Key()] = true
w.SeedFinding(old.Key(), value) w.SeedFinding(old.Key(), value)
body, err := takein.EncodeTakeIn(takein.TakeIn{Kind: takein.TakeInPending, Key: "m.00000000000000000043", What: takein.TakeInRevision, Since: time.Now(), WaitsFor: "src2:x", body, err := takein.EncodeTakeIn(takein.TakeIn{Kind: takein.TakeInPending, Key: "m.00000000000000000043", What: takein.TakeInRevision, Since: time.Now(), Lost: []string{"m.00000000000000000043"},
Merges: []takein.Merge{{Repository: "novox/a", Number: 1, Merge: "c0ffee"}}}) Merges: []takein.Merge{{Repository: "novox/a", Number: 1, Merge: "c0ffee"}}})
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
@@ -136,27 +136,27 @@ func TestTheWatchmansFindings(t *testing.T) {
// began ten minutes ago, and the indexer still holds it. // began ten minutes ago, and the indexer still holds it.
since := c.t.Add(-10 * time.Minute) since := c.t.Add(-10 * time.Minute)
w.SeedTakeIn(takein.PendingAnswer{Waits: []takein.PendingWait{{Key: revKey, What: takein.TakeInRevision, Keys: []string{revKey}, w.SeedTakeIn(takein.PendingAnswer{Waits: []takein.PendingWait{{Key: revKey, What: takein.TakeInRevision, Keys: []string{revKey},
Merges: []takein.Merge{{Repository: "novox/a", Number: 1, Merge: "c0ffee"}}, WaitsFor: "src2:x", Since: since}}}) Merges: []takein.Merge{{Repository: "novox/a", Number: 1, Merge: "c0ffee"}}, Lost: []string{"m.00000000000000000042"}, Since: since}}})
w.TakeInCaughtUp() w.TakeInCaughtUp()
if len(o.findings) != 0 || o.ttls[signalKey("take-in", revKey)] != TakeInRevisionBound-10*time.Minute { if len(o.findings) != 0 || o.ttls[signalKey("take-in", revKey)] != TakeInRevisionBound-10*time.Minute {
t.Fatalf("within its bound: findings %v, ttl %v; want none yet and the rest of the bound", o.findings, o.ttls[signalKey("take-in", revKey)]) t.Fatalf("within its bound: findings %v, ttl %v; want none yet and the rest of the bound", o.findings, o.ttls[signalKey("take-in", revKey)])
} }
expire(w, c, signalKey("take-in", revKey), TakeInRevisionBound-10*time.Minute) expire(w, c, signalKey("take-in", revKey), TakeInRevisionBound-10*time.Minute)
}, map[string]string{FindingKey(ScopeDelivery, revKey, KindTakeInWaits): Warning}}, }, map[string]string{FindingKey(ScopeDelivery, revKey, KindTakeInWaits): Warning}},
{"ADR 0633 §1 (issue 1921): a merge held because the indexer cannot learn its own build is raised past the revision bound from its since, naming why", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { {"ADR 0633 §1 (issue 1921): (A-476 b) a revision held because its message is lost or unread is raised past the revision bound from its since, naming why", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) {
w.TakeInCaughtUp() w.TakeInCaughtUp()
body, err := takein.EncodeTakeIn(takein.TakeIn{Kind: takein.TakeInPending, Key: revKey, What: takein.TakeInRevision, Since: c.t, body, err := takein.EncodeTakeIn(takein.TakeIn{Kind: takein.TakeInPending, Key: revKey, What: takein.TakeInRevision, Since: c.t,
Merges: []takein.Merge{{Repository: "novox/a", Number: 1, Merge: "c0ffee"}}, Unknown: "the engine does not answer"}) Merges: []takein.Merge{{Repository: "novox/a", Number: 1, Merge: "c0ffee"}}, Lost: []string{revKey}})
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
w.TakeIn(takeInEvent(t, takein.TakeInPending, revKey), body) w.TakeIn(takeInEvent(t, takein.TakeInPending, revKey), body)
if o.ttls[signalKey("take-in", revKey)] != TakeInRevisionBound || len(o.findings) != 0 { if o.ttls[signalKey("take-in", revKey)] != TakeInRevisionBound || len(o.findings) != 0 {
t.Fatalf("an unknown own build: ttls %v, findings %v", o.ttls, o.findings) t.Fatalf("a lost message: ttls %v, findings %v", o.ttls, o.findings)
} }
expire(w, c, signalKey("take-in", revKey), TakeInRevisionBound) expire(w, c, signalKey("take-in", revKey), TakeInRevisionBound)
for _, f := range o.findings { for _, f := range o.findings {
if !strings.Contains(f.Summary, "cannot learn the build it runs") { if !strings.Contains(f.Summary, "lost or unread") {
t.Errorf("finding %q does not say why", f.Summary) t.Errorf("finding %q does not say why", f.Summary)
} }
} }
@@ -573,7 +573,7 @@ func takeInBody(t *testing.T, kind, key string, since time.Time) []byte {
if key[0] == 'g' { if key[0] == 'g' {
body.What, body.Group = takein.TakeInGroup, "feat/take-in" body.What, body.Group = takein.TakeInGroup, "feat/take-in"
} else if kind == takein.TakeInPending { } else if kind == takein.TakeInPending {
body.WaitsFor = "src2:abc" body.Lost = []string{key}
} }
b, err := takein.EncodeTakeIn(body) b, err := takein.EncodeTakeIn(body)
if err != nil { if err != nil {
@@ -135,10 +135,7 @@ func (w *Watchman) takeIn(kind, key string, t takein.TakeIn) {
w.unfind(finding) w.unfind(finding)
return return
} }
bound, what := TakeInRevisionBound, "a source revision waits for the indexer build "+t.WaitsFor bound, what := TakeInRevisionBound, "a source revision is held (a message lost or unread, no room for it, or its refusal)"
if t.Unknown != "" {
what = "a source revision is held because the indexer cannot learn the build it runs (" + t.Unknown + ")"
}
if t.What == takein.TakeInGroup { if t.What == takein.TakeInGroup {
bound, what = TakeInGroupBound, "a delivery group is gathered and its end has not come" bound, what = TakeInGroupBound, "a delivery group is gathered and its end has not come"
} }
@@ -152,7 +149,7 @@ func (w *Watchman) takeIn(kind, key string, t takein.TakeIn) {
Headline: "A merged change is held back by the indexer", Headline: "A merged change is held back by the indexer",
Explanation: "The indexer holds " + what + " and has not taken it in within " + bound.String() + ", so what the merge changes does " + Explanation: "The indexer holds " + what + " and has not taken it in within " + bound.String() + ", so what the merge changes does " +
"not reach the machines.", "not reach the machines.",
Needs: "the indexer build it waits for to run, or the take-in by hand (indexer_take_in_pending), or a drop (indexer_drop_pending)"} Needs: "the take-in by hand (indexer_take_in_pending), or a drop (indexer_drop_pending)"}
left := bound - w.now().Sub(t.Since) left := bound - w.now().Sub(t.Since)
w.unfind(finding) w.unfind(finding)
w.unsignal(takeInRow, key) w.unsignal(takeInRow, key)