Merge origin/main into the builder reservation takeover work
This commit is contained in:
@@ -93,6 +93,11 @@ type seatTools struct {
|
||||
// seats is the seat list the bundle reads (seatlist.go): until it is read, propose judges no mesh seat the bootstrapped
|
||||
// definitions do not name as held once (ADR 0636 §1). Nil reads nothing.
|
||||
seatList *seatlist.Part
|
||||
// watched is the tools process's watched copy of the state seat-holders (tools.go wants.watched, which gate.Holders
|
||||
// also reads: A-298), read by the verb assignments with seats instead of a bus get per key (issue 1982: under load
|
||||
// that took about 14 s, where the watched copy answers at once). Nil, or a call it cannot answer, falls back to a
|
||||
// direct listing.
|
||||
watched *stdio.WatchedStates
|
||||
}
|
||||
|
||||
func newSeatTools(st seatStore, ask func(string, any) (json.RawMessage, error)) *seatTools {
|
||||
@@ -554,25 +559,43 @@ func (s *seatTools) tools() []stdio.Tool {
|
||||
return out
|
||||
}
|
||||
|
||||
// seats is the verb assignments with seats (ADR 0571 §4): who holds each seat and on which node, as the state seat-holders
|
||||
// holds it; with node, the keys of seats held on that node.
|
||||
func (s *seatTools) seats(node string) (any, error) {
|
||||
// seatHolders is every key of the state seat-holders and its raw value: from the watched copy (s.watched) when
|
||||
// there is one and it answers, else a direct listing, key by key, as the verb answered before (issue 1982).
|
||||
func (s *seatTools) seatHolders() (map[string][]byte, error) {
|
||||
if s.watched != nil {
|
||||
if vals, err := s.watched.Values(mma.StateSeatHolders); err == nil {
|
||||
return vals, nil
|
||||
}
|
||||
}
|
||||
keys, err := s.st.keys(mma.StateSeatHolders)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("seat-holders cannot be listed: %w", err)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
out := []mma.SeatHolders{}
|
||||
out := make(map[string][]byte, len(keys))
|
||||
for _, key := range keys {
|
||||
raw, _, found, err := s.st.get(mma.StateSeatHolders, key)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("seat-holders %s cannot be read: %w", key, err)
|
||||
}
|
||||
if !found {
|
||||
continue
|
||||
if found {
|
||||
out[key] = raw
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// seats is the verb assignments with seats (ADR 0571 §4): who holds each seat and on which node, as the state seat-holders
|
||||
// holds it; with node, the keys of seats held on that node.
|
||||
func (s *seatTools) seats(node string) (any, error) {
|
||||
raws, err := s.seatHolders()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
keys := slices.Sorted(maps.Keys(raws))
|
||||
out := []mma.SeatHolders{}
|
||||
for _, key := range keys {
|
||||
var sh mma.SeatHolders
|
||||
if json.Unmarshal(raw, &sh) != nil {
|
||||
if json.Unmarshal(raws[key], &sh) != nil {
|
||||
continue
|
||||
}
|
||||
if node != "" && sh.Node != node && !slices.ContainsFunc(sh.Holders, func(h mma.SeatHoldersHoldersItem) bool { return h.Node == node }) {
|
||||
|
||||
@@ -352,6 +352,53 @@ func TestTheSeatsVerbs(t *testing.T) {
|
||||
t.Errorf("seats on ace = %+v, %v", out, err)
|
||||
}
|
||||
})
|
||||
t.Run("issue 1982: assignments with seats reads the tools process's watched copy of seat-holders, the store only while it cannot", func(t *testing.T) {
|
||||
seeded := map[string][]byte{}
|
||||
put := func(seat, node string, holders ...mma.SeatHoldersHoldersItem) {
|
||||
raw, err := json.Marshal(mma.SeatHolders{Seat: seat, Node: node, Holders: holders})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
seeded[seat] = raw
|
||||
}
|
||||
put("mesh-broker", "", mma.SeatHoldersHoldersItem{Module: "nats", Node: "ace"})
|
||||
put("node-shell.g14", "g14")
|
||||
st := &countingSeatStore{memStore: memStore{mma.StateSeatHolders: &memKV{values: seeded}}}
|
||||
sv := newSeatTools(st, nil)
|
||||
watches := 0
|
||||
sv.watched = stdio.WatchedStatesOver(func(state string, handle func(stdio.StateChange) error) error {
|
||||
watches++
|
||||
for k, v := range seeded {
|
||||
if err := handle(stdio.StateChange{Key: k, Op: stdio.OpPut, Value: v, Current: true}); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}, func(state, key string) ([]byte, bool, error) {
|
||||
v, ok := seeded[key]
|
||||
return v, ok, nil
|
||||
})
|
||||
out, err := sv.seats("")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := out.(map[string]any)["seats"].([]mma.SeatHolders); len(got) != 2 || watches != 1 || st.keysCalls != 0 || st.getCalls != 0 {
|
||||
t.Errorf("seats from the watched copy = %+v, watched %d time(s), the store's own keys %d get %d (want none)",
|
||||
got, watches, st.keysCalls, st.getCalls)
|
||||
}
|
||||
// The lead's answer on issue 1982: the verb falls back to the store's own listing only while the watched copy
|
||||
// cannot be read (here, its watch refused); the answer's form does not change.
|
||||
sv.watched = stdio.WatchedStatesOver(func(string, func(stdio.StateChange) error) error {
|
||||
return errors.New("refused")
|
||||
}, func(state, key string) ([]byte, bool, error) { return nil, false, nil })
|
||||
out, err = sv.seats("")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := out.(map[string]any)["seats"].([]mma.SeatHolders); len(got) != 2 || st.keysCalls != 1 || st.getCalls != 2 {
|
||||
t.Errorf("seats falling back to the store = %+v, the store's own keys %d get %d (want 1, 2)", got, st.keysCalls, st.getCalls)
|
||||
}
|
||||
})
|
||||
t.Run("research 072 row 37: a node is written as its keys, and history reads the node key", func(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
kv := &revKV{values: map[string][]byte{}, revs: map[string]uint64{}}
|
||||
@@ -750,6 +797,23 @@ func readManifest() (catalogue.Manifest, error) {
|
||||
return catalogue.Decode(raw)
|
||||
}
|
||||
|
||||
// countingSeatStore is a memStore that counts its calls to keys and get, to prove a reader takes the state from the
|
||||
// watched copy and the store only while that copy cannot be read (issue 1982).
|
||||
type countingSeatStore struct {
|
||||
memStore
|
||||
keysCalls, getCalls int
|
||||
}
|
||||
|
||||
func (c *countingSeatStore) keys(state string) ([]string, error) {
|
||||
c.keysCalls++
|
||||
return c.memStore.keys(state)
|
||||
}
|
||||
|
||||
func (c *countingSeatStore) get(state, key string) ([]byte, uint64, bool, error) {
|
||||
c.getCalls++
|
||||
return c.memStore.get(state, key)
|
||||
}
|
||||
|
||||
// countingKV is a memKV that counts its puts.
|
||||
type countingKV struct {
|
||||
memKV
|
||||
|
||||
@@ -419,6 +419,8 @@ func serveTools() error {
|
||||
wants.watched = stdio.NewWatchedStates() // the large states read whole from one watched copy (ADR 0580 clause 3)
|
||||
// The callers' seat entries are resolved from the assigner's own seat-holders, its watched copy (A-298).
|
||||
gate.Holders(seatlist.HoldersIn(func() (map[string][]byte, error) { return wants.watched.Values(mma.StateSeatHolders) }))
|
||||
// The verb assignments with seats answers from the same watched copy, so every caller of it is fast too (issue 1982).
|
||||
seatVerbs.watched = wants.watched
|
||||
wants.revisions = runtimeObjects{busstate.ObjectsOf(assigngraph.RevisionsObjects)}
|
||||
tools = append(tools, wants.tools()...)
|
||||
keeper := &inputKeeper{vals: stateobjects.ForState(buckets.InputsState), now: time.Now, logf: func(string, ...any) {}}
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
@@ -15,59 +13,43 @@ import (
|
||||
|
||||
// catalogRead reads the indexer's view of the trunk (its modules and module graph, from the newest catalog graph
|
||||
// revision) and its manifests state for the seat's verbs.
|
||||
func catalogRead(modules views.Source, manifests stdio.KeptState, nodes views.Source, seatDefinitions, marks views.Source) index.CatalogRead {
|
||||
func catalogRead(modules views.Source, manifest func(digest string) (string, error), nodes views.Source, seatDefinitions, marks views.Source) index.CatalogRead {
|
||||
return index.CatalogRead{
|
||||
SeatDefinitions: seatDefinitions, Marks: marks,
|
||||
Modules: func() (map[string]meshcatalog.ModulesByModuleValue, error) {
|
||||
keys, err := modules.Keys()
|
||||
values, err := index.SourceValues(modules)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := map[string]meshcatalog.ModulesByModuleValue{}
|
||||
for _, k := range keys {
|
||||
e, ok, err := modules.Get(k)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("modules.%s: %w", k, err)
|
||||
}
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if m, err := meshcatalogclient.DecodeModule(e.Value); err == nil {
|
||||
for k, v := range values {
|
||||
if m, err := meshcatalogclient.DecodeModule(v); err == nil {
|
||||
out[k] = m
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
},
|
||||
Manifest: func(digest string) (string, error) {
|
||||
key, err := meshcatalogclient.ManifestsKey(digest)
|
||||
Module: func(name string) (meshcatalog.ModulesByModuleValue, bool, error) {
|
||||
key, err := meshcatalogclient.ModulesKey(name)
|
||||
if err != nil {
|
||||
return "", err
|
||||
return meshcatalog.ModulesByModuleValue{}, false, nil // a name no key holds is a module the trunk holds none of
|
||||
}
|
||||
e, err := manifests.Get(key)
|
||||
if err != nil {
|
||||
return "", err
|
||||
e, ok, err := modules.Get(key)
|
||||
if err != nil || !ok {
|
||||
return meshcatalog.ModulesByModuleValue{}, false, err
|
||||
}
|
||||
if e == nil {
|
||||
return "", fmt.Errorf("the manifests state holds no %s", digest)
|
||||
}
|
||||
kept, err := meshcatalogclient.DecodeManifest(e.Value)
|
||||
return kept.Manifest, err
|
||||
m, err := meshcatalogclient.DecodeModule(e.Value)
|
||||
return m, err == nil, nil
|
||||
},
|
||||
Manifest: manifest,
|
||||
Graph: func() (graph.Graph, error) {
|
||||
keys, err := nodes.Keys()
|
||||
values, err := index.SourceValues(nodes)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
g := graph.Graph{}
|
||||
for _, k := range keys {
|
||||
e, ok, err := nodes.Get(k)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("graph.%s: %w", k, err)
|
||||
}
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if n, err := graph.Decode(e.Value); err == nil {
|
||||
for k, v := range values {
|
||||
if n, err := graph.Decode(v); err == nil {
|
||||
g[k] = n
|
||||
}
|
||||
}
|
||||
@@ -77,7 +59,27 @@ func catalogRead(modules views.Source, manifests stdio.KeptState, nodes views.So
|
||||
}
|
||||
|
||||
// catalogTools are the seat's verbs over the indexer's own states, served beside the indexer's own tools, since the indexer's manifest claims the seat (ADR 0604 §2).
|
||||
func catalogTools(state func(string) views.Source) []stdio.Tool {
|
||||
return index.CatalogTools(catalogRead(state("modules"), stdio.State("manifests"), state(graph.State),
|
||||
keptSource(index.SeatDefinitionsState), keptSource(views.MarksState)))
|
||||
// Every state a verb reads whole is answered from memory (the view of the trunk held, or a watched copy), so a call costs no
|
||||
// round trip per key: four node-engines ask every module at once.
|
||||
func catalogTools(state func(string) views.Source, kept *index.KeptTrunk, watched *stdio.WatchedStates) []stdio.Tool {
|
||||
return index.CatalogTools(catalogRead(state("modules"), kept.Manifest, state(graph.State),
|
||||
watchedSource{watched, index.SeatDefinitionsState}, keptSource(views.MarksState)))
|
||||
}
|
||||
|
||||
// watchedSource is a state as its watched copy holds it (stdio.WatchedStates), as a Source; it offers every value in one read.
|
||||
type watchedSource struct {
|
||||
w *stdio.WatchedStates
|
||||
state string
|
||||
}
|
||||
|
||||
func (s watchedSource) Keys() ([]string, error) { return s.w.Keys(s.state) }
|
||||
|
||||
func (s watchedSource) Get(key string) (*views.Entry, bool, error) {
|
||||
v, ok, err := s.w.Get(s.state, key)
|
||||
if err != nil || !ok {
|
||||
return nil, false, err
|
||||
}
|
||||
return &views.Entry{Value: v}, true, nil
|
||||
}
|
||||
|
||||
func (s watchedSource) Values() (map[string][]byte, error) { return s.w.Values(s.state) }
|
||||
|
||||
@@ -17,9 +17,11 @@ import (
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/busstate"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
"git.novox.be/novox/mesh-sdk/go/views"
|
||||
|
||||
"mesh-delivery-indexer/internal/index"
|
||||
)
|
||||
@@ -286,13 +288,30 @@ func main() {
|
||||
tools := Tools(asks, time.Now, stdio.State(index.VerdictsState), heldCounts(stdio.State(index.OldSectionsState)), acts)
|
||||
// The modules and the module graph are the newest catalog graph revision's (milestone 47/G): revision.go.
|
||||
revisions := bundleObjects{busstate.ObjectsOf(index.CatalogRevisionsObjects)}
|
||||
trunk := index.TrunkStates{Objects: revisions, Under: keptSource, Kept: &index.KeptTrunk{Say: func(format string, a ...any) {
|
||||
// The view of the trunk is read from the indexer's own objects once when the runtime starts and again each time the seat's
|
||||
// event catalog-revision is heard (design 47 G A-487b), through the bundle's view path.
|
||||
say := func(format string, a ...any) {
|
||||
fmt.Fprintf(os.Stderr, "[mesh-delivery-indexer-tools] "+format+"\n", a...)
|
||||
}}}
|
||||
tools = append(tools, catalogTools(trunk.State)...)
|
||||
}
|
||||
kept := &index.KeptTrunk{Say: say}
|
||||
trunk := index.TrunkStates{Objects: revisions, Under: keptSource, Kept: kept}
|
||||
follow := &views.Reader{Module: os.Getenv(stdio.ServedAs), Logf: say, Acts: acts}
|
||||
if err := follow.Adopt(trunkFollower(kept, revisions), meshcatalog.Seat, meshcatalog.EventCatalogRevision, func(ctx context.Context, data []byte) error {
|
||||
var ev meshcatalog.CatalogRevisionEvent
|
||||
if err := json.Unmarshal(data, &ev); err != nil {
|
||||
return nil // not a revision event: offering it again changes nothing
|
||||
}
|
||||
return kept.Heard(ctx, revisions, ev)
|
||||
}); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "[mesh-delivery-indexer-tools] %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
follow.Start(context.Background())
|
||||
watched := stdio.NewWatchedStates()
|
||||
tools = append(tools, catalogTools(trunk.State, kept, watched)...)
|
||||
// The seat mesh-catalog's verb revision, a first view of the catalog half answered in this runtime of its own to exactly
|
||||
// the parts whose entry declares it (novox/hq ADR 0663).
|
||||
tools = append(tools, index.RevisionTool(revisions, os.Getenv(stdio.ServedAs)), index.EntryTool(revisions, keptRead{stdio.State(index.EntryIndexState)}), index.EntriesTool(revisions, keptRead{stdio.State(index.EntryIndexState)}, keptRead{stdio.State(index.NamedState)}), index.VersionsTool(keptRead{stdio.State(index.VersionsState)}),
|
||||
tools = append(tools, index.RevisionTool(revisions, kept, os.Getenv(stdio.ServedAs)), index.EntryTool(revisions, keptRead{stdio.State(index.EntryIndexState)}), index.EntriesTool(revisions, keptRead{stdio.State(index.EntryIndexState)}, keptRead{stdio.State(index.NamedState)}), index.VersionsTool(keptRead{stdio.State(index.VersionsState)}),
|
||||
index.OutdatedTool(&index.Indexer{Versions: keptRead{stdio.State(index.VersionsState)}, Heard: keptRead{stdio.State(index.HeardState)}, Revisions: revisions}),
|
||||
index.InputsReadTool(keptRead{stdio.State(index.InputsState)}, stateobjects.ForState(index.InputsState).Objects))
|
||||
tools = append(tools, index.CatalogRevisionTools(revisions, asks, time.Now, func(what, why string) error {
|
||||
@@ -318,18 +337,26 @@ func main() {
|
||||
})...)
|
||||
tools = append(tools, viewTools(os.Getenv, trunk.State, time.Now)...)
|
||||
tools = append(tools, index.PendingTools(keptRead{stdio.State(index.PendingState)}, keptRead{stdio.State(index.OwnBuildStateName)}, asks, time.Now)...)
|
||||
tools = append(tools, index.ValidationTools(asks, time.Now, func(what, why string) error {
|
||||
_, err := acts.Record(meshhandactsclient.Request{What: what, Why: why, Cause: index.ValidateAgainTool, Verb: "validate-again"})
|
||||
return err
|
||||
})...)
|
||||
tools = append(tools, index.FingerprintsTools(asks, time.Now, func(what, why string) error {
|
||||
_, err := acts.Record(meshhandactsclient.Request{What: what, Why: why, Cause: index.FingerprintsAgainTool, Verb: "fingerprints-again"})
|
||||
return err
|
||||
}, 10*time.Second)...)
|
||||
tools = append(tools, configReadersTool(stdio.NewWatchedStates(), trunk.State("modules")), busStateTool(trunk.State), requirementsTool(keptSource),
|
||||
tools = append(tools, configReadersTool(watched, trunk.State("modules")), busStateTool(trunk.State), requirementsTool(keptSource),
|
||||
packageUsersTool(trunk.State), seatDefinitionTool(keptSource), outOfStepTool(os.Getenv(stdio.ServedAs), keptSource))
|
||||
if err := stdio.Serve("", tools); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "[mesh-delivery-indexer-tools] %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
// trunkFollower is the bundle's view of the trunk as a follower of the reader: its first view is the one read of the
|
||||
// indexer's objects the start makes, and answers no value (the picture is KeptTrunk's, not the reader's).
|
||||
func trunkFollower(kept *index.KeptTrunk, revisions index.Objects) *views.Follower {
|
||||
return &views.Follower{Owner: meshcatalog.Seat, State: "catalog-half", Tool: "catalog-revisions",
|
||||
Ask: func(ctx context.Context, _ string) (views.Answer, error) {
|
||||
if err := kept.Reread(ctx, revisions); err != nil {
|
||||
return views.Answer{}, err
|
||||
}
|
||||
return views.Answer{V: views.Version, Owner: meshcatalog.Seat, State: "catalog-half", Created: time.Unix(1, 0).UTC(), Values: map[string]json.RawMessage{}, At: time.Now()}, nil
|
||||
}}
|
||||
}
|
||||
|
||||
@@ -22,7 +22,6 @@ import (
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo"
|
||||
sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/composer"
|
||||
|
||||
"github.com/nats-io/nats-server/v2/server"
|
||||
"github.com/nats-io/nats.go"
|
||||
@@ -97,11 +96,7 @@ func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10; 1585/T14, T15
|
||||
// The forge bridge's tool bridge_merged_message (issue 1585): answers the merged message published for a merge, by
|
||||
// repository@merge, as the bridge does; a restarted indexer holds none in memory and asks it.
|
||||
var bridgeMessages sync.Map
|
||||
// The build seat node-build-go's stand-in (design 47 G §712, §713): it hears the indexer's validation asks and ends each with
|
||||
// the manifest validator's verdict; refusing names, by the commit the job runs at, the manifest a revision's change refuses.
|
||||
var validations *standIn
|
||||
var refusing sync.Map
|
||||
js, ctx, candidates, _, restart := startIndexerRun(t, func(ctx context.Context, builds buildLog) {
|
||||
js, ctx, candidates, _, _ := startIndexerRun(t, func(ctx context.Context, builds buildLog) {
|
||||
// The builder's log entries of two builds' ends (ADR 0571 clause 1), as the builder appends them.
|
||||
builds.end(t, "novox/mesh-delivery", "a", "modules/a")
|
||||
builds.end(t, "novox/mesh-catalog", "c", "modules/c")
|
||||
@@ -128,9 +123,8 @@ func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10; 1585/T14, T15
|
||||
if _, err := tc.Put(ctx, key, value); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The trunk holds the indexer's entry, taken in at an earlier commit with its own manifest; the validation names no Go module (A-268).
|
||||
// The trunk holds the indexer's entry, taken in at an earlier commit with its own manifest; the take-in asks no validation (A-476 a).
|
||||
seedBuildModules(t, ctx, js, map[string]meshcatalog.CatalogEntry{testIndexer: trunkIndexerEntry(t, ctx, js)})
|
||||
validations = standInBuildSeat(t, ctx, js, &refusing)
|
||||
// The node-engine's composition on n1 (process.Conn.Call's subjects): the build of the indexer that runs there.
|
||||
answer := func(m *nats.Msg) {
|
||||
raw, _ := json.Marshal(map[string]any{"result": map[string]any{"runs": map[string]string{"fp": *running.Load()}}, "node": "n1"})
|
||||
@@ -281,17 +275,13 @@ func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10; 1585/T14, T15
|
||||
}
|
||||
return takein.TakeIn{}, false
|
||||
}
|
||||
held, ok := said(takein.TakeInPending, m1Key)
|
||||
if !ok || held.What != takein.TakeInRevision || held.WaitsFor == "" || held.WaitsFor == first || len(held.Merges) != 1 {
|
||||
t.Fatalf("take-in.pending.%s = %+v (said %v) within 20 s, want the revision waiting for another build than %s", m1Key, held, ok, first)
|
||||
// A-476 a, b: the revision changing the indexer's own tree is taken in at once, with the build this program runs and no
|
||||
// validation ask: it is never said pending, and the build the revision names is not waited for.
|
||||
if taken, ok := said(takein.TakeInTaken, m1Key); !ok || len(taken.Merges) != 1 || taken.Merges[0].Merge != m1.Merge {
|
||||
t.Fatalf("take-in.taken.%s = %+v (said %v) within 20 s, want the revision taken in at once", m1Key, taken, ok)
|
||||
}
|
||||
named := held.WaitsFor
|
||||
// IX5 (§712): the revision was validated first, by one run ask whose job names the indexer entry's repository, path and
|
||||
// the revision's head, and no heads; accepted, it is never asked again while it waits for the named indexer.
|
||||
if asks := validations.asked(); len(asks) != 1 || asks[0].Run == nil || asks[0].Kind != sdkstep.AskCheck ||
|
||||
asks[0].Run.Program.Repository != m1.Repository || asks[0].Run.Program.Dir != "modules/"+ix || asks[0].Run.Program.Commit != m1.Merge ||
|
||||
len(asks[0].Run.Heads) != 0 || asks[0].Run.Program.Package != "./cmd/mesh-judge" {
|
||||
t.Fatalf("validation asks = %+v, want one run of the indexer's manifest validator at %s", asks, m1.Merge)
|
||||
if held := saidAll(takein.TakeInPending, m1Key); len(held) != 0 {
|
||||
t.Errorf("the revision was said pending: %+v", held)
|
||||
}
|
||||
get := func(bucket, key string) ([]byte, bool) {
|
||||
t.Helper()
|
||||
@@ -312,11 +302,10 @@ func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10; 1585/T14, T15
|
||||
}
|
||||
return e.Value(), true
|
||||
}
|
||||
// Only what building and moving the named indexer needs is written.
|
||||
raw, ok := get(index.ModulesState, ix)
|
||||
own, err := meshcatalogclient.DecodeModule(raw)
|
||||
if !ok || err != nil || own.Fp2 != named || own.Commit != m1.Merge {
|
||||
t.Fatalf("modules.%s = %+v (%v), want the named build %s at the merge", ix, own, err, named)
|
||||
if !ok || err != nil || own.Commit != m1.Merge {
|
||||
t.Fatalf("modules.%s = %+v (%v), want the indexer at the merge", ix, own, err)
|
||||
}
|
||||
if _, ok := get(index.Bucket(testIndexer, index.ManifestsState), own.Manifest); !ok {
|
||||
t.Errorf("manifests.%s is not written", own.Manifest)
|
||||
@@ -325,16 +314,11 @@ func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10; 1585/T14, T15
|
||||
t.Errorf("the verdict on the indexer's manifest %s is not written", own.Manifest)
|
||||
}
|
||||
repoKey, _ := trunk.RepositoryKey(m1.Repository)
|
||||
if _, ok := get(index.ModulesState, "a"); ok {
|
||||
t.Errorf("modules.a is written while the revision is pending")
|
||||
if raw, ok := get(index.ModulesState, "a"); !ok || !strings.Contains(string(raw), m1.Merge) {
|
||||
t.Errorf("modules.a = %s, want it at the merge %s", raw, m1.Merge)
|
||||
}
|
||||
if _, ok := get(index.Bucket(testIndexer, index.TakenState), repoKey); ok {
|
||||
t.Errorf("taken.%s is written while the revision is pending", repoKey)
|
||||
}
|
||||
if e, err := candidates.Get(ctx, "feat/x"); err != nil {
|
||||
t.Errorf("the candidate feat/x is gone while the merge of #12 is pending: %v", err)
|
||||
} else if c, err := index.DecodeCandidate(e.Value()); err != nil || len(c.Pulls) != 2 {
|
||||
t.Errorf("the candidate feat/x = %+v while the merge of #12 is pending, want its two pull requests", c)
|
||||
if _, ok := get(index.Bucket(testIndexer, index.TakenState), repoKey); !ok {
|
||||
t.Errorf("taken.%s is not written", repoKey)
|
||||
}
|
||||
|
||||
// A group's first member arrives meanwhile; its end is still to come (the knowledge base's remark: it stays across the restart).
|
||||
@@ -365,34 +349,13 @@ func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10; 1585/T14, T15
|
||||
}
|
||||
var w []takein.PendingWait
|
||||
for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
|
||||
if w = waits(); len(w) == 2 && w[0].WaitsFor == named && w[1].Said != "" {
|
||||
if w = waits(); len(w) == 1 && w[0].Said != "" {
|
||||
break
|
||||
}
|
||||
}
|
||||
if len(w) != 2 || w[0].Key != m1Key || w[0].WaitsFor != named || !slices.Contains(w[0].Holds, "a") || !slices.Contains(w[0].Holds, ix) ||
|
||||
w[1].Key != groupKey || w[1].What != takein.TakeInGroup || len(w[1].Merges) != 1 || w[1].Said != takein.TakeInPendingEvent {
|
||||
t.Fatalf("indexer_pending = %+v, want the revision %s waiting for %s holding a and the indexer, then the group gathered", w, m1Key, named)
|
||||
}
|
||||
|
||||
// The named indexer starts (the node-engine now runs that build): it takes the revision in, and leaves the group.
|
||||
running.Store(&named)
|
||||
restart()
|
||||
if taken, ok := said(takein.TakeInTaken, m1Key); !ok || len(taken.Merges) != 1 || taken.Merges[0].Merge != m1.Merge {
|
||||
t.Fatalf("take-in.taken.%s = %+v (said %v) within 20 s of the start of %s", m1Key, taken, ok, named)
|
||||
}
|
||||
for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
|
||||
if w = waits(); len(w) == 1 {
|
||||
break
|
||||
}
|
||||
}
|
||||
if raw, ok := get(index.ModulesState, "a"); !ok || !strings.Contains(string(raw), m1.Merge) {
|
||||
t.Errorf("modules.a = %s after the take-in at the start, want it at the merge %s", raw, m1.Merge)
|
||||
}
|
||||
if _, ok := get(index.Bucket(testIndexer, index.TakenState), repoKey); !ok {
|
||||
t.Errorf("taken.%s is not written by the take-in at the start", repoKey)
|
||||
}
|
||||
if len(w) != 1 || w[0].Key != groupKey || len(w[0].Keys) != 1 || w[0].Keys[0] != g1Key {
|
||||
t.Fatalf("indexer_pending after the start = %+v, want only the group gathered, its first member %s", w, g1Key)
|
||||
if len(w) != 1 || w[0].Key != groupKey || w[0].What != takein.TakeInGroup || len(w[0].Merges) != 1 || len(w[0].Keys) != 1 || w[0].Keys[0] != g1Key ||
|
||||
w[0].Said != takein.TakeInPendingEvent {
|
||||
t.Fatalf("indexer_pending = %+v, want only the group gathered, its first member %s", w, g1Key)
|
||||
}
|
||||
if _, ok := get(index.ModulesState, "c"); ok {
|
||||
t.Errorf("modules.c is written while its group is gathered")
|
||||
@@ -425,23 +388,6 @@ func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10; 1585/T14, T15
|
||||
t.Errorf("the group's first member %s was said pending as a revision of its own", g1Key)
|
||||
}
|
||||
|
||||
// IX5 (§712, §713): a merge whose changed manifest gains a problem the trunk case lacks stays pending, refused naming it,
|
||||
// said in take-in.pending; nothing of it is taken in.
|
||||
m3 := forgemsg.Forge{Kind: forgemsg.ForgeMerged, Repository: "novox/mesh-delivery", Branch: "feat/y", Number: 13, Head: h1,
|
||||
Merge: "3333333333333333333333333333333333333333", Root: "r0", Trees: map[string]string{"modules/a": "tree-a-3"},
|
||||
Manifests: map[string]string{"modules/a": `{"module":"a","version":"2.0"}`}}
|
||||
refusing.Store(m3.Merge, "a")
|
||||
m3Key := takein.PendingKey(merged(m3))
|
||||
if refused, ok := said(takein.TakeInPending, m3Key); !ok || !strings.Contains(refused.Refused, "a: the stand-in's problem") || refused.Failed != "" {
|
||||
t.Fatalf("take-in.pending.%s = %+v (said %v) within 20 s, want the revision refused naming a's problem", m3Key, refused, ok)
|
||||
}
|
||||
if w = waits(); len(w) != 1 || w[0].Key != m3Key || w[0].Validation == nil || w[0].Validation.Accepted != 0 || len(validations.asked()) != 3 {
|
||||
t.Fatalf("indexer_pending = %+v, asks %d: want the refused revision pending with its validation, asked once after m1 and the group", w, len(validations.asked()))
|
||||
}
|
||||
if raw, ok := get(index.ModulesState, "a"); !ok || strings.Contains(string(raw), m3.Merge) {
|
||||
t.Errorf("modules.a = %s: the refused revision was taken in", raw)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// trunkIndexerEntry keeps the indexer's own manifest by its digest, accepted, and answers its trunk entry at an earlier commit.
|
||||
@@ -475,63 +421,6 @@ func trunkIndexerEntry(t *testing.T, ctx context.Context, js jetstream.JetStream
|
||||
return e
|
||||
}
|
||||
|
||||
// standIn is the build seat node-build-go as the indexer's validation meets it: the asks it heard, oldest first.
|
||||
type standIn struct {
|
||||
mu sync.Mutex
|
||||
asks []sdkstep.StepAsk
|
||||
}
|
||||
|
||||
func (s *standIn) asked() []sdkstep.StepAsk {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return slices.Clone(s.asks)
|
||||
}
|
||||
|
||||
// standInBuildSeat keeps the seat's check queue as a stream and hears the indexer's asks on it (accept.check.indexer): each is
|
||||
// ended at once with step-ended on the seat's family, its answer inline (A-254), the program's last line: the manifest validator's verdict over both cases,
|
||||
// the change case refusing the manifest refusing names for the job's commit.
|
||||
func standInBuildSeat(t *testing.T, ctx context.Context, js jetstream.JetStream, refusing *sync.Map) *standIn {
|
||||
t.Helper()
|
||||
subject, err := sdkstep.AskSubjectBy(sdkstep.AskGo, sdkstep.AskCheck, index.ValidationAsker)
|
||||
if err != nil || subject != "mesh.seat.node-build-go.accept.check.mesh-catalog" {
|
||||
t.Fatalf("the validation's subject %q %v, want the indexer's seat name its asker (A-163, A-232)", subject, err)
|
||||
}
|
||||
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "SEAT_NODE_BUILD_GO_CHECK", Subjects: []string{"mesh.seat.node-build-go.accept.check.>"}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The indexer's state inputs and its objects, where the validation's request is kept for the build seat's holder (§354).
|
||||
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: index.Bucket(testIndexer, index.InputsState), History: 1}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := js.CreateObjectStore(ctx, jetstream.ObjectStoreConfig{Bucket: catalogue.BusName(testIndexer, catalogue.StateObjectsName(index.InputsState))}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
s := &standIn{}
|
||||
if _, err := js.Conn().Subscribe(subject, func(m *nats.Msg) {
|
||||
a, err := sdkstep.DecodeStepAsk(m.Data)
|
||||
if err != nil || a.Run == nil {
|
||||
t.Errorf("the stand-in build seat heard what is no run ask: %v", err)
|
||||
return
|
||||
}
|
||||
s.mu.Lock()
|
||||
s.asks = append(s.asks, a)
|
||||
s.mu.Unlock()
|
||||
change := composer.CaseVerdict{Name: "change", Manifests: map[string]composer.ManifestRead{}}
|
||||
if module, ok := refusing.Load(a.Run.Program.Commit); ok {
|
||||
change.Manifests[module.(string)] = composer.ManifestRead{Refused: "the stand-in's problem"}
|
||||
}
|
||||
verdict, _ := json.Marshal(composer.Verdict{V: 1, Cases: []composer.CaseVerdict{change, {Name: judge.TrunkCase, Manifests: map[string]composer.ManifestRead{}}}})
|
||||
end, _ := json.Marshal(sdkstep.StepEnded{V: 1, ID: a.ID, Ask: a.ID, Asker: index.ValidationAsker, Node: "n1", Outcome: sdkstep.OutcomePassed,
|
||||
Output: string(verdict), At: time.Now().UTC()})
|
||||
if _, err := js.Publish(ctx, nodebuildgo.SubjectStepEnded("n1"), end); err != nil {
|
||||
t.Errorf("the stand-in build seat could not say step-ended: %v", err)
|
||||
}
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// jsSource is a bucket as a views.Source.
|
||||
type jsSource struct {
|
||||
ctx context.Context
|
||||
@@ -1529,7 +1418,8 @@ func TestIndexerCountsOnlyThePlacedBuildNodesToolchains(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
toolchains = kv
|
||||
for node, version := range map[string]string{"n1": "go1.27.3", "n2": "go1.27.2"} {
|
||||
// A-471: a toolchain is agreed by its language version (go1.27), not its patch; n2 reports another language version.
|
||||
for node, version := range map[string]string{"n1": "go1.27", "n2": "go1.28.1"} {
|
||||
key, _ := index.ToolchainKey(node, "go")
|
||||
value, err := index.EncodeToolchain(nodebuildgo.Toolchain{Language: "go", Version: version, Recipe: "1", At: ptr(time.Now())})
|
||||
if err != nil {
|
||||
@@ -1573,15 +1463,15 @@ func TestIndexerCountsOnlyThePlacedBuildNodesToolchains(t *testing.T) {
|
||||
return got
|
||||
}
|
||||
agreed := func(m meshcatalog.ModulesByModuleValue) bool {
|
||||
return m.Fp2 != "" && m.Toolchains["go"].Version == "go1.27.3"
|
||||
return m.Fp2 != "" && m.Toolchains["go"].Version == "go1.27"
|
||||
}
|
||||
held := read(agreed)
|
||||
if !agreed(held) {
|
||||
t.Fatalf("within 20 s avahi has fp2 %q, none2 %q, toolchains %v: want a fingerprint of n1's go1.27.3 alone", held.Fp2, held.None2, held.Toolchains)
|
||||
t.Fatalf("within 20 s avahi has fp2 %q, none2 %q, toolchains %v: want a fingerprint of n1's go1.27 alone", held.Fp2, held.None2, held.Toolchains)
|
||||
}
|
||||
place(ctx, t, "n2", "mesh-build-go")
|
||||
key, _ := index.ToolchainKey("n2", "go")
|
||||
value, err := index.EncodeToolchain(nodebuildgo.Toolchain{Language: "go", Version: "go1.27.2", Recipe: "1", At: ptr(time.Now().Add(time.Second))})
|
||||
value, err := index.EncodeToolchain(nodebuildgo.Toolchain{Language: "go", Version: "go1.28.1", Recipe: "1", At: ptr(time.Now().Add(time.Second))})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -1604,7 +1494,7 @@ func TestIndexerCountsOnlyThePlacedBuildNodesToolchains(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
disagree := func(m meshcatalog.ModulesByModuleValue) bool {
|
||||
return m.Fp2 == "" && strings.HasPrefix(m.None2, "builders disagree: go1.27.2 recipe 1 on n2")
|
||||
return m.Fp2 == "" && strings.HasPrefix(m.None2, "builders disagree: go1.27 recipe 1 on n1; go1.28 recipe 1 on n2")
|
||||
}
|
||||
if m := read(disagree); !disagree(m) {
|
||||
t.Fatalf("within 20 s of a merge after n2's assignments avahi has fp2 %q, none2 %q: want the builders to disagree", m.Fp2, m.None2)
|
||||
|
||||
@@ -479,11 +479,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
// The take-in step (ADR 0633): its state pending and its stream take-in, both this delivery's, bound when they appear;
|
||||
// until then every merge is taken in at once, as before.
|
||||
Pending: awaitedStore{b: process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, index.PendingState), "no merge is held for the indexer its revision names: each is taken in at once", logger.Printf)},
|
||||
SayTakeIn: sayTakeIn(js, cred),
|
||||
// The validation's run ask (design 47 G §713, A-247, A-248), put under the indexer's own asker.
|
||||
ValidationAsks: buildrun.Asker{Name: index.ValidationAsker, Publisher: buildrun.JetStreamAsks{JS: js}},
|
||||
// The build seat's verb output, for a validation's answer kept as an object (A-254).
|
||||
AskVerb: calls.CallFile}
|
||||
SayTakeIn: sayTakeIn(js, cred)}
|
||||
// The registrations and the changes that wait for their warrant (design 47 G §2, §667): their states are bound when they
|
||||
// appear; a warrant is heard on the router's decided event for this module, below.
|
||||
registry := newRegistry(js, conn, cred, calls.Call,
|
||||
|
||||
@@ -206,6 +206,7 @@ func TestAMeshSDKMergeMovesOnlyTheModulesImportingAChangedPackage(t *testing.T)
|
||||
if pub.Commit != m2 || len(pub.Packages) != 3 {
|
||||
t.Fatalf("packages.%s = %+v within 20 s, want the merge %s with a digest of go, announce and buildstep", key, pub, m2)
|
||||
}
|
||||
// A-476a: mesh-sdk's merge moves importer, of another repository, which imports a package it moved.
|
||||
moved := module("importer", func(m meshcatalog.ModulesByModuleValue) bool { return m.Fp2 != importer.Fp2 })
|
||||
if moved.Packages[sdkGo] != m2 {
|
||||
t.Errorf("importer moved to %+v, want it built against %s", moved.Packages, m2)
|
||||
|
||||
@@ -28,7 +28,6 @@ type takerIn interface {
|
||||
NamedDrop(ctx context.Context, fingerprint, member, by, why string) index.TakenIn
|
||||
Unpin(ctx context.Context, revision, record, by, why string) index.TakenIn
|
||||
DropRunByHand(ctx context.Context, run, by, why string) index.TakenIn
|
||||
ValidateAgainByHand(ctx context.Context, key, by, why string) index.TakenIn
|
||||
RegistryAct(ctx context.Context, a index.TakeInAsk) index.TakenIn
|
||||
}
|
||||
|
||||
@@ -75,9 +74,6 @@ func takenIn(ctx context.Context, ix takerIn, kv jetstream.KeyValue, e jetstream
|
||||
} else if a.Merge != "" {
|
||||
asked, r = "hand take-in of a merge", ix.TakeInMerge(ctx, a.MergeRepository, a.Merge, a.Sequence, a.By, a.Why)
|
||||
a.Module = r.Module
|
||||
} else if a.Pending != "" && a.ValidateAgain {
|
||||
asked, r = "validation asked again", ix.ValidateAgainByHand(ctx, a.Pending, a.By, a.Why)
|
||||
a.Module = a.Pending
|
||||
} else if a.Pending != "" && a.Drop {
|
||||
asked, r = "drop of a pending merge", ix.DropPending(ctx, a.Pending, a.By, a.Why, a.Abandon)
|
||||
a.Module = a.Pending
|
||||
|
||||
@@ -7,8 +7,6 @@ import (
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
@@ -155,15 +153,6 @@ func heardStepEnded(ctx context.Context, ix *index.Indexer, m jetstream.Msg, log
|
||||
_ = m.Ack()
|
||||
return
|
||||
}
|
||||
// The end of a validation ask resumes its pending revision (design 47 G §713); its own fault is given again by the bus.
|
||||
var end buildstep.StepEnded
|
||||
if err := json.Unmarshal(m.Data(), &end); err == nil {
|
||||
if err := ix.HearValidationEnded(ctx, end); err != nil {
|
||||
logf("take-in: the end of the validation ask %s is not taken in, so the bus gives it again: %v", end.Ask, err)
|
||||
leftForBus(m, "the end of the validation ask "+end.Ask, err, logf)
|
||||
return
|
||||
}
|
||||
}
|
||||
if err := ix.HearInputEnded(ctx, e); err != nil {
|
||||
logf("inputs: the input of the ask %s could not be deleted, so the bus gives it again: %v", e.Ask, err)
|
||||
leftForBus(m, "the step-ended of the ask "+e.Ask, err, logf)
|
||||
|
||||
@@ -91,7 +91,7 @@ func (ix *Indexer) TakeInBehind(ctx context.Context) []string {
|
||||
}
|
||||
switch {
|
||||
case held[head]:
|
||||
// A head the pending holds (a member waiting for its group's end or its build) is taken in by that path, whole with
|
||||
// A head the pending holds (a member waiting for its group's end) is taken in by that path, whole with
|
||||
// its group: it is not behind (A-463a).
|
||||
ix.clearUntaken(ctx, key, "the head of "+repository+" waits in the indexer's pending")
|
||||
case merge == head:
|
||||
|
||||
@@ -6,7 +6,6 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
|
||||
"git.novox.be/novox/mesh-sdk/go/stateobjects"
|
||||
"reflect"
|
||||
"slices"
|
||||
@@ -234,12 +233,6 @@ type Indexer struct {
|
||||
Pending Store
|
||||
// SayTakeIn publishes a take-in event (take-in.pending, take-in.taken; ADR 0633 §5); nil says none.
|
||||
SayTakeIn func(ctx context.Context, t takein.TakeIn) error
|
||||
// ValidationAsks puts the validation's run ask (design 47 G §713, A-247: buildrun.Asker, named as the indexer's asker,
|
||||
// A-248); nil validates nothing, and a revision is taken in as before.
|
||||
ValidationAsks ValidationAsks
|
||||
// AskVerb asks a seat verb as the runtime calls it: the build seat's verb output, for a run's answer the seat keeps as an
|
||||
// object (design 47 G A-254, buildstep.StepEnded.ReadOutput); nil reads none, and such an answer could not validate.
|
||||
AskVerb buildstep.AskVerb
|
||||
// Runs is the state runs: the runs on request the indexer holds a candidate for (runcandidates.go, A-174); nil keeps none.
|
||||
Runs Store
|
||||
runMu sync.Mutex // one run event at a time (runcandidates.go)
|
||||
|
||||
@@ -25,6 +25,9 @@ import (
|
||||
type CatalogRead struct {
|
||||
// Modules is every value of the modules state, by module.
|
||||
Modules func() (map[string]meshcatalog.ModulesByModuleValue, error)
|
||||
// Module is the one value of the modules state of that module, and whether the state holds it: the verb module reads
|
||||
// the one module it is asked, never every module (design 47 G A-487).
|
||||
Module func(name string) (meshcatalog.ModulesByModuleValue, bool, error)
|
||||
// Manifest is the manifest kept under a digest in the manifests state, as its JSON.
|
||||
Manifest func(digest string) (string, error)
|
||||
// Graph is the module graph, by module.
|
||||
@@ -51,17 +54,26 @@ func (c CatalogRead) CatalogModulesAnswer() (catalogue.CatalogModules, error) {
|
||||
return catalogue.CatalogModules{Modules: out}, nil
|
||||
}
|
||||
|
||||
// module is one module's value: through Module when set, else out of every module's.
|
||||
func (c CatalogRead) module(name string) (meshcatalog.ModulesByModuleValue, bool, error) {
|
||||
if c.Module != nil {
|
||||
return c.Module(name)
|
||||
}
|
||||
mods, err := c.Modules()
|
||||
m, ok := mods[name]
|
||||
return m, ok, err
|
||||
}
|
||||
|
||||
// CatalogModuleAnswer is the verb module: one module whole, its manifest included.
|
||||
func (c CatalogRead) CatalogModuleAnswer(name string) (catalogue.CatalogOne, error) {
|
||||
name = strings.TrimSpace(name)
|
||||
if name == "" {
|
||||
return catalogue.CatalogOne{}, fmt.Errorf("name the module")
|
||||
}
|
||||
mods, err := c.Modules()
|
||||
m, ok, err := c.module(name)
|
||||
if err != nil {
|
||||
return catalogue.CatalogOne{}, fmt.Errorf("the indexer's modules cannot be read: %w", err)
|
||||
}
|
||||
m, ok := mods[name]
|
||||
if !ok {
|
||||
return catalogue.CatalogOne{}, fmt.Errorf("the trunk holds no module %s", name)
|
||||
}
|
||||
|
||||
@@ -45,7 +45,7 @@ func TestACatalogRevisionIsKeptPerWatchedBranch(t *testing.T) {
|
||||
t.Errorf("said %+v, want main 1, then the escaped branch 1 and 2", said)
|
||||
}
|
||||
answer := func(branch string, revision int64) meshcatalog.RevisionAnswer {
|
||||
a, err := Revision(ctx, ix.Revisions, "indexer", "reader", meshcatalog.RevisionInput{Branch: branch, Revision: revision})
|
||||
a, err := Revision(ctx, ix.Revisions, nil, "indexer", "reader", meshcatalog.RevisionInput{Branch: branch, Revision: revision})
|
||||
if err != nil {
|
||||
t.Fatalf("revision of %q: %v", branch, err)
|
||||
}
|
||||
@@ -66,7 +66,7 @@ func TestACatalogRevisionIsKeptPerWatchedBranch(t *testing.T) {
|
||||
// A-205: `source` answers the branch's revision taken in from that source revision; a candidate's group alone answers its
|
||||
// newest revision, whatever its heads.
|
||||
bySource := func(branch string, source int64) (meshcatalog.RevisionAnswer, error) {
|
||||
return Revision(ctx, ix.Revisions, "indexer", "reader", meshcatalog.RevisionInput{Branch: branch, Source: source})
|
||||
return Revision(ctx, ix.Revisions, nil, "indexer", "reader", meshcatalog.RevisionInput{Branch: branch, Source: source})
|
||||
}
|
||||
if a, err := bySource("main", 5); err != nil || a.Revision.Revision != 1 || a.Entries[0].Commit != "c1" {
|
||||
t.Errorf("main's source 5 answered %+v (%v)", a.Revision, err)
|
||||
@@ -89,10 +89,10 @@ func TestACatalogRevisionIsKeptPerWatchedBranch(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if a, err := Revision(ctx, ix.Revisions, "indexer", "reader", meshcatalog.RevisionInput{Group: "run-ab"}); err != nil || a.Revision.Revision != 4 || a.Revision.Heads != "h2" || a.Entries[0].Commit != "c9" {
|
||||
if a, err := Revision(ctx, ix.Revisions, nil, "indexer", "reader", meshcatalog.RevisionInput{Group: "run-ab"}); err != nil || a.Revision.Revision != 4 || a.Revision.Heads != "h2" || a.Entries[0].Commit != "c9" {
|
||||
t.Errorf("a group alone answered %+v (%v), want its newest revision 4", a.Revision, err)
|
||||
}
|
||||
if _, err := Revision(ctx, ix.Revisions, "indexer", "reader", meshcatalog.RevisionInput{Group: "run-none"}); err == nil {
|
||||
if _, err := Revision(ctx, ix.Revisions, nil, "indexer", "reader", meshcatalog.RevisionInput{Group: "run-none"}); err == nil {
|
||||
t.Error("a candidate never kept was answered")
|
||||
}
|
||||
}
|
||||
@@ -123,7 +123,7 @@ func TestTheNodeEngineReadsTheRevisionAsTheModuleHoldingItsSeat(t *testing.T) {
|
||||
if err := ix.KeepBranchEntries(ctx, "main", 5, map[string]meshcatalog.CatalogEntry{"mesh-host": engine}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err := Revision(ctx, ix.Revisions, "indexer", c.caller, meshcatalog.RevisionInput{Branch: "main"})
|
||||
_, err := Revision(ctx, ix.Revisions, nil, "indexer", c.caller, meshcatalog.RevisionInput{Branch: "main"})
|
||||
if (err == nil) != c.want {
|
||||
t.Errorf("caller %q: err %v, want answered %v", c.caller, err, c.want)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,72 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-sdk/go/views"
|
||||
)
|
||||
|
||||
// listingObjects counts the listings of the bucket: each costs the whole bucket (56 ms at 11 000 objects, more through the runtime).
|
||||
type listingObjects struct {
|
||||
*memObjects
|
||||
listings atomic.Int64
|
||||
}
|
||||
|
||||
func (l *listingObjects) Names(ctx context.Context) ([]string, error) {
|
||||
l.listings.Add(1)
|
||||
return l.memObjects.Names(ctx)
|
||||
}
|
||||
|
||||
// A seat verb that answers from the whole view of the trunk (modules, provides, dependents, module) reads the catalog-revisions
|
||||
// bucket's listing once per call, never once per module: the node-engines ask `module` for every module at once, and 33 asks
|
||||
// in 5 minutes passed the tool runner's 30 s (2026-11-17).
|
||||
func TestTrunkViewIsReadOnceForAWholeAnswer(t *testing.T) {
|
||||
objects := &listingObjects{memObjects: newMemObjects(0)}
|
||||
const modules = 40
|
||||
var refs []meshcatalog.EntryRef
|
||||
for m := 0; m < modules; m++ {
|
||||
name := fmt.Sprintf("mod%02d", m)
|
||||
digest := fmt.Sprintf("%064x", m)
|
||||
e := meshcatalog.CatalogEntry{Module: name, Repository: "o/r", Path: "m/" + name, Commit: "c", Fp: "src2:" + strings.Repeat("a", 64),
|
||||
Manifest: map[string]any{"name": name}, ManifestDigest: "sha256:" + digest}
|
||||
b, _ := json.Marshal(e)
|
||||
if err := objects.Put(context.Background(), EntryObject(digest), b); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
refs = append(refs, meshcatalog.EntryRef{Module: name, Digest: digest})
|
||||
}
|
||||
b, _ := json.Marshal(meshcatalog.CatalogRevision{Kind: "trunk", Revision: 1, Entries: refs})
|
||||
if err := objects.Put(context.Background(), "trunk.1", b); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ts := TrunkStates{Objects: objects, Under: func(string) views.Source { return nil }, Kept: &KeptTrunk{}}
|
||||
for _, state := range []string{"modules", "graph"} {
|
||||
objects.listings.Store(0)
|
||||
values, err := SourceValues(ts.State(state))
|
||||
if err != nil || len(values) != modules {
|
||||
t.Fatalf("the %s of the view = %d values (%v), want %d", state, len(values), err, modules)
|
||||
}
|
||||
if n := objects.listings.Load(); n > 2 {
|
||||
t.Errorf("a whole read of the %s listed the bucket %d times, want at most 2 (once per call, not once per module)", state, n)
|
||||
}
|
||||
}
|
||||
if _, err := meshcatalogclient.DecodeModule(mustValue(t, ts)); err != nil {
|
||||
t.Errorf("a value of the view does not decode: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func mustValue(t *testing.T, ts TrunkStates) []byte {
|
||||
t.Helper()
|
||||
v, err := SourceValues(ts.State("modules"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return v["mod00"]
|
||||
}
|
||||
@@ -182,7 +182,7 @@ func newestOf(ctx context.Context, o Objects, branch string) (meshcatalog.Catalo
|
||||
name := RevisionObject(meshcatalog.CatalogRevision{Branch: branch, Revision: newest})
|
||||
data, err := o.Get(ctx, name)
|
||||
if err != nil {
|
||||
return meshcatalog.CatalogRevision{}, false, err
|
||||
return meshcatalog.CatalogRevision{Branch: branch, Revision: newest}, false, err // the number that failed, for readNewestOf
|
||||
}
|
||||
var rev meshcatalog.CatalogRevision
|
||||
if err := json.Unmarshal(data, &rev); err != nil {
|
||||
@@ -191,6 +191,31 @@ func newestOf(ctx context.Context, o Objects, branch string) (meshcatalog.Catalo
|
||||
return rev, true, nil
|
||||
}
|
||||
|
||||
// readNewestOf is the newest revision of a branch and its entries whole (design 47 G A-487d). The pin judge removes a revision
|
||||
// as soon as a newer one is kept, so a revision whose list or entry is removed under the read is read again as the newest,
|
||||
// while the newest is newer than the one that failed; the same revision failing twice is the read's failure. Progress bounds
|
||||
// the loop. No revision kept is false.
|
||||
func readNewestOf(ctx context.Context, o Objects, branch string) (meshcatalog.CatalogRevision, []meshcatalog.CatalogEntry, bool, error) {
|
||||
var failed int64
|
||||
for {
|
||||
newest, any, err := newestOf(ctx, o, branch)
|
||||
if err == nil && !any {
|
||||
return newest, nil, false, nil
|
||||
}
|
||||
var entries []meshcatalog.CatalogEntry
|
||||
if err == nil {
|
||||
_, entries, err = ReadRevision(ctx, o, RevisionObject(newest))
|
||||
if err == nil {
|
||||
return newest, entries, true, nil
|
||||
}
|
||||
}
|
||||
if !errors.Is(err, ErrNoObject) || newest.Revision == 0 || newest.Revision <= failed {
|
||||
return newest, nil, false, err
|
||||
}
|
||||
failed = newest.Revision
|
||||
}
|
||||
}
|
||||
|
||||
// changedEntries are the modules whose entry differs between two revisions' lists, one gone or added included, by name.
|
||||
func changedEntries(before, after []meshcatalog.EntryRef) []string {
|
||||
was := map[string]string{}
|
||||
@@ -438,15 +463,17 @@ func TrunkView(ctx context.Context, o Objects) (modules, nodes Store, at int64,
|
||||
// BranchView is TrunkView of a watched branch: the view read from the newest revision the bucket keeps of that branch
|
||||
// (design 47 G §291, A-93), empty and at 0 when it keeps none.
|
||||
func BranchView(ctx context.Context, o Objects, branch string) (modules, nodes Store, at int64, err error) {
|
||||
view, held := newOverlay(nil), newOverlay(nil)
|
||||
newest, any, err := newestOf(ctx, o, branch)
|
||||
newest, entries, any, err := readNewestOf(ctx, o, branch)
|
||||
if err != nil || !any {
|
||||
return view, held, 0, err
|
||||
}
|
||||
_, entries, err := ReadRevision(ctx, o, RevisionObject(newest))
|
||||
if err != nil {
|
||||
return nil, nil, 0, err
|
||||
return newOverlay(nil), newOverlay(nil), 0, err
|
||||
}
|
||||
return viewOfEntries(newest.Revision, entries)
|
||||
}
|
||||
|
||||
// viewOfEntries is the view of the entries of one trunk revision, as BranchView reads them.
|
||||
func viewOfEntries(revision int64, entries []meshcatalog.CatalogEntry) (modules, nodes Store, at int64, err error) {
|
||||
view, held := newOverlay(nil), newOverlay(nil)
|
||||
ctx, newest := context.Background(), meshcatalog.CatalogRevision{Revision: revision}
|
||||
for _, e := range entries {
|
||||
key, err := meshcatalogclient.ModulesKey(e.Module)
|
||||
if err != nil {
|
||||
@@ -496,14 +523,106 @@ type TrunkStates struct {
|
||||
Kept *KeptTrunk
|
||||
}
|
||||
|
||||
// KeptTrunk is the view of the trunk as last read, at its revision, shared by every read of a TrunkStates; Say, when set,
|
||||
// says each read of a revision and how long it took.
|
||||
// KeptTrunk is the view of the newest trunk revision, held in memory (design 47 G A-487, A-487b): it reads the newest
|
||||
// revision from the indexer's own objects catalog-revisions once when the runtime starts (Reread) and again each time it
|
||||
// hears the seat's event catalog-revision of a newer trunk revision (Heard); a read answers from the view held and never
|
||||
// lists the bucket. Say, when set, says each read of a revision with its number, its module count and how long it took.
|
||||
type KeptTrunk struct {
|
||||
Say func(format string, a ...any)
|
||||
mu sync.Mutex
|
||||
at int64
|
||||
mods Store
|
||||
held Store
|
||||
Say func(format string, a ...any)
|
||||
|
||||
read sync.Mutex // one re-read at a time; a read of the view never holds it
|
||||
mu sync.Mutex
|
||||
started bool
|
||||
at int64
|
||||
mods Store
|
||||
held Store
|
||||
rev meshcatalog.CatalogRevision
|
||||
entries []meshcatalog.CatalogEntry
|
||||
// manifests are the manifests the revision's entries carry whole, as JSON, by their digest as a module's value names it
|
||||
// (A-487c: the verbs module and provides answer them from here, never from the manifests state).
|
||||
manifests map[string]string
|
||||
}
|
||||
|
||||
// Reread reads the newest trunk revision of the bucket and holds its view; a bucket keeping none holds an empty view at 0.
|
||||
func (k *KeptTrunk) Reread(ctx context.Context, o Objects) error {
|
||||
k.read.Lock()
|
||||
defer k.read.Unlock()
|
||||
return k.reread(ctx, o)
|
||||
}
|
||||
|
||||
func (k *KeptTrunk) reread(ctx context.Context, o Objects) error {
|
||||
began := time.Now()
|
||||
fail := func(err error) error {
|
||||
if k.Say != nil {
|
||||
k.Say("the view of the trunk is BEHIND, its re-read failed after %s (the view held stays, the next heard catalog-revision reads again): %v", time.Since(began).Round(time.Millisecond), err)
|
||||
}
|
||||
return err
|
||||
}
|
||||
newest, entries, any, err := readNewestOf(ctx, o, TrunkBranch)
|
||||
if err != nil {
|
||||
return fail(err)
|
||||
}
|
||||
mods, held, at := Store(newOverlay(nil)), Store(newOverlay(nil)), int64(0)
|
||||
if any {
|
||||
if mods, held, at, err = viewOfEntries(newest.Revision, entries); err != nil {
|
||||
return fail(err)
|
||||
}
|
||||
}
|
||||
manifests := map[string]string{}
|
||||
for _, e := range entries {
|
||||
if d := meshcatalog.BareManifestDigest(e.ManifestDigest); d != "" && e.Manifest != nil {
|
||||
if raw, err := json.Marshal(e.Manifest); err == nil {
|
||||
manifests[d] = string(raw)
|
||||
}
|
||||
}
|
||||
}
|
||||
k.mu.Lock()
|
||||
was := k.at
|
||||
k.mods, k.held, k.at, k.started, k.rev, k.entries, k.manifests = mods, held, at, true, newest, entries, manifests
|
||||
k.mu.Unlock()
|
||||
if k.Say != nil {
|
||||
n, _ := mods.Keys(ctx)
|
||||
k.Say("the view of the trunk: read revision %d (was %d), %d modules, in %s", at, was, len(n), time.Since(began).Round(time.Millisecond))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Newest is the newest trunk revision held, with a copy of its entries; false when the bucket kept none at the last read.
|
||||
func (k *KeptTrunk) Newest(ctx context.Context, o Objects) (meshcatalog.CatalogRevision, []meshcatalog.CatalogEntry, bool, error) {
|
||||
if _, _, _, err := k.current(ctx, o); err != nil {
|
||||
return meshcatalog.CatalogRevision{}, nil, false, err
|
||||
}
|
||||
k.mu.Lock()
|
||||
defer k.mu.Unlock()
|
||||
return k.rev, slices.Clone(k.entries), k.at != 0, nil
|
||||
}
|
||||
|
||||
// Manifest is the manifest of that digest (as a module's value names it) the held revision's entries carry.
|
||||
func (k *KeptTrunk) Manifest(digest string) (string, error) {
|
||||
k.mu.Lock()
|
||||
defer k.mu.Unlock()
|
||||
raw, ok := k.manifests[digest]
|
||||
if !ok {
|
||||
return "", fmt.Errorf("the newest trunk revision's entries carry no manifest %s", digest)
|
||||
}
|
||||
return raw, nil
|
||||
}
|
||||
|
||||
// Heard is the seat's event catalog-revision: one of a trunk revision newer than the one held wakes one re-read of the
|
||||
// objects; any other event is no change. A failed re-read is an error, so the event is offered again.
|
||||
func (k *KeptTrunk) Heard(ctx context.Context, o Objects, e meshcatalog.CatalogRevisionEvent) error {
|
||||
if e.Kind != "trunk" || !isTrunkBranch(e.Branch) {
|
||||
return nil
|
||||
}
|
||||
k.read.Lock()
|
||||
defer k.read.Unlock()
|
||||
k.mu.Lock()
|
||||
held := k.started && e.Revision <= k.at
|
||||
k.mu.Unlock()
|
||||
if held {
|
||||
return nil
|
||||
}
|
||||
return k.reread(ctx, o)
|
||||
}
|
||||
|
||||
// read is the view's modules and nodes, and the revision they were read at; none kept yet is ErrNoRevision.
|
||||
@@ -518,36 +637,37 @@ func (t TrunkStates) read(nodes bool) (Store, int64, error) {
|
||||
return mods, at, err
|
||||
}
|
||||
|
||||
// view is the view of the newest trunk revision: the one kept when it is still the newest, else read whole and kept.
|
||||
// view is the view of the trunk: the one held while the follower's half is still at its revision, else made from the
|
||||
// half and held.
|
||||
func (t TrunkStates) view() (modules, nodes Store, at int64, err error) {
|
||||
ctx := context.Background()
|
||||
if t.Kept == nil {
|
||||
return TrunkView(ctx, t.Objects)
|
||||
return TrunkView(context.Background(), t.Objects)
|
||||
}
|
||||
k := t.Kept
|
||||
newest, any, err := newestOf(ctx, t.Objects, TrunkBranch)
|
||||
if err != nil || !any {
|
||||
return newOverlay(nil), newOverlay(nil), 0, err
|
||||
return t.Kept.current(context.Background(), t.Objects)
|
||||
}
|
||||
|
||||
// current is the view held; before the start's read it reads the newest revision once.
|
||||
func (k *KeptTrunk) current(ctx context.Context, o Objects) (Store, Store, int64, error) {
|
||||
k.mu.Lock()
|
||||
started := k.started
|
||||
k.mu.Unlock()
|
||||
if !started {
|
||||
k.read.Lock()
|
||||
k.mu.Lock()
|
||||
started = k.started
|
||||
k.mu.Unlock()
|
||||
var err error
|
||||
if !started {
|
||||
err = k.reread(ctx, o)
|
||||
}
|
||||
k.read.Unlock()
|
||||
if err != nil {
|
||||
return nil, nil, 0, err
|
||||
}
|
||||
}
|
||||
k.mu.Lock()
|
||||
defer k.mu.Unlock()
|
||||
if k.mods != nil && k.at == newest.Revision {
|
||||
return k.mods, k.held, k.at, nil
|
||||
}
|
||||
started := time.Now()
|
||||
mods, held, at, err := TrunkView(ctx, t.Objects)
|
||||
if err != nil {
|
||||
if k.Say != nil {
|
||||
k.Say("the view of the trunk: revision %d cannot be read after %s: %v", newest.Revision, time.Since(started).Round(time.Millisecond), err)
|
||||
}
|
||||
return nil, nil, 0, err
|
||||
}
|
||||
if k.Say != nil {
|
||||
n, _ := mods.Keys(ctx)
|
||||
k.Say("the view of the trunk: read revision %d (was %d), %d modules, in %s", at, k.at, len(n), time.Since(started).Round(time.Millisecond))
|
||||
}
|
||||
k.mods, k.held, k.at = mods, held, at
|
||||
return mods, held, at, nil
|
||||
return k.mods, k.held, k.at, nil
|
||||
}
|
||||
|
||||
// state is the source of a state by its name: modules and graph from the revision, their marks at its number, every
|
||||
@@ -588,6 +708,57 @@ func (r revisionSource) Get(key string) (*views.Entry, bool, error) {
|
||||
return &views.Entry{Value: value, Revision: uint64(at)}, true, nil
|
||||
}
|
||||
|
||||
// Values is every value of the modules (or the module graph) of the newest trunk revision from one read of it: a seat verb
|
||||
// that answers from the whole view asks which revision is newest once, never once per key (each ask lists the whole
|
||||
// catalog-revisions bucket: 6 s for the 110 modules of a bucket of 11 000 objects, and 35 s with 12 callers at once).
|
||||
func (r revisionSource) Values() (map[string][]byte, error) {
|
||||
s, _, err := r.t.read(r.nodes)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ctx := context.Background()
|
||||
keys, err := s.Keys(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make(map[string][]byte, len(keys))
|
||||
for _, k := range keys {
|
||||
value, _, found, err := s.Get(ctx, k)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if found {
|
||||
out[k] = value
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// SourceValues is every key's value of a source: in one read where the source offers it (the view of the trunk, a
|
||||
// watched copy of a state), else a get per key.
|
||||
func SourceValues(src views.Source) (map[string][]byte, error) {
|
||||
if all, ok := src.(interface {
|
||||
Values() (map[string][]byte, error)
|
||||
}); ok {
|
||||
return all.Values()
|
||||
}
|
||||
keys, err := src.Keys()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make(map[string][]byte, len(keys))
|
||||
for _, k := range keys {
|
||||
e, ok, err := src.Get(k)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s: %w", k, err)
|
||||
}
|
||||
if ok && e != nil {
|
||||
out[k] = e.Value
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Page is the values in key order after a key from one read of the revision, as many as fit maxBytes and at least one
|
||||
// (views.Pager): a first view's page reads the revision once, never once per key.
|
||||
func (r revisionSource) Page(after string, maxBytes int, _ time.Duration) ([]views.KeyedEntry, string, error) {
|
||||
|
||||
@@ -60,7 +60,7 @@ func (ix *Indexer) FingerprintsAgain(ctx context.Context, module string) TakenIn
|
||||
ix.freshMu.Unlock()
|
||||
// The path of a merge's recompute: the modules, then the candidates their fingerprints reach (recomputeWith), then the
|
||||
// pending merges a changed fingerprint settles (PendingAgain, as after a toolchain report).
|
||||
err = ix.recomputeWith(ctx, nil, nil)
|
||||
err = ix.recomputeWith(ctx, nil, nil, "")
|
||||
ix.freshMu.Lock()
|
||||
ix.fresh = nil
|
||||
ix.freshMu.Unlock()
|
||||
|
||||
@@ -0,0 +1,325 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-sdk/go/views"
|
||||
)
|
||||
|
||||
// listedObjects counts the listings of its bucket.
|
||||
type listedObjects struct {
|
||||
*memObjects
|
||||
n atomic.Int64
|
||||
fail atomic.Bool
|
||||
}
|
||||
|
||||
func (l *listedObjects) Names(ctx context.Context) ([]string, error) {
|
||||
l.n.Add(1)
|
||||
if l.fail.Load() {
|
||||
return nil, errors.New("the bucket does not answer")
|
||||
}
|
||||
return l.memObjects.Names(ctx)
|
||||
}
|
||||
|
||||
func keptDigest(name string) string {
|
||||
return "sha256:" + fmt.Sprintf("%064x", len(name))
|
||||
}
|
||||
|
||||
// A-487, A-487b, A-487c: the view of the trunk is read from the bucket once at the start and once for each heard event of a newer
|
||||
// trunk revision; 100 reads between events list the bucket zero times.
|
||||
func TestTheKeptTrunkIsReadAtTheStartAndOncePerHeardRevision(t *testing.T) {
|
||||
objects := &listedObjects{memObjects: newMemObjects(0)}
|
||||
keep := func(revision int64, modules ...string) {
|
||||
t.Helper()
|
||||
rev, entries := meshcatalog.CatalogRevision{Revision: revision, Kind: "trunk"}, map[string][]byte{}
|
||||
for _, name := range modules {
|
||||
data, _ := json.Marshal(meshcatalog.CatalogEntry{Module: name, Repository: catalogueRepo, None: "not built", ManifestDigest: keptDigest(name), Manifest: map[string]any{"module": name}})
|
||||
digest := fmt.Sprintf("%d-%s", revision, name)
|
||||
entries[digest] = data
|
||||
rev.Entries = append(rev.Entries, meshcatalog.EntryRef{Module: name, Digest: digest})
|
||||
}
|
||||
if err := KeepRevision(context.Background(), objects, rev, entries); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
said := []string{}
|
||||
kept := &KeptTrunk{Say: func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) }}
|
||||
modules := TrunkStates{Objects: objects, Under: func(string) views.Source { return nil }, Kept: kept}.State("modules")
|
||||
keep(1, "avahi")
|
||||
if err := kept.Reread(context.Background(), objects); err != nil { // the start's read
|
||||
t.Fatal(err)
|
||||
}
|
||||
listed := objects.n.Load()
|
||||
for i := 0; i < 50; i++ {
|
||||
if _, _, err := modules.Get("avahi"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := modules.Keys(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if got := objects.n.Load() - listed; got != 0 {
|
||||
t.Errorf("100 reads listed the bucket %d times, want none", got)
|
||||
}
|
||||
// a newer revision kept is no reason to read: only the event says it
|
||||
keep(2, "avahi", "ca-trust")
|
||||
listed = objects.n.Load()
|
||||
if keys, _ := modules.Keys(); len(keys) != 1 {
|
||||
t.Errorf("the view holds %v before the event, want revision 1's one module", keys)
|
||||
}
|
||||
for _, e := range []meshcatalog.CatalogRevisionEvent{{Kind: "candidate", Revision: 9}, {Kind: "trunk", Branch: "other", Revision: 9}, {Kind: "trunk", Revision: 1}} {
|
||||
if err := kept.Heard(context.Background(), objects, e); err != nil {
|
||||
t.Errorf("%+v: %v", e, err)
|
||||
}
|
||||
}
|
||||
if got := objects.n.Load() - listed; got != 0 {
|
||||
t.Errorf("events of a candidate, another branch and the revision held listed the bucket %d times, want none", got)
|
||||
}
|
||||
if err := kept.Heard(context.Background(), objects, meshcatalog.CatalogRevisionEvent{Kind: "trunk", Branch: "main", Revision: 2}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := objects.n.Load() - listed; got != 1 {
|
||||
t.Errorf("a heard catalog-revision listed the bucket %d times, want exactly one re-read", got)
|
||||
}
|
||||
if e, ok, err := modules.Get("ca-trust"); err != nil || !ok || e.Revision != 2 {
|
||||
t.Errorf("after the event ca-trust = %+v %v (%v), want it at revision 2", e, ok, err)
|
||||
}
|
||||
// A-487c: the manifests by digest and the verb revision asked of the trunk's newest answer from the view held.
|
||||
listed = objects.n.Load()
|
||||
if raw, err := kept.Manifest(meshcatalog.BareManifestDigest(keptDigest("ca-trust"))); err != nil || raw != `{"module":"ca-trust"}` {
|
||||
t.Errorf("the held manifest = %q (%v), want the entry's manifest", raw, err)
|
||||
}
|
||||
if a, err := Revision(context.Background(), objects, kept, "avahi", "avahi", meshcatalog.RevisionInput{}); err != nil || a.Revision.Revision != 2 || len(a.Entries) != 2 {
|
||||
t.Errorf("revision of the newest answered %+v (%v), want revision 2 whole", a, err)
|
||||
}
|
||||
if got := objects.n.Load() - listed; got != 0 {
|
||||
t.Errorf("the manifest and the verb revision listed the bucket %d times, want none", got)
|
||||
}
|
||||
if len(said) != 2 || !strings.Contains(said[1], "read revision 2 (was 1), 2 modules, in ") {
|
||||
t.Errorf("said %q, want each read with its revision, module count and time", said)
|
||||
}
|
||||
// A-487b: a re-read that fails is said loudly, the view held stays, and the next heard event reads again.
|
||||
keep(3, "avahi", "ca-trust", "dbus")
|
||||
objects.fail.Store(true)
|
||||
if err := kept.Heard(context.Background(), objects, meshcatalog.CatalogRevisionEvent{Kind: "trunk", Revision: 3}); err == nil {
|
||||
t.Error("a re-read that failed was taken")
|
||||
}
|
||||
if last := said[len(said)-1]; !strings.Contains(last, "BEHIND") {
|
||||
t.Errorf("a failed re-read said %q, want it said loudly", last)
|
||||
}
|
||||
if keys, _ := modules.Keys(); len(keys) != 2 {
|
||||
t.Errorf("the view held after a failed re-read has %v, want revision 2's two modules", keys)
|
||||
}
|
||||
objects.fail.Store(false)
|
||||
if err := kept.Heard(context.Background(), objects, meshcatalog.CatalogRevisionEvent{Kind: "trunk", Revision: 3}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if keys, _ := modules.Keys(); len(keys) != 3 {
|
||||
t.Errorf("the next heard event left the view at %v, want revision 3's three modules", keys)
|
||||
}
|
||||
}
|
||||
|
||||
// A-487: the verb module reads the one module it is asked.
|
||||
func TestTheVerbModuleReadsOneModule(t *testing.T) {
|
||||
c := CatalogRead{
|
||||
Modules: func() (map[string]meshcatalog.ModulesByModuleValue, error) {
|
||||
return nil, errors.New("every module was read")
|
||||
},
|
||||
Module: func(name string) (meshcatalog.ModulesByModuleValue, bool, error) {
|
||||
return meshcatalog.ModulesByModuleValue{Manifest: "d1"}, name == "avahi", nil
|
||||
},
|
||||
Manifest: func(string) (string, error) { return `{"module":"avahi"}`, nil },
|
||||
}
|
||||
if one, err := c.CatalogModuleAnswer("avahi"); err != nil || one.Module.Module != "avahi" {
|
||||
t.Errorf("module avahi = %+v (%v)", one, err)
|
||||
}
|
||||
if _, err := c.CatalogModuleAnswer("nope"); err == nil || !strings.Contains(err.Error(), "holds no module nope") {
|
||||
t.Errorf("module nope = %v, want the trunk holds none", err)
|
||||
}
|
||||
}
|
||||
|
||||
// goneUnderRead is a bucket whose pin judge runs while the first entry of the revision named newest is read: a newer revision
|
||||
// is kept and the one named is removed, its list and its entries, as the indexer's judge does.
|
||||
type goneUnderRead struct {
|
||||
*memObjects
|
||||
once atomic.Bool
|
||||
prune func()
|
||||
on string // when set, only the read of this entry runs the pruning
|
||||
}
|
||||
|
||||
func (g *goneUnderRead) Get(ctx context.Context, name string) ([]byte, error) {
|
||||
if strings.HasPrefix(name, "entry.") && (g.on == "" || name == g.on) && g.once.CompareAndSwap(false, true) {
|
||||
g.prune()
|
||||
}
|
||||
return g.memObjects.Get(ctx, name)
|
||||
}
|
||||
|
||||
// A revision removed while it is read (take-ins keep one every few seconds, each removing the last) is read again as the newest
|
||||
// one, never failed: on novox the first view of the catalog half never came while revisions 487 to 493 were kept.
|
||||
func TestTheKeptTrunkReadsTheNewerRevisionWhenTheOneNamedIsRemovedUnderIt(t *testing.T) {
|
||||
mem := newMemObjects(0)
|
||||
keep := func(revision int64) {
|
||||
data, _ := json.Marshal(meshcatalog.CatalogEntry{Module: "avahi", Repository: catalogueRepo, None: "not built"})
|
||||
digest := fmt.Sprintf("d%d", revision)
|
||||
rev := meshcatalog.CatalogRevision{Revision: revision, Kind: "trunk", Entries: []meshcatalog.EntryRef{{Module: "avahi", Digest: digest}}}
|
||||
if err := KeepRevision(context.Background(), mem, rev, map[string][]byte{digest: data}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
keep(1)
|
||||
objects := &goneUnderRead{memObjects: mem}
|
||||
objects.prune = func() {
|
||||
keep(2)
|
||||
_ = mem.Delete(context.Background(), RevisionObject(meshcatalog.CatalogRevision{Revision: 1, Kind: "trunk"}))
|
||||
_ = mem.Delete(context.Background(), EntryObject("d1"))
|
||||
}
|
||||
kept := &KeptTrunk{}
|
||||
if err := kept.Reread(context.Background(), objects); err != nil {
|
||||
t.Fatalf("the read failed when the revision named was removed under it: %v", err)
|
||||
}
|
||||
if rev, _, ok, err := kept.Newest(context.Background(), objects); err != nil || !ok || rev.Revision != 2 {
|
||||
t.Errorf("the view holds revision %d (%v, %v), want 2", rev.Revision, ok, err)
|
||||
}
|
||||
}
|
||||
|
||||
// A-487d: the same revision failing twice is the read's failure, said as it is; a read by number never reads another.
|
||||
func TestTheKeptTrunkFailsWhenTheSameRevisionIsGoneTwice(t *testing.T) {
|
||||
mem := newMemObjects(0)
|
||||
data, _ := json.Marshal(meshcatalog.CatalogEntry{Module: "avahi", Repository: catalogueRepo, None: "not built"})
|
||||
rev := meshcatalog.CatalogRevision{Revision: 1, Kind: "trunk", Entries: []meshcatalog.EntryRef{{Module: "avahi", Digest: "d1"}}}
|
||||
if err := KeepRevision(context.Background(), mem, rev, map[string][]byte{"d1": data}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_ = mem.Delete(context.Background(), EntryObject("d1")) // revision 1 stays listed, its entry lost, no newer revision
|
||||
for name, read := range map[string]func() error{
|
||||
"kept trunk": func() error { return (&KeptTrunk{}).Reread(context.Background(), mem) },
|
||||
"TrunkView": func() error { _, _, _, err := TrunkView(context.Background(), mem); return err },
|
||||
"Outdated": func() error {
|
||||
_, err := outdatedOfAvahi(mem).Outdated(context.Background(), meshcatalog.OutdatedInput{})
|
||||
return err
|
||||
},
|
||||
} {
|
||||
if err := read(); !errors.Is(err, ErrNoObject) {
|
||||
t.Errorf("%s: %v, want the lost entry said (no such object)", name, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A-487d: BranchView and TrunkView name the newest again when the one named is removed under the read.
|
||||
func TestTrunkViewReadsTheNewerRevisionWhenTheOneNamedIsRemovedUnderIt(t *testing.T) {
|
||||
mem := newMemObjects(0)
|
||||
keep := func(revision int64) {
|
||||
data, _ := json.Marshal(meshcatalog.CatalogEntry{Module: "avahi", Repository: catalogueRepo, None: "not built"})
|
||||
digest := fmt.Sprintf("d%d", revision)
|
||||
rev := meshcatalog.CatalogRevision{Revision: revision, Kind: "trunk", Entries: []meshcatalog.EntryRef{{Module: "avahi", Digest: digest}}}
|
||||
if err := KeepRevision(context.Background(), mem, rev, map[string][]byte{digest: data}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
keep(1)
|
||||
objects := &goneUnderRead{memObjects: mem}
|
||||
objects.prune = func() {
|
||||
keep(2)
|
||||
_ = mem.Delete(context.Background(), RevisionObject(meshcatalog.CatalogRevision{Revision: 1, Kind: "trunk"}))
|
||||
_ = mem.Delete(context.Background(), EntryObject("d1"))
|
||||
}
|
||||
if _, _, at, err := TrunkView(context.Background(), objects); err != nil || at != 2 {
|
||||
t.Errorf("TrunkView read revision %d (%v), want 2", at, err)
|
||||
}
|
||||
}
|
||||
|
||||
// A-487d: every reader of the newest revision in the indexer program reads the newest again when the revision it named is
|
||||
// removed under it (ModuleInNewestView, SeedEntryHeld, entryOf, SeedModulesFrom, FillVersions, the tool indexer_catalog_revision
|
||||
// and the verb revision, trunk's and a watched branch's).
|
||||
func TestTheNewestRevisionReadersReadAgainWhenTheRevisionIsRemovedUnderThem(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cases := map[string]struct {
|
||||
branch string
|
||||
read func(ix *Indexer) (bool, error)
|
||||
}{
|
||||
"ModuleInNewestView": {"", func(ix *Indexer) (bool, error) { return ix.ModuleInNewestView(ctx, "avahi") }},
|
||||
"SeedEntryHeld": {"", func(ix *Indexer) (bool, error) { return ix.SeedEntryHeld(ctx, "avahi", "fp2") }},
|
||||
"entryOf": {"", func(ix *Indexer) (bool, error) {
|
||||
e, err := ix.entryOf(ctx, "avahi")
|
||||
return e.Fp == "fp2", err
|
||||
}},
|
||||
"SeedModulesFrom": {"", func(ix *Indexer) (bool, error) { _, err := ix.SeedModulesFrom(ctx, nil, nil); return err == nil, err }},
|
||||
"FillVersions": {"", func(ix *Indexer) (bool, error) {
|
||||
ix.Versions, ix.Heard = MemoryStore(), MemoryStore()
|
||||
err := ix.FillVersions(ctx, func(context.Context, string, string) (BuilderVersionsAnswer, error) {
|
||||
return BuilderVersionsAnswer{}, nil
|
||||
})
|
||||
return err == nil, err
|
||||
}},
|
||||
"indexer_catalog_revision": {"", func(ix *Indexer) (bool, error) {
|
||||
var tool map[string]any
|
||||
for _, tl := range CatalogRevisionTools(ix.Revisions, nil, time.Now, nil) {
|
||||
if tl.Name == CatalogRevisionTool {
|
||||
got, err := tl.Run(map[string]any{})
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
tool = got.(map[string]any)
|
||||
}
|
||||
}
|
||||
return tool != nil && tool["name"] == "trunk.2", nil
|
||||
}},
|
||||
"Outdated": {"", func(ix *Indexer) (bool, error) {
|
||||
a, err := outdatedOfAvahi(ix.Revisions).Outdated(ctx, meshcatalog.OutdatedInput{})
|
||||
return len(a.Behind) == 1 && a.Behind[0].Kind == 5 && strings.Contains(a.Behind[0].Newest, "fp2"), err
|
||||
}},
|
||||
"Revision": {"", func(ix *Indexer) (bool, error) {
|
||||
a, err := Revision(ctx, ix.Revisions, nil, "avahi", "avahi", meshcatalog.RevisionInput{})
|
||||
return a.Revision != nil && a.Revision.Revision == 2, err
|
||||
}},
|
||||
"Revision of a branch": {"dev", func(ix *Indexer) (bool, error) {
|
||||
a, err := Revision(ctx, ix.Revisions, nil, "avahi", "avahi", meshcatalog.RevisionInput{Branch: "dev"})
|
||||
return a.Revision != nil && a.Revision.Revision == 2 && len(a.Entries) == 1 && a.Entries[0].Fp == "fp2", err
|
||||
}},
|
||||
}
|
||||
for name, c := range cases {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
mem := newMemObjects(0)
|
||||
keep := func(branch string, revision int64) {
|
||||
data, _ := json.Marshal(meshcatalog.CatalogEntry{Module: "avahi", Repository: catalogueRepo, None: "not built", Fp: fmt.Sprintf("fp%d", revision)})
|
||||
digest := fmt.Sprintf("%s-d%d", branch, revision)
|
||||
rev := meshcatalog.CatalogRevision{Branch: branch, Revision: revision, Kind: "trunk", Entries: []meshcatalog.EntryRef{{Module: "avahi", Digest: digest}}}
|
||||
if err := KeepRevision(ctx, mem, rev, map[string][]byte{digest: data}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
keep("", 1)
|
||||
if c.branch != "" { // the trunk stays at revision 1; the branch's revision 1 is removed under the read
|
||||
keep("", 2)
|
||||
keep(c.branch, 1)
|
||||
}
|
||||
objects := &goneUnderRead{memObjects: mem}
|
||||
pruned := c.branch
|
||||
objects.on = EntryObject(fmt.Sprintf("%s-d1", pruned))
|
||||
objects.prune = func() {
|
||||
keep(pruned, 2)
|
||||
_ = mem.Delete(ctx, RevisionObject(meshcatalog.CatalogRevision{Branch: pruned, Revision: 1, Kind: "trunk"}))
|
||||
_ = mem.Delete(ctx, objects.on)
|
||||
}
|
||||
if ok, err := c.read(&Indexer{Revisions: objects}); err != nil || !ok {
|
||||
t.Errorf("answered %v (%v), want revision 2's entry", ok, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// outdatedOfAvahi is an indexer whose state versions holds avahi at a kind-5 edge, so Outdated reads the trunk's entry of avahi.
|
||||
func outdatedOfAvahi(o Objects) *Indexer {
|
||||
ix := &Indexer{Versions: MemoryStore(), Heard: MemoryStore(), Revisions: o}
|
||||
data, _ := json.Marshal(meshcatalog.VersionEntry{V: 1, Module: "avahi", Version: "0.1.1", Edges: []meshcatalog.VersionEdge{{Kind: 5, Part: "actor:program"}}})
|
||||
_ = ix.Versions.Create(context.Background(), meshcatalog.VersionsByModuleVersionKey("avahi", "0.1.1"), data)
|
||||
return ix
|
||||
}
|
||||
@@ -144,7 +144,7 @@ func (ix *Indexer) applyMoving(ctx context.Context, f forgemsg.Forge, drop bool,
|
||||
ix.logf("%s at %s: its packages are not published: %v", f.Repository, f.Merge, err)
|
||||
}
|
||||
// What this merge moved reaches the modules standing on it (a base's tree, build-agent's fingerprint), in the graph's order.
|
||||
return w, ix.recomputeWith(ctx, overlay, w)
|
||||
return w, ix.recomputeWith(ctx, overlay, w, f.Repository)
|
||||
}
|
||||
|
||||
// baseFP answers the new-form fingerprint of a base module as modules.<base> holds it, or why it has none.
|
||||
|
||||
@@ -3,7 +3,6 @@ package index
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
@@ -51,14 +50,10 @@ func (ix *Indexer) SeedEntryHeld(ctx context.Context, module, fp string) (bool,
|
||||
if ctx.Err() != nil || ix.Revisions == nil {
|
||||
return false, ctx.Err()
|
||||
}
|
||||
rev, found, err := newestTrunk(ctx, ix.Revisions)
|
||||
_, entries, found, err := readNewestOf(ctx, ix.Revisions, TrunkBranch)
|
||||
if err != nil || !found {
|
||||
return false, err
|
||||
}
|
||||
_, entries, err := ReadRevision(ctx, ix.Revisions, fmt.Sprintf("trunk.%d", rev.Revision))
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return slices.ContainsFunc(entries, func(e meshcatalog.CatalogEntry) bool { return e.Module == module && e.Fp == fp }), nil
|
||||
}
|
||||
|
||||
@@ -67,14 +62,10 @@ func (ix *Indexer) ModuleInNewestView(ctx context.Context, module string) (bool,
|
||||
if ctx.Err() != nil || ix.Revisions == nil {
|
||||
return false, ctx.Err()
|
||||
}
|
||||
rev, found, err := newestTrunk(ctx, ix.Revisions)
|
||||
_, entries, found, err := readNewestOf(ctx, ix.Revisions, TrunkBranch)
|
||||
if err != nil || !found {
|
||||
return false, err
|
||||
}
|
||||
_, entries, err := ReadRevision(ctx, ix.Revisions, fmt.Sprintf("trunk.%d", rev.Revision))
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return slices.ContainsFunc(entries, func(e meshcatalog.CatalogEntry) bool { return e.Module == module }), nil
|
||||
}
|
||||
|
||||
|
||||
@@ -13,7 +13,6 @@ import (
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein"
|
||||
@@ -23,13 +22,10 @@ import (
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
"mesh-delivery-indexer/judge"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/graph"
|
||||
)
|
||||
|
||||
// The take-in step (novox/hq ADR 0633): a source revision is taken in only by the indexer it names. A merged message is
|
||||
// kept in the indexer's own state pending, one key per merge, until the take-in that ends it; a revision naming another
|
||||
// build of the indexer writes only what building and moving that indexer needs, and the rest stays pending for it.
|
||||
// The take-in step (novox/hq ADR 0633, A-476): a merged message is kept in the indexer's own state pending, one key per
|
||||
// merge, until the take-in that ends it; a revision is taken in by the running indexer whatever build of the indexer it names.
|
||||
const (
|
||||
// PendingState is the indexer's state of held merges: the bucket mesh-delivery-indexer_pending, keyed
|
||||
// takein.PendingKey, so the key order is the merge order. An old `state` entry (the indexer is foundation, ADR
|
||||
@@ -48,10 +44,6 @@ var errClaimLost = errors.New("another take-in claimed the merges first, so this
|
||||
// runs such a build, so the revision waits until a later merge gives it one or a hand take-in (indexer_take_in_pending).
|
||||
const NoBuild = "no build: "
|
||||
|
||||
// ErrOwnBuildUnknown is what a merged message answers while the indexer cannot learn the build it runs (ADR 0633 §1):
|
||||
// the message is left unacknowledged, so the bus delivers it again, and nothing is written.
|
||||
var ErrOwnBuildUnknown = errors.New("the build of the indexer this program runs is not known")
|
||||
|
||||
// PendingRef is what a pending value keeps of its merged message instead of a copy (a merge of a repository with many
|
||||
// modules is about 0.5 MB, over the state's 256 KiB value cap; issue 1585): the small fields the waits are worked out
|
||||
// from. The message itself is read from the forge bridge's stream by Pending.Seq when a take-in needs it.
|
||||
@@ -98,8 +90,6 @@ type Pending struct {
|
||||
// Held is why a ready merge is not taken in although its revision names this build: objects full (HeldObjectsFull),
|
||||
// until room is said (RoomAgain).
|
||||
Held string `json:"held,omitempty"`
|
||||
// Validation is the revision's validation (design 47 G §713, A-247), on the revision's newest value alone.
|
||||
Validation *takein.Validation `json:"validation,omitempty"`
|
||||
}
|
||||
|
||||
// PendingEntry is one key of the pending state as read: its key, revision, value and the merged message decoded. For a
|
||||
@@ -389,26 +379,13 @@ func (ix *Indexer) pendingOnBus(ctx context.Context) bool {
|
||||
}
|
||||
|
||||
// ownBuildNow is the build this program runs, as FixOwnBuild fixed it at the start: empty while unknown, and never asked again.
|
||||
// It names only the writer of what is written (LastWriter); nothing waits for it (A-476 b).
|
||||
func (ix *Indexer) ownBuildNow(context.Context) string {
|
||||
ix.ownMu.Lock()
|
||||
defer ix.ownMu.Unlock()
|
||||
return ix.ownFP
|
||||
}
|
||||
|
||||
// sayOwnBuildUnknown says, for a merge this process cannot take in because it does not know its own build (ADR 0633 §1), a
|
||||
// take-in.pending that names why and since when, so the watchman's take-in-waits bound covers it. The merge is not written to
|
||||
// pending; the stream delivers it again to the process that knows its build.
|
||||
func (ix *Indexer) sayOwnBuildUnknown(ctx context.Context, seq uint64, f forgemsg.Forge) {
|
||||
ix.ownMu.Lock()
|
||||
since, why := ix.ownSince, ix.ownWhy
|
||||
ix.ownMu.Unlock()
|
||||
if since.IsZero() { // never fixed: the start did not run (no engine wired)
|
||||
since, why = ix.now(), "the build this program runs was never asked"
|
||||
}
|
||||
ix.sayTakeIn(ctx, takein.TakeIn{Kind: takein.TakeInPending, Key: takein.PendingKey(seq), What: takein.TakeInRevision,
|
||||
Merges: []takein.Merge{{Repository: f.Repository, Number: f.Number, Merge: f.Merge}}, Since: since, Unknown: why})
|
||||
}
|
||||
|
||||
// sayTakeIn publishes a take-in event and keeps what it said on the entries, so indexer_pending reads the last event of
|
||||
// each wait from the state. A publish that fails is said in the log: the next evaluation says it again.
|
||||
func (ix *Indexer) sayTakeIn(ctx context.Context, t takein.TakeIn) bool {
|
||||
@@ -456,10 +433,6 @@ func (ix *Indexer) Arrive(ctx context.Context, seq uint64, f forgemsg.Forge) (Ap
|
||||
return a, "", err
|
||||
}
|
||||
own := ix.ownBuildNow(ctx)
|
||||
if own == "" {
|
||||
ix.sayOwnBuildUnknown(ctx, seq, f)
|
||||
return Applied{}, "", ErrOwnBuildUnknown
|
||||
}
|
||||
ix.pendingMu.Lock()
|
||||
defer ix.pendingMu.Unlock()
|
||||
key := takein.PendingKey(seq)
|
||||
@@ -502,9 +475,6 @@ func (ix *Indexer) EndGroup(ctx context.Context, end forgemsg.ForgeGroupEnd) (Ap
|
||||
return Applied{}, "no pending state: the group's merges were taken in one by one", nil
|
||||
}
|
||||
own := ix.ownBuildNow(ctx)
|
||||
if own == "" {
|
||||
return Applied{}, "", ErrOwnBuildUnknown
|
||||
}
|
||||
ix.pendingMu.Lock()
|
||||
defer ix.pendingMu.Unlock()
|
||||
entries, err := ReadPending(ctx, ix.Pending, ix.logf)
|
||||
@@ -677,21 +647,15 @@ func (ix *Indexer) takenAlready(ctx context.Context, m forgemsg.GroupMerge) (boo
|
||||
}
|
||||
|
||||
// TakeInAtStart is the take-in at the start (ADR 0633 §2), before the consumer on the forge bridge's stream starts:
|
||||
// it evaluates pending once with the build this program runs, taking in the whole revisions that name this build; a
|
||||
// group still being gathered stays, and a revision naming yet another build is held for that one. A claim left by an
|
||||
// indexer that ran before is void here. When the build is not known (the node-engine did not answer), it waits for
|
||||
// nothing: it says so and the consumer starts all the same; a merged message is then left unacknowledged until the
|
||||
// build is known (clause 1), and its redelivery, or a toolchain report (ToolchainReported), asks again. No timer.
|
||||
// it evaluates pending once, taking in the whole revisions whatever build of the indexer they name (A-476 b); a group
|
||||
// still being gathered stays. A claim left by an indexer that ran before is void here. The build this program runs, known
|
||||
// or not, only names the writer. No timer.
|
||||
func (ix *Indexer) TakeInAtStart(ctx context.Context) (Applied, string, error) {
|
||||
if ix.Pending == nil || !ix.pendingOnBus(ctx) {
|
||||
// Nothing can wait in a state not on the bus: the start asks no build and does not wait for one.
|
||||
return Applied{}, "the pending state is not on the bus yet: nothing waits", nil
|
||||
}
|
||||
own := ix.ownBuildNow(ctx)
|
||||
if own == "" {
|
||||
ix.logf("take-in: the build this program runs is not known at its start, so nothing pending is taken in now; the forge consumer starts, and a merged message left unacknowledged asks it again")
|
||||
return Applied{}, "the build this program runs is not known yet: nothing pending is taken in at the start", nil
|
||||
}
|
||||
ix.pendingMu.Lock()
|
||||
defer ix.pendingMu.Unlock()
|
||||
a, words, err := ix.evaluate(ctx, own, true)
|
||||
@@ -699,8 +663,8 @@ func (ix *Indexer) TakeInAtStart(ctx context.Context) (Applied, string, error) {
|
||||
return a, "the pending state is not on the bus yet: nothing waits", nil
|
||||
}
|
||||
if err == nil && len(a.Written) == 0 {
|
||||
// A start that takes nothing in says so, with the build it learnt (the wait's own build is in the line holdFor says).
|
||||
ix.logf("take-in: at the start, this program runs build %s and took nothing in: %s", own, words)
|
||||
// A start that takes nothing in says so.
|
||||
ix.logf("take-in: at the start, this program (build %q) took nothing in: %s", own, words)
|
||||
}
|
||||
return a, words, err
|
||||
}
|
||||
@@ -731,16 +695,12 @@ func (ix *Indexer) UnreadAgain(ctx context.Context) error {
|
||||
}
|
||||
|
||||
// PendingAgain evaluates pending again after something a fingerprint is computed from changed, a toolchain report
|
||||
// (ADR 0633 §1, §3), or the verdicts state bound (issue 1966): a revision that now names the running build is taken in, and one that names another build is held
|
||||
// for that one. It does nothing while the pending state is not on the bus or the build this program runs is not known.
|
||||
// (ADR 0633 §3), or the verdicts state bound (issue 1966): what waits is taken in. It does nothing while the pending state is not on the bus.
|
||||
func (ix *Indexer) PendingAgain(ctx context.Context) error {
|
||||
if ix.Pending == nil || !ix.pendingOnBus(ctx) {
|
||||
return nil
|
||||
}
|
||||
own := ix.ownBuildNow(ctx)
|
||||
if own == "" {
|
||||
return nil
|
||||
}
|
||||
ix.pendingMu.Lock()
|
||||
defer ix.pendingMu.Unlock()
|
||||
_, words, err := ix.evaluate(ctx, own, false)
|
||||
@@ -786,31 +746,37 @@ func (ix *Indexer) evaluate(ctx context.Context, own string, start bool) (Applie
|
||||
if len(lost) > 0 {
|
||||
return Applied{}, ix.waitingLine(w), ix.holdLost(ctx, w.Ready, lost, reasons, unread)
|
||||
}
|
||||
if wait, words, err := ix.validationGate(ctx, ready, own); wait || err != nil {
|
||||
return Applied{}, words, err
|
||||
}
|
||||
names, dry, err := ix.computeOver(ctx, ready)
|
||||
dry, err := ix.computeOver(ctx, ready)
|
||||
if err != nil {
|
||||
return Applied{}, "", err
|
||||
}
|
||||
if words, refused, err := ix.refuseCycle(ctx, ready, dry); refused || err != nil {
|
||||
return Applied{}, words, err
|
||||
}
|
||||
if names == own {
|
||||
a, err := ix.takeInWhole(ctx, ready, own, "")
|
||||
if errors.Is(err, errClaimLost) {
|
||||
return a, "claimed by another take-in", nil
|
||||
}
|
||||
if errors.Is(err, errVerdictsNotYet) {
|
||||
return a, "the verdicts state is not on the bus yet: the revision waits for it, taken in when it binds", nil
|
||||
}
|
||||
if errors.Is(err, ErrObjectsFull) {
|
||||
ix.holdFull(ctx, ready)
|
||||
return a, ix.waitingLine(w) + "; held: " + HeldObjectsFull, nil
|
||||
}
|
||||
return a, ix.waitingLine(w), err
|
||||
// The indexer takes in with the build it runs, whatever build of the indexer the revision names (A-476 b).
|
||||
a, err := ix.takeInWhole(ctx, ready, own, "")
|
||||
if errors.Is(err, errClaimLost) {
|
||||
return a, "claimed by another take-in", nil
|
||||
}
|
||||
return Applied{}, ix.waitingLine(w), ix.holdFor(ctx, ready, names, own, dry)
|
||||
if errors.Is(err, errVerdictsNotYet) {
|
||||
return a, "the verdicts state is not on the bus yet: the revision waits for it, taken in when it binds", nil
|
||||
}
|
||||
if errors.Is(err, ErrObjectsFull) {
|
||||
ix.holdFull(ctx, ready)
|
||||
return a, ix.waitingLine(w) + "; held: " + HeldObjectsFull, nil
|
||||
}
|
||||
return a, ix.waitingLine(w), err
|
||||
}
|
||||
|
||||
// newestOfRevision is the index in ready of the revision's newest merge.
|
||||
func newestOfRevision(ready []PendingEntry) int {
|
||||
n := 0
|
||||
for i, e := range ready {
|
||||
if e.Value.Seq > ready[n].Value.Seq {
|
||||
n = i
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// refuseCycle refuses the revision whose catalog graph would hold a cycle with a build edge the take-in refuses (design 47 G
|
||||
@@ -951,7 +917,6 @@ func (ix *Indexer) dry() *dryRun {
|
||||
x := &Indexer{Module: ix.Module, Manifests: ix.Manifests, Packages: ix.Packages, JudgeID: ix.JudgeID, Files: ix.Files, GoSum: ix.GoSum,
|
||||
GoMod: ix.GoMod, Now: ix.Now, Builds: ix.Builds, Named: ix.Named, Trunk: ix.Trunk, BranchHead: ix.BranchHead, Read: ix.Read, Contains: ix.Contains,
|
||||
ReadOpen: nil, OwnBuild: nil, Message: ix.Message, SayRefused: nil, SayAccepted: nil, SayTakeIn: nil, Pending: nil,
|
||||
ValidationAsks: nil, AskVerb: nil, // a dry run asks no validation (§713)
|
||||
// The registry milestone's fields: what a take-in would write on the bus stays out of a dry run, so the states of the
|
||||
// requirements, findings, seat definitions, holds and their bounds are not overlaid, nothing is said, and no hold is
|
||||
// weighed (Holds and Counted nil hold nothing); ReadFile only reads.
|
||||
@@ -991,16 +956,15 @@ func (ix *Indexer) dry() *dryRun {
|
||||
}
|
||||
|
||||
// computeOver applies the ready merges, in merge order, to a dry indexer, as one overlay: each later merge's trees
|
||||
// govern (ADR 0633 §3). It answers the fingerprint the indexer's own module has once they are applied (none is "",
|
||||
// with its reason), and the dry run, whose overlays hold what the take-in would write. A merge that can never be read
|
||||
// govern (ADR 0633 §3). It answers the dry run, whose overlays hold what the take-in would write. A merge that can never be read
|
||||
// is left out, as its take-in will drop it.
|
||||
func (ix *Indexer) computeOver(ctx context.Context, ready []PendingEntry) (string, *dryRun, error) {
|
||||
func (ix *Indexer) computeOver(ctx context.Context, ready []PendingEntry) (*dryRun, error) {
|
||||
d := ix.dry()
|
||||
// The manifests are judged against the seat definitions the revision carries (seatsof.go, ADR 0633 §1), so the indexer
|
||||
// it names is not refused for a verb or an event the built-in list lacks.
|
||||
seats, err := ix.revisionSeats(ctx, ready)
|
||||
if err != nil {
|
||||
return "", d, fmt.Errorf("reading the seat definitions of %s: %w", ready[0].Key, err)
|
||||
return d, fmt.Errorf("reading the seat definitions of %s: %w", ready[0].Key, err)
|
||||
}
|
||||
var applyErr error
|
||||
judge.WithSeats(seats, func() {
|
||||
@@ -1018,24 +982,9 @@ func (ix *Indexer) computeOver(ctx context.Context, ready []PendingEntry) (strin
|
||||
}
|
||||
})
|
||||
if applyErr != nil {
|
||||
return "", d, applyErr
|
||||
return d, applyErr
|
||||
}
|
||||
key, _ := meshcatalogclient.ModulesKey(ix.Module)
|
||||
value, _, found, err := d.ix.Modules.Get(ctx, key)
|
||||
if err != nil {
|
||||
return "", d, err
|
||||
}
|
||||
if !found {
|
||||
return NoBuild + "the indexer holds no record of itself", d, nil
|
||||
}
|
||||
m, err := meshcatalogclient.DecodeModule(value)
|
||||
if err != nil {
|
||||
return "", d, nil
|
||||
}
|
||||
if m.Fp2 == "" {
|
||||
return NoBuild + m.None2, d, nil
|
||||
}
|
||||
return m.Fp2, d, nil
|
||||
return d, nil
|
||||
}
|
||||
|
||||
// claim marks the entries as taken in by this take-in, by compare-and-set on the revisions read. A lost one (another
|
||||
@@ -1238,8 +1187,7 @@ func (ix *Indexer) nextSource(branch string) int64 {
|
||||
// takenWaits are the taken events of a take-in: every wait the entries were said under (saidWaits), every wait
|
||||
// indexer_pending names the claimed entries by now (WaitsOf: a revision by its oldest merge's key, a group by its g. key) and
|
||||
// the key of every claimed entry, since indexer_pending named it by that key earlier when it waited behind a group. So a wait
|
||||
// the watchman learned of without the indexer marking it (seeded from indexer_pending, or said by the own-build-unknown
|
||||
// path) ends too (design 47, Taken in by the indexer it names). No key is said twice.
|
||||
// the watchman learned of without the indexer marking it (seeded from indexer_pending) ends too. No key is said twice.
|
||||
func takenWaits(entries []PendingEntry) []takein.TakeIn {
|
||||
out := saidWaits(entries)
|
||||
said := map[string]bool{}
|
||||
@@ -1298,83 +1246,6 @@ func saidWaits(entries []PendingEntry) []takein.TakeIn {
|
||||
return out
|
||||
}
|
||||
|
||||
// holdFor keeps the ready merges pending for the indexer build they name and writes only what building and moving
|
||||
// that indexer needs (ADR 0633 §1): the indexer's own module entry, its key of the module graph and the verdict on its
|
||||
// manifest, from the dry run; the manifests and the packages are written by the dry run itself. The revision is said
|
||||
// pending under its oldest merge's key, and a wait it was said under before by another key is said taken.
|
||||
func (ix *Indexer) holdFor(ctx context.Context, ready []PendingEntry, names, own string, d *dryRun) error {
|
||||
key, _ := meshcatalogclient.ModulesKey(ix.Module)
|
||||
if err := copyKey(ctx, d.modules, ix.Modules, key); err != nil {
|
||||
return fmt.Errorf("the indexer's own module entry: %w", err)
|
||||
}
|
||||
if gk, err := graph.Key(ix.Module); err == nil {
|
||||
if err := copyKey(ctx, d.graph, ix.Graph, gk); err != nil && !errors.Is(err, process.ErrNotYet) {
|
||||
return fmt.Errorf("the indexer's key of the module graph: %w", err)
|
||||
}
|
||||
}
|
||||
// Why the manifest of the indexer the revision names was refused, when it was (judged with the seat definitions the
|
||||
// revision carries): said with the wait, as a merge the forge bridge could not give is (holdLost).
|
||||
refused := ""
|
||||
if value, _, found, err := d.ix.Modules.Get(ctx, key); err == nil && found {
|
||||
if m, err := meshcatalogclient.DecodeModule(value); err == nil && m.Manifest != "" {
|
||||
if vk, err := meshcatalogclient.VerdictsKey(m.Manifest); err == nil {
|
||||
if err := copyKey(ctx, d.verdicts, ix.Verdicts, vk); err != nil && !errors.Is(err, process.ErrNotYet) {
|
||||
return fmt.Errorf("the verdict on the indexer's manifest: %w", err)
|
||||
}
|
||||
if vv, _, ok, err := d.ix.Verdicts.Get(ctx, vk); err == nil && ok {
|
||||
if v, err := meshcatalogclient.DecodeVerdict(vv); err == nil && meshcatalogclient.VerdictRefuses(v) {
|
||||
refused = strings.Join(v.Problems, "; ")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// The indexer's own entry the held revision names is kept as the trunk revision after the newest, every other entry as
|
||||
// it was (milestone 47/G W4.7: a merged revision naming a newer indexer replaces only the indexer's entry).
|
||||
if err := ix.KeepView(ctx); err != nil {
|
||||
return fmt.Errorf("the revision with the indexer's own entry: %w", err)
|
||||
}
|
||||
t := revisionWait(takein.TakeInPending, ready, names)
|
||||
t.Refused = refused
|
||||
var ended, unmarked []PendingEntry
|
||||
said, changed := false, false
|
||||
for _, e := range ready {
|
||||
switch {
|
||||
case e.Value.Said == t.Key && e.Value.SaidKind == takein.TakeInPending:
|
||||
said = true
|
||||
changed = changed || e.Value.WaitsFor != names || e.Value.Refused != refused
|
||||
if e.Value.WaitsFor != names || e.Value.Refused != refused {
|
||||
unmarked = append(unmarked, e)
|
||||
}
|
||||
case e.Value.Said != "" && e.Value.SaidKind == takein.TakeInPending:
|
||||
ended = append(ended, e)
|
||||
unmarked = append(unmarked, e)
|
||||
default:
|
||||
unmarked = append(unmarked, e)
|
||||
}
|
||||
}
|
||||
for _, old := range saidWaits(ended) {
|
||||
ix.sayTakeIn(ctx, old)
|
||||
}
|
||||
if said && !changed {
|
||||
// Nothing to say on the bus, but the start decided nothing and a reader of the journal sees why: the build learnt, the build the wait names.
|
||||
ix.logf("take-in: %d merges, oldest %s, still name the indexer build %s, this program runs %s: they wait, as already said", len(ready), t.Key, names, own)
|
||||
}
|
||||
if !said || changed {
|
||||
ix.logf("take-in: %d merges, oldest %s, name the indexer build %s, not %s, which runs: they wait for it", len(ready), t.Key, names, own)
|
||||
if refused != "" {
|
||||
ix.logf("take-in: the manifest of the indexer they name is refused, judged with the seat definitions they carry, so no such indexer runs: %s", refused)
|
||||
}
|
||||
if !ix.sayTakeIn(ctx, t) {
|
||||
ix.markSaid(ctx, unmarked, nil, func(p *Pending) { p.WaitsFor, p.LastWriter, p.Refused = names, own, refused })
|
||||
return nil
|
||||
}
|
||||
unmarked = ready
|
||||
}
|
||||
ix.markSaid(ctx, unmarked, &t, func(p *Pending) { p.WaitsFor, p.LastWriter, p.Refused = names, own, refused })
|
||||
return nil
|
||||
}
|
||||
|
||||
// copyKey writes what the overlay holds of key to the store, when it wrote or deleted it.
|
||||
func copyKey(ctx context.Context, from *overlayStore, to Store, key string) error {
|
||||
if from == nil || to == nil {
|
||||
@@ -1439,9 +1310,6 @@ func waitOf(entries []PendingEntry, what, behind string) takein.PendingWait {
|
||||
if e.Value.Refused != "" {
|
||||
pw.Refused = e.Value.Refused
|
||||
}
|
||||
if e.Value.Validation != nil {
|
||||
pw.Validation = e.Value.Validation
|
||||
}
|
||||
if e.Value.Said != "" && e.Value.SaidAt.After(pw.SaidAt) {
|
||||
pw.Said, pw.SaidAt = "take-in."+e.Value.SaidKind, e.Value.SaidAt
|
||||
}
|
||||
@@ -1450,9 +1318,9 @@ func waitOf(entries []PendingEntry, what, behind string) takein.PendingWait {
|
||||
return pw
|
||||
}
|
||||
|
||||
// TakeInPendingByHand is the hand take-in (indexer_take_in_pending, ADR 0633 §5): the wait named by its key, the
|
||||
// ready revision (its m. key) or a group gathered so far (its g. key), is taken in with this indexer and its judge,
|
||||
// whatever build it names, for when the named indexer can never run or the group's end never comes. by and why are
|
||||
// TakeInPendingByHand is the hand take-in (indexer_take_in_pending, ADR 0633 §5, A-476, principle 4): the pending revision (its
|
||||
// m. key) or the group gathered so far (its g. key) is taken in now with this indexer and its judge,
|
||||
// for a revision whose message is unread or lost or held for room, or a group whose end never comes. by and why are
|
||||
// said in the journal and on the take-in.taken event. What waited behind it is evaluated again after.
|
||||
func (ix *Indexer) TakeInPendingByHand(ctx context.Context, key, by, why string) TakenIn {
|
||||
r := TakenIn{Module: key}
|
||||
|
||||
@@ -106,6 +106,16 @@ func (b *takeInBed) merge(number int, c, avahi, indexer string) forgemsg.Forge {
|
||||
return f
|
||||
}
|
||||
|
||||
// arriveHeld arrives f while the verdicts state is not on the bus, so the take-in holds it (ADR 0668 §4): the way a merge is
|
||||
// left pending now that no merge waits for the indexer build it names (A-476 b).
|
||||
func (b *takeInBed) arriveHeld(t *testing.T, seq uint64, f forgemsg.Forge) {
|
||||
t.Helper()
|
||||
bound := b.ix.Verdicts
|
||||
b.ix.Verdicts = &memStore{unbound: process.ErrNotYet}
|
||||
defer func() { b.ix.Verdicts = bound }()
|
||||
b.arrive(t, seq, f)
|
||||
}
|
||||
|
||||
func (b *takeInBed) arrive(t *testing.T, seq uint64, f forgemsg.Forge) {
|
||||
t.Helper()
|
||||
b.publish(t, f)
|
||||
@@ -176,63 +186,21 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633)
|
||||
})
|
||||
}
|
||||
})
|
||||
t.Run("T6 a merge while the own build is unknown writes nothing", func(t *testing.T) {
|
||||
t.Run("T6 a merge while the own build is unknown is taken in all the same (A-476 b)", func(t *testing.T) {
|
||||
b := newTakeInBed(t)
|
||||
b.ix.ownFP, b.ix.ownFixed = "", false // this process is started again and asks the engine, which does not answer
|
||||
b.ix.OwnBuild = func(context.Context) (string, error) { return "", errors.New("the engine does not answer") }
|
||||
b.ix.FixOwnBuild(ctx)
|
||||
b.said = nil
|
||||
_, _, err := b.ix.Arrive(ctx, 1, b.merge(20, sha('b'), "tree-avahi-2", ""))
|
||||
if !errors.Is(err, ErrOwnBuildUnknown) || len(b.keys()) != 0 || b.commit(t, "avahi") == sha('b') {
|
||||
t.Errorf("err %v, pending %v, avahi at %s: want ErrOwnBuildUnknown and nothing written", err, b.keys(), b.commit(t, "avahi"))
|
||||
if _, _, err := b.ix.Arrive(ctx, 1, b.merge(20, sha('b'), "tree-avahi-2", "tree-indexer-2")); err != nil || len(b.keys()) != 0 || b.commit(t, "avahi") != sha('b') {
|
||||
t.Errorf("err %v, pending %v, avahi at %s: want the merge taken in at once", err, b.keys(), b.commit(t, "avahi"))
|
||||
}
|
||||
// The merge is said pending with why and since when (so the watchman's bound covers it), and indexer_pending says the
|
||||
// build is unknown, from the own-build state.
|
||||
if len(b.said) != 1 || b.said[0].Kind != takein.TakeInPending || b.said[0].Key != takein.PendingKey(1) || !strings.Contains(b.said[0].Unknown, "does not answer") ||
|
||||
b.said[0].Since.IsZero() || b.said[0].WaitsFor != "" {
|
||||
t.Errorf("said %+v, want one take-in.pending of m.1 saying why the own build is unknown", b.said)
|
||||
} else if _, err := takein.EncodeTakeIn(b.said[0]); err != nil {
|
||||
t.Errorf("the unknown-build event is refused by its own contract: %v", err)
|
||||
if len(b.said) != 1 || b.said[0].Kind != takein.TakeInTaken {
|
||||
t.Errorf("said %+v, want one take-in.taken and no wait said", b.said)
|
||||
}
|
||||
answer, err := PendingTools(b.pending, b.ix.OwnBuildState, nil, nil)[0].Run(map[string]any{})
|
||||
if a, _ := answer.(takein.PendingAnswer); err != nil || !strings.Contains(a.OwnBuildUnknown, "does not answer") || a.OwnBuildSince.IsZero() {
|
||||
t.Errorf("indexer_pending = %+v %v, want the build unknown with why and since", answer, err)
|
||||
}
|
||||
// Before the pending state is on the bus nothing waits: the start asks no build, and the merge is taken in at once,
|
||||
// as before ADR 0633, though the build is unknown (TestIndexerWhole found the start waiting on it for ever).
|
||||
b.ix.Pending = &memStore{unbound: process.ErrNotYet}
|
||||
if _, _, err := b.ix.TakeInAtStart(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, _, err := b.ix.Arrive(ctx, 1, b.merge(20, sha('b'), "tree-avahi-2", "")); err != nil || b.commit(t, "avahi") != sha('b') {
|
||||
t.Errorf("no pending state: err %v, avahi at %s: want the merge taken in at once", err, b.commit(t, "avahi"))
|
||||
}
|
||||
})
|
||||
t.Run("1966/AC195-2 AC195-3 a merge said pending with its build unknown and taken in on redelivery ends that wait under its key; a revision wait is said once", func(t *testing.T) {
|
||||
b := newTakeInBed(t)
|
||||
ownBuild := b.ix.OwnBuild
|
||||
b.ix.ownFP, b.ix.ownFixed = "", false
|
||||
b.ix.OwnBuild = func(context.Context) (string, error) { return "", errors.New("the engine does not answer") }
|
||||
b.ix.FixOwnBuild(ctx)
|
||||
b.said = nil
|
||||
if _, _, err := b.ix.Arrive(ctx, 7, b.merge(30, sha('b'), "tree-avahi-2", "")); !errors.Is(err, ErrOwnBuildUnknown) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b.ix.OwnBuild, b.ix.ownFP, b.ix.ownFixed = ownBuild, "", false
|
||||
b.ix.FixOwnBuild(ctx)
|
||||
b.arrive(t, 7, b.merge(30, sha('b'), "tree-avahi-2", "")) // the stream delivers it again
|
||||
if want := []string{"pending " + takein.PendingKey(7), "taken " + takein.PendingKey(7)}; !slices.Equal(b.events(), want) || len(b.keys()) != 0 {
|
||||
t.Fatalf("said %v, pending %v: want %v and nothing pending", b.events(), b.keys(), want)
|
||||
}
|
||||
// Two merges said pending under the oldest's key (waiting for a named indexer), then taken in together: one taken, under
|
||||
// that key, and no entry's key said twice.
|
||||
b.said = nil
|
||||
b.arrive(t, 8, b.merge(31, sha('c'), "tree-avahi-3", "tree-indexer-2"))
|
||||
b.arrive(t, 9, b.merge(32, sha('d'), "tree-avahi-4", "tree-indexer-2"))
|
||||
b.said = nil
|
||||
b.arrive(t, 10, b.merge(33, sha('e'), "tree-avahi-5", b.base.Trees["modules/"+testSelf]))
|
||||
if want := []string{"taken " + takein.PendingKey(8), "taken " + takein.PendingKey(9), "taken " + takein.PendingKey(10)}; !slices.Equal(b.events(), want) || len(b.keys()) != 0 {
|
||||
t.Errorf("said %v, pending %v: want %v, the wait's key (m.8, said pending) once and each other entry's own", b.events(), b.keys(), want)
|
||||
if a, _ := answer.(takein.PendingAnswer); err != nil || a.OwnBuild != "" || a.OwnBuildSince.IsZero() || len(a.Waits) != 0 {
|
||||
t.Errorf("indexer_pending = %+v %v, want no wait and no known build", answer, err)
|
||||
}
|
||||
})
|
||||
t.Run("T7 and T3 a merge alone, an indexer change, a merge behind it and a revert", func(t *testing.T) {
|
||||
@@ -259,42 +227,28 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633)
|
||||
t.Fatalf("alone: avahi %s (was %s), pending %v, said %v: want taken in, said taken under its key (1966/AC195-1)", avahi1, avahi0, b.keys(), b.events())
|
||||
}
|
||||
b.said = nil
|
||||
avahi1, verdicts := b.fp(t, "avahi"), len(b.ix.Verdicts.(*memStore).vals)
|
||||
// T4: a change to the indexer writes only its entry (with its graph key and verdict); the rest is pending.
|
||||
avahi1 := b.fp(t, "avahi")
|
||||
// A-476 b: a merge naming another indexer build is taken in at once, with the build this program runs: the indexer's entry
|
||||
// and avahi are written, nothing is held and said pending.
|
||||
b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2"))
|
||||
named := b.fp(t, testSelf)
|
||||
if named == own || b.fp(t, "avahi") != avahi1 || b.commit(t, "avahi") != sha('b') {
|
||||
t.Fatalf("indexer change: indexer %s (own %s), avahi %s at %s: want only the indexer written", named, own, b.fp(t, "avahi"), b.commit(t, "avahi"))
|
||||
if named == own || b.fp(t, "avahi") == avahi1 || b.commit(t, "avahi") != sha('c') || len(b.keys()) != 0 {
|
||||
t.Fatalf("indexer change: indexer %s (own %s), avahi %s at %s, pending %v: want the whole revision taken in at once", named, own, b.fp(t, "avahi"), b.commit(t, "avahi"), b.keys())
|
||||
}
|
||||
if g := graphOf(t, b.ix, testSelf); g.FP != named {
|
||||
t.Errorf("graph of the indexer names %s, want %s", g.FP, named)
|
||||
}
|
||||
if n := len(b.ix.Verdicts.(*memStore).vals); n > verdicts+1 {
|
||||
t.Errorf("verdicts %d, want at most %d: only the new indexer manifest's", n, verdicts+1)
|
||||
if want := []string{"taken " + takein.PendingKey(2)}; !slices.Equal(b.events(), want) {
|
||||
t.Errorf("said %v, want %v: nothing said pending", b.events(), want)
|
||||
}
|
||||
if want := []string{takein.PendingKey(2)}; !reflect.DeepEqual(b.keys(), want) || !slices.Equal(b.events(), []string{"pending " + takein.PendingKey(2)}) || b.said[0].WaitsFor != named {
|
||||
t.Fatalf("pending %v, said %+v: want m.2 pending for %s", b.keys(), b.said, named)
|
||||
}
|
||||
// T5: the merge delivered again writes no second key and says nothing again.
|
||||
b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2"))
|
||||
if len(b.keys()) != 1 || len(b.said) != 1 {
|
||||
t.Errorf("redelivered: pending %v, said %v", b.keys(), b.events())
|
||||
}
|
||||
// A later merge joins: the newest governs and still names the new indexer.
|
||||
// A later merge is taken in at once too, and a revert to the first indexer.
|
||||
b.arrive(t, 3, b.merge(22, sha('d'), "tree-avahi-4", "tree-indexer-2"))
|
||||
if len(b.keys()) != 2 || b.commit(t, "avahi") != sha('b') || len(b.said) != 1 {
|
||||
t.Fatalf("behind: pending %v, avahi at %s, said %v: want both pending and nothing said again", b.keys(), b.commit(t, "avahi"), b.events())
|
||||
}
|
||||
// A revert brings the indexer back to the running build: everything is taken in, oldest first.
|
||||
b.arrive(t, 4, b.merge(23, sha('e'), "tree-avahi-5", b.base.Trees["modules/"+testSelf]))
|
||||
if len(b.keys()) != 0 || b.fp(t, testSelf) != own || b.commit(t, "avahi") != sha('e') {
|
||||
t.Fatalf("revert: pending %v, indexer %s, avahi at %s: want all taken in", b.keys(), b.fp(t, testSelf), b.commit(t, "avahi"))
|
||||
}
|
||||
if want := []string{"pending " + takein.PendingKey(2), "taken " + takein.PendingKey(2), "taken " + takein.PendingKey(3), "taken " + takein.PendingKey(4)}; !slices.Equal(b.events(), want) {
|
||||
t.Errorf("said %v, want %v", b.events(), want)
|
||||
}
|
||||
})
|
||||
t.Run("IX22 a merged revision naming a newer indexer is the previous catalog graph revision with only the indexer's entry replaced", func(t *testing.T) { // design 47 G §273, §237
|
||||
t.Run("IX22 a merged revision naming a newer indexer is a catalog graph revision with its entry and the merge's others replaced", func(t *testing.T) { // design 47 G §273, §237
|
||||
b := newTakeInBed(t)
|
||||
store := newMemObjects(0)
|
||||
b.ix.Revisions = store
|
||||
@@ -305,14 +259,14 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633)
|
||||
if err != nil || before.Revision != 1 {
|
||||
t.Fatalf("the view's revision %+v (%v), want revision 1", before, err)
|
||||
}
|
||||
// The merge changes avahi and the indexer: only the indexer is written, the revision is made at once.
|
||||
// The merge changes avahi and the indexer: both are taken in at once (A-476 b), in a revision after the first.
|
||||
b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2"))
|
||||
after, _, err := newestTrunk(ctx, store)
|
||||
if err != nil || after.Revision != 2 {
|
||||
t.Fatalf("after the merge naming a newer indexer the newest revision is %+v (%v), want revision 2", after, err)
|
||||
if err != nil || after.Revision <= before.Revision {
|
||||
t.Fatalf("after the merge naming a newer indexer the newest revision is %+v (%v), want one after revision %d", after, err, before.Revision)
|
||||
}
|
||||
if changed := changedEntries(before.Entries, after.Entries); !slices.Equal(changed, []string{testSelf}) {
|
||||
t.Errorf("the revision differs from the previous in %v, want the indexer's entry alone", changed)
|
||||
if changed := changedEntries(before.Entries, after.Entries); !slices.Equal(changed, []string{"avahi", testSelf}) && !slices.Equal(changed, []string{testSelf, "avahi"}) {
|
||||
t.Errorf("the revision differs from the previous in %v, want avahi and the indexer", changed)
|
||||
}
|
||||
})
|
||||
t.Run("P1 a revision that adds a Go module its own indexer uses is computed against that module's packages at its merge", func(t *testing.T) {
|
||||
@@ -353,71 +307,13 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633)
|
||||
t.Fatal(err)
|
||||
}
|
||||
b.arrive(t, 2, f)
|
||||
if len(b.said) != 1 || !strings.HasPrefix(b.said[0].WaitsFor, "src2:") {
|
||||
t.Errorf("said %+v, want m.2 waiting for the indexer build its merge's packages give, not for a version no take-in has published", b.said)
|
||||
if len(b.said) != 1 || b.said[0].Kind != takein.TakeInTaken || b.said[0].WaitsFor != "" || len(b.keys()) != 0 {
|
||||
t.Errorf("said %+v, pending %v, want m.2 taken in at once", b.said, b.keys())
|
||||
}
|
||||
if _, ok := b.ix.publishedAll(ctx)[sibling]; !ok {
|
||||
t.Errorf("the packages the revision publishes at its merge lack %s", sibling)
|
||||
}
|
||||
})
|
||||
t.Run("K1 the indexer a revision names is judged with the seat definitions the revision carries", func(t *testing.T) {
|
||||
// A successor's manifest invokes a verb of the node-engine's seat that the seat list lacks and the revision's
|
||||
// definitions add (ADR 0633 §1, §6): refused without them, with its reason said on the wait and in indexer_pending;
|
||||
// accepted with them, and the revision waits for the build it names.
|
||||
const verb, contracts = "revision-verb", "modules/seat-contracts"
|
||||
// The third row's definer is the Delivery domain's contracts module defining a Core seat: not taken (ADR 0636 §5).
|
||||
for _, row := range []struct {
|
||||
name string
|
||||
defined bool
|
||||
refused bool
|
||||
module string
|
||||
}{{"definitions the revision does not carry", false, true, ""}, {"definitions the revision carries", true, false, "seat-contracts"},
|
||||
{"definitions of a seat outside the carrier's domain", true, true, "mesh-delivery-contracts"}} {
|
||||
b := newTakeInBed(t)
|
||||
f := b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2")
|
||||
f.Manifests = map[string]string{}
|
||||
for dir, raw := range b.base.Manifests {
|
||||
f.Manifests[dir] = raw
|
||||
}
|
||||
dir := "modules/" + testSelf
|
||||
f.Manifests[dir] = strings.Replace(f.Manifests[dir], `"module": "`+testSelf+`"`,
|
||||
`"module": "`+testSelf+`", "invokes": ["seat:node-engine.`+verb+`"]`, 1)
|
||||
if row.defined {
|
||||
f.Trees[contracts] = "tree-seat-contracts"
|
||||
f.Locks = map[string]map[string]string{}
|
||||
for d, l := range b.base.Locks {
|
||||
f.Locks[d] = l
|
||||
}
|
||||
f.Locks[contracts] = map[string]string{}
|
||||
f.Manifests[contracts] = `{"module":"` + row.module + `","version":"1.0","build":{"artifacts":[{"name":"go-seats","kind":"package","language":"go","paths":["go"],"series":"0.1","generated-from":"."}]}}`
|
||||
b.ix.Files = func(_ context.Context, repository, commit string, dirs []string) (map[string]map[string]string, error) {
|
||||
if repository != f.Repository || commit != f.Merge || !slices.Equal(dirs, []string{contracts + "/seats"}) {
|
||||
t.Errorf("listed %v of %s at %s", dirs, repository, commit)
|
||||
}
|
||||
return map[string]map[string]string{dirs[0]: {"node-engine/seat.json": "1", "node-engine/fixtures/a.json": "2"}}, nil
|
||||
}
|
||||
b.ix.ReadFile = func(_ context.Context, _, _, p string) ([]byte, error) {
|
||||
switch p {
|
||||
case contracts + "/seats/node-engine/seat.json":
|
||||
return []byte(`{"format":1,"seat":"node-engine","scope":"node","grain":"seat-list","decision":"d","domain":"Core","verbs":[{"name":"` + verb + `"}]}`), nil
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
}
|
||||
b.arrive(t, 2, f)
|
||||
waits := PendingWaits(readPending(t, b))
|
||||
refused := ""
|
||||
if len(b.said) > 0 {
|
||||
refused = b.said[0].Refused
|
||||
}
|
||||
if got := strings.Contains(refused, `serves no verb "`+verb+`"`); got != row.refused || (len(waits) == 1 && waits[0].Refused != refused) {
|
||||
t.Errorf("%s: said %q, indexer_pending %+v: want a refusal naming the verb %v", row.name, refused, waits, row.refused)
|
||||
}
|
||||
if _, ok := catalogue.SeatNamed("node-engine"); !ok || judgeSeatsHave("node-engine", verb) {
|
||||
t.Errorf("%s: the seats are not put back after the dry run", row.name)
|
||||
}
|
||||
}
|
||||
})
|
||||
t.Run("A-326 a revision whose catalog graph holds a cycle with a build edge no build seat breaks is refused, naming it", func(t *testing.T) {
|
||||
// avahi's artifact needs the indexer's (kind 2), and the indexer is built on avahi (kind 3): avahi holds no build seat, so
|
||||
// no running version breaks the cycle, and the revision is refused at take-in naming every artifact on it (design 47 G §49).
|
||||
@@ -443,15 +339,15 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633)
|
||||
t.Errorf("indexer_pending %+v, want the refusal naming the cycle", waits)
|
||||
}
|
||||
})
|
||||
t.Run("T10 the named indexer takes the revision in at its start; a gathered group stays", func(t *testing.T) {
|
||||
t.Run("T10 a start takes what waits in whatever its build; a gathered group stays", func(t *testing.T) {
|
||||
b := newTakeInBed(t)
|
||||
b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2"))
|
||||
b.arriveHeld(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2"))
|
||||
g := b.merge(30, sha('f'), "", "")
|
||||
g.Repository, g.Trees, g.Manifests, g.Locks, g.Group = "novox/mesh-sdk", map[string]string{}, nil, nil, "feat/x"
|
||||
g.Members = []meshdeliveryclient.Head{{Repository: "novox/mesh-sdk", Number: 30, Head: g.Head}, {Repository: "novox/mesh-host", Number: 4, Head: sha('9')}}
|
||||
b.arrive(t, 5, g)
|
||||
named := b.fp(t, testSelf)
|
||||
// The new indexer starts; its build is not known (the engine does not answer at its start): a hand ask is answered
|
||||
b.arriveHeld(t, 5, g)
|
||||
own, _ := b.ix.OwnBuild(ctx)
|
||||
// The indexer starts; its build is not known (the engine does not answer at its start): a hand ask is answered
|
||||
// meanwhile (R5), and the start waits for nothing. It is never asked again, so it stays unknown until it is started again.
|
||||
mk := func(ask func(context.Context) (string, error)) *Indexer {
|
||||
n := &Indexer{Module: testSelf, Store: b.ix.Store, Modules: b.ix.Modules, Manifests: b.ix.Manifests, Graph: b.ix.Graph, Verdicts: b.ix.Verdicts,
|
||||
@@ -462,39 +358,21 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633)
|
||||
n.FixOwnBuild(ctx)
|
||||
return n
|
||||
}
|
||||
// A start that decides nothing says so (issue 1921): the build it learnt, the build the held wait names, also when the
|
||||
// wait was said before; and indexer_pending names the known build and since when.
|
||||
oldBuild, _ := b.ix.OwnBuild(ctx)
|
||||
oldStart := mk(func(context.Context) (string, error) { return oldBuild, nil })
|
||||
b.logs.Reset()
|
||||
if _, words, err := oldStart.TakeInAtStart(ctx); err != nil || len(b.keys()) != 2 || !strings.Contains(b.logs.String(), "still name the indexer build") ||
|
||||
!strings.Contains(b.logs.String(), "at the start, this program runs build") {
|
||||
t.Fatalf("a start that holds: %q %v, journal %q: want the build learnt and the build the wait names said", words, err, b.logs.String())
|
||||
}
|
||||
if answer, err := PendingTools(b.pending, b.ix.OwnBuildState, nil, nil)[0].Run(map[string]any{}); err != nil || answer.(takein.PendingAnswer).OwnBuild != oldBuild || answer.(takein.PendingAnswer).OwnBuildSince.IsZero() {
|
||||
t.Errorf("indexer_pending = %+v %v, want the known build and since", answer, err)
|
||||
}
|
||||
asked := 0
|
||||
next := mk(func(context.Context) (string, error) { asked++; return "", errors.New("not yet") })
|
||||
// The build this program runs is not known (the engine does not answer at its start): nothing waits for it, so the start
|
||||
// takes the revision in all the same (A-476 b), and a hand ask names what waits.
|
||||
next := mk(func(context.Context) (string, error) { return "", errors.New("not yet") })
|
||||
if r := next.TakeInPendingByHand(ctx, "m.nothing", "lead@shanks", "a test"); r.Stored || !strings.Contains(r.Answer, "nothing waits") {
|
||||
t.Errorf("a hand ask while the own build is unknown: %+v", r)
|
||||
t.Errorf("a hand ask for what does not wait: %+v", r)
|
||||
}
|
||||
if _, words, err := next.TakeInAtStart(ctx); err != nil || len(b.keys()) != 2 || !strings.Contains(words, "not known") {
|
||||
t.Fatalf("a start with its build unknown: %q %v, pending %v: want it to answer at once and take nothing in", words, err, b.keys())
|
||||
}
|
||||
// The engine answers now, and the same process does not ask it again: it learnt nothing it did not know at its start.
|
||||
next.OwnBuild = func(context.Context) (string, error) { asked++; return named, nil }
|
||||
if _, words, err := next.TakeInAtStart(ctx); err != nil || asked != 1 || len(b.keys()) != 2 || !strings.Contains(words, "not known") {
|
||||
t.Fatalf("the window of a replacement: %q %v, asked %d, pending %v: want the build fixed at the start, never learnt later", words, err, asked, b.keys())
|
||||
}
|
||||
// The module started again fixes the build at its start and takes the revision in.
|
||||
next = mk(func(context.Context) (string, error) { return named, nil })
|
||||
if _, _, err := next.TakeInAtStart(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(b.keys(), []string{takein.PendingKey(5)}) || b.commit(t, "avahi") != sha('c') {
|
||||
t.Errorf("start: pending %v, avahi at %s: want the revision taken in, the group left", b.keys(), b.commit(t, "avahi"))
|
||||
}
|
||||
if answer, err := PendingTools(b.pending, b.ix.OwnBuildState, nil, nil)[0].Run(map[string]any{}); err != nil || answer.(takein.PendingAnswer).OwnBuild != own {
|
||||
t.Errorf("indexer_pending = %+v %v, want the build the original program knows", answer, err)
|
||||
}
|
||||
})
|
||||
t.Run("T8 T9 T11 a group gathered until its end, a merge behind it, one alone, an end again", func(t *testing.T) {
|
||||
b := newTakeInBed(t)
|
||||
@@ -544,7 +422,7 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633)
|
||||
if _, words, err := b.ix.EndGroup(ctx, end); err != nil || !strings.Contains(words, "taken in already") {
|
||||
t.Errorf("an end again: %q %v", words, err)
|
||||
}
|
||||
// T11 (the lead's row): a group whose revision changes the indexer: the group's wait is taken, then the revision pending.
|
||||
// T11: a group whose revision changes the indexer: the group's wait is taken and the revision taken in at once (A-476 b).
|
||||
b.said = nil
|
||||
g := member(catalogueRepo, 40, sha('c'))
|
||||
g.Group, g.Trees = "feat/two", b.merge(0, "", "", "tree-indexer-7").Trees
|
||||
@@ -553,7 +431,7 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633)
|
||||
if _, _, err := b.ix.EndGroup(ctx, forgemsg.ForgeGroupEnd{Group: "feat/two", Members: []forgemsg.GroupMerge{{Repository: catalogueRepo, Number: 40, Merge: sha('c')}}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if want := []string{"pending g.feat/two", "taken g.feat/two", "pending " + takein.PendingKey(9)}; !slices.Equal(b.events(), want) {
|
||||
if want := []string{"pending g.feat/two", "taken g.feat/two", "taken " + takein.PendingKey(9)}; !slices.Equal(b.events(), want) {
|
||||
t.Errorf("a group naming another indexer: said %v, want %v", b.events(), want)
|
||||
}
|
||||
})
|
||||
@@ -593,11 +471,8 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633)
|
||||
}
|
||||
held := func() *takeInBed {
|
||||
b := newTakeInBed(t)
|
||||
b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2"))
|
||||
b.arrive(t, 3, b.merge(22, sha('d'), "tree-avahi-4", "tree-indexer-2"))
|
||||
b.ix.ownMu.Lock()
|
||||
b.ix.ownFP = b.fp(t, testSelf)
|
||||
b.ix.ownMu.Unlock()
|
||||
b.arriveHeld(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2"))
|
||||
b.arriveHeld(t, 3, b.merge(22, sha('d'), "tree-avahi-4", "tree-indexer-2"))
|
||||
return b
|
||||
}
|
||||
// A probe that fails keeps the group's wait and stops nothing else: an unrelated merge is still taken in.
|
||||
@@ -719,7 +594,7 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633)
|
||||
})
|
||||
t.Run("T5 two take-ins of one pending revision: the one that loses the claim writes nothing", func(t *testing.T) {
|
||||
b := newTakeInBed(t)
|
||||
b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2"))
|
||||
b.arriveHeld(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2"))
|
||||
named := b.fp(t, testSelf)
|
||||
b.ix.ownMu.Lock()
|
||||
b.ix.ownFP = named
|
||||
@@ -774,8 +649,8 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633)
|
||||
{"a take-in that fails", func(b *takeInBed) { b.mods.failWrite = once(func(string) bool { return true }) }},
|
||||
} {
|
||||
b := newTakeInBed(t)
|
||||
b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2"))
|
||||
b.arrive(t, 3, b.merge(22, sha('d'), "tree-avahi-4", "tree-indexer-2"))
|
||||
b.arriveHeld(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2"))
|
||||
b.arriveHeld(t, 3, b.merge(22, sha('d'), "tree-avahi-4", "tree-indexer-2"))
|
||||
named := b.fp(t, testSelf)
|
||||
b.ix.ownMu.Lock()
|
||||
b.ix.ownFP = named
|
||||
@@ -794,7 +669,7 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633)
|
||||
})
|
||||
t.Run("W4.6 a full store holds the merge with the reason objects full, says full once, and room takes it in", func(t *testing.T) {
|
||||
b := newTakeInBed(t)
|
||||
b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2"))
|
||||
b.arriveHeld(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2"))
|
||||
named := b.fp(t, testSelf)
|
||||
b.ix.ownMu.Lock()
|
||||
b.ix.ownFP = named
|
||||
@@ -826,7 +701,7 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633)
|
||||
// The same take-in into a store it fills to 85 % of its cap says filling once, and is taken in.
|
||||
kept := store.used()
|
||||
b2 := newTakeInBed(t)
|
||||
b2.arrive(t, 2, b2.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2"))
|
||||
b2.arriveHeld(t, 2, b2.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2"))
|
||||
b2.ix.ownMu.Lock()
|
||||
b2.ix.ownFP = b2.fp(t, testSelf)
|
||||
b2.ix.ownMu.Unlock()
|
||||
@@ -845,7 +720,7 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633)
|
||||
t.Run("W4.12 a candidate's half dropped by hand frees its own entries, says room and takes the held merge in; the read tools list it; a recompute that moves nothing is equal", func(t *testing.T) {
|
||||
bed := func(store *memObjects) *takeInBed {
|
||||
b := newTakeInBed(t)
|
||||
b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2"))
|
||||
b.arriveHeld(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2"))
|
||||
b.ix.ownMu.Lock()
|
||||
b.ix.ownFP = b.fp(t, testSelf)
|
||||
b.ix.ownMu.Unlock()
|
||||
@@ -930,40 +805,10 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633)
|
||||
t.Errorf("indexer_recompute = %+v (%v): want equal to the kept revision 2", got, err)
|
||||
}
|
||||
})
|
||||
t.Run("T7 a toolchain report evaluates pending again: the revision then names the running build and is taken in", func(t *testing.T) {
|
||||
b := newTakeInBed(t)
|
||||
b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2"))
|
||||
entries, err := ReadPending(ctx, b.pending, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The build the revision names under the next toolchain, worked out on a dry indexer; this one runs it.
|
||||
x := b.ix.dry().ix
|
||||
for _, node := range []string{"novox", "ace"} {
|
||||
key, _ := ToolchainKey(node, "go")
|
||||
x.PutToolchain("mesh-build-go", key, toolchainValue(t, "go", "go1.28.0", "1"), false)
|
||||
}
|
||||
ready, _, _, _ := b.ix.loadNewest(ctx, WaitsOf(entries).Ready)
|
||||
next, _, err := x.computeOver(ctx, ready)
|
||||
if err != nil || next == "" || next == b.said[0].WaitsFor {
|
||||
t.Fatalf("the build under the next toolchain: %q %v (the revision names %s)", next, err, b.said[0].WaitsFor)
|
||||
}
|
||||
b.ix.ownMu.Lock()
|
||||
b.ix.ownFP = next
|
||||
b.ix.ownMu.Unlock()
|
||||
reportToolchain(t, b.ix, "mesh-build-go", "novox", "go", "go1.28.0")
|
||||
if len(b.keys()) != 1 {
|
||||
t.Fatalf("one node's report: pending %v, want the revision still held (the nodes disagree)", b.keys())
|
||||
}
|
||||
reportToolchain(t, b.ix, "mesh-build-go", "ace", "go", "go1.28.0")
|
||||
if len(b.keys()) != 0 || b.commit(t, "avahi") != sha('c') {
|
||||
t.Errorf("after the toolchain report: pending %v, avahi at %s: want the revision taken in", b.keys(), b.commit(t, "avahi"))
|
||||
}
|
||||
})
|
||||
t.Run("T12 T13 indexer_pending names the wait; a hand take-in and a drop say what they ended", func(t *testing.T) {
|
||||
b := newTakeInBed(t)
|
||||
b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2"))
|
||||
b.arrive(t, 3, b.merge(22, sha('d'), "tree-avahi-4", "tree-indexer-2"))
|
||||
b.arriveHeld(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2"))
|
||||
b.arriveHeld(t, 3, b.merge(22, sha('d'), "tree-avahi-4", "tree-indexer-2"))
|
||||
var answer takein.PendingAnswer
|
||||
for _, tool := range PendingTools(b.pending, nil, nil, nil) {
|
||||
if tool.Name == PendingTool {
|
||||
@@ -975,7 +820,7 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633)
|
||||
}
|
||||
}
|
||||
if len(answer.Waits) != 1 || answer.Waits[0].Key != takein.PendingKey(2) || !slices.Equal(answer.Waits[0].Holds, []string{testSelf, "avahi"}) ||
|
||||
answer.Waits[0].Said != takein.TakeInPendingEvent || len(answer.Waits[0].Keys) != 2 {
|
||||
len(answer.Waits[0].Keys) != 2 {
|
||||
t.Fatalf("indexer_pending = %+v", answer)
|
||||
}
|
||||
// ADR 0668 §6: a later merge covers a pending one only when its commit has the pending merge's as an ancestor.
|
||||
@@ -990,32 +835,47 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633)
|
||||
}
|
||||
history = "cd"
|
||||
b.said = nil
|
||||
bound := b.ix.Verdicts
|
||||
b.ix.Verdicts = &memStore{unbound: process.ErrNotYet} // m.3 stays held while m.2 is dropped
|
||||
if r := b.ix.DropPending(ctx, takein.PendingKey(2), "lead@shanks", "replaced by m.3", false); !r.Stored {
|
||||
t.Fatalf("drop: %+v", r)
|
||||
}
|
||||
if len(b.said) < 2 || !b.said[0].Dropped || b.said[0].Key != takein.PendingKey(2) || b.said[0].By != "lead@shanks" || b.said[1].Key != takein.PendingKey(3) {
|
||||
t.Fatalf("drop said %+v: want m.2 taken (dropped, by) and m.3 pending", b.said)
|
||||
b.ix.Verdicts = bound
|
||||
if len(b.keys()) != 1 || b.keys()[0] != takein.PendingKey(3) {
|
||||
t.Fatalf("pending %v: want m.2 dropped and m.3 pending", b.keys())
|
||||
}
|
||||
// Issue 1966 (#650), ADR 0633 §5: the newest pending merge of its repository, which no taken merge covers, is not
|
||||
// dropped; the answer names the read again. abandon drops it all the same (below).
|
||||
// dropped; the answer names abandon, which drops it all the same (below).
|
||||
if r := b.ix.DropPending(ctx, takein.PendingKey(3), "lead@shanks", "lost message", false); r.Stored ||
|
||||
!strings.Contains(r.Answer, TakeInPendingTool) || len(b.keys()) != 1 {
|
||||
!strings.Contains(r.Answer, "abandon") || len(b.keys()) != 1 {
|
||||
t.Fatalf("a drop nothing covers: %+v, pending %v", r, b.keys())
|
||||
}
|
||||
// A pending revision is taken in by the tool, by its key (A-476, principle 4).
|
||||
if _, _, err := NewPendingAsk(takein.PendingKey(3), "x", "lead@shanks", false, false, time.Unix(0, 0)); err != nil {
|
||||
t.Fatalf("the ask to take a pending revision in: %v", err)
|
||||
}
|
||||
if r := b.ix.TakeInPendingByHand(ctx, takein.PendingKey(3), "lead@shanks", "its message was unread"); !r.Stored || len(b.keys()) != 0 || b.commit(t, "avahi") != sha('d') {
|
||||
t.Fatalf("hand take-in of a revision: %+v, pending %v, avahi at %s", r, b.keys(), b.commit(t, "avahi"))
|
||||
}
|
||||
// A delivery group gathered is taken in by hand (A-476 b: only a group is).
|
||||
b.ix.Pins = newMem()
|
||||
b.ix.Revisions = newMemObjects(0) // A-448: the taken event names the catalog graph revision the take-in made
|
||||
gm := b.merge(24, sha('f'), "tree-avahi-6", "")
|
||||
gm.Group, gm.Head = "feat/h", sha('1')
|
||||
gm.Members = []meshdeliveryclient.Head{{Repository: catalogueRepo, Number: 24, Head: sha('1')}, {Repository: "novox/mesh-sdk", Number: 25, Head: sha('2')}}
|
||||
b.arrive(t, 5, gm)
|
||||
b.said = nil
|
||||
if r := b.ix.TakeInPendingByHand(ctx, takein.PendingKey(3), "lead@shanks", "the named indexer can never run"); !r.Stored {
|
||||
if r := b.ix.TakeInPendingByHand(ctx, takein.GroupTakeInKey("feat/h"), "lead@shanks", "the group's end never comes"); !r.Stored {
|
||||
t.Fatalf("hand take-in: %+v", r)
|
||||
}
|
||||
if len(b.keys()) != 0 || b.commit(t, "avahi") != sha('d') || len(b.said) != 1 || b.said[0].Kind != takein.TakeInTaken || b.said[0].By != "lead@shanks" {
|
||||
if len(b.keys()) != 0 || b.commit(t, "avahi") != sha('f') || len(b.said) < 1 || b.said[0].Kind != takein.TakeInTaken || b.said[0].By != "lead@shanks" {
|
||||
t.Errorf("hand take-in: pending %v, avahi at %s, said %+v", b.keys(), b.commit(t, "avahi"), b.said)
|
||||
}
|
||||
// §476, §493: a hand take-in's taken event carries the branch, the source revision it makes and its entries.
|
||||
if len(b.said) == 1 {
|
||||
if len(b.said) >= 1 {
|
||||
got := b.said[0]
|
||||
if got.Branch != TrunkBranch || got.Source <= 0 || len(got.Entries) != 1 || got.Entries[0].Module != "avahi" ||
|
||||
got.Entries[0].Commit != sha('d') || got.Entries[0].Fingerprint == "" {
|
||||
if got.Branch != TrunkBranch || got.Source <= 0 || len(got.Entries) != 2 || got.Entries[1].Module != "avahi" ||
|
||||
got.Entries[1].Commit != sha('f') || got.Entries[1].Fingerprint == "" {
|
||||
t.Errorf("hand take-in said branch %q source %d entries %+v: want main, a source revision and avahi at its commit", got.Branch, got.Source, got.Entries)
|
||||
} else if newest, any, _ := newestTrunk(ctx, b.ix.Revisions); !any || got.Revision != newest.Revision {
|
||||
t.Errorf("hand take-in said revision %d, want the revision it made, %d (A-448)", got.Revision, newest.Revision)
|
||||
@@ -1025,7 +885,7 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633)
|
||||
t.Errorf("the taken event is refused by its own contract: %v", err)
|
||||
}
|
||||
}
|
||||
b.arrive(t, 4, b.merge(23, sha('e'), "tree-avahi-5", "tree-indexer-2"))
|
||||
b.arriveHeld(t, 4, b.merge(23, sha('e'), "tree-avahi-5", "tree-indexer-2"))
|
||||
if r := b.ix.DropPending(ctx, takein.PendingKey(4), "lead@shanks", "abandoned on the operator's word", true); !r.Stored || len(b.keys()) != 0 {
|
||||
t.Errorf("an abandoned drop: %+v, pending %v", r, b.keys())
|
||||
}
|
||||
@@ -1062,8 +922,8 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633)
|
||||
if _, _, found, _ := b.ix.Store.Get(ctx, key); !found {
|
||||
t.Fatal("the group has no candidate")
|
||||
}
|
||||
b.arrive(t, 2, big(b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2")))
|
||||
b.arrive(t, 3, big(b.merge(22, sha('d'), "tree-avahi-4", "tree-indexer-2")))
|
||||
b.arriveHeld(t, 2, big(b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2")))
|
||||
b.arriveHeld(t, 3, big(b.merge(22, sha('d'), "tree-avahi-4", "tree-indexer-2")))
|
||||
raw, _, _, _ := b.pending.Get(ctx, takein.PendingKey(2))
|
||||
if len(b.keys()) != 2 || len(raw) > 4096 || b.commit(t, "avahi") != sha('b') {
|
||||
t.Fatalf("a big merge held: pending %v, value %d bytes, avahi at %s: want two small values", b.keys(), len(raw), b.commit(t, "avahi"))
|
||||
@@ -1139,7 +999,7 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633)
|
||||
b := newTakeInBed(t)
|
||||
f := b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2")
|
||||
f.Words, f.Branch = strings.Repeat("x", 300*1024), "feat/g"
|
||||
b.arrive(t, 2, f) // held by reference, for an indexer build that is not this one
|
||||
b.arriveHeld(t, 2, f) // held by reference, with the verdicts state not on the bus
|
||||
b.ix.Taken = newMem()
|
||||
b.ix.Contains = func(context.Context, string, string, string) (bool, error) { return c.behind, nil }
|
||||
data, _ := takein.EncodeTaken(takein.Taken{Repository: trunk.RepositoryName(catalogueRepo), Merge: sha('d')})
|
||||
|
||||
@@ -4,7 +4,6 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein"
|
||||
@@ -44,15 +43,13 @@ func PendingTools(pending, ownBuild Store, ask func(key string, value []byte) (T
|
||||
return []stdio.Tool{
|
||||
{Name: PendingTool,
|
||||
Description: "What waits for a take-in (ADR 0633): merges the indexer holds in its pending state instead of taking them in. " +
|
||||
"A revision (what revision, its key its oldest merge's m.<sequence>) waits for the build of the indexer it names " +
|
||||
"(waits-for): the running indexer wrote only what building and moving that indexer needs, and the new indexer takes " +
|
||||
"it in at its start. A group (what group, key g.<group>) is a delivery group whose members are gathered until the " +
|
||||
"A revision (what revision, its key its oldest merge's m.<sequence>) waits only while its message is unread or lost or the " +
|
||||
"objects are full. A group (what group, key g.<group>) is a delivery group whose members are gathered until the " +
|
||||
"forge bridge says its end; a merge in one of its members' repositories waits behind it (behind). Each names its " +
|
||||
"merges, since when it waits, the modules it holds back (holds), the merges whose message the forge bridge's stream no " +
|
||||
"longer holds (lost: they are not taken in; indexer_drop_pending drops them) and the last take-in event said of it. Empty waits: " +
|
||||
"every merge is taken in. The watchman raises take-in-waits past 60 minutes for a revision, 30 for a group; " +
|
||||
"indexer_take_in_pending and indexer_drop_pending steer a wait. own-build says which build the indexer knows it runs and own-build-since since when; own-build-unknown, when set, says the indexer cannot learn the build it " +
|
||||
"runs (since when): it fixes it once at its start, so every merge waits (a take-in.pending that says so) until the module is started again.",
|
||||
"indexer_take_in_pending and indexer_drop_pending steer a wait. own-build says which build the indexer knows it runs and own-build-since since when; it only names the writer, and no merge waits for it.",
|
||||
Input: map[string]any{"type": "object", "properties": map[string]any{}},
|
||||
Run: func(map[string]any) (any, error) {
|
||||
var unread []string
|
||||
@@ -63,7 +60,7 @@ func PendingTools(pending, ownBuild Store, ask func(key string, value []byte) (T
|
||||
return nil, err
|
||||
}
|
||||
answer := takein.PendingAnswer{Waits: PendingWaits(entries), Unread: unread}
|
||||
// Whether this indexer knows the build it runs: when it does not, every merge waits, and this says since when and why.
|
||||
// The build this indexer knows it runs, and since when.
|
||||
if ownBuild != nil {
|
||||
if v, _, found, err := ownBuild.Get(context.Background(), OwnBuildKey); err == nil && found {
|
||||
var o OwnBuild
|
||||
@@ -71,8 +68,6 @@ func PendingTools(pending, ownBuild Store, ask func(key string, value []byte) (T
|
||||
answer.OwnBuildSince = o.Since
|
||||
if o.Known {
|
||||
answer.OwnBuild = o.FP
|
||||
} else {
|
||||
answer.OwnBuildUnknown = o.Why
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -80,10 +75,10 @@ func PendingTools(pending, ownBuild Store, ask func(key string, value []byte) (T
|
||||
return answer, nil
|
||||
}},
|
||||
{Name: TakeInPendingTool,
|
||||
Description: "Take a wait in by hand with the running indexer and its judge (ADR 0633 §5): the revision under its m. key, " +
|
||||
"whatever build of the indexer it names, or a delivery group under its g. key with the members gathered so far. For " +
|
||||
"when the named indexer can never run, or a group's end never comes. A hand act: who and why are said in the " +
|
||||
"indexer's journal and on the take-in.taken event, which clears take-in-waits. indexer_pending names the keys.",
|
||||
Description: "Take in now a pending revision (its m.<sequence> key), for a merge whose message is unread or lost or that is held " +
|
||||
"for room, or a delivery group (its g.<group> key) with the members gathered so far, for when its end never comes (ADR 0633 §5). " +
|
||||
"A hand act: who and why are said in the indexer's journal and on the take-in.taken event, which clears take-in-waits. " +
|
||||
"indexer_pending names the keys.",
|
||||
Input: map[string]any{"type": "object", "required": []string{"key", "why"}, "properties": map[string]any{
|
||||
"key": map[string]any{"type": "string", "description": "the wait's key, m.<sequence> or g.<group>, as indexer_pending names it"},
|
||||
"why": map[string]any{"type": "string", "description": "why it is taken in by hand, in plain words"}}},
|
||||
@@ -112,33 +107,3 @@ func PendingTools(pending, ownBuild Store, ask func(key string, value []byte) (T
|
||||
Run: merge},
|
||||
}
|
||||
}
|
||||
|
||||
// ValidationTools is indexer_validate_again (class act, design 47 G §713): the tools bundle writes an ask in takeins and the
|
||||
// indexer asks the revision's validation again; record says the hand act in the hand-act log before it is asked.
|
||||
func ValidationTools(ask func(key string, value []byte) (TakeInAsk, error), now func() time.Time, record func(what, why string) error) []stdio.Tool {
|
||||
return []stdio.Tool{{Name: ValidateAgainTool,
|
||||
Description: "Ask again the validation of a pending revision that could not validate (design 47 G §712, §713): the manifest " +
|
||||
"validator's run on the build seat node-build-go ended could not run, exited other than 0, or answered not both cases. " +
|
||||
"The revision is asked under its next try; the answer says the validation's state (the ask, its try, accepted, refused, " +
|
||||
"or why it could not validate). A revision accepted, refused or whose ask is out is asked nothing: a refusal is asked " +
|
||||
"again only by a later merge of its branch. A hand act, recorded through mesh-hand-acts before it is asked. " +
|
||||
"indexer_pending names the keys and shows the validation whole.",
|
||||
Input: map[string]any{"type": "object", "required": []string{"key", "why"}, "properties": map[string]any{
|
||||
"key": map[string]any{"type": "string", "description": "the revision's wait key, m.<sequence>, as indexer_pending names it"},
|
||||
"why": map[string]any{"type": "string", "description": "why it is asked again, in plain words"}}},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
str := func(k string) string { s, _ := args[k].(string); return s }
|
||||
key, value, err := NewValidateAgainAsk(str("key"), str("why"), stdio.CallerOf(args), now())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := record("asked the validation of the pending revision "+strings.TrimSpace(str("key"))+" again by hand", strings.TrimSpace(str("why"))); err != nil {
|
||||
return nil, fmt.Errorf("the hand act was not recorded through mesh-hand-acts, so nothing was asked: %w", err)
|
||||
}
|
||||
a, err := ask(key, value)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return a.Result, nil
|
||||
}}}
|
||||
}
|
||||
|
||||
@@ -324,24 +324,28 @@ func CatalogRevisionTools(o Objects, ask func(key string, value []byte) (TakeInA
|
||||
ctx := context.Background()
|
||||
n, _ := args["revision"].(float64)
|
||||
name := ""
|
||||
var rev meshcatalog.CatalogRevision
|
||||
var entries []meshcatalog.CatalogEntry
|
||||
switch {
|
||||
case str(args, "group") != "" || str(args, "heads") != "":
|
||||
name = RevisionObject(meshcatalog.CatalogRevision{Kind: "candidate", Group: str(args, "group"), Heads: str(args, "heads")})
|
||||
case n > 0:
|
||||
name = "trunk." + strconv.FormatInt(int64(n), 10)
|
||||
default:
|
||||
newest, ok, err := newestTrunk(ctx, o)
|
||||
newest, newestEntries, ok, err := readNewestOf(ctx, o, TrunkBranch)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the catalog graph revisions cannot be read: %w", err)
|
||||
}
|
||||
if !ok {
|
||||
return nil, ErrNoRevision
|
||||
}
|
||||
name = RevisionObject(newest)
|
||||
name, rev, entries = RevisionObject(newest), newest, newestEntries
|
||||
}
|
||||
rev, entries, err := ReadRevision(ctx, o, name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
if entries == nil {
|
||||
var err error
|
||||
if rev, entries, err = ReadRevision(ctx, o, name); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return map[string]any{"name": name, "revision": rev, "entries": entries}, nil
|
||||
}},
|
||||
|
||||
@@ -65,9 +65,20 @@ const ToolRunner = "node-tools"
|
||||
// answered only when the newest trunk revision's entry of its module declares the verb in invokes, and the holder is
|
||||
// refused too. With no trunk revision kept yet the answer is `none-yet` alone (design 47 G §530), never an empty
|
||||
// catalog and never an error.
|
||||
func Revision(ctx context.Context, o Objects, holder, caller string, in meshcatalog.RevisionInput) (meshcatalog.RevisionAnswer, error) {
|
||||
//
|
||||
// kept, when not nil, is the trunk's newest revision held in memory: the trunk's newest answers from it with no read of
|
||||
// the bucket (A-487c); a candidate, a branch and an older number read the bucket.
|
||||
func Revision(ctx context.Context, o Objects, kept *KeptTrunk, holder, caller string, in meshcatalog.RevisionInput) (meshcatalog.RevisionAnswer, error) {
|
||||
var out meshcatalog.RevisionAnswer
|
||||
newest, ok, err := newestTrunk(ctx, o)
|
||||
var newest meshcatalog.CatalogRevision
|
||||
var ok bool
|
||||
var trunkEntries, branchEntries []meshcatalog.CatalogEntry
|
||||
var err error
|
||||
if kept != nil {
|
||||
newest, trunkEntries, ok, err = kept.Newest(ctx, o)
|
||||
} else {
|
||||
newest, trunkEntries, ok, err = readNewestOf(ctx, o, TrunkBranch)
|
||||
}
|
||||
if err != nil {
|
||||
return out, fmt.Errorf("the catalog graph revisions cannot be read: %w", err)
|
||||
}
|
||||
@@ -75,10 +86,6 @@ func Revision(ctx context.Context, o Objects, holder, caller string, in meshcata
|
||||
// An answer is told by its fields (design 47 G §530): no trunk revision kept yet is `none-yet`, no error.
|
||||
return meshcatalog.RevisionAnswer{NoneYet: true}, nil
|
||||
}
|
||||
_, trunkEntries, err := ReadRevision(ctx, o, RevisionObject(newest))
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
caller = CallerModule(trunkEntries, caller)
|
||||
if err := views.Refusal(meshcatalog.Seat+"."+meshcatalog.VerbRevision, RevisionCallers(trunkEntries, holder), caller); err != nil {
|
||||
return out, fmt.Errorf("%w (novox/hq ADR 0663 §2: a part reads it by declaring %s in invokes)", err, RevisionInvoke)
|
||||
@@ -90,14 +97,14 @@ func Revision(ctx context.Context, o Objects, holder, caller string, in meshcata
|
||||
return out, fmt.Errorf("the branch %q is no escaped name: %w", in.Branch, err)
|
||||
}
|
||||
if !isTrunkBranch(branch) {
|
||||
other, found, err := newestOf(ctx, o, branch)
|
||||
other, entries, found, err := readNewestOf(ctx, o, branch)
|
||||
if err != nil {
|
||||
return out, fmt.Errorf("the catalog graph revisions of %s cannot be read: %w", branch, err)
|
||||
}
|
||||
if !found {
|
||||
return meshcatalog.RevisionAnswer{NoneYet: true}, nil
|
||||
}
|
||||
newest = other
|
||||
newest, branchEntries = other, entries
|
||||
}
|
||||
name := RevisionObject(newest)
|
||||
switch {
|
||||
@@ -121,7 +128,9 @@ func Revision(ctx context.Context, o Objects, holder, caller string, in meshcata
|
||||
}
|
||||
rev, entries := newest, trunkEntries
|
||||
if !isTrunkBranch(branch) && in.Group == "" && in.Heads == "" {
|
||||
if rev, entries, err = ReadRevision(ctx, o, name); err != nil {
|
||||
if name == RevisionObject(newest) {
|
||||
entries = branchEntries // the newest of the branch, read by readNewestOf (A-487d)
|
||||
} else if rev, entries, err = ReadRevision(ctx, o, name); err != nil {
|
||||
return out, err
|
||||
}
|
||||
} else if name != RevisionObject(newest) {
|
||||
@@ -152,13 +161,13 @@ func Revision(ctx context.Context, o Objects, holder, caller string, in meshcata
|
||||
|
||||
// RevisionTool is the verb revision as the tool mesh-catalog.revision of the indexer's own runtime (ADR 0663 §1): the
|
||||
// tool runner describes it with the seat's definition, so it carries no description or input of its own.
|
||||
func RevisionTool(o Objects, holder string) stdio.Tool {
|
||||
func RevisionTool(o Objects, kept *KeptTrunk, holder string) stdio.Tool {
|
||||
return stdio.Tool{Name: meshcatalog.Seat + "." + meshcatalog.VerbRevision, Run: func(args map[string]any) (any, error) {
|
||||
var in meshcatalog.RevisionInput
|
||||
if raw, err := json.Marshal(args); err == nil {
|
||||
_ = json.Unmarshal(raw, &in)
|
||||
}
|
||||
return Revision(context.Background(), o, holder, stdio.CallerOf(args), in)
|
||||
return Revision(context.Background(), o, kept, holder, stdio.CallerOf(args), in)
|
||||
}}
|
||||
}
|
||||
|
||||
|
||||
@@ -63,17 +63,13 @@ func (ix *Indexer) SeedModulesFrom(ctx context.Context, defs map[string]seatdef.
|
||||
if ix.Revisions == nil {
|
||||
return nil, fmt.Errorf("the catalog graph is not kept here: no trunk revision to seed from")
|
||||
}
|
||||
rev, found, err := newestTrunk(ctx, ix.Revisions)
|
||||
_, entries, found, err := readNewestOf(ctx, ix.Revisions, TrunkBranch)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !found {
|
||||
return nil, fmt.Errorf("the indexer keeps no trunk revision yet: the seed waits for its first")
|
||||
}
|
||||
_, entries, err := ReadRevision(ctx, ix.Revisions, fmt.Sprintf("trunk.%d", rev.Revision))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
seatOf := map[string]string{} // artifact reference -> its build seat, all found before any call is made
|
||||
unbuilt := map[string][]string{} // module -> each artifact of it no build seat builds, with its language or kind
|
||||
for _, e := range entries {
|
||||
|
||||
@@ -51,9 +51,6 @@ type TakeInAsk struct {
|
||||
// Abandon, with Drop, drops a pending merge no later merge covers (ADR 0633 §5 asks a later merge; issue 1966, ADR
|
||||
// 0668 §5): the hand act says why it is abandoned all the same.
|
||||
Abandon bool `json:"abandon,omitempty"`
|
||||
// ValidateAgain, with Pending, asks the validation of the revision under that key again (indexer_validate_again, design 47
|
||||
// G §713): one that could not validate.
|
||||
ValidateAgain bool `json:"validate-again,omitempty"`
|
||||
// Merge asks the merge of MergeRepository at this commit to be taken in by hand (indexer_take_in_merge): its merged message
|
||||
// is asked of the forge bridge and run through the take-in as a delivered one, under its stream sequence Sequence.
|
||||
Merge string `json:"merge,omitempty"`
|
||||
@@ -167,20 +164,6 @@ func NewMergeAsk(repository, merge string, sequence uint64, why, by string, at t
|
||||
return "merge." + merge, data, err
|
||||
}
|
||||
|
||||
// NewValidateAgainAsk is the value indexer_validate_again writes: its key in takeins, `pending.validate-again.<key>`, and the
|
||||
// value. key is a revision's wait key, m.<sequence>, as indexer_pending names it.
|
||||
func NewValidateAgainAsk(key, why, by string, at time.Time) (string, []byte, error) {
|
||||
key = strings.TrimSpace(key)
|
||||
if !strings.HasPrefix(key, "m.") {
|
||||
return "", nil, errors.New("name the revision whose validation is asked again by its key, m.<sequence>, as indexer_pending names it")
|
||||
}
|
||||
if strings.TrimSpace(why) == "" {
|
||||
return "", nil, errors.New("say why (why): a hand act is recorded through mesh-hand-acts")
|
||||
}
|
||||
data, err := json.Marshal(TakeInAsk{V: 1, Pending: key, ValidateAgain: true, Why: strings.TrimSpace(why), By: by, At: at.UTC()})
|
||||
return "pending.validate-again." + key, data, err
|
||||
}
|
||||
|
||||
// NewPendingAsk is the value indexer_take_in_pending (drop false) or indexer_drop_pending (drop true) writes: its key
|
||||
// in takeins, `pending.<take-in|drop>.<key>`, and the value. key is a wait's key as indexer_pending names it, for a
|
||||
// drop one pending merge's; abandon drops one no later merge covers.
|
||||
|
||||
@@ -13,6 +13,7 @@ import (
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/trunk"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
|
||||
"mesh-delivery-indexer/judge"
|
||||
@@ -127,24 +128,26 @@ type known struct {
|
||||
// the order the bucket lists them in; modules that stand on each other have a cycle as their reason. A module is written
|
||||
// only when its fingerprint or its reason changes, and only if its manifest is still the one computed from. A module with
|
||||
// no kept inputs (a message that carried no manifest) is left as it is.
|
||||
func (ix *Indexer) Recompute(ctx context.Context) error { return ix.recomputeWith(ctx, nil, nil) }
|
||||
func (ix *Indexer) Recompute(ctx context.Context) error { return ix.recomputeWith(ctx, nil, nil, "") }
|
||||
|
||||
// recomputeWith is Recompute for a merge as well: the records the merge gives (overlay, by module key) stand in place
|
||||
// of what the bucket holds, are computed with the others, and are written once, with their final fingerprint or reason
|
||||
// (issue 1444: a merge writes each module once). A record of the overlay that has no kept inputs is written as it is. w is
|
||||
// the merge's write (order.go), nil for none; it is told each module written.
|
||||
func (ix *Indexer) recomputeWith(ctx context.Context, overlay map[string]meshcatalog.ModulesByModuleValue, w *mergeWrite) error {
|
||||
// the merge's write (order.go), nil for none; it is told each module written. A merge's recompute names its repository
|
||||
// (A-476 c, A-476a): the modules registered for it are moved, and those of another repository only when they import a package it moved; "" moves every module (a toolchain
|
||||
// report, a hand recompute).
|
||||
func (ix *Indexer) recomputeWith(ctx context.Context, overlay map[string]meshcatalog.ModulesByModuleValue, w *mergeWrite, repository string) error {
|
||||
if err := ix.fillPaths(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := ix.recomputeModules(ctx, overlay, w); err != nil {
|
||||
if err := ix.recomputeModules(ctx, overlay, w, repository); err != nil {
|
||||
return err
|
||||
}
|
||||
// What the modules moved to reaches the candidates' fingerprints too (a base, the builder, a toolchain).
|
||||
return ix.RecomputeCandidates(ctx)
|
||||
}
|
||||
|
||||
func (ix *Indexer) recomputeModules(ctx context.Context, overlay map[string]meshcatalog.ModulesByModuleValue, w *mergeWrite) error {
|
||||
func (ix *Indexer) recomputeModules(ctx context.Context, overlay map[string]meshcatalog.ModulesByModuleValue, w *mergeWrite, repository string) error {
|
||||
if ix.Modules == nil {
|
||||
return nil
|
||||
}
|
||||
@@ -153,11 +156,11 @@ func (ix *Indexer) recomputeModules(ctx context.Context, overlay map[string]mesh
|
||||
}
|
||||
ix.recompute.Lock()
|
||||
defer ix.recompute.Unlock()
|
||||
return ix.recomputeLocked(ctx, overlay, w)
|
||||
return ix.recomputeLocked(ctx, overlay, w, repository)
|
||||
}
|
||||
|
||||
// recomputeLocked is recomputeModules with ix.recompute held by the caller (a hold that moves, holds.go).
|
||||
func (ix *Indexer) recomputeLocked(ctx context.Context, overlay map[string]meshcatalog.ModulesByModuleValue, w *mergeWrite) error {
|
||||
func (ix *Indexer) recomputeLocked(ctx context.Context, overlay map[string]meshcatalog.ModulesByModuleValue, w *mergeWrite, repository string) error {
|
||||
if ix.Manifests == nil {
|
||||
return ix.writeOverlay(ctx, overlay, nil, w)
|
||||
}
|
||||
@@ -298,6 +301,14 @@ func (ix *Indexer) recomputeLocked(ctx context.Context, overlay map[string]meshc
|
||||
r.tcs, r.pkgs = toolchainsUsed(toolchains), f.versions
|
||||
}
|
||||
}
|
||||
if _, merged := overlay[module]; !merged && repository != "" && !trunk.SameRepository(k.cur.Repository, repository) &&
|
||||
maps.Equal(k.held.Packages, map[string]string(r.pkgs)) {
|
||||
// A module registered for another repository is moved by this repository's commit only when it imports a package the
|
||||
// commit moved (A-476a, ADR 0593 §3); any other stands as held (A-476 c), and so it is for a base of this repository's.
|
||||
r = result{fp: k.held.Fp2, none: k.held.None2, tcs: k.held.Toolchains, pkgs: sdkstep.PackageVersions(k.held.Packages)}
|
||||
done[module] = r
|
||||
return r
|
||||
}
|
||||
done[module] = r
|
||||
k.cur.Inputs2 = &kept
|
||||
return r
|
||||
|
||||
@@ -645,22 +645,22 @@ func TestBasesAreComputedFirstAndNoFingerprintTakesTheBuilder(t *testing.T) { //
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if got, err := Revision(context.Background(), verbObjects, testSelf, "n1.node-tools", meshcatalog.RevisionInput{}); err != nil || !got.NoneYet || got.Revision != nil || len(got.Entries) != 0 {
|
||||
if got, err := Revision(context.Background(), verbObjects, nil, testSelf, "n1.node-tools", meshcatalog.RevisionInput{}); err != nil || !got.NoneYet || got.Revision != nil || len(got.Entries) != 0 {
|
||||
t.Errorf("revision with none kept answered %+v, %v, want none-yet alone with no error (design 47 G §530)", got, err)
|
||||
}
|
||||
keep(meshcatalog.CatalogRevision{Revision: 3, Kind: "trunk"}, map[string]bool{"node-tools": true, testSelf: true})
|
||||
keep(meshcatalog.CatalogRevision{Revision: 4, Kind: "candidate", Group: "feat/x", Heads: "0123456789ab"}, map[string]bool{"node-tools": true, "mesh-grants": true})
|
||||
for caller, answered := range map[string]bool{"n1.node-tools": true, "n1.avahi": true, "n1.mesh-grants": true, "n1." + testSelf: true, "person.someone": false, "unproven": false} {
|
||||
got, err := Revision(context.Background(), verbObjects, testSelf, caller, meshcatalog.RevisionInput{})
|
||||
got, err := Revision(context.Background(), verbObjects, nil, testSelf, caller, meshcatalog.RevisionInput{})
|
||||
if (err == nil) != answered || (answered && (got.Revision.Revision != 3 || len(got.Entries) != 4 || got.Next != "")) {
|
||||
t.Errorf("revision to %s answered %+v (%v), want answered %v with trunk revision 3 whole", caller, got, err, answered)
|
||||
}
|
||||
}
|
||||
if got, err := Revision(context.Background(), verbObjects, testSelf, "n1.node-tools", meshcatalog.RevisionInput{Group: "feat/x", Heads: "0123456789ab", After: "mesh-grants"}); err != nil ||
|
||||
if got, err := Revision(context.Background(), verbObjects, nil, testSelf, "n1.node-tools", meshcatalog.RevisionInput{Group: "feat/x", Heads: "0123456789ab", After: "mesh-grants"}); err != nil ||
|
||||
got.Revision.Kind != "candidate" || len(got.Entries) != 1 || got.Entries[0].Module != "node-tools" {
|
||||
t.Errorf("a candidate's revision after mesh-grants answered %+v (%v), want its one entry node-tools", got, err)
|
||||
}
|
||||
if _, err := Revision(context.Background(), verbObjects, testSelf, "n1.node-tools", meshcatalog.RevisionInput{Revision: 9}); err == nil {
|
||||
if _, err := Revision(context.Background(), verbObjects, nil, testSelf, "n1.node-tools", meshcatalog.RevisionInput{Revision: 9}); err == nil {
|
||||
t.Error("a trunk revision not kept was answered")
|
||||
}
|
||||
// ADR 0614 §2, §3: the graph names what each module invokes and serves, and leaves out, over the whole graph, the
|
||||
@@ -832,6 +832,8 @@ func TestAModuleKeepsTheToolchainsItsFingerprintWasComputedWith(t *testing.T) {
|
||||
if err := ix.Apply(context.Background(), f); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// A-476a: a commit of mesh-sdk moves avahi, of another repository, which imports its package, and leaves mesh-tools,
|
||||
// which imports nothing it moved.
|
||||
a, _ := moduleOf(t, mods, "avahi")
|
||||
if a.Fp2 == "" || !reflect.DeepEqual(a.Packages, map[string]string{sdk: commit}) {
|
||||
t.Fatalf("after mesh-sdk's merge %.8s: fp2 %q none2 %q packages %v", commit, a.Fp2, a.None2, a.Packages)
|
||||
@@ -841,12 +843,24 @@ func TestAModuleKeepsTheToolchainsItsFingerprintWasComputedWith(t *testing.T) {
|
||||
}
|
||||
return a
|
||||
}
|
||||
// A-476 c: mesh-tools, of another repository than mesh-sdk, holds a fingerprint a recompute would move (here a stale one); a
|
||||
// commit of mesh-sdk, which moved nothing it imports, leaves it as held, and only a full recompute moves it.
|
||||
stale := tools
|
||||
stale.Fp2 = "src2:" + strings.Repeat("5", 64)
|
||||
mods.put("mesh-tools", mustEncode(t, stale))
|
||||
tools = stale
|
||||
first := sdkMerge(strings.Repeat("1", 40), "tree-sdk-1")
|
||||
second := sdkMerge(strings.Repeat("2", 40), "tree-sdk-2")
|
||||
if second.Fp2 == first.Fp2 {
|
||||
t.Errorf("a mesh-sdk trunk merge kept avahi's fingerprint %q", first.Fp2)
|
||||
}
|
||||
n = second
|
||||
if err := ix.Recompute(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if o, _ := moduleOf(t, mods, "mesh-tools"); o.Fp2 == stale.Fp2 {
|
||||
t.Errorf("a full recompute left mesh-tools at %q: the check above proves nothing", o.Fp2)
|
||||
}
|
||||
n, _ = moduleOf(t, mods, "avahi") // moved by the recompute too: its base mesh-tools moved
|
||||
// Nodes that disagree keep the fingerprint already computed, with its toolchains (issue 1940, 2026-10-25 20:56: a
|
||||
// stale report blanked every module and the controller's take-in refused modules already built); only a new
|
||||
// fingerprint needs the agreement: a merge that moves the module's tree gives it none, naming the disagreement.
|
||||
|
||||
@@ -1,264 +0,0 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"maps"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/composer"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/trunk"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
|
||||
"mesh-delivery-indexer/judge"
|
||||
)
|
||||
|
||||
// The validation before take-in (design 47 G §712, A-213, A-232): for a pending revision of a watched branch the running
|
||||
// indexer asks the seat node-build-go a `run` of its own manifest validator over two cases, change and trunk, and refuses only
|
||||
// what the revision causes (ADR 0579 §2). The ask rides the check queue under the indexer's name, its kind `run` in the body.
|
||||
const (
|
||||
// validationProgram is the manifest validator's program, by its directory under the indexer entry's Go module: the indexer's
|
||||
// own code, so no repository or module name is fixed here (ADR 0666 §5).
|
||||
validationProgram = "cmd/mesh-judge"
|
||||
// ValidationAsker is the asker the ask is put under (buildrun.Asker.Name, A-248): the indexer's seat name, mesh-catalog
|
||||
// (design 47 G A-163, A-232).
|
||||
ValidationAsker = meshcatalog.Seat
|
||||
// validationCatalogSeat is the seat the indexer entry claims.
|
||||
validationCatalogSeat = "mesh-catalog"
|
||||
// The cases of the request, as the manifest validator names them.
|
||||
validationChange = "change"
|
||||
validationTrunk = judge.TrunkCase
|
||||
)
|
||||
|
||||
// validationInputVerb is where the build seat's holder asks the input: the seat mesh-catalog's verb input (§354).
|
||||
var validationInputVerb = "seat:" + meshcatalog.Seat + "." + meshcatalog.VerbInput
|
||||
|
||||
// validationRequest is the manifest validator's request for the ready merges of branch over the branch's current revision
|
||||
// (§712): case change, every module of the revision by its entry's manifest, each one the merges carry replaced by the merged
|
||||
// message's bytes (the newest merge governs) and one added for a new directory; case trunk, every module as it stands; the
|
||||
// seat definitions the merges' trees carry; the packages and the predefined list as the take-in's own validator has them;
|
||||
// no machines. It answers the indexer entry of the change case too, the one that claims the seat mesh-catalog.
|
||||
func (ix *Indexer) validationRequest(ctx context.Context, branch string, ready []PendingEntry) (composer.Request, meshcatalog.CatalogEntry, error) {
|
||||
var indexer meshcatalog.CatalogEntry
|
||||
if ix.Revisions == nil {
|
||||
return composer.Request{}, indexer, errors.New("the catalog graph is not kept here: no revision to validate over")
|
||||
}
|
||||
rev, found, err := newestOf(ctx, ix.Revisions, branch)
|
||||
if err != nil {
|
||||
return composer.Request{}, indexer, err
|
||||
}
|
||||
var entries []meshcatalog.CatalogEntry
|
||||
if found {
|
||||
if _, entries, err = ReadRevision(ctx, ix.Revisions, RevisionObject(rev)); err != nil {
|
||||
return composer.Request{}, indexer, err
|
||||
}
|
||||
}
|
||||
trunkCase := composer.Case{Name: validationTrunk, Manifests: map[string]composer.Manifest{}}
|
||||
byPlace := map[string]string{} // <repository> <path> -> module
|
||||
of := map[string]meshcatalog.CatalogEntry{}
|
||||
for _, e := range entries {
|
||||
raw := ix.manifestBytes(ctx, e.ManifestDigest)
|
||||
if raw == "" {
|
||||
b, err := json.Marshal(e.Manifest)
|
||||
if err != nil {
|
||||
return composer.Request{}, indexer, fmt.Errorf("the manifest of %s: %w", e.Module, err)
|
||||
}
|
||||
raw = string(b)
|
||||
}
|
||||
trunkCase.Manifests[e.Module] = composer.Manifest{Raw: json.RawMessage(raw)}
|
||||
byPlace[placeKey(e.Repository, e.Path)] = e.Module
|
||||
of[e.Module] = e
|
||||
}
|
||||
change := composer.Case{Name: validationChange, Manifests: maps.Clone(trunkCase.Manifests)}
|
||||
for _, e := range ready {
|
||||
if e.Covered {
|
||||
continue
|
||||
}
|
||||
for _, dir := range slices.Sorted(maps.Keys(e.Forge.Manifests)) {
|
||||
raw := e.Forge.Manifests[dir]
|
||||
module, known := byPlace[placeKey(e.Forge.Repository, dir)]
|
||||
if !known {
|
||||
m, err := judge.ForInputs([]byte(raw))
|
||||
if err != nil || m.Module == "" {
|
||||
module = e.Forge.Repository + "/" + dir // a new directory whose manifest names no module: judged under its place
|
||||
} else {
|
||||
module = m.Module
|
||||
}
|
||||
byPlace[placeKey(e.Forge.Repository, dir)] = module
|
||||
of[module] = meshcatalog.CatalogEntry{Module: module, Repository: e.Forge.Repository, Path: dir}
|
||||
}
|
||||
change.Manifests[module] = composer.Manifest{Raw: json.RawMessage(raw)}
|
||||
}
|
||||
}
|
||||
for _, module := range slices.Sorted(maps.Keys(change.Manifests)) {
|
||||
m, err := judge.ForInputs(change.Manifests[module].Raw)
|
||||
if err != nil || !slices.ContainsFunc(m.Claims, func(c catalogue.Claim) bool { return c.Name == validationCatalogSeat }) {
|
||||
continue
|
||||
}
|
||||
indexer = of[module]
|
||||
if indexer.Module == "" {
|
||||
indexer.Module = module
|
||||
}
|
||||
break
|
||||
}
|
||||
seats, err := ix.revisionDefinitions(ctx, ready)
|
||||
if err != nil {
|
||||
return composer.Request{}, indexer, err
|
||||
}
|
||||
req := composer.Request{V: composer.Version, Input: json.RawMessage("{}"), Cases: []composer.Case{change, trunkCase}, Seats: seats,
|
||||
Predefined: ix.predefinedList(ctx)}
|
||||
if known := ix.knownPackages(ctx); known != nil {
|
||||
req.Packages = slices.Sorted(maps.Keys(known))
|
||||
}
|
||||
return req, indexer, nil
|
||||
}
|
||||
|
||||
func placeKey(repository, path string) string {
|
||||
return strings.ToLower(trunk.RepositoryName(repository)) + " " + path
|
||||
}
|
||||
|
||||
// revisionDefinitions are the validation's seat definitions, raw (§712 `seats`, A-241): the branch's revision's seat
|
||||
// definitions with the definitions the merges' trees carry laid over them, so a revision that touches no definition is still
|
||||
// judged against the whole seat list. A revision's own definitions are not kept yet (A-234), so the revision's are the
|
||||
// trunk's, read from the state seat-definitions as the verb seat-definitions answers them.
|
||||
func (ix *Indexer) revisionDefinitions(ctx context.Context, ready []PendingEntry) ([]json.RawMessage, error) {
|
||||
defs, err := ix.trunkDefinitions(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
merged, err := ix.revisionSeatDefinitions(ctx, ready)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for seat, d := range merged {
|
||||
raw, err := json.Marshal(d)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the definition of %s: %w", seat, err)
|
||||
}
|
||||
defs[seat] = raw
|
||||
}
|
||||
var out []json.RawMessage
|
||||
for _, seat := range slices.Sorted(maps.Keys(defs)) {
|
||||
out = append(out, defs[seat])
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// trunkDefinitions are the trunk's seat definitions of the state seat-definitions, raw, by seat: every `<seat>.trunk` entry
|
||||
// that holds a definition; none where the indexer keeps no such state.
|
||||
func (ix *Indexer) trunkDefinitions(ctx context.Context) (map[string]json.RawMessage, error) {
|
||||
defs := map[string]json.RawMessage{}
|
||||
if ix.SeatDefinitions == nil {
|
||||
return defs, nil
|
||||
}
|
||||
keys, err := ix.SeatDefinitions.Keys(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the %s state cannot be listed: %w", SeatDefinitionsState, err)
|
||||
}
|
||||
for _, key := range keys {
|
||||
seat, ok := strings.CutSuffix(key, "."+SourceTrunk)
|
||||
if !ok || strings.HasPrefix(key, outOfStepPrefix) {
|
||||
continue
|
||||
}
|
||||
v, _, found, err := ix.SeatDefinitions.Get(ctx, key)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s.%s cannot be read: %w", SeatDefinitionsState, key, err)
|
||||
}
|
||||
var e SeatDefinitionEntry
|
||||
if !found || json.Unmarshal(v, &e) != nil || e.Digest == "" || len(e.Definition) == 0 {
|
||||
continue // an entry that holds no definition is no seat of the list
|
||||
}
|
||||
defs[seat] = e.Definition
|
||||
}
|
||||
return defs, nil
|
||||
}
|
||||
|
||||
// validationJob is the run of the manifest validator for the request (§712): the program cmd/mesh-judge, its directory under
|
||||
// the Go module the go.mod at the entry's path names (read by the build, A-268), at the entry's path, built at head, the revision's head of the entry's repository; no
|
||||
// heads and no arguments; the input the request's bytes, served by the seat mesh-catalog's verb input.
|
||||
func validationJob(indexer meshcatalog.CatalogEntry, head, branch string, seq uint64, input []byte) (buildstep.RunJob, error) {
|
||||
switch {
|
||||
case indexer.Repository == "":
|
||||
return buildstep.RunJob{}, fmt.Errorf("no entry of the revision claims the seat %s", validationCatalogSeat)
|
||||
case head == "":
|
||||
return buildstep.RunJob{}, fmt.Errorf("the revision names no head of %s, the repository of the indexer entry %s", indexer.Repository, indexer.Module)
|
||||
}
|
||||
in, err := buildstep.RunInputOf(validationInputVerb, input)
|
||||
if err != nil {
|
||||
return buildstep.RunJob{}, err
|
||||
}
|
||||
dir := indexer.Path
|
||||
if dir == "" {
|
||||
dir = "."
|
||||
}
|
||||
return buildstep.RunJob{
|
||||
Program: buildstep.RunProgram{Repository: indexer.Repository, Commit: head, Dir: dir, Package: "./" + validationProgram},
|
||||
Input: in,
|
||||
Reader: fmt.Sprintf("manifest validator of %s at %d", branchName(branch), seq),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func branchName(branch string) string {
|
||||
if isTrunkBranch(branch) {
|
||||
return TrunkBranch
|
||||
}
|
||||
return branch
|
||||
}
|
||||
|
||||
// ValidationRefusal is what a validation's verdict refuses (ADR 0579 §2, §712): of each manifest the change case refuses that
|
||||
// the revision adds or changes (its bytes differ from the trunk case's), each problem the trunk case lacks; and each problem
|
||||
// between manifests the trunk case lacks. A problem only in unchanged manifests refuses nothing. It is empty when the revision
|
||||
// is accepted. An answer that holds not both cases is no verdict: it answers an error, so the run could not validate.
|
||||
func ValidationRefusal(req composer.Request, v composer.Verdict) (map[string][]string, error) {
|
||||
cases := map[string]composer.CaseVerdict{}
|
||||
for _, c := range v.Cases {
|
||||
cases[c.Name] = c
|
||||
}
|
||||
change, okChange := cases[validationChange]
|
||||
trunkV, okTrunk := cases[validationTrunk]
|
||||
if !okChange || !okTrunk {
|
||||
return nil, fmt.Errorf("the manifest validator's answer holds not both cases %s and %s", validationChange, validationTrunk)
|
||||
}
|
||||
raws := map[string]map[string]string{}
|
||||
for _, c := range req.Cases {
|
||||
raws[c.Name] = map[string]string{}
|
||||
for name, m := range c.Manifests {
|
||||
raws[c.Name][name] = InputDigest(m.Raw)
|
||||
}
|
||||
}
|
||||
refused := map[string][]string{}
|
||||
for _, module := range slices.Sorted(maps.Keys(change.Manifests)) {
|
||||
r := change.Manifests[module]
|
||||
if r.Accepted || r.Refused == "" {
|
||||
continue
|
||||
}
|
||||
before, had := raws[validationTrunk][module]
|
||||
if module != judge.Unnamed && had && before == raws[validationChange][module] {
|
||||
continue // unchanged: its problems refuse nothing, and its entry keeps its verdict (ADR 0579 §1)
|
||||
}
|
||||
old := map[string]bool{}
|
||||
for _, p := range strings.Split(trunkV.Manifests[module].Refused, "\n") {
|
||||
old[p] = true
|
||||
}
|
||||
for _, p := range strings.Split(r.Refused, "\n") {
|
||||
if p != "" && !old[p] {
|
||||
refused[module] = append(refused[module], p)
|
||||
}
|
||||
}
|
||||
}
|
||||
return refused, nil
|
||||
}
|
||||
|
||||
// RefusedWords is a refusal as the pending value's `refused` says it: each manifest and its problems, one manifest a line.
|
||||
func RefusedWords(refused map[string][]string) string {
|
||||
var lines []string
|
||||
for _, module := range slices.Sorted(maps.Keys(refused)) {
|
||||
lines = append(lines, module+": "+strings.Join(refused[module], "; "))
|
||||
}
|
||||
return strings.Join(lines, "\n")
|
||||
}
|
||||
@@ -1,130 +0,0 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/composer"
|
||||
|
||||
"mesh-delivery-indexer/judge"
|
||||
)
|
||||
|
||||
// The validation refuses only what the revision causes (design 47 G §712, ADR 0579 §2), and its run names the indexer entry's
|
||||
// repository, path and the revision's head, and no heads.
|
||||
func TestValidationRefusesWhatTheRevisionCauses(t *testing.T) {
|
||||
manifests := func(m map[string]string) map[string]composer.Manifest {
|
||||
out := map[string]composer.Manifest{}
|
||||
for k, v := range m {
|
||||
out[k] = composer.Manifest{Raw: json.RawMessage(v)}
|
||||
}
|
||||
return out
|
||||
}
|
||||
req := composer.Request{Cases: []composer.Case{
|
||||
{Name: "change", Manifests: manifests(map[string]string{"a": `{"module":"a","v":2}`, "b": `{"module":"b"}`})},
|
||||
{Name: "trunk", Manifests: manifests(map[string]string{"a": `{"module":"a"}`, "b": `{"module":"b"}`})},
|
||||
}}
|
||||
verdict := func(change, trunk map[string]string) composer.Verdict {
|
||||
read := func(m map[string]string) map[string]composer.ManifestRead {
|
||||
out := map[string]composer.ManifestRead{"a": {Accepted: true}, "b": {Accepted: true}}
|
||||
for k, v := range m {
|
||||
out[k] = composer.ManifestRead{Refused: v}
|
||||
}
|
||||
return out
|
||||
}
|
||||
return composer.Verdict{Cases: []composer.CaseVerdict{{Name: "change", Manifests: read(change)}, {Name: "trunk", Manifests: read(trunk)}}}
|
||||
}
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
v composer.Verdict
|
||||
want, wantErr string
|
||||
}{
|
||||
{"a changed manifest gains a problem", verdict(map[string]string{"a": "new\nold"}, map[string]string{"a": "old"}), "a: new", ""},
|
||||
{"an unchanged manifest's problem refuses nothing", verdict(map[string]string{"b": "bad"}, nil), "", ""},
|
||||
{"a problem between manifests the trunk lacks", verdict(map[string]string{judge.Unnamed: "x\ny"}, map[string]string{judge.Unnamed: "y"}), judge.Unnamed + ": x", ""},
|
||||
{"an answer with one case is no verdict", composer.Verdict{Cases: verdict(nil, nil).Cases[:1]}, "", "not both cases"},
|
||||
} {
|
||||
got, err := ValidationRefusal(req, c.v)
|
||||
if c.wantErr != "" {
|
||||
if err == nil || !strings.Contains(err.Error(), c.wantErr) {
|
||||
t.Errorf("%s: %v, want %q", c.name, err, c.wantErr)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if err != nil || RefusedWords(got) != c.want {
|
||||
t.Errorf("%s: %q, %v, want %q", c.name, RefusedWords(got), err, c.want)
|
||||
}
|
||||
}
|
||||
head := strings.Repeat("a", 40)
|
||||
entry := meshcatalog.CatalogEntry{Module: "mesh-delivery-indexer", Repository: "novox/mesh-delivery", Path: "modules/mesh-delivery-indexer",
|
||||
Artifacts: []meshcatalog.Artifact{{Language: "go", Kind: "bundle"}}}
|
||||
job, err := validationJob(entry, head, "main", 7, []byte(`{"v":1}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := buildstep.RunProgram{Repository: "novox/mesh-delivery", Commit: head, Dir: "modules/mesh-delivery-indexer", Package: "./cmd/mesh-judge"}
|
||||
if job.Program != want || len(job.Heads) != 0 || len(job.Args) != 0 || job.Input.Verb != "seat:mesh-catalog.input" || job.Reader != "manifest validator of main at 7" {
|
||||
t.Errorf("the job %+v", job)
|
||||
}
|
||||
if _, err := validationJob(meshcatalog.CatalogEntry{}, head, "main", 7, nil); err == nil || !strings.Contains(err.Error(), "claims the seat mesh-catalog") {
|
||||
t.Errorf("no indexer entry: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// The validation's seats are the branch's revision's seat definitions with the merges' changed ones laid over them (A-241): a
|
||||
// revision that touches no definition is judged against the whole seat list, and a merge's definition replaces its seat's.
|
||||
func TestValidationSeatsAreTheRevisionsWithTheMergesOver(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
defs := newMem()
|
||||
for seat, raw := range map[string]string{"node-engine": `{"seat":"node-engine","verbs":[{"name":"old"}]}`, "git": `{"seat":"git"}`} {
|
||||
v, _ := json.Marshal(SeatDefinitionEntry{Seat: seat, Source: SourceTrunk, Digest: "d-" + seat, Definition: json.RawMessage(raw)})
|
||||
defs.put(SeatDefinitionKey(seat, SourceTrunk, "", 0), v)
|
||||
}
|
||||
v, _ := json.Marshal(SeatDefinitionEntry{Seat: "pulled", Source: SourcePull, Pull: 3, Digest: "d", Definition: json.RawMessage(`{"seat":"pulled"}`)})
|
||||
defs.put(SeatDefinitionKey("pulled", SourcePull, "r", 3), v)
|
||||
ix := &Indexer{Module: testSelf, SeatDefinitions: defs}
|
||||
seatsOf := func(raws []json.RawMessage) map[string]string {
|
||||
out := map[string]string{}
|
||||
for _, raw := range raws {
|
||||
var d struct {
|
||||
Seat string `json:"seat"`
|
||||
Verbs []struct {
|
||||
Name string `json:"name"`
|
||||
} `json:"verbs"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &d); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out[d.Seat] = ""
|
||||
if len(d.Verbs) > 0 {
|
||||
out[d.Seat] = d.Verbs[0].Name
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
got, err := ix.revisionDefinitions(ctx, nil)
|
||||
if err != nil || !reflect.DeepEqual(seatsOf(got), map[string]string{"node-engine": "old", "git": ""}) {
|
||||
t.Errorf("a revision touching no definition: %v (%v), want the trunk's whole list and no pull request's", seatsOf(got), err)
|
||||
}
|
||||
const contracts = "modules/seat-contracts"
|
||||
f := forgemsg.Forge{Repository: "r", Merge: strings.Repeat("c", 40), Manifests: map[string]string{
|
||||
contracts: `{"module":"seat-contracts","version":"1.0","build":{"artifacts":[{"name":"go-seats","kind":"package","language":"go","paths":["go"],"series":"0.1","generated-from":"."}]}}`}}
|
||||
ix.Files = func(_ context.Context, _, _ string, dirs []string) (map[string]map[string]string, error) {
|
||||
return map[string]map[string]string{dirs[0]: {"node-engine/seat.json": "1"}}, nil
|
||||
}
|
||||
ix.ReadFile = func(_ context.Context, _, _, p string) ([]byte, error) {
|
||||
if p == contracts+"/seats/node-engine/seat.json" {
|
||||
return []byte(`{"format":1,"seat":"node-engine","scope":"node","grain":"seat-list","decision":"d","domain":"Core","verbs":[{"name":"new"}]}`), nil
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
got, err = ix.revisionDefinitions(ctx, []PendingEntry{{Forge: f}})
|
||||
if err != nil || !reflect.DeepEqual(seatsOf(got), map[string]string{"node-engine": "new", "git": ""}) {
|
||||
t.Errorf("a merge changing a definition: %v (%v), want its definition over the trunk's and the rest kept", seatsOf(got), err)
|
||||
}
|
||||
}
|
||||
@@ -1,329 +0,0 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/composer"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein"
|
||||
)
|
||||
|
||||
// The pending revision's wait for its validation (design 47 G §713, A-232, A-247): the take-in puts the run ask of §712, writes
|
||||
// `validation` on the revision's newest pending value and ends there; the build seat's step-ended of that ask, heard through the
|
||||
// indexer's own durable consumer, resumes it: accepted takes the revision in as before, refused keeps it pending with `refused`,
|
||||
// and an end that could not validate keeps it pending with `failed`, asked again by a later merge of the branch or by hand.
|
||||
|
||||
// ValidationAsks puts a run ask on the build seat (buildrun.Asker).
|
||||
type ValidationAsks interface {
|
||||
Put(ctx context.Context, a buildstep.StepAsk) error
|
||||
}
|
||||
|
||||
// newestOfRevision is the index in ready of the revision's newest merge, the value that carries `validation` (A-247).
|
||||
func newestOfRevision(ready []PendingEntry) int {
|
||||
n := 0
|
||||
for i, e := range ready {
|
||||
if e.Value.Seq > ready[n].Value.Seq {
|
||||
n = i
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// branchOfRevision is the watched branch the revision's merges landed on, its newest merged message's base (design 47 G §714);
|
||||
// a message that names none is the trunk's. A pending value's own branch is its pull request's, never the branch validated.
|
||||
func branchOfRevision(ready []PendingEntry) string {
|
||||
if base := ready[newestOfRevision(ready)].Forge.Base; base != "" {
|
||||
return base
|
||||
}
|
||||
return TrunkBranch
|
||||
}
|
||||
|
||||
// validationGate says whether the ready revision waits for its validation (§713): it goes on to the take-in once its newest
|
||||
// value's validation was accepted at that value's sequence; it waits while its ask is out, while it is refused, and while it
|
||||
// could not validate; otherwise it puts a new ask, the next try, and waits. Without an asker nothing is validated.
|
||||
func (ix *Indexer) validationGate(ctx context.Context, ready []PendingEntry, own string) (bool, string, error) {
|
||||
if ix.ValidationAsks == nil {
|
||||
return false, "", nil
|
||||
}
|
||||
newest := ready[newestOfRevision(ready)]
|
||||
seq := newest.Value.Seq
|
||||
if v := newest.Value.Validation; v != nil && v.Seq == seq {
|
||||
switch {
|
||||
case v.Accepted == seq:
|
||||
return false, "", nil
|
||||
case v.Failed != "":
|
||||
return true, fmt.Sprintf("the validation of the revision at %d could not validate: %s", seq, v.Failed), nil
|
||||
case newest.Value.Refused != "":
|
||||
return true, fmt.Sprintf("the revision at %d is refused: %s", seq, newest.Value.Refused), nil
|
||||
case v.Ask != "":
|
||||
return true, fmt.Sprintf("the revision at %d waits for its validation, the ask %s", seq, v.Ask), nil
|
||||
}
|
||||
}
|
||||
return true, "", ix.askValidation(ctx, ready, own)
|
||||
}
|
||||
|
||||
// askValidation puts the revision's validation ask (§712) and writes it on its newest value, the try after every earlier try
|
||||
// of the revision; an older value's validation goes (A-247: on the newest alone). A request or job that cannot be made could not
|
||||
// validate: it is written as `failed`, said, and no ask is put.
|
||||
func (ix *Indexer) askValidation(ctx context.Context, ready []PendingEntry, own string) error {
|
||||
n := newestOfRevision(ready)
|
||||
newest := ready[n]
|
||||
try := 0
|
||||
for _, e := range ready {
|
||||
if v := e.Value.Validation; v != nil && v.Try > try {
|
||||
try = v.Try
|
||||
}
|
||||
}
|
||||
v := &takein.Validation{Seq: newest.Value.Seq, Try: try + 1, Asked: ix.now()}
|
||||
branch := branchOfRevision(ready)
|
||||
ask := buildstep.StepAsk{ID: buildstep.NewID(), Kind: buildstep.AskCheck, Language: buildstep.AskGo, Asker: ValidationAsker, At: ix.now().UTC()}
|
||||
req, entry, err := ix.validationRequest(ctx, branch, ready)
|
||||
var raw []byte
|
||||
if err == nil {
|
||||
raw, err = json.Marshal(req)
|
||||
}
|
||||
var job buildstep.RunJob
|
||||
if err == nil {
|
||||
job, err = validationJob(entry, revisionHead(ready, entry.Repository, entry.Commit), branch, v.Seq, raw)
|
||||
}
|
||||
if err == nil {
|
||||
v.Input, err = ix.KeepInput(ctx, ask.ID, raw)
|
||||
}
|
||||
if err != nil {
|
||||
v.Failed = "the validation could not be asked: " + err.Error()
|
||||
} else {
|
||||
v.Ask, ask.Run = ask.ID, &job
|
||||
}
|
||||
if err := ix.writeValidation(ctx, ready, n, v, own); err != nil {
|
||||
return err
|
||||
}
|
||||
if v.Failed != "" {
|
||||
ix.sayValidation(ctx, ready, v)
|
||||
return nil
|
||||
}
|
||||
if err := ix.ValidationAsks.Put(ctx, ask); err != nil {
|
||||
return fmt.Errorf("the validation ask %s of the revision at %d could not be put: %w", ask.ID, v.Seq, err)
|
||||
}
|
||||
ix.logf("take-in: the revision at %d is validated by the ask %s (try %d); it waits for its end", v.Seq, ask.ID, v.Try)
|
||||
return nil
|
||||
}
|
||||
|
||||
// revisionHead is the revision's head of repository: the newest merge of it the revision covers, else the commit its entry stands at.
|
||||
func revisionHead(ready []PendingEntry, repository, commit string) string {
|
||||
var seq uint64
|
||||
head := commit
|
||||
for _, e := range ready {
|
||||
if e.Forge.Merge != "" && strings.EqualFold(e.Forge.Repository, repository) && e.Value.Seq >= seq {
|
||||
seq, head = e.Value.Seq, e.Forge.Merge
|
||||
}
|
||||
}
|
||||
return head
|
||||
}
|
||||
|
||||
// writeValidation writes v on ready[n] and takes it from every other value, each by compare-and-set; a new validation clears the
|
||||
// value's refusal. A value lost under it is an error: the evaluation is made again.
|
||||
func (ix *Indexer) writeValidation(ctx context.Context, ready []PendingEntry, n int, v *takein.Validation, own string) error {
|
||||
for i, e := range ready {
|
||||
if i != n && e.Value.Validation == nil {
|
||||
continue
|
||||
}
|
||||
p := e.Value
|
||||
p.Validation, p.LastWriter = nil, own
|
||||
if i == n {
|
||||
p.Validation, p.Refused = v, ""
|
||||
}
|
||||
data, err := encodePending(p)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := ix.Pending.Update(ctx, e.Key, data, e.Revision); err != nil {
|
||||
return fmt.Errorf("%s.%s: %w", PendingState, e.Key, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// sayValidation says the revision's wait in take-in.pending with its refusal or why it could not validate.
|
||||
func (ix *Indexer) sayValidation(ctx context.Context, ready []PendingEntry, v *takein.Validation) {
|
||||
t := revisionWait(takein.TakeInPending, ready, "")
|
||||
t.Failed = v.Failed
|
||||
t.Refused = ready[newestOfRevision(ready)].Value.Refused
|
||||
ix.sayTakeIn(ctx, t)
|
||||
}
|
||||
|
||||
// errNoValidationAnswer is an end whose answer is no verdict of both cases.
|
||||
var errNoValidationAnswer = errors.New("no answer")
|
||||
|
||||
// HearValidationEnded takes in the end of a validation ask (§713), heard on the build seat's step-ended through the indexer's own
|
||||
// durable consumer: only an end of the indexer's asks outside a pipeline; an end whose ask no pending value names now is dropped,
|
||||
// written in the journal. The answer is read from the event's own output or output object (A-247).
|
||||
func (ix *Indexer) HearValidationEnded(ctx context.Context, e buildstep.StepEnded) error {
|
||||
if e.Asker != ValidationAsker || e.Pipeline != "" || e.Ask == "" || ix.Pending == nil || !ix.pendingOnBus(ctx) {
|
||||
return nil
|
||||
}
|
||||
own := ix.ownBuildNow(ctx)
|
||||
ix.pendingMu.Lock()
|
||||
defer ix.pendingMu.Unlock()
|
||||
entries, err := ReadPending(ctx, ix.Pending, ix.logf)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ready := WaitsOf(entries).Ready
|
||||
n := -1
|
||||
for i, p := range ready {
|
||||
if p.Value.Validation != nil && p.Value.Validation.Ask == e.Ask {
|
||||
n = i
|
||||
}
|
||||
}
|
||||
if n < 0 {
|
||||
ix.logf("take-in: the end of the validation ask %s (%s on %s) is dropped: no pending revision names that ask now", e.Ask, e.Outcome, e.Node)
|
||||
return nil
|
||||
}
|
||||
ready, lost, _, _ := ix.loadNewest(ctx, ready)
|
||||
if len(lost) > 0 {
|
||||
return fmt.Errorf("the revision the validation ask %s ended for holds a merge that cannot be read", e.Ask)
|
||||
}
|
||||
n = newestOfRevision(ready)
|
||||
v := *ready[n].Value.Validation
|
||||
refused, failed := ix.validationAnswer(ctx, ready, e, v)
|
||||
p := ready[n].Value
|
||||
switch {
|
||||
case failed != "":
|
||||
v.Failed = failed
|
||||
case refused != "":
|
||||
p.Refused = refused
|
||||
default:
|
||||
v.Accepted = v.Seq
|
||||
}
|
||||
p.Validation, p.LastWriter = &v, own
|
||||
data, err := encodePending(p)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := ix.Pending.Update(ctx, ready[n].Key, data, ready[n].Revision); err != nil {
|
||||
return fmt.Errorf("%s.%s: %w", PendingState, ready[n].Key, err)
|
||||
}
|
||||
if v.Accepted == 0 {
|
||||
ready[n].Value = p
|
||||
ix.sayValidation(ctx, ready, &v)
|
||||
ix.logf("take-in: the validation ask %s of the revision at %d ended: refused %q, could not validate %q", e.Ask, v.Seq, refused, failed)
|
||||
return nil
|
||||
}
|
||||
ix.logf("take-in: the validation ask %s accepted the revision at %d", e.Ask, v.Seq)
|
||||
if own == "" {
|
||||
return nil // the take-in waits for the build this program runs, as every take-in does
|
||||
}
|
||||
_, _, err = ix.evaluate(ctx, own, false)
|
||||
return err
|
||||
}
|
||||
|
||||
// validationAnswer reads the end's answer: the refusal of the revision, or why it could not validate (the step did not pass, the
|
||||
// program exited other than 0, its answer is no verdict of both cases, or its request is no longer the one asked).
|
||||
func (ix *Indexer) validationAnswer(ctx context.Context, ready []PendingEntry, e buildstep.StepEnded, v takein.Validation) (string, string) {
|
||||
if e.Outcome != buildstep.OutcomePassed {
|
||||
return "", strings.TrimSpace(e.Outcome + ": " + e.Said)
|
||||
}
|
||||
// The answer is the program's last line of standard output, inline or kept by the seat as an object (A-254); a program that
|
||||
// exited other than 0 ended the step failed, read above.
|
||||
out, err := e.ReadOutput(buildstep.AskGo, ix.AskVerb)
|
||||
if err != nil {
|
||||
return "", "the run's answer could not be read: " + err.Error()
|
||||
}
|
||||
var verdict composer.Verdict
|
||||
if err := json.Unmarshal(out, &verdict); err != nil {
|
||||
return "", fmt.Sprintf("%v: the manifest validator's output is no verdict: %v", errNoValidationAnswer, err)
|
||||
}
|
||||
req, _, err := ix.validationRequest(ctx, branchOfRevision(ready), ready)
|
||||
var raw []byte
|
||||
if err == nil {
|
||||
raw, err = json.Marshal(req)
|
||||
}
|
||||
if err != nil {
|
||||
return "", "the request cannot be made again to read the verdict: " + err.Error()
|
||||
}
|
||||
if runDigest(raw) != v.Input {
|
||||
return "", "the revision's request is no longer the one the ask " + v.Ask + " was asked with"
|
||||
}
|
||||
refused, err := ValidationRefusal(req, verdict)
|
||||
if err != nil {
|
||||
return "", err.Error()
|
||||
}
|
||||
return RefusedWords(refused), ""
|
||||
}
|
||||
|
||||
// ValidateAgainTool asks the validation of a revision that could not validate again by hand (§713).
|
||||
const ValidateAgainTool = "indexer_validate_again"
|
||||
|
||||
// validationWords is the validation's state as the hand ask answers it.
|
||||
func validationWords(v *takein.Validation, refused string) string {
|
||||
switch {
|
||||
case v == nil:
|
||||
return "no validation is asked yet"
|
||||
case v.Accepted != 0:
|
||||
return fmt.Sprintf("accepted at %d (try %d, the ask %s)", v.Accepted, v.Try, v.Ask)
|
||||
case v.Failed != "":
|
||||
return fmt.Sprintf("could not validate at %d (try %d): %s", v.Seq, v.Try, v.Failed)
|
||||
case refused != "":
|
||||
return fmt.Sprintf("refused at %d (try %d): %s", v.Seq, v.Try, refused)
|
||||
default:
|
||||
return fmt.Sprintf("the ask %s is out since %s (try %d at %d)", v.Ask, v.Asked.UTC().Format(time.RFC3339), v.Try, v.Seq)
|
||||
}
|
||||
}
|
||||
|
||||
// ValidateAgainByHand is indexer_validate_again answered (§713): the ready revision under key (its oldest merge's m. key, as
|
||||
// indexer_pending names it) whose validation could not validate is asked again, the next try; its answer says the ask's state.
|
||||
// A revision accepted, refused or whose ask is out is asked nothing: a refusal is asked again only by a later merge of its branch.
|
||||
func (ix *Indexer) ValidateAgainByHand(ctx context.Context, key, by, why string) TakenIn {
|
||||
r := TakenIn{Module: key}
|
||||
if ix.Pending == nil || ix.ValidationAsks == nil {
|
||||
r.Answer = "this indexer asks no validation: nothing asked"
|
||||
return r
|
||||
}
|
||||
own := ix.ownBuildNow(ctx)
|
||||
ix.pendingMu.Lock()
|
||||
defer ix.pendingMu.Unlock()
|
||||
entries, err := ReadPending(ctx, ix.Pending, ix.logf)
|
||||
if err != nil {
|
||||
r.Answer = err.Error()
|
||||
return r
|
||||
}
|
||||
ready := WaitsOf(entries).Ready
|
||||
if len(ready) == 0 || ready[0].Key != key {
|
||||
r.Answer = fmt.Sprintf("no revision waits under %s: indexer_pending names the waits", key)
|
||||
return r
|
||||
}
|
||||
ready, lost, reasons, _ := ix.loadNewest(ctx, ready)
|
||||
if len(lost) > 0 {
|
||||
r.Answer = fmt.Sprintf("nothing asked: the message of %s is not read (%s)", lost[0].Key, reasons[lost[0].Key])
|
||||
return r
|
||||
}
|
||||
newest := ready[newestOfRevision(ready)]
|
||||
v := newest.Value.Validation
|
||||
if v == nil || v.Seq != newest.Value.Seq || v.Failed == "" {
|
||||
r.Answer = "nothing asked: the validation is " + validationWords(v, newest.Value.Refused) + "; only one that could not validate is asked again by hand"
|
||||
return r
|
||||
}
|
||||
ix.logf("take-in: the validation of %s is asked again by hand by %s: %s", key, by, why)
|
||||
if err := ix.askValidation(ctx, ready, own); err != nil {
|
||||
r.Answer = "the validation was not asked again: " + err.Error()
|
||||
return r
|
||||
}
|
||||
after, err := ReadPending(ctx, ix.Pending, ix.logf)
|
||||
if err == nil {
|
||||
ready = WaitsOf(after).Ready
|
||||
}
|
||||
if err != nil || len(ready) == 0 {
|
||||
r.Answer = "asked again; its state is not read back"
|
||||
return r
|
||||
}
|
||||
n := ready[newestOfRevision(ready)]
|
||||
r.Stored = n.Value.Validation != nil && n.Value.Validation.Failed == ""
|
||||
r.Answer = fmt.Sprintf("asked again by %s (%s): the validation is %s", by, why, validationWords(n.Value.Validation, n.Value.Refused))
|
||||
return r
|
||||
}
|
||||
@@ -1,113 +0,0 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/composer"
|
||||
)
|
||||
|
||||
type askRecorder struct{ asks []buildstep.StepAsk }
|
||||
|
||||
func (r *askRecorder) Put(_ context.Context, a buildstep.StepAsk) error {
|
||||
r.asks = append(r.asks, a)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Design 47 G §713, A-247: a pending revision names its ask on its newest value and resumes on that ask's end; an end of a
|
||||
// replaced ask is dropped; a run that could not validate is no refusal and never passed; the start asks nothing again while
|
||||
// the ask is out.
|
||||
func TestAPendingRevisionWaitsForItsValidation(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
b := newTakeInBed(t)
|
||||
asks := &askRecorder{}
|
||||
b.ix.ValidationAsks, b.ix.Inputs, b.ix.Revisions = asks, newMem(), newMemObjects(0)
|
||||
// No entry of the revision claims the seat mesh-catalog: the validation could not be asked, so it is failed, said, and
|
||||
// nothing is taken in.
|
||||
b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", ""))
|
||||
entries := readPending(t, b)
|
||||
if len(entries) != 1 || entries[0].Value.Validation == nil || !strings.Contains(entries[0].Value.Validation.Failed, "claims the seat mesh-catalog") || len(asks.asks) != 0 {
|
||||
t.Fatalf("pending %+v, asks %d: want m.2 pending with failed and no ask", entries, len(asks.asks))
|
||||
}
|
||||
if last := b.said[len(b.said)-1]; last.Kind != takein.TakeInPending || last.Failed == "" || last.Refused != "" {
|
||||
t.Errorf("said %+v, want take-in.pending naming failed, never a refusal", last)
|
||||
}
|
||||
if w := PendingWaits(entries); len(w) != 1 || w[0].Validation == nil || w[0].Validation.Try != 1 {
|
||||
t.Errorf("indexer_pending %+v, want the validation whole", w)
|
||||
}
|
||||
// An ask out: the newest value names it; the start asks nothing new while it is out.
|
||||
loaded, _, _, _ := b.ix.loadNewest(ctx, entries)
|
||||
req, _, err := b.ix.validationRequest(ctx, TrunkBranch, loaded)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
raw, _ := json.Marshal(req)
|
||||
setValidation := func(v takein.Validation) {
|
||||
e := readPending(t, b)[0]
|
||||
p := e.Value
|
||||
p.Validation = &v
|
||||
data, _ := encodePending(p)
|
||||
if err := b.pending.Update(ctx, e.Key, data, e.Revision); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
setValidation(takein.Validation{Ask: "mvl0frv18d1957", Input: runDigest(raw), Seq: 2, Try: 2})
|
||||
if _, _, err := b.ix.TakeInAtStart(ctx); err != nil || len(asks.asks) != 0 || len(b.keys()) != 1 {
|
||||
t.Fatalf("the start: %v, asks %d, pending %v: want it waiting for its ask", err, len(asks.asks), b.keys())
|
||||
}
|
||||
end := func(ask, outcome, said string, answer any) buildstep.StepEnded {
|
||||
out, _ := json.Marshal(answer)
|
||||
return buildstep.StepEnded{Ask: ask, ID: ask, Asker: ValidationAsker, Node: "ace", Outcome: outcome, Said: said, Output: string(out)}
|
||||
}
|
||||
// The answer is the manifest validator's verdict itself, the program's last line (A-254).
|
||||
accepted := composer.Verdict{Cases: []composer.CaseVerdict{{Name: "change"}, {Name: "trunk"}}}
|
||||
// The end of a replaced ask is dropped, written in the journal.
|
||||
if err := b.ix.HearValidationEnded(ctx, end("mvl0frv18d1950", buildstep.OutcomePassed, "", accepted)); err != nil || len(b.keys()) != 1 || !strings.Contains(b.logs.String(), "is dropped") {
|
||||
t.Fatalf("a replaced ask's end: %v, pending %v", err, b.keys())
|
||||
}
|
||||
// could not run: pending with failed, never passed.
|
||||
if err := b.ix.HearValidationEnded(ctx, end("mvl0frv18d1957", buildstep.OutcomeCouldNotRun, "no build node", nil)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if v := readPending(t, b)[0].Value.Validation; v == nil || v.Failed != "could not run: no build node" || v.Accepted != 0 {
|
||||
t.Fatalf("after could not run: %+v", v)
|
||||
}
|
||||
// indexer_validate_again (§713): its hand act recorded first, then the next try asked; the answer says the ask's state.
|
||||
var recorded []string
|
||||
tools := ValidationTools(func(key string, value []byte) (TakeInAsk, error) {
|
||||
a, err := DecodeTakeIn(value)
|
||||
if err != nil || key != "pending.validate-again."+takein.PendingKey(2) || !a.ValidateAgain {
|
||||
t.Fatalf("asked %s %+v %v", key, a, err)
|
||||
}
|
||||
r := b.ix.ValidateAgainByHand(ctx, a.Pending, a.By, a.Why)
|
||||
a.Result = &r
|
||||
return a, nil
|
||||
}, b.ix.now, func(what, why string) error { recorded = append(recorded, what); return nil })
|
||||
if len(tools) != 1 || tools[0].Name != ValidateAgainTool {
|
||||
t.Fatalf("tools = %v", tools)
|
||||
}
|
||||
if _, err := tools[0].Run(map[string]any{"key": takein.PendingKey(2)}); err == nil || len(recorded) != 0 {
|
||||
t.Fatalf("asked again with no why: %v, %v", err, recorded)
|
||||
}
|
||||
got, err := tools[0].Run(map[string]any{"key": takein.PendingKey(2), "why": "the build node is back"})
|
||||
if v := readPending(t, b)[0].Value.Validation; err != nil || len(recorded) != 1 || v == nil || v.Try != 3 || v.Failed == "could not run: no build node" ||
|
||||
!strings.Contains(got.(*TakenIn).Answer, "try 3") {
|
||||
t.Fatalf("asked again: %v, recorded %v, validation %+v, answer %+v: want the try after 2 asked", err, recorded, v, got)
|
||||
}
|
||||
setValidation(takein.Validation{Ask: "mvl0frv18d1959", Input: runDigest(raw), Seq: 2, Try: 3})
|
||||
if r := b.ix.ValidateAgainByHand(ctx, takein.PendingKey(2), "lead", "again"); r.Stored || !strings.Contains(r.Answer, "is out") || readPending(t, b)[0].Value.Validation.Try != 3 {
|
||||
t.Fatalf("an ask out was asked again: %+v", r)
|
||||
}
|
||||
// Asked again (by hand or by a later merge) and accepted: the revision is taken in, and its keys go.
|
||||
setValidation(takein.Validation{Ask: "mvl0frv18d1958", Input: runDigest(raw), Seq: 2, Try: 3})
|
||||
if err := b.ix.HearValidationEnded(ctx, end("mvl0frv18d1958", buildstep.OutcomePassed, "", accepted)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(b.keys()) != 0 || b.commit(t, "avahi") != sha('c') {
|
||||
t.Fatalf("accepted: pending %v %+v, avahi at %s: want it taken in\n%s", b.keys(), readPending(t, b)[0].Value, b.commit(t, "avahi"), b.logs.String())
|
||||
}
|
||||
}
|
||||
@@ -506,8 +506,8 @@ func (ix *Indexer) writeVerdict(ctx context.Context, digest string, v meshcatalo
|
||||
}
|
||||
|
||||
// OwnBuildState is the indexer's state of the build it runs (the bucket mesh-delivery-indexer_own-build, one key, OwnBuildKey):
|
||||
// written once, at the process's start by FixOwnBuild, so indexer_pending says whether this process knows its build and, when it
|
||||
// does not, since when and why.
|
||||
// written once, at the process's start by FixOwnBuild, so indexer_pending says which build this process knows it runs, and since
|
||||
// when. The build only names the writer of what is written (A-476 b).
|
||||
const (
|
||||
OwnBuildStateName = "own-build"
|
||||
OwnBuildKey = "state"
|
||||
@@ -526,8 +526,7 @@ type OwnBuild struct {
|
||||
// seat (OwnBuild; never the trunk's value of the indexer, which names a build that may be merged and not delivered), before
|
||||
// this process hears a merge. It is never asked again: while the node-engine replaces the indexer the seat already names the
|
||||
// successor, and an old process that learnt it later would take revisions in with its old judge. Where the start's ask is not
|
||||
// answered the build stays unknown for the life of this process: every merge waits, loudly (a take-in.pending that says so, the
|
||||
// own-build state, indexer_pending), and the module is started again to learn it.
|
||||
// answered the build stays unknown for the life of this process, said in the own-build state and the journal; nothing waits for it.
|
||||
func (ix *Indexer) FixOwnBuild(ctx context.Context) {
|
||||
ix.ownMu.Lock()
|
||||
if ix.ownFixed || ix.OwnBuild == nil {
|
||||
@@ -543,7 +542,7 @@ func (ix *Indexer) FixOwnBuild(ctx context.Context) {
|
||||
st := OwnBuild{V: 1, Since: ix.now().UTC()}
|
||||
if err != nil {
|
||||
st.Why = err.Error()
|
||||
ix.logf("judge: the build this program runs is not known (%v), so every merge waits and a manifest with a field it does not know waits and is not refused; it is learnt only at the start, so the indexer is started again", err)
|
||||
ix.logf("judge: the build this program runs is not known (%v); merges are taken in all the same, and it is learnt only at the start", err)
|
||||
} else {
|
||||
st.Known, st.FP = true, fp
|
||||
ix.logf("judge: this program runs the indexer build %s, fixed here at its start", fp)
|
||||
|
||||
@@ -177,17 +177,13 @@ func (ix *Indexer) entryOf(ctx context.Context, module string) (meshcatalog.Cata
|
||||
if ix.Revisions == nil {
|
||||
return meshcatalog.CatalogEntry{}, errors.New("the catalog graph is not kept here: no entry names the module's artifacts")
|
||||
}
|
||||
rev, found, err := newestTrunk(ctx, ix.Revisions)
|
||||
_, entries, found, err := readNewestOf(ctx, ix.Revisions, TrunkBranch)
|
||||
if err != nil {
|
||||
return meshcatalog.CatalogEntry{}, err
|
||||
}
|
||||
if !found {
|
||||
return meshcatalog.CatalogEntry{}, errors.New("the indexer keeps no trunk revision yet")
|
||||
}
|
||||
_, entries, err := ReadRevision(ctx, ix.Revisions, fmt.Sprintf("trunk.%d", rev.Revision))
|
||||
if err != nil {
|
||||
return meshcatalog.CatalogEntry{}, err
|
||||
}
|
||||
for _, e := range entries {
|
||||
if e.Module == module {
|
||||
return e, nil
|
||||
|
||||
@@ -194,11 +194,13 @@ func (ix *Indexer) Outdated(ctx context.Context, in meshcatalog.OutdatedInput) (
|
||||
}
|
||||
trunk := map[string]meshcatalog.CatalogEntry{} // the newest trunk revision's entries, read where a kind needs them
|
||||
if ix.Revisions != nil {
|
||||
if rev, found, err := newestTrunk(ctx, ix.Revisions); err == nil && found {
|
||||
if _, entries, err := ReadRevision(ctx, ix.Revisions, fmt.Sprintf("trunk.%d", rev.Revision)); err == nil {
|
||||
for _, e := range entries {
|
||||
trunk[e.Module] = e
|
||||
}
|
||||
_, entries, found, err := readNewestOf(ctx, ix.Revisions, TrunkBranch)
|
||||
if err != nil {
|
||||
return out, fmt.Errorf("the trunk's entries, which the kinds 3 and 5 are told by, cannot be read: %w", err)
|
||||
}
|
||||
if found {
|
||||
for _, e := range entries {
|
||||
trunk[e.Module] = e
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -79,17 +79,13 @@ func (ix *Indexer) FillVersions(ctx context.Context, ask VersionsAsker) error {
|
||||
if ix.Revisions == nil {
|
||||
return errors.New("the catalog graph is not kept here: no entry names the modules' artifacts")
|
||||
}
|
||||
rev, found, err := newestTrunk(ctx, ix.Revisions)
|
||||
_, entries, found, err := readNewestOf(ctx, ix.Revisions, TrunkBranch)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !found {
|
||||
return errors.New("the indexer keeps no trunk revision yet: the first fill of the versions waits for its first")
|
||||
}
|
||||
_, entries, err := ReadRevision(ctx, ix.Revisions, fmt.Sprintf("trunk.%d", rev.Revision))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
type found1 struct {
|
||||
v BuilderVersion
|
||||
name string
|
||||
|
||||
@@ -2476,24 +2476,14 @@ func invokeProblems(m catalogue.Manifest) []string {
|
||||
return problems
|
||||
}
|
||||
|
||||
// ownSecretOwnerProblems is what is wrong with an own secret's declared owner (novox/hq ADR 0436 §1): naming
|
||||
// an account the module makes — a `user` resource of that name — never root, which a secret file owned by it
|
||||
// would make readable by anything, and never the operator account's placeholder (`${machine:account}`), which
|
||||
// the mesh fills per machine and a manifest cannot say safely (novox/hq ADR 0436, settled by the lead).
|
||||
// ownSecretOwnerProblems is what is wrong with an own secret's declared owner (novox/hq ADR 0680 §1, §2, which
|
||||
// ends ADR 0436 §1's refusal of root and the operator account here): an account the module makes — a `user`
|
||||
// resource of that name — the operator account's placeholder (`${machine:account}`) or root. Any other account
|
||||
// the module does not make is still refused.
|
||||
func ownSecretOwnerProblems(m catalogue.Manifest, name, owner string) []string {
|
||||
if owner == "" {
|
||||
if owner == "" || owner == "root" || owner == "${machine:account}" {
|
||||
return nil
|
||||
}
|
||||
if owner == "root" {
|
||||
return []string{fmt.Sprintf(
|
||||
"%s names %q as the owner of its secret %q; an own secret's owner is an account the module "+
|
||||
"makes, never root (novox/hq ADR 0436 §1)", m.Module, owner, name)}
|
||||
}
|
||||
if owner == "${machine:account}" {
|
||||
return []string{fmt.Sprintf(
|
||||
"%s names %q as the owner of its secret %q; an own secret's owner is an account the module "+
|
||||
"makes, never the operator account (novox/hq ADR 0436 §1)", m.Module, owner, name)}
|
||||
}
|
||||
for _, r := range m.Resources {
|
||||
if fmt.Sprint(r["type"]) == "user" && fmt.Sprint(r["name"]) == owner {
|
||||
return nil
|
||||
@@ -2501,7 +2491,7 @@ func ownSecretOwnerProblems(m catalogue.Manifest, name, owner string) []string {
|
||||
}
|
||||
return []string{fmt.Sprintf(
|
||||
"%s names %q as the owner of its secret %q, which it does not make: a user resource named %q "+
|
||||
"(novox/hq ADR 0436 §1)", m.Module, owner, name, owner)}
|
||||
"(novox/hq ADR 0680 §2)", m.Module, owner, name, owner)}
|
||||
}
|
||||
|
||||
// ownSecretPromptProblems is what is wrong with an own secret's declared prompt, in the module check's words:
|
||||
@@ -3218,6 +3208,9 @@ func parseManifest(raw []byte, predefined catalogue.PredefinedList) (catalogue.M
|
||||
problems = append(problems, ownSecretPromptProblems(m, name, own.Prompt, catalogue.IsFamily(name))...)
|
||||
problems = append(problems, ownSecretOwnerProblems(m, name, own.Owner)...)
|
||||
}
|
||||
// The module's secrets-owner is held to the same accounts as an own secret's owner (novox/hq ADR 0680 §2).
|
||||
problems = append(problems, ownSecretOwnerProblems(m, "(its secrets-owner)", m.SecretsOwner)...)
|
||||
problems = append(problems, ownSecretReadProblems(m)...)
|
||||
localOf := map[string]string{}
|
||||
for _, to := range m.SecretRequirements() {
|
||||
if _, plain := m.Secrets[to]; plain {
|
||||
@@ -4204,25 +4197,19 @@ func passableDir(m catalogue.Manifest, id string, depth int) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// processAccountProblems is what is wrong with one process's bus account (novox/hq ADR 0436 §1, §2): an
|
||||
// account the module makes — a `user` resource of that name — never root and never the operator account,
|
||||
// which a manifest cannot say safely (a node with no account has nothing to resolve it to, novox/hq ADR
|
||||
// 0436, settled by the lead).
|
||||
// processAccountProblems is what is wrong with one process's bus account (novox/hq ADR 0680 §1, §2, which ends ADR
|
||||
// 0436 §1's refusal of root and the operator account here): an account the module makes — a `user` resource of
|
||||
// that name — the operator account's placeholder (`${machine:account}`) or root. A process that declares bus
|
||||
// and names no account, or one the module does not make, is refused.
|
||||
func processAccountProblems(m catalogue.Manifest, processID, user string) []string {
|
||||
switch {
|
||||
case user == "":
|
||||
switch user {
|
||||
case "":
|
||||
return []string{fmt.Sprintf(
|
||||
"%s's process %s declares bus and names no account; a process account is a user resource the "+
|
||||
"module makes, never root and never the operator account (novox/hq ADR 0436 §1, §2)",
|
||||
"module makes, the operator account or root (novox/hq ADR 0680 §2)",
|
||||
m.Module, processID)}
|
||||
case user == "root":
|
||||
return []string{fmt.Sprintf(
|
||||
"%s's process %s runs as root; a process account is the module's own, never root (novox/hq ADR "+
|
||||
"0436 §1, §2)", m.Module, processID)}
|
||||
case user == "${machine:account}":
|
||||
return []string{fmt.Sprintf(
|
||||
"%s's process %s runs as the operator account; a process account is the module's own, never the "+
|
||||
"operator's (novox/hq ADR 0436 §1, §2)", m.Module, processID)}
|
||||
case "root", "${machine:account}":
|
||||
return nil
|
||||
}
|
||||
for _, r := range m.Resources {
|
||||
if fmt.Sprint(r["type"]) == "user" && fmt.Sprint(r["name"]) == user {
|
||||
@@ -4230,8 +4217,75 @@ func processAccountProblems(m catalogue.Manifest, processID, user string) []stri
|
||||
}
|
||||
}
|
||||
return []string{fmt.Sprintf(
|
||||
"%s's process %s runs as %s, which it does not make: a user resource named %s (novox/hq ADR 0436 §1, "+
|
||||
"§2)", m.Module, processID, user, user)}
|
||||
"%s's process %s runs as %s, which it does not make: a user resource named %s (novox/hq ADR 0680 §2)",
|
||||
m.Module, processID, user, user)}
|
||||
}
|
||||
|
||||
// ownSecretReadProblems is what is wrong with a process that names an own secret it cannot read (novox/hq ADR
|
||||
// 0680 §3): a process resource whose `run` or `env` names an own secret's path must run as that secret's
|
||||
// resolved owner (its `owner`, else the module's `secrets-owner`, else root) or as root. A module with a
|
||||
// container is excepted: a container reads its secrets as its own account. A process with no `user` runs as
|
||||
// root, as the service manager runs it (lead answer A-485 a, b: a container module is any module with a
|
||||
// `container` resource). A word names a secret only when it is the secret's path (a family's: begins with the path
|
||||
// up to its `*`).
|
||||
func ownSecretReadProblems(m catalogue.Manifest) []string {
|
||||
for _, r := range m.Resources {
|
||||
if fmt.Sprint(r["type"]) == "container" {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
var problems []string
|
||||
for _, r := range m.Resources {
|
||||
if fmt.Sprint(r["type"]) != "process" {
|
||||
continue
|
||||
}
|
||||
user, _ := r["user"].(string)
|
||||
if user == "" || user == "root" {
|
||||
continue
|
||||
}
|
||||
var words []string
|
||||
if run, ok := r["run"].([]any); ok {
|
||||
for _, w := range run {
|
||||
words = append(words, fmt.Sprint(w))
|
||||
}
|
||||
}
|
||||
if env, ok := r["env"].(map[string]any); ok {
|
||||
for _, v := range env {
|
||||
words = append(words, fmt.Sprint(v))
|
||||
}
|
||||
}
|
||||
for name, own := range m.OwnSecrets {
|
||||
path, _, _ := strings.Cut(own.Path, "*")
|
||||
if path == "" {
|
||||
continue
|
||||
}
|
||||
named := false
|
||||
for _, w := range words {
|
||||
if w == own.Path || (strings.Contains(own.Path, "*") && strings.HasPrefix(w, path)) {
|
||||
named = true
|
||||
}
|
||||
}
|
||||
owner := own.Owner
|
||||
if owner == "" {
|
||||
owner = m.SecretsOwner
|
||||
}
|
||||
if named && owner != user {
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s's process %v runs as %s and names its secret %q, which belongs to %s; a process runs as "+
|
||||
"the owner of an own secret it names, or as root (novox/hq ADR 0680 §3)",
|
||||
m.Module, r["id"], user, name, ownerWord(owner)))
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.Strings(problems)
|
||||
return problems
|
||||
}
|
||||
|
||||
func ownerWord(owner string) string {
|
||||
if owner == "" {
|
||||
return "root"
|
||||
}
|
||||
return owner
|
||||
}
|
||||
|
||||
// busKeeps says the process resource r's bus declares `keeps` (novox/hq ADR 0626 §1): read from the resource as
|
||||
|
||||
@@ -2113,19 +2113,79 @@ func caseAnOwnSecretSaysWhenAnOutsidePartyIssuesIt(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// 1170/AC2: an owner naming an account the module does not make, root, or the operator account's placeholder is
|
||||
// refused, naming the secret and the account.
|
||||
// ADR 0680 §2, §4: an own secret's owner of root or the operator account's placeholder is accepted; an owner naming an
|
||||
// account the module does not make is refused, naming the secret and the account.
|
||||
func caseAnOwnSecretsOwnerIsRefusedWhenItIsNotAnAccountTheModuleMakes(t *testing.T) {
|
||||
for name, c := range map[string]struct{ owner, says string }{
|
||||
"root": {"root", "root"},
|
||||
"the operator account": {"${machine:account}", "operator account"},
|
||||
"an account not declared": {"not-made", "not-made"},
|
||||
for name, c := range map[string]struct {
|
||||
owner string
|
||||
ok bool
|
||||
}{
|
||||
"root": {"root", true},
|
||||
"the operator account": {"${machine:account}", true},
|
||||
"an account the module makes": {"mesh-mail-send", true},
|
||||
"an account not declared": {"not-made", false},
|
||||
} {
|
||||
_, err := ParseManifest([]byte(`{"module":"mesh-mail","version":"1.0","own-secrets":{` +
|
||||
`"mail-credential-*":{"path":"/var/lib/mesh-mail/mail-credential-*.secret","issued-by":"outside","owner":"` +
|
||||
c.owner + `"}},"resources":[{"id":"account","type":"user","name":"mesh-mail-send"}]}`))
|
||||
if err == nil || !strings.Contains(err.Error(), "mail-credential-*") || !strings.Contains(err.Error(), c.says) {
|
||||
t.Errorf("%s: %v; want a refusal naming the secret and %q", name, err, c.says)
|
||||
switch {
|
||||
case c.ok && err != nil:
|
||||
t.Errorf("%s: refused: %v", name, err)
|
||||
case !c.ok && (err == nil || !strings.Contains(err.Error(), "mail-credential-*") || !strings.Contains(err.Error(), c.owner)):
|
||||
t.Errorf("%s: %v; want a refusal naming the secret and %q", name, err, c.owner)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ADR 0680 §2, §4: the module's secrets-owner is held to the same accounts as an own secret's owner.
|
||||
func caseASecretsOwnerIsRefusedWhenItIsNotAnAccountTheModuleMakes(t *testing.T) {
|
||||
for name, c := range map[string]struct {
|
||||
owner string
|
||||
ok bool
|
||||
}{
|
||||
"root": {"root", true},
|
||||
"the operator account": {"${machine:account}", true},
|
||||
"an account the module makes": {"mesh-mail-send", true},
|
||||
"an account not declared": {"not-made", false},
|
||||
} {
|
||||
_, err := ParseManifest([]byte(`{"module":"mesh-mail","version":"1.0","secrets-owner":"` + c.owner +
|
||||
`","resources":[{"id":"account","type":"user","name":"mesh-mail-send"}]}`))
|
||||
switch {
|
||||
case c.ok && err != nil:
|
||||
t.Errorf("%s: refused: %v", name, err)
|
||||
case !c.ok && (err == nil || !strings.Contains(err.Error(), c.owner)):
|
||||
t.Errorf("%s: %v; want a refusal naming %q", name, err, c.owner)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ADR 0680 §3, §4: a process naming an own secret runs as its resolved owner (its owner, else the module's
|
||||
// secrets-owner) or as root; any other account is refused; a module with a container is excepted.
|
||||
func caseAProcessNamingAnOwnSecretMustBeAbleToReadIt(t *testing.T) {
|
||||
const users = `{"id":"u1","type":"user","name":"alice"},{"id":"u2","type":"user","name":"bob"}`
|
||||
for name, c := range map[string]struct {
|
||||
secret, extra, proc string
|
||||
ok bool
|
||||
}{
|
||||
"the resolved owner": {`"path":"/var/lib/x/k","owner":"alice"`, ``, `"user":"alice","run":["./x","--key","/var/lib/x/k"]`, true},
|
||||
"the owner by secrets-owner": {`"path":"/var/lib/x/k"`, `"secrets-owner":"alice",`, `"user":"alice","env":{"KEY":"/var/lib/x/k"}`, true},
|
||||
"root": {`"path":"/var/lib/x/k","owner":"alice"`, ``, `"user":"root","run":["./x","/var/lib/x/k"]`, true},
|
||||
"the operator account": {`"path":"/var/lib/x/k","owner":"${machine:account}"`, ``, `"user":"${machine:account}","run":["./x","/var/lib/x/k"]`, true},
|
||||
"a secret it does not name": {`"path":"/var/lib/x/k","owner":"alice"`, ``, `"user":"bob","run":["./x"]`, true},
|
||||
"another account": {`"path":"/var/lib/x/k","owner":"alice"`, ``, `"user":"bob","run":["./x","/var/lib/x/k"]`, false},
|
||||
"another account by secrets-owner": {`"path":"/var/lib/x/k"`, `"secrets-owner":"alice",`, `"user":"bob","env":{"KEY":"/var/lib/x/k"}`, false},
|
||||
"a secret owned by root": {`"path":"/var/lib/x/k"`, ``, `"user":"bob","run":["./x","/var/lib/x/k"]`, false},
|
||||
"a longer word holding the path": {`"path":"/var/lib/x/k","owner":"alice"`, ``, `"user":"bob","env":{"KEY":"/var/lib/x/keyring"}`, true},
|
||||
"a container module": {`"path":"/var/lib/x/k","owner":"alice"`, ``, `"user":"bob","run":["./x","/var/lib/x/k"]},{"id":"c","type":"container","image":"x"`, true},
|
||||
} {
|
||||
raw := `{"module":"x","version":"1.0",` + c.extra + `"own-secrets":{"k":{` + c.secret + `}},"resources":[` + users +
|
||||
`,{"id":"p","type":"process","artifact":"x",` + c.proc + `}]}`
|
||||
_, err := ParseManifest([]byte(raw))
|
||||
switch {
|
||||
case c.ok && err != nil:
|
||||
t.Errorf("%s: refused: %v", name, err)
|
||||
case !c.ok && (err == nil || !strings.Contains(err.Error(), "ADR 0680 §3")):
|
||||
t.Errorf("%s: %v; want the ADR 0680 §3 refusal", name, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -2419,13 +2479,18 @@ func caseAPortWithNoSourceIsRefused(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// 1179/AC2: a bus on a process with no user, user root, or the operator account is refused naming the
|
||||
// process and the account rule.
|
||||
// ADR 0680 §2, §4: a bus on a process with no user, or a user the module does not make, is refused naming the
|
||||
// process; root and the operator account are accepted.
|
||||
func caseAProcessBusAccountMustBeTheModulesOwn(t *testing.T) {
|
||||
for name, c := range map[string]struct{ user, says string }{
|
||||
"no user": {"", "send"},
|
||||
"user root": {"root", "root"},
|
||||
"the operator account": {"${machine:account}", "operator"},
|
||||
for name, c := range map[string]struct {
|
||||
user string
|
||||
ok bool
|
||||
}{
|
||||
"no user": {"", false},
|
||||
"an account not declared": {"not-made", false},
|
||||
"user root": {"root", true},
|
||||
"the operator account": {"${machine:account}", true},
|
||||
"the module's own": {"mesh-mail-send", true},
|
||||
} {
|
||||
raw := `{"module":"mesh-mail","version":"1.0","emits":["mail.send.sent"],"state":["sends"],` +
|
||||
`"streams":[{"name":"mail-sends","events":["mail.send.approved"],"retention":"work-queue",` +
|
||||
@@ -2434,10 +2499,13 @@ func caseAProcessBusAccountMustBeTheModulesOwn(t *testing.T) {
|
||||
if c.user != "" {
|
||||
raw += `,"user":"` + c.user + `"`
|
||||
}
|
||||
raw += `,"bus":{"emits":["mail.send.sent"]}}]}`
|
||||
raw += `,"bus":{"emits":["mail.send.sent"],"queues":["mail-sends"]}}]}`
|
||||
_, err := ParseManifest([]byte(raw))
|
||||
if err == nil || !strings.Contains(err.Error(), "send") || !strings.Contains(err.Error(), c.says) {
|
||||
t.Errorf("%s: %v; want a refusal naming the process and %q", name, err, c.says)
|
||||
switch {
|
||||
case c.ok && err != nil:
|
||||
t.Errorf("%s: refused: %v", name, err)
|
||||
case !c.ok && (err == nil || !strings.Contains(err.Error(), "send")):
|
||||
t.Errorf("%s: %v; want a refusal naming the process", name, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -6414,6 +6482,8 @@ func TestRulesSecrets(t *testing.T) {
|
||||
t.Run("AnOffersIdentityIsParsedStrictly", caseAnOffersIdentityIsParsedStrictly)
|
||||
t.Run("AnOwnSecretSaysWhenAnOutsidePartyIssuesIt", caseAnOwnSecretSaysWhenAnOutsidePartyIssuesIt)
|
||||
t.Run("AnOwnSecretsOwnerIsRefusedWhenItIsNotAnAccountTheModuleMakes", caseAnOwnSecretsOwnerIsRefusedWhenItIsNotAnAccountTheModuleMakes)
|
||||
t.Run("ASecretsOwnerIsRefusedWhenItIsNotAnAccountTheModuleMakes", caseASecretsOwnerIsRefusedWhenItIsNotAnAccountTheModuleMakes)
|
||||
t.Run("AProcessNamingAnOwnSecretMustBeAbleToReadIt", caseAProcessNamingAnOwnSecretMustBeAbleToReadIt)
|
||||
t.Run("AnOwnSecretTakenSomeOtherWayIsRefused", caseAnOwnSecretTakenSomeOtherWayIsRefused)
|
||||
t.Run("AnUnknownAccessReferenceIsRefusedAtParse", caseAnUnknownAccessReferenceIsRefusedAtParse)
|
||||
t.Run("AnUnknownPlaceholderKeyIsRefused", caseAnUnknownPlaceholderKeyIsRefused)
|
||||
|
||||
@@ -165,7 +165,7 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build
|
||||
// "Reads FORGE" is a consumes of the indexer's stream: the broker grants only buckets by reads (mesh-control
|
||||
// internal/broker/state.go:62-93).
|
||||
// The registry seats' published fill each (§413: no consume of version.numbered) package's series (DLV-I5, design 51 *The versions*).
|
||||
if want := []string{"go-package-registry.published", "mesh-build-go.toolchain.reported", "mesh-build-node.toolchain.reported", "mesh-builds.package-published", "mesh-builds.toolchain-agreed", "mesh-conditions.findings-asked", "mesh-delivery-checker.verdict", "mesh-delivery-forge-bridge.forge.**",
|
||||
if want := []string{"go-package-registry.published", "mesh-build-go.toolchain.reported", "mesh-build-node.toolchain.reported", "mesh-builds.package-published", "mesh-builds.toolchain-agreed", "mesh-catalog.catalog-revision", "mesh-conditions.findings-asked", "mesh-delivery-checker.verdict", "mesh-delivery-forge-bridge.forge.**",
|
||||
"mesh-delivery.freed", "mesh-delivery.heads-replaced", "mesh-delivery.pinned", "mesh-delivery.pins-stated", "mesh-delivery.source-moved", "mesh-pipeline-runner.pipeline-ended", "mesh-pipeline-runner.run-started", "mesh-wants.version-delivered", "mesh-wants.wait-ended", "mesh-wants.wanted", "node-build-go.step-ended", "node-engine.running", "npm-package-registry.published", "operator-channel.decided", "seat:mesh-module-assigner.views.assignments", "seat:mesh-module-assigner.views.seat-holders"}; !reflect.DeepEqual(sorted(m.Consumes), want) {
|
||||
t.Errorf("consumes = %v, want %v", m.Consumes, want)
|
||||
}
|
||||
@@ -179,8 +179,6 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build
|
||||
// ADR 0499 clauses 2 and 3, ADR 0495 clause 1; and the module directories of the trunk, issue 1900) and whether a
|
||||
// commit is in another (issue 1900).
|
||||
if want := []string{"seat:git.trees", "seat:git.file", "seat:git.contains", "seat:git.open-pulls", "seat:git.branch", "mesh-delivery-forge-bridge.bridge_merged_message", "seat:node-engine.composition", "seat:mesh-delivery.source", "seat:mesh-module-assigner.assignments", "seat:mesh-module-assigner.placement", "seat:mesh-hand-acts.hand-act", "seat:mesh-watchdogs.presence", "seat:node-build-go.steps",
|
||||
"seat:node-build-go.accept.check", // design 47 G A-231: the validation's run ask (§712) on the check queue
|
||||
"seat:node-build-go.output", // design 47 G A-254: a validation's answer kept as an object
|
||||
"seat:node-build-node.steps", "seat:node-build-image.steps", "mesh-delivery-builder.builder_versions"}; !reflect.DeepEqual(m.Invokes, want) {
|
||||
t.Errorf("invokes = %v, want %v", m.Invokes, want)
|
||||
}
|
||||
@@ -204,9 +202,8 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build
|
||||
// milestone 47/G W4.12 (design 47, the indexer's tools): the catalog graph revisions read and steered.
|
||||
index.CatalogRevisionTool, index.CatalogRevisionsTool, index.CandidateGraphsTool, index.RecomputeTool, index.DropCandidateGraphTool,
|
||||
index.EntryIndexTool, index.NamedDropTool, // design 47 G §675
|
||||
index.ProposalsTool, index.ProposalDropTool, index.SeedAgainTool, index.InputsTool, index.PinsTool, index.UnpinTool, index.ConsumerTool, index.RunsTool, index.RunDropTool, // design 47 G §667, §8, §579, §64 (A-194)
|
||||
index.ValidateAgainTool) // design 47 G §713
|
||||
if !reflect.DeepEqual(m.Tools, want) || len(want) != 52 || !slices.Contains(m.Tools, "modules") {
|
||||
index.ProposalsTool, index.ProposalDropTool, index.SeedAgainTool, index.InputsTool, index.PinsTool, index.UnpinTool, index.ConsumerTool, index.RunsTool, index.RunDropTool) // design 47 G §667, §8, §579, §64 (A-194)
|
||||
if !reflect.DeepEqual(m.Tools, want) || len(want) != 51 || !slices.Contains(m.Tools, "modules") {
|
||||
t.Errorf("tools = %v, want %v", m.Tools, want)
|
||||
}
|
||||
// Design 47 G §2, §9: every version of every registration is kept with no age limit, 64 MiB, on the old streams section.
|
||||
|
||||
@@ -77,8 +77,7 @@
|
||||
"indexer_unpin",
|
||||
"indexer_consumer",
|
||||
"indexer_runs",
|
||||
"indexer_run_drop",
|
||||
"indexer_validate_again"
|
||||
"indexer_run_drop"
|
||||
],
|
||||
"consumes": [
|
||||
"go-package-registry.published",
|
||||
@@ -94,6 +93,7 @@
|
||||
"mesh-wants.wait-ended",
|
||||
"mesh-delivery-checker.verdict",
|
||||
"mesh-delivery-forge-bridge.forge.**",
|
||||
"mesh-catalog.catalog-revision",
|
||||
"mesh-delivery.source-moved",
|
||||
"mesh-delivery.pinned",
|
||||
"mesh-delivery.freed",
|
||||
@@ -179,8 +179,6 @@
|
||||
"seat:mesh-hand-acts.hand-act",
|
||||
"seat:mesh-watchdogs.presence",
|
||||
"seat:node-build-go.steps",
|
||||
"seat:node-build-go.accept.check",
|
||||
"seat:node-build-go.output",
|
||||
"seat:node-build-node.steps",
|
||||
"seat:node-build-image.steps",
|
||||
"mesh-delivery-builder.builder_versions"
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
// Package takein is the indexer's take-in contract (novox/hq ADR 0633 §1, §5): the events it publishes on its own stream
|
||||
// `take-in` while a source revision waits for the indexer it names or a delivery group is gathered, their keys, and the
|
||||
// `take-in` while a source revision waits (its message unread, lost or held for room) or a delivery group is gathered, their keys, and the
|
||||
// answer of its tool indexer_pending. The indexer writes them; the watchman and the view read them. It is the indexer's
|
||||
// own package: one module owns these types, so they live with it and not in mesh-sdk (the operator, 2026-11-04).
|
||||
package takein
|
||||
@@ -118,7 +118,7 @@ func checkTakeInKey(key string) error {
|
||||
}
|
||||
|
||||
// TakeIn is the body of a take-in event. Kind is the event's kind and Key its key. What says what waits: a revision
|
||||
// (Key its oldest merge's PendingKey, WaitsFor the indexer build it names) or a group being gathered (Key its
|
||||
// (Key its oldest merge's PendingKey) or a group being gathered (Key its
|
||||
// GroupTakeInKey, Members every approved member with its head, Merges the members held so far). Merges are the merges
|
||||
// it holds, oldest first; Since is when the wait began. A `taken` event names the same once the wait ended; By and Why
|
||||
// name a hand act (a hand take-in, or a drop, Dropped), and are carried by nothing else.
|
||||
@@ -138,16 +138,9 @@ type TakeIn struct {
|
||||
// Lost are the keys of merges whose message the forge bridge's stream no longer holds: they are not taken in, and the
|
||||
// revision waits until indexer_drop_pending drops them (issue 1585).
|
||||
Lost []string `json:"lost,omitempty"`
|
||||
// Refused is why the manifest of the indexer the revision names was refused when the revision was worked out: with
|
||||
// the seat definitions the revision carries (ADR 0633 §1), the manifest validator's problems. The revision waits for
|
||||
// an indexer whose manifest is accepted, and says why here, as it says what it lost.
|
||||
// Refused is why the revision is refused: the cycle with a build edge its catalog graph would hold (design 47 G §49). The
|
||||
// revision waits, and says why here, as it says what it lost.
|
||||
Refused string `json:"refused,omitempty"`
|
||||
// Failed is why the revision's validation could not validate (design 47 G §713): the build seat's words, never a refusal
|
||||
// and never passed; it is asked again by a later merge of the branch and by indexer_validate_again.
|
||||
Failed string `json:"failed,omitempty"`
|
||||
// Unknown is why the indexer holds a merge it does not take in although no indexer build is named: it cannot learn the build
|
||||
// it runs itself (ADR 0633 §1), so Since is when it began not to know. A pending revision names WaitsFor or Unknown.
|
||||
Unknown string `json:"own-build-unknown,omitempty"`
|
||||
// Branch, Source and Entries are what a take-in took in (design 47 G §476, §493, §648): the watched branch, the source
|
||||
// revision it makes of that branch, and one entry per module it wrote. Every taken event of one take-in carries the same;
|
||||
// a taken event that ends a wait without a take-in (a group's end, a drop) carries none and starts no pipeline.
|
||||
@@ -184,8 +177,8 @@ func (t TakeIn) check() error {
|
||||
if !strings.HasPrefix(t.Key, "m.") {
|
||||
return fmt.Errorf("delivery: a revision's take-in key is its oldest merge's m.<sequence>, not %q", t.Key)
|
||||
}
|
||||
if t.Kind == TakeInPending && t.WaitsFor == "" && t.Unknown == "" && len(t.Lost) == 0 && t.Refused == "" && t.Failed == "" {
|
||||
return fmt.Errorf("delivery: a pending revision %s names the indexer build it waits for, why the indexer does not know its own, the merges it lost, its refusal or why it could not validate", t.Key)
|
||||
if t.Kind == TakeInPending && len(t.Lost) == 0 && t.Refused == "" {
|
||||
return fmt.Errorf("delivery: a pending revision %s names the merges it lost or its refusal", t.Key)
|
||||
}
|
||||
case TakeInGroup:
|
||||
if t.Group == "" {
|
||||
@@ -266,32 +259,15 @@ type PendingWait struct {
|
||||
Unread []string `json:"unread,omitempty"`
|
||||
// Refused is why the manifest of the indexer it names was refused (TakeIn.Refused).
|
||||
Refused string `json:"refused,omitempty"`
|
||||
// Validation is the revision's validation whole, as its newest pending value keeps it (design 47 G §713, A-247).
|
||||
Validation *Validation `json:"validation,omitempty"`
|
||||
}
|
||||
|
||||
// Validation is a pending revision's validation (design 47 G §712, §713, A-232, A-247), kept on the revision's newest pending
|
||||
// value alone: the run ask's id, when it was asked, its input's digest, the sequence of the newest merge it covers and the try
|
||||
// number for it, why it could not validate, and the sequence it was accepted at (never asked again once set).
|
||||
type Validation struct {
|
||||
Ask string `json:"ask,omitempty"`
|
||||
Asked time.Time `json:"asked,omitzero"`
|
||||
Input string `json:"input,omitempty"`
|
||||
Seq uint64 `json:"seq"`
|
||||
Try int `json:"try"`
|
||||
Failed string `json:"failed,omitempty"`
|
||||
Accepted uint64 `json:"accepted,omitempty"`
|
||||
}
|
||||
|
||||
// PendingAnswer is indexer_pending's answer: every wait, oldest first, and the pending keys that do not read.
|
||||
type PendingAnswer struct {
|
||||
Waits []PendingWait `json:"waits"`
|
||||
Unread []string `json:"unread,omitempty"`
|
||||
// OwnBuildUnknown is why the running indexer does not know its own build, and since when: while it is set, every merge waits
|
||||
// (a take-in.pending with the same words) and the indexer is started again to learn it. Empty when the build is known.
|
||||
OwnBuildUnknown string `json:"own-build-unknown,omitempty"`
|
||||
OwnBuildSince time.Time `json:"own-build-since,omitzero"`
|
||||
// OwnBuild is the build the running indexer knows it runs, fixed at its start; OwnBuildSince is since when. Empty when unknown.
|
||||
Waits []PendingWait `json:"waits"`
|
||||
Unread []string `json:"unread,omitempty"`
|
||||
OwnBuildSince time.Time `json:"own-build-since,omitzero"`
|
||||
// OwnBuild is the build the running indexer knows it runs, fixed at its start; OwnBuildSince is since when. Empty when unknown:
|
||||
// it names only the writer of what the indexer writes, and nothing waits for it.
|
||||
OwnBuild string `json:"own-build,omitempty"`
|
||||
}
|
||||
|
||||
|
||||
@@ -16,15 +16,12 @@ import (
|
||||
func sampleTakeIns(since time.Time) []TakeIn {
|
||||
m := []Merge{{Repository: "novox/mesh-delivery", Number: 9, Merge: "eeee"}}
|
||||
return []TakeIn{
|
||||
{Kind: TakeInPending, Key: PendingKey(12), What: TakeInRevision, Merges: m, WaitsFor: "src2:abc", Since: since},
|
||||
{Kind: TakeInPending, Key: PendingKey(12), What: TakeInRevision, Merges: m, Lost: []string{"m.00000000000000000011"}, Since: since},
|
||||
{Kind: TakeInPending, Key: GroupTakeInKey("feat/take-in"), What: TakeInGroup, Merges: m, Group: "feat/take-in",
|
||||
Members: []meshdeliveryclient.Head{{Repository: "novox/mesh-delivery", Number: 9, Head: "ffff"}, {Repository: "novox/mesh-sdk", Number: 37, Head: "dddd"}}, Since: since},
|
||||
{Kind: TakeInTaken, Key: PendingKey(12), What: TakeInRevision, Merges: m, WaitsFor: "src2:abc", Since: since,
|
||||
By: "lead-47@shanks", Why: "the named indexer can never run"},
|
||||
{Kind: TakeInPending, Key: PendingKey(13), What: TakeInRevision, Merges: m, Unknown: "the engine does not answer", Since: since}, // the indexer cannot learn its own build
|
||||
// design 47 G §713: a revision refused by its validation, and one that could not validate.
|
||||
{Kind: TakeInPending, Key: PendingKey(14), What: TakeInRevision, Merges: m, Refused: "a: names no version", Since: since},
|
||||
{Kind: TakeInPending, Key: PendingKey(15), What: TakeInRevision, Merges: m, Failed: "could not run: no build node", Since: since},
|
||||
{Kind: TakeInPending, Key: PendingKey(14), What: TakeInRevision, Merges: m, Refused: "a cycle with a build edge", Since: since},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -73,7 +70,7 @@ func TestTakeInEvents(t *testing.T) {
|
||||
}{
|
||||
{"group take-in without its group", "names its group", 1, func(t *TakeIn) { t.Group = "" }},
|
||||
{"group take-in under another group's key", "g.other", 1, func(t *TakeIn) { t.Key = GroupTakeInKey("other") }},
|
||||
{"pending revision waiting for no build", "lost", 0, func(t *TakeIn) { t.WaitsFor = "" }},
|
||||
{"pending revision naming neither a lost merge nor a refusal", "lost", 0, func(t *TakeIn) { t.Lost = nil }},
|
||||
{"pending with a hand act", "only a taken event", 0, func(t *TakeIn) { t.By = "someone@node" }},
|
||||
{"hand act without its by", "by", 2, func(t *TakeIn) { t.By = "" }},
|
||||
{"take-in without merges", "no merge", 0, func(t *TakeIn) { t.Merges = nil }},
|
||||
|
||||
@@ -15,6 +15,8 @@ mesh-catalog b5dbfbb784507e211372a7c9640d91f7d6dadd84 modules/*/module.json, m
|
||||
mesh-host 233bb23bbcb1a829ce2a6164a31a5c5cc4f3d950 internal/apply/placement_guard.go
|
||||
mesh-catalog 2d5358b8 (milestone 47/G's AM-K1) modules/zsh/module.json alone: the login shell uses operator-channel for
|
||||
its seat's marked execute (design 46 approval mark §21), which the validator requires (§4)
|
||||
mesh-catalog cd3b8fe8 (restic's secrets-owner, ADR 0680) modules/restic/module.json and modules/restic/files/** alone: its
|
||||
processes run as the operator account, so the validator (ADR 0680 §3) needs the secrets-owner it names
|
||||
|
||||
To move them, from mesh-delivery's modules/mesh-delivery-indexer directory, with the two repositories checked out
|
||||
beside mesh-delivery:
|
||||
|
||||
Vendored
+1
@@ -0,0 +1 @@
|
||||
${config:put-off-bound}
|
||||
Vendored
+131
-59
@@ -1,6 +1,9 @@
|
||||
{
|
||||
"module": "restic",
|
||||
"version": "1",
|
||||
"version": "0.1",
|
||||
"packages": {
|
||||
"git.novox.be/novox/mesh-catalog/modules/disk-load/diskreadings": "0.1"
|
||||
},
|
||||
"moved": true,
|
||||
"claims": [
|
||||
{
|
||||
@@ -13,13 +16,27 @@
|
||||
]
|
||||
}
|
||||
],
|
||||
"invokes": [
|
||||
"seat:mesh-module-assigner.assignments",
|
||||
"seat:mesh-broker.show"
|
||||
],
|
||||
"consumes": [
|
||||
"mesh-recovery.restore-started",
|
||||
"mesh-recovery.restored",
|
||||
"mesh-conditions.findings-asked"
|
||||
],
|
||||
"emits": [
|
||||
"backed-up"
|
||||
],
|
||||
"tools": [
|
||||
"backup_health",
|
||||
"backup_delete_retired",
|
||||
"backup_deleted"
|
||||
],
|
||||
"secrets-owner": "${machine:account}",
|
||||
"own-secrets": {
|
||||
"repository": "${dir:state}/repository.secret"
|
||||
"repository": "${dir:state}/repository.secret",
|
||||
"broker": "${dir:state}/broker"
|
||||
},
|
||||
"data": {
|
||||
"own": [
|
||||
@@ -38,40 +55,48 @@
|
||||
}
|
||||
]
|
||||
},
|
||||
"config": {
|
||||
"keep-daily": {
|
||||
"type": "number",
|
||||
"description": "How many nightly restore points are kept, a count from 1 to 9999: two weeks undo a mistake noticed within a fortnight. Read by the night, which thins nothing while it does not read (novox/hq issue 416); lowering it deletes the restore points beyond the new count at the next night, and that cannot be undone.",
|
||||
"minimum": 1,
|
||||
"maximum": 9999,
|
||||
"trusted": true,
|
||||
"default": 14
|
||||
},
|
||||
"keep-weekly": {
|
||||
"type": "number",
|
||||
"description": "How many weekly restore points are kept behind the dailies, a count from 1 to 9999, refused by the night like keep-daily; lowering it deletes the restore points beyond the new count at the next night, and that cannot be undone.",
|
||||
"minimum": 1,
|
||||
"maximum": 9999,
|
||||
"trusted": true,
|
||||
"default": 8
|
||||
},
|
||||
"keep-monthly": {
|
||||
"type": "number",
|
||||
"description": "How many monthly restore points are kept behind the weeklies, a count from 1 to 9999, refused by the night like keep-daily; lowering it deletes the restore points beyond the new count at the next night, and that cannot be undone.",
|
||||
"minimum": 1,
|
||||
"maximum": 9999,
|
||||
"trusted": true,
|
||||
"default": 6
|
||||
},
|
||||
"keep-hand-taken-days": {
|
||||
"type": "number",
|
||||
"description": "How many days a restore point taken by hand (node-backup.now, or the mesh's before a module moves) is kept at least, a count from 1 to 9999; it is kept until a later good night of its module exists, and this many days in any case, so the next thinning never takes the one taken against a risky change (novox/hq issue 453). Lowering it lets go of the older ones taken by hand at the next thinning, and that cannot be undone.",
|
||||
"minimum": 1,
|
||||
"maximum": 9999,
|
||||
"trusted": true,
|
||||
"default": 7
|
||||
}
|
||||
"config": {
|
||||
"keep-daily": {
|
||||
"type": "number",
|
||||
"description": "How many nightly restore points are kept, a count from 1 to 9999: two weeks undo a mistake noticed within a fortnight. Read by the night, which thins nothing while it does not read (novox/hq issue 416); lowering it deletes the restore points beyond the new count at the next night, and that cannot be undone.",
|
||||
"minimum": 1,
|
||||
"maximum": 9999,
|
||||
"trusted": true,
|
||||
"default": 14
|
||||
},
|
||||
"keep-weekly": {
|
||||
"type": "number",
|
||||
"description": "How many weekly restore points are kept behind the dailies, a count from 1 to 9999, refused by the night like keep-daily; lowering it deletes the restore points beyond the new count at the next night, and that cannot be undone.",
|
||||
"minimum": 1,
|
||||
"maximum": 9999,
|
||||
"trusted": true,
|
||||
"default": 8
|
||||
},
|
||||
"keep-monthly": {
|
||||
"type": "number",
|
||||
"description": "How many monthly restore points are kept behind the weeklies, a count from 1 to 9999, refused by the night like keep-daily; lowering it deletes the restore points beyond the new count at the next night, and that cannot be undone.",
|
||||
"minimum": 1,
|
||||
"maximum": 9999,
|
||||
"trusted": true,
|
||||
"default": 6
|
||||
},
|
||||
"keep-hand-taken-days": {
|
||||
"type": "number",
|
||||
"description": "How many days a restore point taken by hand (node-backup.now, or the mesh's before a module moves) is kept at least, a count from 1 to 9999; it is kept until a later good night of its module exists, and this many days in any case, so the next thinning never takes the one taken against a risky change (novox/hq issue 453). Lowering it lets go of the older ones taken by hand at the next thinning, and that cannot be undone.",
|
||||
"minimum": 1,
|
||||
"maximum": 9999,
|
||||
"trusted": true,
|
||||
"default": 7
|
||||
},
|
||||
"put-off-bound": {
|
||||
"type": "number",
|
||||
"description": "minutes an item put off for a restore of the recovery module is waited for before the finding backup-waits is written and the wait ends, so the night never holds its lock without a bound; timed by a bound key of the state put-off (design 47 G sections 561, 557, A-155).",
|
||||
"minimum": 1,
|
||||
"maximum": 10080,
|
||||
"trusted": false,
|
||||
"default": 60
|
||||
}
|
||||
},
|
||||
"resources": [
|
||||
{
|
||||
"id": "state",
|
||||
@@ -108,6 +133,13 @@
|
||||
"trusted": true,
|
||||
"template": "files/retention.conf"
|
||||
},
|
||||
{
|
||||
"id": "put-off-bound",
|
||||
"type": "file",
|
||||
"path": "${dir:state}/put-off-bound",
|
||||
"mode": "0600",
|
||||
"template": "files/put-off-bound"
|
||||
},
|
||||
{
|
||||
"id": "tool",
|
||||
"type": "package",
|
||||
@@ -133,7 +165,32 @@
|
||||
"MESH_BACKUP_REPOSITORY": "${dir:repository}",
|
||||
"MESH_BACKUP_PASSWORD_FILE": "${dir:state}/repository.secret",
|
||||
"MESH_BACKUP_STATE": "${dir:state}",
|
||||
"MESH_BACKUP_DATA": "${dir:state}/data.conf"
|
||||
"MESH_BACKUP_DATA": "${dir:state}/data.conf",
|
||||
"MESH_BROKER_FILE": "${dir:state}/broker",
|
||||
"MESH_BACKUP_PUT_OFF_BOUND_FILE": "${dir:state}/put-off-bound"
|
||||
},
|
||||
"health": {
|
||||
"kind": "unit"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "awake",
|
||||
"type": "process",
|
||||
"name": "restic-awake",
|
||||
"artifact": "tools",
|
||||
"run": [
|
||||
"./restic-backups",
|
||||
"awake"
|
||||
],
|
||||
"user": "${machine:account}",
|
||||
"env": {
|
||||
"MESH_BACKUP_DECLARED": "${dir:state}/backups.conf",
|
||||
"MESH_BACKUP_REPOSITORY": "${dir:repository}",
|
||||
"MESH_BACKUP_PASSWORD_FILE": "${dir:state}/repository.secret",
|
||||
"MESH_BACKUP_STATE": "${dir:state}",
|
||||
"MESH_BACKUP_DATA": "${dir:state}/data.conf",
|
||||
"MESH_BROKER_FILE": "${dir:state}/broker",
|
||||
"MESH_BACKUP_PUT_OFF_BOUND_FILE": "${dir:state}/put-off-bound"
|
||||
},
|
||||
"health": {
|
||||
"kind": "tool",
|
||||
@@ -143,27 +200,6 @@
|
||||
"looks": 2,
|
||||
"grace": "60s"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "measure",
|
||||
"type": "process",
|
||||
"name": "restic-measure",
|
||||
"artifact": "tools",
|
||||
"run": [
|
||||
"./restic-backups",
|
||||
"measure"
|
||||
],
|
||||
"user": "${machine:account}",
|
||||
"env": {
|
||||
"MESH_BACKUP_DECLARED": "${dir:state}/backups.conf",
|
||||
"MESH_BACKUP_REPOSITORY": "${dir:repository}",
|
||||
"MESH_BACKUP_PASSWORD_FILE": "${dir:state}/repository.secret",
|
||||
"MESH_BACKUP_STATE": "${dir:state}",
|
||||
"MESH_BACKUP_DATA": "${dir:state}/data.conf"
|
||||
},
|
||||
"health": {
|
||||
"kind": "unit"
|
||||
}
|
||||
}
|
||||
],
|
||||
"build": {
|
||||
@@ -183,7 +219,8 @@
|
||||
"MESH_BACKUP_REPOSITORY": "${dir:repository}",
|
||||
"MESH_BACKUP_PASSWORD_FILE": "${dir:state}/repository.secret",
|
||||
"MESH_BACKUP_STATE": "${dir:state}",
|
||||
"MESH_BACKUP_DATA": "${dir:state}/data.conf"
|
||||
"MESH_BACKUP_DATA": "${dir:state}/data.conf",
|
||||
"MESH_BACKUP_PUT_OFF_BOUND_FILE": "${dir:state}/put-off-bound"
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -192,5 +229,40 @@
|
||||
"from": "files"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"reads": [
|
||||
"disk-load.readings"
|
||||
],
|
||||
"state": [
|
||||
"good"
|
||||
],
|
||||
"requires": [
|
||||
{
|
||||
"name": "put-off",
|
||||
"storage": "key-value",
|
||||
"template": "state",
|
||||
"options": {
|
||||
"key-ttl": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "findings",
|
||||
"storage": "key-value",
|
||||
"template": "state"
|
||||
},
|
||||
{
|
||||
"name": "finding-events",
|
||||
"storage": "stream",
|
||||
"template": "events",
|
||||
"options": {
|
||||
"events": [
|
||||
"finding-written.*",
|
||||
"finding-cleared.*",
|
||||
"findings-stated.*"
|
||||
],
|
||||
"max-age-days": 7,
|
||||
"max-mib": 16
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@@ -152,6 +152,7 @@ func main() {
|
||||
tools = append(tools, UnloggedTools(stdio.State(seat.StatePipelines), stdio.Log(seat.StateStepLog), meshhandactsclient.Recorder{Ask: stdio.Ask},
|
||||
func(f string, a ...any) { fmt.Fprintf(os.Stderr, "[mesh-delivery-pipeline-tools] "+f+"\n", a...) })...)
|
||||
tools = append(tools, CancelTool(r.List, stdio.Ask), RerunTool(stdio.Ask))
|
||||
tools = append(tools, VerbTools(stdio.Ask)...)
|
||||
tools = append(tools, TestRunsTools(stdio.Ask)...)
|
||||
tools = append(tools, HealthTool(r.Pipelines, kept{stdio.State(pipeline.StateConsumers)}, time.Now))
|
||||
if err := stdio.Serve("", tools); err != nil {
|
||||
|
||||
@@ -32,13 +32,16 @@ func TestTools(t *testing.T) {
|
||||
names = append(names, tl.Name)
|
||||
}
|
||||
names = append(names, CancelTool(nil, nil).Name, RerunTool(nil).Name)
|
||||
for _, tl := range VerbTools(nil) {
|
||||
names = append(names, tl.Name)
|
||||
}
|
||||
// RN25 (§468, §76; A-164): the health check is the bundle's runner_health.
|
||||
names = append(names, HealthTool(none{}, none{}, nil).Name)
|
||||
// §606, §549: the unlogged entries are listed and appended by hand.
|
||||
for _, tl := range UnloggedTools(stdio.State("pipelines"), stdio.Log("step-log"), nil, nil) {
|
||||
names = append(names, tl.Name)
|
||||
}
|
||||
want := "mesh-pipeline-runner.show mesh-pipeline-runner.list mesh-pipeline-runner.stands mesh-pipeline-runner.why-carried mesh-pipeline-runner.kept runner_first_views runner_first_view_again runner_cancel runner_rerun runner_health runner_unlogged runner_unlogged_append"
|
||||
want := "mesh-pipeline-runner.show mesh-pipeline-runner.list mesh-pipeline-runner.stands mesh-pipeline-runner.why-carried mesh-pipeline-runner.kept runner_first_views runner_first_view_again runner_cancel runner_rerun runner_start runner_run runner_accept runner_ask_again runner_health runner_unlogged runner_unlogged_append"
|
||||
if got := strings.Join(names, " "); got != want {
|
||||
t.Errorf("tools %q, want %q", got, want)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,81 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
|
||||
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
)
|
||||
|
||||
// VerbTools are runner_start, runner_run, runner_accept and runner_ask_again: the runner's seat verbs start, run, accept and
|
||||
// ask-again, reachable as tools (design 47 G §83, §482), as runner_rerun reaches rerun. Each asks its verb over the bus with
|
||||
// the arguments it was given and answers the verb's answer; the verb records the hand act and refuses what it refuses.
|
||||
func VerbTools(ask func(key string, body any) (json.RawMessage, error)) []stdio.Tool {
|
||||
str := map[string]any{"type": "string"}
|
||||
obj := func(required []string, props map[string]any) map[string]any {
|
||||
return map[string]any{"type": "object", "properties": props, "required": required}
|
||||
}
|
||||
return []stdio.Tool{
|
||||
verbTool[seat.StartInput](ask, "start", seat.VerbStart,
|
||||
"Start by hand the delivering pipeline of the head of a watched branch for a module, by the seat verb start: the branch, the module and why. A head whose pipeline is written already is refused by the verb, naming it.",
|
||||
obj([]string{"branch", "module", "why"}, map[string]any{"branch": str, "module": str, "why": str}),
|
||||
func(in seat.StartInput) string {
|
||||
if in.Branch == "" || in.Module == "" {
|
||||
return "start names the watched branch and the module"
|
||||
}
|
||||
return whyOf(in.Why)
|
||||
}),
|
||||
verbTool[seat.RunInput](ask, "run", seat.VerbRun,
|
||||
"Start a run on request by the seat verb run: the members (module, branch, commit as the seat's Member) and optionally why.",
|
||||
obj([]string{"members"}, map[string]any{"members": map[string]any{"type": "array", "items": map[string]any{"type": "object"}}, "why": str}),
|
||||
func(in seat.RunInput) string {
|
||||
if len(in.Members) == 0 {
|
||||
return "run names its members"
|
||||
}
|
||||
return ""
|
||||
}),
|
||||
verbTool[seat.AcceptInput](ask, "accept", seat.VerbAccept,
|
||||
"Accept one step of a pipeline by hand by the seat verb accept: the pipeline's key, the step, the artifact where the step is per artifact, and why.",
|
||||
obj([]string{"pipeline", "step", "why"}, map[string]any{"pipeline": str, "step": str, "artifact": str, "why": str}),
|
||||
func(in seat.AcceptInput) string {
|
||||
if in.Pipeline == "" || in.Step == "" {
|
||||
return "accept names the pipeline and the step"
|
||||
}
|
||||
return whyOf(in.Why)
|
||||
}),
|
||||
verbTool[seat.AskAgainInput](ask, "ask_again", seat.VerbAskAgain,
|
||||
"Ask a declined approval step of a pipeline once more by the seat verb ask-again: the pipeline's key and why.",
|
||||
obj([]string{"pipeline", "why"}, map[string]any{"pipeline": str, "why": str}),
|
||||
func(in seat.AskAgainInput) string {
|
||||
if in.Pipeline == "" {
|
||||
return "ask-again names the pipeline"
|
||||
}
|
||||
return whyOf(in.Why)
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
func whyOf(why string) string {
|
||||
if strings.TrimSpace(why) == "" {
|
||||
return "a hand act names why"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// verbTool is one tool over one seat verb: it decodes the arguments into the verb's input, refuses what the check refuses
|
||||
// before asking, and answers the verb's answer.
|
||||
func verbTool[T any](ask func(string, any) (json.RawMessage, error), name, verb, description string, input map[string]any, check func(T) string) stdio.Tool {
|
||||
return stdio.Tool{Name: Prefix + "_" + name, Description: description, Input: input,
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
in, err := as[T](args)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if msg := check(in); msg != "" {
|
||||
return nil, errors.New(msg)
|
||||
}
|
||||
return ask("seat:"+seat.Seat+"."+verb, in)
|
||||
}}
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||
)
|
||||
|
||||
// Each tool asks its seat verb with the arguments it was given and refuses, before asking, a call the verb would refuse.
|
||||
func TestVerbTools(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
name, tool, key, refusal string
|
||||
args map[string]any
|
||||
body any
|
||||
}{
|
||||
{"start", "runner_start", "seat:mesh-pipeline-runner.start", "", map[string]any{"branch": "main", "module": "i3", "why": "cancelled"},
|
||||
seat.StartInput{Branch: "main", Module: "i3", Why: "cancelled"}},
|
||||
{"start no why", "runner_start", "", "names why", map[string]any{"branch": "main", "module": "i3"}, nil},
|
||||
{"start no module", "runner_start", "", "branch and the module", map[string]any{"branch": "main", "why": "x"}, nil},
|
||||
{"run", "runner_run", "seat:mesh-pipeline-runner.run", "", map[string]any{"members": []any{map[string]any{}}, "why": "x"},
|
||||
seat.RunInput{Members: []seat.Member{{}}, Why: "x"}},
|
||||
{"run no members", "runner_run", "", "members", map[string]any{}, nil},
|
||||
{"accept", "runner_accept", "seat:mesh-pipeline-runner.accept", "", map[string]any{"pipeline": "p", "step": "lint", "artifact": "a", "why": "x"},
|
||||
seat.AcceptInput{Pipeline: "p", Step: "lint", Artifact: "a", Why: "x"}},
|
||||
{"accept no step", "runner_accept", "", "pipeline and the step", map[string]any{"pipeline": "p", "why": "x"}, nil},
|
||||
{"ask-again", "runner_ask_again", "seat:mesh-pipeline-runner.ask-again", "", map[string]any{"pipeline": "p", "why": "x"},
|
||||
seat.AskAgainInput{Pipeline: "p", Why: "x"}},
|
||||
{"ask-again no why", "runner_ask_again", "", "names why", map[string]any{"pipeline": "p"}, nil},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
var key string
|
||||
var body any
|
||||
ask := func(k string, b any) (json.RawMessage, error) {
|
||||
key, body = k, b
|
||||
return json.RawMessage(`{"ok":1}`), nil
|
||||
}
|
||||
var run func(map[string]any) (any, error)
|
||||
for _, tl := range VerbTools(ask) {
|
||||
if tl.Name == c.tool {
|
||||
run = tl.Run
|
||||
}
|
||||
}
|
||||
out, err := run(c.args)
|
||||
if c.refusal != "" {
|
||||
if err == nil || !strings.Contains(err.Error(), c.refusal) || key != "" {
|
||||
t.Fatalf("want refusal %q and no ask, got %v (asked %q)", c.refusal, err, key)
|
||||
}
|
||||
return
|
||||
}
|
||||
want, _ := json.Marshal(c.body)
|
||||
got, _ := json.Marshal(body)
|
||||
if err != nil || key != c.key || string(got) != string(want) || out == nil {
|
||||
t.Fatalf("asked %q with %s, want %s, err %v", key, got, want, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -127,10 +127,17 @@ func place(p seat.Pipeline, reg catalog.Registration, entries map[string]catalog
|
||||
steps = append(steps, s)
|
||||
continue
|
||||
}
|
||||
// The ask names the language of the seat it is asked of, never the artifact's word (design 47 G A-488).
|
||||
language, err := sdkstep.AskLanguageOfSeat(bs)
|
||||
if err != nil {
|
||||
s.Outcome, s.Started, s.Ended, s.Why = seat.StepResultOutcomeCouldNotRun, &now, &now, err.Error()
|
||||
steps = append(steps, s)
|
||||
continue
|
||||
}
|
||||
s.Seat, s.Outcome, s.Started = bs, seat.StepResultOutcomeRunning, &now
|
||||
s.Ask = AskID(startOf(p, now), p.Pipeline, a.Ref, step)
|
||||
body := stepAsk(p, entry, a, step, s.Ask, asker, now)
|
||||
ask := Asked{Seat: bs, ID: s.Ask, Ask: body}
|
||||
body := stepAsk(p, entry, a, language, step, s.Ask, asker, now)
|
||||
ask := Asked{Seat: bs, ID: s.Ask, Ask: body, Toolchain: a.Language}
|
||||
if share && BoundSeat(bs) {
|
||||
digest := ShareDigest(p.Fingerprint, step, status, body.Step.Fetch)
|
||||
run := TestRunKey(candidate, step, digest)
|
||||
@@ -189,17 +196,20 @@ type Asked struct {
|
||||
// Run is the test run the ask asks, its key in test-runs, with its level, its share digest and the steps of the pipeline
|
||||
// reading it, the asking step first (§724, §725); empty for an ask of no test run.
|
||||
Run, Level, Digest string
|
||||
Readers []seat.TestRunReader
|
||||
// Toolchain is the language of the artifact's toolchain as the catalog keys it (go, typescript; design 47 G A-104, A-488b),
|
||||
// never the ask's language, which names the seat; empty for an artifact with none (an image).
|
||||
Toolchain string
|
||||
Readers []seat.TestRunReader
|
||||
}
|
||||
|
||||
func stepAsk(p seat.Pipeline, e catalog.CatalogEntry, a catalog.Artifact, step, id, asker string, now time.Time) nodebuildgo.StepAsk {
|
||||
func stepAsk(p seat.Pipeline, e catalog.CatalogEntry, a catalog.Artifact, language, step, id, asker string, now time.Time) nodebuildgo.StepAsk {
|
||||
// A step is for its module alone (principle 10, A-447): the fetch always names the module's directory, and the
|
||||
// artifact's named paths in it when the entry has them.
|
||||
fetch := nodebuildgo.Fetch{Repository: p.Repository, Commit: p.Commit, Dir: e.Path}
|
||||
if paths := artifactPaths(e, a.Ref); len(paths) > 0 {
|
||||
fetch.Paths = paths
|
||||
}
|
||||
return nodebuildgo.StepAsk{V: 1, ID: id, Asker: asker, Kind: "step", Language: a.Language, At: &now,
|
||||
return nodebuildgo.StepAsk{V: 1, ID: id, Asker: asker, Kind: "step", Language: language, At: &now,
|
||||
Step: &nodebuildgo.StepRun{Artifact: a.Ref, Branch: p.Branch, Commit: p.Commit, Fetch: fetch, Fingerprint: p.Fingerprint,
|
||||
Module: p.Module, Pipeline: p.Pipeline, Revision: p.Source, Step: step}}
|
||||
}
|
||||
|
||||
@@ -242,7 +242,9 @@ func TestAStepAskCarriesTheKeptResultsAndTheToolchain(t *testing.T) {
|
||||
reg, entries := placeFixture()
|
||||
e := entries["app"]
|
||||
e.Artifacts[0].Package = &catalog.ArtifactPackage{Ecosystem: "go", Name: "git.example/o/app"}
|
||||
e.Artifacts = append(e.Artifacts, catalog.Artifact{Ref: "app:lib", Language: "typescript"})
|
||||
entries["app"] = e
|
||||
entries["mesh-build-node"] = catalog.CatalogEntry{Module: "mesh-build-node", Acts: []catalog.ActingRule{{Seat: "node-build-node", Step: "build", On: "typescript"}}}
|
||||
store, asks := created{}, &stepAsks{}
|
||||
reads := 0
|
||||
s := &Starter{Pipelines: store, Say: func(string, any) error { return nil }, Steps: asks, Name: "the-runner",
|
||||
@@ -260,7 +262,7 @@ func TestAStepAskCarriesTheKeptResultsAndTheToolchain(t *testing.T) {
|
||||
if _, err := s.Start(Begin{Branch: "main", Source: 4, Module: "app", Commit: "c1", Fp: "src2:aa", Repository: "o/app", Entries: entries}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var program, tools *nodebuildgo.StepRun
|
||||
var program, tools, lib *nodebuildgo.StepRun
|
||||
for _, a := range asks.asks {
|
||||
if ask, ok := a.(nodebuildgo.StepAsk); ok {
|
||||
switch {
|
||||
@@ -268,6 +270,8 @@ func TestAStepAskCarriesTheKeptResultsAndTheToolchain(t *testing.T) {
|
||||
program = ask.Step
|
||||
case ask.Step.Artifact == "app:tools" && ask.Step.Step == "unit":
|
||||
tools = ask.Step
|
||||
case ask.Step.Artifact == "app:lib" && ask.Step.Step == "unit":
|
||||
lib = ask.Step
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -277,6 +281,10 @@ func TestAStepAskCarriesTheKeptResultsAndTheToolchain(t *testing.T) {
|
||||
if tools == nil || len(tools.Kept) != 0 || tools.Toolchain == nil {
|
||||
t.Errorf("an artifact with no package carries no kept result and its language's toolchain: %+v", tools)
|
||||
}
|
||||
// Design 47 G A-488b: the toolchain is read by the artifact's toolchain language (typescript), never by the ask's (node).
|
||||
if lib == nil || lib.Toolchain == nil || lib.Toolchain.Language != "typescript" {
|
||||
t.Errorf("the TypeScript artifact's ask carries the toolchain %+v, want the one of typescript", lib)
|
||||
}
|
||||
for pkg, want := range map[string]bool{"a/b": true, "a/b/c": true, "a/bc": false, "a": false} {
|
||||
if InPackage(pkg, "a/b") != want {
|
||||
t.Errorf("InPackage(%s, a/b) = %v, want %v", pkg, !want, want)
|
||||
@@ -295,8 +303,10 @@ func TestAnArtifactWithoutALanguageIsPlacedByItsKind(t *testing.T) {
|
||||
reg := catalog.Registration{Module: "app", Policy: catalog.Policy{Steps: catalog.StepStatuses{Lint: "required", Unit: "required", Integration: "required"}}}
|
||||
entries := map[string]catalog.CatalogEntry{
|
||||
"app": {Module: "app", Artifacts: []catalog.Artifact{{Ref: "app:server", Kind: "image"}, {Ref: "app:files", Kind: "archive"},
|
||||
{Ref: "app:mirror", Kind: "upstream"}, {Ref: "app:web", Language: "rust"}}},
|
||||
{Ref: "app:mirror", Kind: "upstream"}, {Ref: "app:web", Language: "rust"}, {Ref: "app:lib", Language: "typescript"}, {Ref: "app:cli", Language: "go"}}},
|
||||
"mesh-build-go": {Module: "mesh-build-go", Acts: []catalog.ActingRule{{Seat: "node-build-go", Step: "build", On: "go"}}},
|
||||
"mesh-build-image": {Module: "mesh-build-image", Acts: []catalog.ActingRule{{Seat: "node-build-image", Step: "build", On: seatdef.OnKindImage}}},
|
||||
"mesh-build-node": {Module: "mesh-build-node", Acts: []catalog.ActingRule{{Seat: "node-build-node", Step: "build", On: "typescript"}}},
|
||||
}
|
||||
steps, asks := Place(seat.Pipeline{Pipeline: "main.4.app", Module: "app"}, reg, entries, time.Unix(0, 0), "the-runner")
|
||||
want := map[string]struct{ outcome, seat, status string }{
|
||||
@@ -304,6 +314,8 @@ func TestAnArtifactWithoutALanguageIsPlacedByItsKind(t *testing.T) {
|
||||
"app:files": {seat.StepResultOutcomeBypassed, "", seat.StepResultStatusBypass},
|
||||
"app:mirror": {seat.StepResultOutcomeBypassed, "", seat.StepResultStatusBypass},
|
||||
"app:web": {seat.StepResultOutcomeCouldNotRun, "", "required"},
|
||||
"app:lib": {seat.StepResultOutcomeRunning, "node-build-node", "required"},
|
||||
"app:cli": {seat.StepResultOutcomeRunning, "node-build-go", "required"},
|
||||
}
|
||||
n := 0
|
||||
for _, s := range steps {
|
||||
@@ -316,16 +328,21 @@ func TestAnArtifactWithoutALanguageIsPlacedByItsKind(t *testing.T) {
|
||||
t.Errorf("%s %s is %s/%s at %q (%s), want %s/%s at %q", s.Artifact, s.Step, s.Status, s.Outcome, s.Seat, s.Why, w.status, w.outcome, w.seat)
|
||||
}
|
||||
}
|
||||
if n != 12 {
|
||||
t.Errorf("placed %d build steps of the four artifacts, want 12", n)
|
||||
if n != 18 {
|
||||
t.Errorf("placed %d build steps of the six artifacts, want 18", n)
|
||||
}
|
||||
for _, a := range asks {
|
||||
if a.Seat != builds.Seat && a.Seat != wants.Seat && a.Seat != "node-build-image" {
|
||||
t.Errorf("asked %s, want only the build, the composition and the image's build seat", a.Seat)
|
||||
if a.Seat != builds.Seat && a.Seat != wants.Seat && a.Seat != "node-build-image" && a.Seat != "node-build-node" && a.Seat != "node-build-go" {
|
||||
t.Errorf("asked %s, want only the build, the composition and the image's and the node's build seats", a.Seat)
|
||||
}
|
||||
// Design 47 G A-488: a step's ask names the language of the seat it is asked of (node-build-node node, node-build-image
|
||||
// image), never the artifact's word (typescript).
|
||||
if ask, ok := a.Ask.(nodebuildgo.StepAsk); ok && ask.Language != strings.TrimPrefix(a.Seat, "node-build-") {
|
||||
t.Errorf("the ask %s on %s names the language %q, want %q", a.ID, a.Seat, ask.Language, strings.TrimPrefix(a.Seat, "node-build-"))
|
||||
}
|
||||
}
|
||||
if len(asks) != 5 {
|
||||
t.Errorf("asked %d, want the build, the image's three steps and the composition", len(asks))
|
||||
if len(asks) != 11 {
|
||||
t.Errorf("asked %d, want the build, the image's and the node's three steps each and the composition", len(asks))
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -483,16 +483,16 @@ func (s *Starter) carry(key string, asks []Asked) {
|
||||
if a.Package != "" {
|
||||
ask.Step.Kept = kept[a.Package]
|
||||
}
|
||||
if s.Toolchain == nil || ask.Language == "" {
|
||||
if s.Toolchain == nil || a.Toolchain == "" {
|
||||
continue
|
||||
}
|
||||
t, done := chains[ask.Language]
|
||||
t, done := chains[a.Toolchain]
|
||||
if !done {
|
||||
var err error
|
||||
if t, err = s.Toolchain(ask.Language); err != nil {
|
||||
s.logf("the toolchain of %s for the steps of %s is not read: %v", ask.Language, key, err)
|
||||
if t, err = s.Toolchain(a.Toolchain); err != nil {
|
||||
s.logf("the toolchain of %s for the steps of %s is not read: %v", a.Toolchain, key, err)
|
||||
}
|
||||
chains[ask.Language] = t
|
||||
chains[a.Toolchain] = t
|
||||
}
|
||||
ask.Step.Toolchain = t
|
||||
}
|
||||
|
||||
@@ -114,6 +114,7 @@ func TestManifest(t *testing.T) {
|
||||
{"uses operator-channel", equalSets(m.Uses, []string{"operator-channel"})},
|
||||
// RN25 (§539): the runner's tools carry the prefix runner_.
|
||||
{"first-view tools", equalSets(m.Tools, []string{"runner_first_views", "runner_first_view_again", "runner_health", "runner_unlogged", "runner_unlogged_append",
|
||||
"runner_start", "runner_run", "runner_accept", "runner_ask_again",
|
||||
"runner_test_runs", "runner_test_runs_drop"})},
|
||||
// A-305 (ADR 0227 rule 8, ADR 0240 rule 2): the process is judged by its unit, as every process of mesh-delivery,
|
||||
// never by a tool of its own; runner_health stays a tool.
|
||||
|
||||
@@ -30,6 +30,10 @@
|
||||
"runner_health",
|
||||
"runner_unlogged",
|
||||
"runner_unlogged_append",
|
||||
"runner_start",
|
||||
"runner_run",
|
||||
"runner_accept",
|
||||
"runner_ask_again",
|
||||
"runner_test_runs",
|
||||
"runner_test_runs_drop"
|
||||
],
|
||||
|
||||
@@ -141,6 +141,19 @@ func BuildSeat(language string) (string, error) {
|
||||
return "node-build-" + language, nil
|
||||
}
|
||||
|
||||
// AskLanguageOfSeat is the language of the asks a build seat takes: node-build-go go, node-build-node node, node-build-image
|
||||
// image (BuildSeat's inverse). A pipeline step's ask names it, never the artifact's language word (design 47 G A-488).
|
||||
func AskLanguageOfSeat(seat string) (string, error) {
|
||||
language, ok := strings.CutPrefix(seat, "node-build-")
|
||||
if !ok {
|
||||
return "", fmt.Errorf("buildstep: %q is no build seat", seat)
|
||||
}
|
||||
if err := checkAskLanguage(language); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return language, nil
|
||||
}
|
||||
|
||||
// AskSubject is the subject a build seat accepts a queue's asks on: the seat's accept subject of main's form, which names no
|
||||
// asker (design 47 G §600, §483).
|
||||
func AskSubject(language, queue string) (string, error) {
|
||||
|
||||
@@ -108,7 +108,7 @@ func tiersToolSaysEachModulesWait(t *testing.T) {
|
||||
t.Fatalf("view_tiers with waits = %+v, %v", out, err)
|
||||
}
|
||||
web = held.Tiers[1][0]
|
||||
if last := web.Waits[len(web.Waits)-1]; last.Step != view.TakeInStep || !last.Since.Equal(heldSince) || !strings.Contains(last.Words, "src2:abc") {
|
||||
if last := web.Waits[len(web.Waits)-1]; last.Step != view.TakeInStep || !last.Since.Equal(heldSince) || !strings.Contains(last.Words, "unread or lost") {
|
||||
t.Errorf("web = %+v, want its last wait on the edge take-in since %s", web, heldSince)
|
||||
}
|
||||
if lib := held.Tiers[0][1]; lib.Module != "lib" || lib.Waits[len(lib.Waits)-1].Step != view.TakeInStep || !strings.Contains(lib.Waits[len(lib.Waits)-1].Words, "group feat") {
|
||||
|
||||
@@ -219,7 +219,7 @@ func (r Reader) heldBack(out *Tiers) (map[string]TierWait, []string) {
|
||||
return held, nil
|
||||
}
|
||||
for _, w := range a.Waits {
|
||||
words := "the indexer holds back " + w.Key + ", which waits for the indexer build " + w.WaitsFor
|
||||
words := "the indexer holds back " + w.Key + " (its message is unread or lost, or it is held for room)"
|
||||
if w.What == takein.TakeInGroup {
|
||||
words = "the indexer gathers the group " + w.Group + " and its end has not come"
|
||||
} else if w.Behind != "" {
|
||||
|
||||
@@ -132,7 +132,7 @@ func TestRunNeedsItsCredential(t *testing.T) { // 1135/AC6
|
||||
}
|
||||
out.findings[old.Key()] = true
|
||||
w.SeedFinding(old.Key(), value)
|
||||
body, err := takein.EncodeTakeIn(takein.TakeIn{Kind: takein.TakeInPending, Key: "m.00000000000000000043", What: takein.TakeInRevision, Since: time.Now(), WaitsFor: "src2:x",
|
||||
body, err := takein.EncodeTakeIn(takein.TakeIn{Kind: takein.TakeInPending, Key: "m.00000000000000000043", What: takein.TakeInRevision, Since: time.Now(), Lost: []string{"m.00000000000000000043"},
|
||||
Merges: []takein.Merge{{Repository: "novox/a", Number: 1, Merge: "c0ffee"}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
|
||||
@@ -136,27 +136,27 @@ func TestTheWatchmansFindings(t *testing.T) {
|
||||
// began ten minutes ago, and the indexer still holds it.
|
||||
since := c.t.Add(-10 * time.Minute)
|
||||
w.SeedTakeIn(takein.PendingAnswer{Waits: []takein.PendingWait{{Key: revKey, What: takein.TakeInRevision, Keys: []string{revKey},
|
||||
Merges: []takein.Merge{{Repository: "novox/a", Number: 1, Merge: "c0ffee"}}, WaitsFor: "src2:x", Since: since}}})
|
||||
Merges: []takein.Merge{{Repository: "novox/a", Number: 1, Merge: "c0ffee"}}, Lost: []string{"m.00000000000000000042"}, Since: since}}})
|
||||
w.TakeInCaughtUp()
|
||||
if len(o.findings) != 0 || o.ttls[signalKey("take-in", revKey)] != TakeInRevisionBound-10*time.Minute {
|
||||
t.Fatalf("within its bound: findings %v, ttl %v; want none yet and the rest of the bound", o.findings, o.ttls[signalKey("take-in", revKey)])
|
||||
}
|
||||
expire(w, c, signalKey("take-in", revKey), TakeInRevisionBound-10*time.Minute)
|
||||
}, map[string]string{FindingKey(ScopeDelivery, revKey, KindTakeInWaits): Warning}},
|
||||
{"ADR 0633 §1 (issue 1921): a merge held because the indexer cannot learn its own build is raised past the revision bound from its since, naming why", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) {
|
||||
{"ADR 0633 §1 (issue 1921): (A-476 b) a revision held because its message is lost or unread is raised past the revision bound from its since, naming why", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) {
|
||||
w.TakeInCaughtUp()
|
||||
body, err := takein.EncodeTakeIn(takein.TakeIn{Kind: takein.TakeInPending, Key: revKey, What: takein.TakeInRevision, Since: c.t,
|
||||
Merges: []takein.Merge{{Repository: "novox/a", Number: 1, Merge: "c0ffee"}}, Unknown: "the engine does not answer"})
|
||||
Merges: []takein.Merge{{Repository: "novox/a", Number: 1, Merge: "c0ffee"}}, Lost: []string{revKey}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
w.TakeIn(takeInEvent(t, takein.TakeInPending, revKey), body)
|
||||
if o.ttls[signalKey("take-in", revKey)] != TakeInRevisionBound || len(o.findings) != 0 {
|
||||
t.Fatalf("an unknown own build: ttls %v, findings %v", o.ttls, o.findings)
|
||||
t.Fatalf("a lost message: ttls %v, findings %v", o.ttls, o.findings)
|
||||
}
|
||||
expire(w, c, signalKey("take-in", revKey), TakeInRevisionBound)
|
||||
for _, f := range o.findings {
|
||||
if !strings.Contains(f.Summary, "cannot learn the build it runs") {
|
||||
if !strings.Contains(f.Summary, "lost or unread") {
|
||||
t.Errorf("finding %q does not say why", f.Summary)
|
||||
}
|
||||
}
|
||||
@@ -573,7 +573,7 @@ func takeInBody(t *testing.T, kind, key string, since time.Time) []byte {
|
||||
if key[0] == 'g' {
|
||||
body.What, body.Group = takein.TakeInGroup, "feat/take-in"
|
||||
} else if kind == takein.TakeInPending {
|
||||
body.WaitsFor = "src2:abc"
|
||||
body.Lost = []string{key}
|
||||
}
|
||||
b, err := takein.EncodeTakeIn(body)
|
||||
if err != nil {
|
||||
|
||||
@@ -135,10 +135,7 @@ func (w *Watchman) takeIn(kind, key string, t takein.TakeIn) {
|
||||
w.unfind(finding)
|
||||
return
|
||||
}
|
||||
bound, what := TakeInRevisionBound, "a source revision waits for the indexer build "+t.WaitsFor
|
||||
if t.Unknown != "" {
|
||||
what = "a source revision is held because the indexer cannot learn the build it runs (" + t.Unknown + ")"
|
||||
}
|
||||
bound, what := TakeInRevisionBound, "a source revision is held (a message lost or unread, no room for it, or its refusal)"
|
||||
if t.What == takein.TakeInGroup {
|
||||
bound, what = TakeInGroupBound, "a delivery group is gathered and its end has not come"
|
||||
}
|
||||
@@ -152,7 +149,7 @@ func (w *Watchman) takeIn(kind, key string, t takein.TakeIn) {
|
||||
Headline: "A merged change is held back by the indexer",
|
||||
Explanation: "The indexer holds " + what + " and has not taken it in within " + bound.String() + ", so what the merge changes does " +
|
||||
"not reach the machines.",
|
||||
Needs: "the indexer build it waits for to run, or the take-in by hand (indexer_take_in_pending), or a drop (indexer_drop_pending)"}
|
||||
Needs: "the take-in by hand (indexer_take_in_pending), or a drop (indexer_drop_pending)"}
|
||||
left := bound - w.now().Sub(t.Since)
|
||||
w.unfind(finding)
|
||||
w.unsignal(takeInRow, key)
|
||||
|
||||
Reference in New Issue
Block a user