IX15: the indexer reads a branch's source revision through the seat mesh-delivery's verb source and its event source-moved
The catch-up of the heads the source names runs before a source-moved is read, once per branch; the gate's source bucket is no longer read. internal/index compiles against the new contracts again (SourceByMainValue is gone); tests of earlier subtasks that the broken compile hid are fixed (the validator's MAJOR.MINOR version in their manifests). Builds: design 47 G §15, §115, §362 Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ
This commit is contained in:
@@ -873,16 +873,16 @@ func TestIndexerGivesACandidateItsFingerprintWhenTheCheckerCreatesThePreviewsBuc
|
||||
if _, err := js.CreateOrUpdateConsumer(ctx, PreviewsStream, jetstream.ConsumerConfig{Durable: "n1_mesh-delivery-indexer", AckPolicy: jetstream.AckExplicitPolicy}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
source, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: index.SourceBucket, History: 4})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
// The seat mesh-delivery's verb source, as the gate serves it (design 47 G §115): main names the catalogue's trunk head.
|
||||
answer := func(m *nats.Msg) {
|
||||
raw, _ := json.Marshal(map[string]any{"result": meshdelivery.SourceAnswer{Branch: "main", Group: "g", Revision: 5,
|
||||
Repositories: []meshdelivery.SourceHead{{Repository: "novox/mesh-catalog", Head: trunk, Trunk: "main"}}}, "node": "n1"})
|
||||
_ = m.Respond(raw)
|
||||
}
|
||||
data, err := meshdeliveryclient.EncodeSource(meshdelivery.SourceByMainValue{Group: "g", Heads: map[string]string{"novox/mesh-catalog": trunk}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := source.Put(ctx, meshdeliveryclient.SourceMainKey, data); err != nil {
|
||||
t.Fatal(err)
|
||||
for _, subject := range []string{"mesh.seat.mesh-delivery.call.source.>", "mesh.seat.mesh-delivery.tool.source"} {
|
||||
if _, err := js.Conn().Subscribe(subject, answer); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
})
|
||||
pull, err := forgemsg.EncodeForge(forgemsg.Forge{Kind: forgemsg.ForgePull, Repository: "novox/mesh-catalog", Branch: "feat/x", Number: 301, Head: head, Action: forgemsg.PullUpdated,
|
||||
|
||||
@@ -22,6 +22,7 @@ import (
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery"
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
@@ -470,22 +471,11 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
"indexed from it, until the indexer starts again", builds.BuildLogRead, err)
|
||||
}
|
||||
started.Store(true)
|
||||
// The gate's source bucket (mesh-delivery-gate_source) comes with the gate (release 3e, novox/hq ADR 0450 §1): its key
|
||||
// main is the mesh's source revision, which stamps the modules (issue 1344). Where it does not exist yet the indexer does
|
||||
// not wait for it (that needs a timer, ADR 0437 §8): it says so, and is started again with the gate.
|
||||
sourceErrs := make(chan error, 1)
|
||||
if source := process.OptionalBucket(ctx, js, index.SourceBucket, "no source revision is read until the indexer starts again", logger.Printf); source != nil {
|
||||
each := func(e jetstream.KeyValueEntry) {
|
||||
if opOf(e) {
|
||||
applySource(ctx, ix, e.Key(), e.Revision(), e.Value(), logger)
|
||||
}
|
||||
}
|
||||
if sw, err := settle(ctx, source, each); err != nil {
|
||||
process.NotWatched(logger.Printf, index.SourceBucket, err, "no source revision is read until the indexer starts again")
|
||||
} else {
|
||||
go func() { sourceErrs <- follow(ctx, sw, each) }()
|
||||
}
|
||||
}
|
||||
// The source revision of a watched branch (design 47 G §15, §115, §362): asked of the seat mesh-delivery's verb source as a
|
||||
// kept call, once at the start, before the handler of its event source-moved reads anything; a source-moved heard
|
||||
// meanwhile is read after.
|
||||
source := &sourceFollow{read: sourceVia(calls.CallKept), apply: ix.ApplySource, logf: logger.Printf}
|
||||
go source.catchUp(ctx)
|
||||
// Only the toolchain reports of nodes the catalogue places the build module on count (issue 1940, ADR 0549 §3): the
|
||||
// assignments is settled first, so the toolchains settle through the gate already knowing it.
|
||||
gate := buildrun.NewToolchainGate(sortedModules(), ix.PutToolchain, logger.Printf)
|
||||
@@ -668,6 +658,9 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := hub.Handle(meshdelivery.SubjectSourceMovedAll, func(m jetstream.Msg) { source.heard(ctx, m) }); err != nil {
|
||||
return err
|
||||
}
|
||||
hub.OnBound(reads.atBind) // what a node's engine said before the consumer was bound (ADR 0668 §4)
|
||||
versions := func(m jetstream.Msg) {
|
||||
if !heardVersion(ctx, ix, m, logger.Printf) {
|
||||
@@ -753,8 +746,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
return nil
|
||||
case err := <-errs:
|
||||
return err
|
||||
case err := <-sourceErrs:
|
||||
return err
|
||||
case err := <-assignmentsErrs:
|
||||
return err
|
||||
case err := <-tools.errs:
|
||||
|
||||
@@ -5,7 +5,9 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -16,7 +18,6 @@ import (
|
||||
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
|
||||
mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
|
||||
"git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshwatchdogs"
|
||||
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient"
|
||||
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
@@ -237,24 +238,64 @@ type opener struct{ err error }
|
||||
|
||||
func (o opener) KeyValue(context.Context, string) (jetstream.KeyValue, error) { return nil, o.err }
|
||||
|
||||
func TestTheSourceMainKeyIsTakenAtItsRevision(t *testing.T) { // 1344/AC1 (the watch's side: key, revision, value)
|
||||
var logs bytes.Buffer
|
||||
logger := log.New(&logs, "", 0)
|
||||
mods := &memStore{vals: map[string][]byte{}}
|
||||
ix := &index.Indexer{Store: &memStore{vals: map[string][]byte{}}, Modules: mods, Builds: index.NewBuilds()}
|
||||
value, err := meshdeliveryclient.EncodeSource(meshdelivery.SourceByMainValue{Group: "g", Heads: map[string]string{"novox/mesh-delivery": head}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
func TestTheSourceIsReadFromTheSeatAndASourceMovedDuringTheCatchUpIsReadAfterIt(t *testing.T) { // design 47 G §15, §115, §362
|
||||
var order []string
|
||||
revision := int64(4)
|
||||
var logs []string
|
||||
var s *sourceFollow
|
||||
s = &sourceFollow{
|
||||
read: func(branch string) (meshdelivery.SourceAnswer, error) {
|
||||
order = append(order, "read "+branch)
|
||||
if revision == 0 {
|
||||
return meshdelivery.SourceAnswer{}, fmt.Errorf("source.main is not readable")
|
||||
}
|
||||
if len(order) == 1 { // a source-moved is heard while the catch-up reads
|
||||
s.moved(context.Background(), "main")
|
||||
s.moved(context.Background(), "main")
|
||||
s.moved(context.Background(), "beta")
|
||||
}
|
||||
return meshdelivery.SourceAnswer{Branch: branch, Revision: revision}, nil
|
||||
},
|
||||
apply: func(_ context.Context, a meshdelivery.SourceAnswer) error {
|
||||
order = append(order, fmt.Sprintf("apply %s %d", a.Branch, a.Revision))
|
||||
return nil
|
||||
},
|
||||
logf: func(f string, a ...any) { logs = append(logs, fmt.Sprintf(f, a...)) },
|
||||
}
|
||||
applySource(context.Background(), ix, "other", 3, value, logger) // not the key of the mesh's source
|
||||
applySource(context.Background(), ix, meshdeliveryclient.SourceMainKey, 4, []byte(`{"v":1`), logger)
|
||||
if !strings.Contains(logs.String(), "source") {
|
||||
t.Errorf("a source that cannot be read was not said: %q", logs.String())
|
||||
s.catchUp(context.Background())
|
||||
want := []string{"read main", "apply main 4", "read main", "apply main 4"} // once, after the catch-up; beta is left
|
||||
if !reflect.DeepEqual(order, want) {
|
||||
t.Errorf("order = %v, want %v", order, want)
|
||||
}
|
||||
logs.Reset()
|
||||
applySource(context.Background(), ix, meshdeliveryclient.SourceMainKey, 5, value, logger)
|
||||
if logs.Len() != 0 {
|
||||
t.Errorf("a good source said %q", logs.String())
|
||||
s.moved(context.Background(), "main") // after the catch-up: read at once
|
||||
if len(order) != 6 {
|
||||
t.Errorf("a source-moved after the catch-up was not read at once: %v", order)
|
||||
}
|
||||
order, revision = nil, 0
|
||||
s.moved(context.Background(), "main")
|
||||
if len(order) != 1 || len(logs) != 1 || !strings.Contains(logs[0], "not readable") {
|
||||
t.Errorf("a source that cannot be read: order %v, said %v, want it said and nothing applied", order, logs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheSeatsSourceAnswerIsReadAndItsErrorIsAnError(t *testing.T) { // design 47 G §115
|
||||
var asked any
|
||||
read := sourceVia(func(key string, body any) (json.RawMessage, error) {
|
||||
asked = body
|
||||
if key != "seat:mesh-delivery.source" {
|
||||
t.Errorf("asked %q", key)
|
||||
}
|
||||
if body.(meshdelivery.SourceInput).Branch == "bad" {
|
||||
return json.RawMessage(`{"error":"source.bad is not readable"}`), nil
|
||||
}
|
||||
return json.RawMessage(`{"branch":"main","revision":7,"repositories":[{"repository":"novox/mesh-delivery","head":"` + head + `","trunk":"main"}]}`), nil
|
||||
})
|
||||
a, err := read("main")
|
||||
if err != nil || a.Revision != 7 || len(a.Repositories) != 1 || asked.(meshdelivery.SourceInput).Branch != "main" {
|
||||
t.Errorf("source = %+v, %v", a, err)
|
||||
}
|
||||
if _, err := read("bad"); err == nil || !strings.Contains(err.Error(), "not readable") {
|
||||
t.Errorf("an answer with an error was no error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -2,26 +2,100 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient"
|
||||
|
||||
"mesh-delivery-indexer/internal/index"
|
||||
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
)
|
||||
|
||||
// applySource takes one put of the gate's bucket source into the indexer: the key main, at its revision, which is the mesh's
|
||||
// source revision (mesh-sdk go/delivery SourceMainKey, Source). Any other key is not the mesh's source. A value that is no
|
||||
// source is said; so is a write that failed, and it is tried again only at the next revision (novox/hq ADR 0437 §8).
|
||||
func applySource(ctx context.Context, ix *index.Indexer, key string, rev uint64, value []byte, logger *log.Logger) {
|
||||
if key != meshdeliveryclient.SourceMainKey {
|
||||
// sourceInvoke is the seat mesh-delivery's verb source as the manifest's invokes names it: a watched branch's source
|
||||
// revision, asked of the seat and never read from the gate's state (design 47 G §15, §115; ADR 0596 §2).
|
||||
const sourceInvoke = "seat:" + meshdelivery.Seat + "." + meshdelivery.VerbSource
|
||||
|
||||
// sourceVia asks the seat for one branch's source revision through call (process.Conn.CallKept, a read) and reads its
|
||||
// declared answer; an answer that carries an error is that error.
|
||||
func sourceVia(call toolCaller) func(branch string) (meshdelivery.SourceAnswer, error) {
|
||||
return func(branch string) (meshdelivery.SourceAnswer, error) {
|
||||
raw, err := call(sourceInvoke, meshdelivery.SourceInput{Branch: branch})
|
||||
if err != nil {
|
||||
return meshdelivery.SourceAnswer{}, err
|
||||
}
|
||||
var a meshdelivery.SourceAnswer
|
||||
if err := json.Unmarshal(raw, &a); err != nil {
|
||||
return a, fmt.Errorf("the answer of %s is not a source revision: %w", sourceInvoke, err)
|
||||
}
|
||||
if a.Error != "" {
|
||||
return a, fmt.Errorf("%s: %s", sourceInvoke, a.Error)
|
||||
}
|
||||
return a, nil
|
||||
}
|
||||
}
|
||||
|
||||
// sourceFollow takes the seat's source revisions into the indexer: first the catch-up of the heads the source names, then
|
||||
// each source-moved the seat says (design 47 G §362). A source-moved heard before the catch-up ended is applied after it,
|
||||
// once per branch.
|
||||
type sourceFollow struct {
|
||||
read func(branch string) (meshdelivery.SourceAnswer, error)
|
||||
apply func(context.Context, meshdelivery.SourceAnswer) error
|
||||
logf func(string, ...any)
|
||||
|
||||
mu sync.Mutex
|
||||
caught bool
|
||||
waiting map[string]bool
|
||||
}
|
||||
|
||||
// catchUp reads the trunk branch's source revision and applies it, then the branches a source-moved named meanwhile. A
|
||||
// read that fails is said; the catch-up ends all the same, and the next source-moved reads again.
|
||||
func (s *sourceFollow) catchUp(ctx context.Context) {
|
||||
s.take(ctx, trunkBranch)
|
||||
s.mu.Lock()
|
||||
s.caught = true
|
||||
branches := s.waiting
|
||||
s.waiting = nil
|
||||
s.mu.Unlock()
|
||||
for branch := range branches {
|
||||
s.take(ctx, branch)
|
||||
}
|
||||
}
|
||||
|
||||
// moved is the handler of one source-moved on branch: held until the catch-up ends, read at once after.
|
||||
func (s *sourceFollow) moved(ctx context.Context, branch string) {
|
||||
s.mu.Lock()
|
||||
if !s.caught {
|
||||
if s.waiting == nil {
|
||||
s.waiting = map[string]bool{}
|
||||
}
|
||||
s.waiting[branch] = true
|
||||
s.mu.Unlock()
|
||||
return
|
||||
}
|
||||
s, err := meshdeliveryclient.DecodeSource(value)
|
||||
s.mu.Unlock()
|
||||
s.take(ctx, branch)
|
||||
}
|
||||
|
||||
// take reads one branch's source revision and applies it. A branch other than the trunk's is left until the indexer keeps
|
||||
// a catalog graph per watched branch.
|
||||
func (s *sourceFollow) take(ctx context.Context, branch string) {
|
||||
if branch != trunkBranch {
|
||||
return
|
||||
}
|
||||
a, err := s.read(branch)
|
||||
if err != nil {
|
||||
logger.Printf("source.%s at revision %d is not a source, and is left: %v", key, rev, err)
|
||||
s.logf("source.%s is not read, and is read again at its next source-moved: %v", branch, err)
|
||||
return
|
||||
}
|
||||
if err := s.apply(ctx, a); err != nil {
|
||||
s.logf("source.%s at revision %d could not be taken into the modules: %v", branch, a.Revision, err)
|
||||
}
|
||||
if err := ix.ApplySource(ctx, rev, s); err != nil {
|
||||
logger.Printf("source.%s at revision %d could not be taken into the modules: %v", key, rev, err)
|
||||
}
|
||||
|
||||
// heard is the hub's handler of source-moved: the branch is the last token of the subject. The message is acknowledged at once.
|
||||
func (s *sourceFollow) heard(ctx context.Context, m jetstream.Msg) {
|
||||
defer func() { _ = m.Ack() }()
|
||||
if branch := m.Subject()[strings.LastIndex(m.Subject(), ".")+1:]; branch != "" {
|
||||
s.moved(ctx, branch)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user