Compare commits
13
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
74ab66ca81 | ||
|
|
7f8ee816a9 | ||
|
|
ac7bf01abf | ||
|
|
fd8aba59c3 | ||
|
|
cb8cab8719 | ||
|
|
92b6dccbec | ||
|
|
7a86c62492 | ||
|
|
afed658c69 | ||
|
|
3f60eb95a0 | ||
|
|
23507fb3ef | ||
|
|
cd0ad47ad6 | ||
|
|
d5737f2c6d | ||
|
|
04276bfda1 |
@@ -18,11 +18,9 @@ import (
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/announce"
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/compose/judge"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
"git.novox.be/novox/mesh-sdk/go/provider"
|
||||
@@ -72,11 +70,7 @@ func startBed(t *testing.T, leaveOut string) *bed {
|
||||
if name == leaveOut {
|
||||
continue
|
||||
}
|
||||
cfg := jetstream.KeyValueConfig{Bucket: name, History: 5}
|
||||
if name == delivery.PlacementPulsesBucket {
|
||||
cfg.History, cfg.LimitMarkerTTL = 1, time.Minute // a time to live per key, as the controller makes it (key-ttl)
|
||||
}
|
||||
kv, err := js.CreateKeyValue(ctx, cfg)
|
||||
kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: name, History: 5})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -135,7 +129,9 @@ func (b *bed) startStoppable(t *testing.T) (<-chan error, func()) {
|
||||
|
||||
// 1297/B2b: started whole on its credential, the assigner says on the bus every state its manifest declares, each readable,
|
||||
// having opened every bucket it names. The states are read from module.json, so a state added there is expected here too.
|
||||
func saysWhatItKeeps(t *testing.T, b *bed, done <-chan error) {
|
||||
func TestTheAssignerStartsAndSaysWhatItKeeps(t *testing.T) {
|
||||
b := startBed(t, "")
|
||||
done := b.start(t)
|
||||
events, err := b.js.Stream(b.ctx, "EVENTS")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -227,8 +223,8 @@ func TestTheAssignerDoesNotStartWithoutABucketItReads(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// Without the seat's placement state it starts all the same, and once the bucket is made (the controller makes
|
||||
// it from the manifest) it is read with no restart: a module only the placement names is
|
||||
// Review of L2 (D1): without the indexer's placement state it starts all the same, and once the bucket is made (the
|
||||
// indexer and the assigner come in one delivery) it is read with no restart: a module only the placement names is
|
||||
// said as placed by one source only.
|
||||
pb := startBed(t, delivery.PlacementBucket)
|
||||
pdone := pb.start(t)
|
||||
@@ -301,6 +297,10 @@ func (b *bed) manifest(t *testing.T, module, body string) string {
|
||||
// 1297/AC1: against the real bus, a fixture change touching one module on one machine is composed and answered within
|
||||
// seconds under the checker's key of the check, naming the machine composed and the mesh-sdk it composed with.
|
||||
func TestAChangeIsComposedOnTheBus(t *testing.T) {
|
||||
// The placement answers' bound (ADR 0559 §5), shortened before the program starts and put back after it stops.
|
||||
bound := assign.AnswerBound
|
||||
t.Cleanup(func() { assign.AnswerBound = bound })
|
||||
assign.AnswerBound = 1500 * time.Millisecond
|
||||
b := startBed(t, "")
|
||||
trunk := b.manifest(t, "web", `{"module":"web","version":"1"}`)
|
||||
changed := b.manifest(t, "web", `{"module":"web","version":"2"}`)
|
||||
@@ -352,6 +352,63 @@ func TestAChangeIsComposedOnTheBus(t *testing.T) {
|
||||
} else if info, err := st.Info(b.ctx); err != nil || info.State.Msgs != 0 {
|
||||
t.Errorf("a group without a mesh-sdk member put %d asks on the Go build seat (%v)", info.State.Msgs, err)
|
||||
}
|
||||
|
||||
// ADR 0559 §5 (issue 1126): a catalogue pull request that places g14 for the first time (the seed's case) is composed
|
||||
// with its candidate placement, touches g14, and waits for g14's answer at its head; g14 answering the same passes it.
|
||||
// A node that does not answer within the bound fails the check, named, at the bound's end with no other event; its later
|
||||
// answer at the same head passes the same heads' composition.
|
||||
record, _ := delivery.EncodeHaveNode(delivery.HaveNode{Record: &delivery.NodeRecord{Node: "g14"}})
|
||||
b.putJSON(t, delivery.HaveBucket, "g14.node", record)
|
||||
placed := func(group string, number int, head string) (string, uint64) {
|
||||
t.Helper()
|
||||
value, err := delivery.EncodePlacementCandidate(delivery.PlacementCandidate{Group: group,
|
||||
Pull: delivery.Head{Repository: delivery.CatalogueRepository, Number: number, Head: head},
|
||||
Placement: &delivery.Placement{Node: "g14", Modules: []delivery.PlacedModule{{Module: "web"}}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
key, _ := delivery.PlacementCandidateKey(group, "g14")
|
||||
rev, err := b.buckets[delivery.PlacementCandidatesBucket].Put(b.ctx, key, value)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pull := []delivery.CandidatePull{{Head: delivery.Head{Repository: delivery.CatalogueRepository, Number: number, Head: head}, Modules: []string{}}}
|
||||
hash, _ := delivery.HeadsHash([]delivery.Head{pull[0].Head})
|
||||
cand, err := delivery.EncodeCandidate(delivery.Candidate{Group: group, HeadsHash: hash, Pulls: pull, FPSet: delivery.FingerprintSet(pull)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ckey, _ := delivery.CandidatesKey(group)
|
||||
b.putJSON(t, delivery.CandidatesBucket, ckey, cand)
|
||||
key, _ = delivery.CompositionKey(group, hash)
|
||||
return key, rev
|
||||
}
|
||||
answer := func(group, head string, rev uint64) {
|
||||
t.Helper()
|
||||
have, err := delivery.EncodeHaveNode(delivery.HaveNode{Record: &delivery.NodeRecord{Node: "g14"},
|
||||
Candidates: map[string]delivery.CandidateAnswer{group: {Head: head, Revision: rev, Verdict: delivery.CandidateSame, Same: []string{"web"}}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b.putJSON(t, delivery.HaveBucket, "g14.node", have)
|
||||
}
|
||||
seedHead, lateHead := strings.Repeat("5", 40), strings.Repeat("6", 40)
|
||||
seed, seedRev := placed("feat/seed", 20, seedHead)
|
||||
waiting := b.verdict(t, seed, func(v delivery.CompositionVerdict) bool { return v.State == delivery.CompositionWaits })
|
||||
if !strings.Contains(waiting.Words, "g14") {
|
||||
t.Errorf("the seed's composition waits saying %q, want g14 named", waiting.Words)
|
||||
}
|
||||
answer("feat/seed", seedHead, seedRev)
|
||||
if v := b.verdict(t, seed, func(v delivery.CompositionVerdict) bool { return v.State == delivery.CompositionPass }); !slices.Contains(v.Machines, "g14") {
|
||||
t.Errorf("the seed's composition = %+v, want g14 composed", v)
|
||||
}
|
||||
late, lateRev := placed("feat/late", 21, lateHead)
|
||||
failed := b.verdict(t, late, func(v delivery.CompositionVerdict) bool { return v.State == delivery.CompositionFail })
|
||||
if failed.Machine != "g14" || !strings.Contains(failed.Words, delivery.CompositionUnanswered) {
|
||||
t.Errorf("a node silent past its bound: %+v", failed)
|
||||
}
|
||||
answer("feat/late", lateHead, lateRev)
|
||||
b.verdict(t, late, func(v delivery.CompositionVerdict) bool { return v.State == delivery.CompositionPass })
|
||||
}
|
||||
|
||||
// get waits for a key of one of the bed's buckets and answers its value.
|
||||
@@ -1005,196 +1062,3 @@ func TestAHoldWrittenAgainAfterALiftAndARestartIsAskedAgain(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// bedStore is the runtime's states over the bed's buckets, as the seat's verbs read and write them: the assigner's own by
|
||||
// their local names, another module's `<module>.<state>`.
|
||||
type bedStore struct{ b *bed }
|
||||
|
||||
func (s bedStore) kv(state string) jetstream.KeyValue {
|
||||
if strings.Contains(state, ".") {
|
||||
return s.b.buckets[strings.Replace(state, ".", "_", 1)]
|
||||
}
|
||||
return s.b.buckets[delivery.AssignerModule+"_"+state]
|
||||
}
|
||||
|
||||
func (s bedStore) get(state, key string) ([]byte, uint64, bool, error) {
|
||||
e, err := s.kv(state).Get(s.b.ctx, key)
|
||||
if err != nil {
|
||||
return nil, 0, false, nil
|
||||
}
|
||||
return e.Value(), e.Revision(), true, nil
|
||||
}
|
||||
|
||||
func (s bedStore) put(state, key string, value []byte) error {
|
||||
_, err := s.kv(state).Put(s.b.ctx, key, value)
|
||||
return err
|
||||
}
|
||||
|
||||
func (s bedStore) keys(state string) ([]string, error) {
|
||||
keys, err := s.kv(state).Keys(s.b.ctx)
|
||||
if err != nil && strings.Contains(err.Error(), "no keys") {
|
||||
return nil, nil
|
||||
}
|
||||
return keys, err
|
||||
}
|
||||
|
||||
func (s bedStore) del(state, key string) error { return s.kv(state).Delete(s.b.ctx, key) }
|
||||
|
||||
func (s bedStore) history(string, string) ([]stdio.StateRevision, error) { return nil, nil }
|
||||
|
||||
// 1126 (ADR 0567 §2-§6), the seat mesh-placement whole on the bus: a proposal moving web from the anchor to the laptop is
|
||||
// proposed through the verb, the assigner finds both nodes must answer, they answer the same in their haves, the
|
||||
// operator is asked and approves, and each key is written at a new revision while placement-changed is heard on the
|
||||
// assigner's stream naming both nodes. A second proposal laid over the old revision of a key moved since is laid again,
|
||||
// and declined, writes nothing and is withdrawn.
|
||||
func TestAProposalIsAnsweredApprovedAndWritten(t *testing.T) {
|
||||
b := startBed(t, "")
|
||||
if _, err := b.js.CreateStream(b.ctx, jetstream.StreamConfig{Name: "MOD_" + delivery.AssignerModule + "_placement",
|
||||
Subjects: []string{process.ModuleStreamSubject(delivery.AssignerModule, ">")}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
heard := make(chan delivery.PlacementEvent, 16)
|
||||
nc, err := nats.Connect(b.srv.ClientURL())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(nc.Close)
|
||||
if _, err := nc.Subscribe(process.ModuleStreamSubject(delivery.AssignerModule, delivery.PlacementChangedEvent), func(m *nats.Msg) {
|
||||
var e delivery.PlacementEvent
|
||||
if json.Unmarshal(m.Data, &e) == nil {
|
||||
heard <- e
|
||||
}
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, node := range []string{"anchor", "laptop"} {
|
||||
rec, _ := delivery.EncodeHaveNode(delivery.HaveNode{Record: &delivery.NodeRecord{Node: node}})
|
||||
b.putJSON(t, delivery.HaveBucket, node+".node", rec)
|
||||
}
|
||||
b.putJSON(t, delivery.ModulesBucket, "web", []byte(`{"v":1}`))
|
||||
b.putJSON(t, delivery.ModulesBucket, "shell", []byte(`{"v":1}`))
|
||||
place := func(node string, modules ...string) uint64 {
|
||||
p := delivery.Placement{Node: node, Modules: []delivery.PlacedModule{}}
|
||||
for _, m := range modules {
|
||||
p.Modules = append(p.Modules, delivery.PlacedModule{Module: m})
|
||||
}
|
||||
raw, _ := delivery.EncodePlacement(p)
|
||||
key, _ := delivery.PlacementKey(node)
|
||||
rev, err := b.buckets[delivery.PlacementBucket].Put(b.ctx, key, raw)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return rev
|
||||
}
|
||||
anchorRev, laptopRev := place("anchor", "web"), place("laptop", "shell")
|
||||
bound := judge.AnswerBound
|
||||
t.Cleanup(func() { judge.AnswerBound = bound })
|
||||
judge.AnswerBound = 2 * time.Second
|
||||
saysWhatItKeeps(t, b, b.start(t))
|
||||
verbs := newSeatTools(bedStore{b}, func(string, any) (json.RawMessage, error) { return json.RawMessage(`{"ok":true}`), nil })
|
||||
propose := func() delivery.PlacementProposal {
|
||||
t.Helper()
|
||||
got, err := verbs.doPropose(map[string]any{"why": "move web", "changes": `[{"op":"take-off","node":"anchor","module":"web"},` +
|
||||
`{"op":"place","node":"laptop","module":"web"}]`}, "operator@laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return got.(proposed).Proposal
|
||||
}
|
||||
proposalNow := func(id, what string, ok func(delivery.PlacementProposal) bool) delivery.PlacementProposal {
|
||||
t.Helper()
|
||||
var last delivery.PlacementProposal
|
||||
for deadline := time.Now().Add(15 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
|
||||
if e, err := b.buckets[delivery.PlacementProposalsBucket].Get(b.ctx, id); err == nil {
|
||||
if p, err := delivery.DecodePlacementProposal(e.Value()); err == nil {
|
||||
if last = p; ok(p) {
|
||||
return p
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
t.Fatalf("the proposal %s never was %s: %+v", id, what, last)
|
||||
return last
|
||||
}
|
||||
answerAll := func(p delivery.PlacementProposal) {
|
||||
t.Helper()
|
||||
for _, node := range p.Must {
|
||||
key, _ := delivery.PlacementCandidateKey(p.ID, node)
|
||||
e, err := b.buckets[delivery.PlacementCandidatesBucket].Get(b.ctx, key)
|
||||
if err != nil {
|
||||
t.Fatalf("no candidate %s: %v", key, err)
|
||||
}
|
||||
have, _ := delivery.EncodeHaveNode(delivery.HaveNode{Record: &delivery.NodeRecord{Node: node}, Candidates: map[string]delivery.CandidateAnswer{
|
||||
p.ID: {Proposal: p.ID, Revision: e.Revision(), Verdict: delivery.CandidateSame, Same: []string{"node"}}}})
|
||||
b.putJSON(t, delivery.HaveBucket, node+".node", have)
|
||||
}
|
||||
}
|
||||
|
||||
p := propose()
|
||||
p = proposalNow(p.ID, "told who must answer", func(p delivery.PlacementProposal) bool { return len(p.Must) > 0 })
|
||||
if !slices.Equal(p.Must, []string{"anchor", "laptop"}) {
|
||||
t.Fatalf("must answer = %v, want both nodes", p.Must)
|
||||
}
|
||||
answerAll(p)
|
||||
p = proposalNow(p.ID, "asked", func(p delivery.PlacementProposal) bool { return p.State == delivery.ProposalAsked })
|
||||
akey, _ := delivery.PlacementAskKey(p.ID)
|
||||
b.answer(t, akey, assign.OptionYes)
|
||||
select {
|
||||
case e := <-heard:
|
||||
if e.Proposal.ID != p.ID || len(e.Nodes) != 2 || e.Nodes[0].Before != anchorRev || e.Nodes[1].Before != laptopRev ||
|
||||
e.Nodes[0].After <= anchorRev || e.Warrant == nil || !slices.Equal(e.Nodes[0].Removed, []string{"web"}) {
|
||||
t.Errorf("placement-changed = %+v", e)
|
||||
}
|
||||
case <-time.After(15 * time.Second):
|
||||
t.Fatal("no placement-changed heard")
|
||||
}
|
||||
for node, want := range map[string][]string{"anchor": {}, "laptop": {"shell", "web"}} {
|
||||
key, _ := delivery.PlacementKey(node)
|
||||
e, err := b.buckets[delivery.PlacementBucket].Get(b.ctx, key)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got, err := delivery.DecodePlacement(e.Value()); err != nil || !slices.Equal(got.Assigned(), want) {
|
||||
t.Errorf("%s = %v, %v, want %v", node, got.Assigned(), err, want)
|
||||
}
|
||||
}
|
||||
if b.hasKey(delivery.PlacementProposalsBucket, p.ID) {
|
||||
t.Error("an applied proposal stays in the state")
|
||||
}
|
||||
|
||||
// A proposal whose key moves before its warrant is laid again; then declined, it writes nothing and ends.
|
||||
q, err := verbs.doPropose(map[string]any{"why": "back", "changes": `[{"op":"take-off","node":"laptop","module":"web"}]`}, "op")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
moved := q.(proposed).Proposal
|
||||
place("laptop", "shell", "web") // another writer moved the key
|
||||
relaid := proposalNow(moved.ID, "laid again", func(p delivery.PlacementProposal) bool {
|
||||
return strings.Contains(p.Words, "laid again") && len(p.Must) > 0
|
||||
})
|
||||
answerAll(relaid)
|
||||
proposalNow(moved.ID, "asked", func(p delivery.PlacementProposal) bool { return p.State == delivery.ProposalAsked })
|
||||
akey, _ = delivery.PlacementAskKey(moved.ID)
|
||||
b.answer(t, akey, assign.OptionNo)
|
||||
for deadline := time.Now().Add(15 * time.Second); b.hasKey(delivery.PlacementProposalsBucket, moved.ID); time.Sleep(50 * time.Millisecond) {
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatal("a declined proposal was not ended")
|
||||
}
|
||||
}
|
||||
key, _ := delivery.PlacementKey("laptop")
|
||||
if e, _ := b.buckets[delivery.PlacementBucket].Get(b.ctx, key); e == nil || !strings.Contains(string(e.Value()), "web") {
|
||||
t.Error("a declined proposal wrote its key")
|
||||
}
|
||||
|
||||
// ADR 0567 §4: a node that does not answer fails the verdict once its pulse key's time to live ends, heard on the bus
|
||||
// with no timer of the assigner's, naming it.
|
||||
q, err = verbs.doPropose(map[string]any{"why": "again", "changes": `[{"op":"take-off","node":"laptop","module":"web"}]`}, "op")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
silent := q.(proposed).Proposal
|
||||
failed := proposalNow(silent.ID, "failed for a silent node", func(p delivery.PlacementProposal) bool { return p.State == delivery.ProposalFailed })
|
||||
if failed.Verdict == nil || !strings.Contains(strings.Join(failed.Verdict.Fails, "; "), "laptop (offline) did not answer") {
|
||||
t.Errorf("the silent node's verdict = %+v", failed.Verdict)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,7 +13,7 @@ import (
|
||||
"log"
|
||||
"os"
|
||||
"os/signal"
|
||||
"slices"
|
||||
"sync"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
@@ -22,7 +22,6 @@ import (
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/announce"
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/buscompose"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
@@ -50,7 +49,6 @@ func main() {
|
||||
// readBuckets are the buckets of other parts the assigner reads, each named in its manifest's reads (the manifest test
|
||||
// holds the two lists equal), so the controller's broker grants exactly these.
|
||||
var readBuckets = []string{
|
||||
buscompose.ResolvedBucket,
|
||||
provider.AssignmentsBucket,
|
||||
delivery.BuildsBucket,
|
||||
delivery.ControllerGrantsBucket,
|
||||
@@ -60,6 +58,8 @@ var readBuckets = []string{
|
||||
delivery.CandidatesBucket,
|
||||
delivery.ManifestsBucket,
|
||||
delivery.ModulesBucket,
|
||||
delivery.PlacementBucket,
|
||||
delivery.PlacementCandidatesBucket,
|
||||
delivery.PresenceBucket,
|
||||
delivery.HaveBucket,
|
||||
delivery.StepsBucket(goBuildModule),
|
||||
@@ -69,26 +69,14 @@ var readBuckets = []string{
|
||||
// their steps in its steps state (novox/hq ADR 0482 clause 4).
|
||||
const goBuildModule = "mesh-build-go"
|
||||
|
||||
// ownBuckets are the assigner's own states, as its manifest declares them: the seat mesh-placement's three among them
|
||||
// (ADR 0567 §1), named after the assigner as every module state is.
|
||||
// ownBuckets are the assigner's own states, as its manifest declares them.
|
||||
var ownBuckets = []string{delivery.AssignerAsksBucket, delivery.AssignerCompositionsBucket, delivery.AssignerDifferencesBucket,
|
||||
delivery.AssignerPutsBucket, delivery.AssignerWantsBucket, delivery.PlacementBucket, delivery.PlacementProposalsBucket,
|
||||
delivery.PlacementCandidatesBucket, delivery.PlacementPulsesBucket}
|
||||
|
||||
// seatBuckets are the seat's states, new with it: until the controller has made them from the manifest and the bus grants
|
||||
// them, the assigner starts without them and binds each once the bus says it was created (issue 1790's kind).
|
||||
var seatBuckets = []string{delivery.PlacementBucket, delivery.PlacementProposalsBucket, delivery.PlacementCandidatesBucket,
|
||||
delivery.PlacementPulsesBucket}
|
||||
delivery.AssignerPutsBucket, delivery.AssignerWantsBucket}
|
||||
|
||||
// keptStates is what the assigner keeps and who may read it (ADR 0495 §2-§3): its wants by the watchman and the view
|
||||
// (and the node agent from release 6), its differences by the view, its compositions by the checker, its asks by the view
|
||||
// and the teller, its puts by whoever follows one (ADR 0546), and the seat mesh-placement's placement, proposals and
|
||||
// candidates by the node agents, the grants part, mesh-config and the view (ADR 0567 §9).
|
||||
// and the teller, its puts by whoever follows one (ADR 0546).
|
||||
var keptStates = []announce.State{
|
||||
{Name: delivery.PlacementState, Readable: true},
|
||||
{Name: delivery.PlacementProposalsState, Readable: true},
|
||||
{Name: delivery.PlacementCandidatesState, Readable: true},
|
||||
{Name: delivery.PlacementPulsesState, Readable: true},
|
||||
{Name: "asks", Readable: true},
|
||||
{Name: "compositions", Readable: true},
|
||||
{Name: "differences", Readable: true},
|
||||
@@ -111,10 +99,10 @@ func open(ctx context.Context, js process.BucketOpener, logf func(string, ...any
|
||||
logf("the bucket %s cannot be opened (%v): no put is taken further until the assigner starts again", name, err)
|
||||
continue
|
||||
}
|
||||
if slices.Contains(seatBuckets, name) && err != nil {
|
||||
// The seat's states come in the same delivery as this assigner (ADR 0567 §1): without placement the controller's
|
||||
// assignments alone name where modules run, as before any placement was written (ADR 0549 §5).
|
||||
logf("the bucket %s cannot be opened (%v): it is bound once the bus says it was made", name, err)
|
||||
if name == delivery.PlacementBucket && err != nil {
|
||||
// The indexer's placement state comes with the second milestone (ADR 0549 §2): without it the controller's
|
||||
// assignments alone name where modules run, as before any placement was read (ADR 0549 §5).
|
||||
logf("the bucket %s cannot be opened (%v): the controller's assignments alone are read until the assigner starts again", name, err)
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
@@ -151,14 +139,10 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
if err := said.Keeps(keptStates...); err != nil {
|
||||
logger.Printf("what the assigner keeps could not be said on the bus, so its readers are not granted until it is: %v", err)
|
||||
}
|
||||
thisNode = account.Node
|
||||
logger.Printf("started on the account %s, reading %d buckets", account.User, len(kvs))
|
||||
return follow(ctx, js, conn, kvs, sayVia(ctx, js), emitting{said}, logger)
|
||||
}
|
||||
|
||||
// thisNode is the node this process runs on, as its credential names it: said with each of the seat's events.
|
||||
var thisNode string
|
||||
|
||||
// emitting is the process connection as the puts' caller: its calls, and its events with no options.
|
||||
type emitting struct{ *process.Conn }
|
||||
|
||||
@@ -167,7 +151,6 @@ func (e emitting) Emit(eventType string, body any) error { return e.Conn.Emit(ev
|
||||
// watched are the buckets whose every value a pass reads; the manifests are read by digest when a pass needs one, and
|
||||
// watched only for keys, so a manifest that arrives wakes a composition that waits for it.
|
||||
var watched = []string{
|
||||
buscompose.ResolvedBucket,
|
||||
provider.AssignmentsBucket,
|
||||
delivery.ControllerGrantsBucket,
|
||||
delivery.ControllerComposedBucket,
|
||||
@@ -176,9 +159,7 @@ var watched = []string{
|
||||
delivery.CandidatesBucket,
|
||||
delivery.ModulesBucket,
|
||||
delivery.PlacementBucket,
|
||||
delivery.PlacementProposalsBucket,
|
||||
delivery.PlacementCandidatesBucket,
|
||||
delivery.PlacementPulsesBucket,
|
||||
delivery.PresenceBucket,
|
||||
delivery.HaveBucket,
|
||||
delivery.AssignerAsksBucket,
|
||||
@@ -275,7 +256,7 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
|
||||
}
|
||||
started := 0
|
||||
for _, name := range watched {
|
||||
if kvs[name] == nil && slices.Contains(seatBuckets, name) {
|
||||
if kvs[name] == nil && (name == delivery.PlacementBucket || name == delivery.PlacementCandidatesBucket) {
|
||||
continue // not on the bus at this start (open said so)
|
||||
}
|
||||
started++
|
||||
@@ -290,24 +271,24 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
|
||||
if err := watch(delivery.ManifestsBucket, jetstream.MetaOnly()); err != nil {
|
||||
return err
|
||||
}
|
||||
// The seat's states arrive with the same delivery as this assigner (ADR 0567 §1): one not on the bus at this start is
|
||||
// bound once the bus says it was created, and watched from then on, with no restart (issue 1790's kind). Until then the
|
||||
// controller's assignments alone name where modules run (ADR 0549 §5).
|
||||
// The indexer's placement state arrives with the same delivery as this assigner (ADR 0549 §2): when it is not on the
|
||||
// bus at this start it is bound once the bus says it was created, and watched from then on, with no restart (issue
|
||||
// 1790's kind). Until then the controller's assignments alone name where modules run (ADR 0549 §5).
|
||||
type appeared struct {
|
||||
name string
|
||||
kv jetstream.KeyValue
|
||||
}
|
||||
placementAppeared := make(chan appeared, len(seatBuckets))
|
||||
for _, name := range seatBuckets {
|
||||
if kvs[name] == nil && js != nil {
|
||||
process.AwaitBucket(ctx, js, nc, name, "the seat mesh-placement waits for it",
|
||||
logger.Printf).Ready(func(kv jetstream.KeyValue) { placementAppeared <- appeared{name, kv} })
|
||||
}
|
||||
placementAppeared := make(chan appeared, 2)
|
||||
if kvs[delivery.PlacementBucket] == nil && js != nil {
|
||||
process.AwaitBucket(ctx, js, nc, delivery.PlacementBucket, "the controller's assignments alone are read",
|
||||
logger.Printf).Ready(func(kv jetstream.KeyValue) { placementAppeared <- appeared{delivery.PlacementBucket, kv} })
|
||||
}
|
||||
// So are the placement candidates (ADR 0559 §3), with the same delivery as the indexer that writes them.
|
||||
if kvs[delivery.PlacementCandidatesBucket] == nil && js != nil {
|
||||
process.AwaitBucket(ctx, js, nc, delivery.PlacementCandidatesBucket, "no change's placement is composed",
|
||||
logger.Printf).Ready(func(kv jetstream.KeyValue) { placementAppeared <- appeared{delivery.PlacementCandidatesBucket, kv} })
|
||||
}
|
||||
takePlacement := func(a appeared) error {
|
||||
if kvs[a.name] != nil {
|
||||
return nil // bound already, by the other of its two events
|
||||
}
|
||||
kvs[a.name] = a.kv
|
||||
logger.Printf("the bucket %s is open and watched", a.name)
|
||||
return watch(a.name)
|
||||
@@ -322,16 +303,11 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
|
||||
}
|
||||
}
|
||||
p := &passer{kvs: kvs, w: w, logf: logger.Printf, say: say, said: map[string]bool{}, answeredAt: map[string]uint64{}}
|
||||
p.wakeAt = wakeAtVia(ctx, wake)
|
||||
p.asks = newComposeAsks(ctx, buildrun.Asker{Publisher: buildrun.JetStreamAsks{JS: js}, Waiter: buildrun.JetStreamAsks{JS: js}}, wake, logger.Printf)
|
||||
if bus != nil && kvs[delivery.AssignerPutsBucket] != nil {
|
||||
p.acts = newPutActs(bus, wake, logger.Printf)
|
||||
}
|
||||
if bus != nil {
|
||||
p.seat = newSeatActs(bus, wake, logger.Printf)
|
||||
}
|
||||
if js != nil {
|
||||
p.publish = streamPublisher(ctx, js, thisNode)
|
||||
}
|
||||
wake()
|
||||
for {
|
||||
select {
|
||||
@@ -346,28 +322,36 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
|
||||
}
|
||||
case <-dirty:
|
||||
reopenPuts(p)
|
||||
// The grants part makes the seat's buckets in the pass that seals the list granting them, so their creation can
|
||||
// be said before this credential may hear it: every pass, itself started by a watched change, binds each one the
|
||||
// bus now holds, and says the seat waits while one is missing.
|
||||
var unbound []string
|
||||
for _, name := range seatBuckets {
|
||||
if kvs[name] != nil || js == nil {
|
||||
continue
|
||||
}
|
||||
if kv, err := js.KeyValue(ctx, name); err == nil {
|
||||
if err := takePlacement(appeared{name, kv}); err != nil {
|
||||
return err
|
||||
}
|
||||
continue
|
||||
}
|
||||
unbound = append(unbound, name)
|
||||
}
|
||||
p.sayWaits(unbound)
|
||||
p.pass(ctx)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// wakeAtVia asks wake once at each time it is given, as a context's deadline (ADR 0437 §8: no timer), each time once.
|
||||
func wakeAtVia(ctx context.Context, wake func()) func(time.Time) {
|
||||
var mu sync.Mutex
|
||||
asked := map[int64]bool{}
|
||||
return func(at time.Time) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if asked[at.UnixNano()] {
|
||||
return
|
||||
}
|
||||
asked[at.UnixNano()] = true
|
||||
go func() {
|
||||
bound, cancel := context.WithDeadline(ctx, at)
|
||||
defer cancel()
|
||||
<-bound.Done()
|
||||
mu.Lock()
|
||||
delete(asked, at.UnixNano())
|
||||
mu.Unlock()
|
||||
if ctx.Err() == nil {
|
||||
wake()
|
||||
}
|
||||
}()
|
||||
}
|
||||
}
|
||||
|
||||
// PutsReopenEvery is how often, at most, a pass opens the puts' bucket again while it is not open.
|
||||
var PutsReopenEvery = time.Minute
|
||||
|
||||
|
||||
@@ -24,10 +24,6 @@ import (
|
||||
// 1297/B2a: the buckets the program opens are exactly the manifest's reads and states, as the broker names them
|
||||
// (`<module>.<state>` read as the bucket `<module>_<state>`), so it is granted what it reads and reads nothing else.
|
||||
func TestOpensWhatItsManifestNames(t *testing.T) {
|
||||
// 1297/B2a: without a credential the program does not start, and says why.
|
||||
if err := run(context.Background(), func(string) string { return "" }, io.Discard); err == nil || !strings.Contains(err.Error(), "MESH_BROKER_FILE") {
|
||||
t.Errorf("run without a credential = %v", err)
|
||||
}
|
||||
raw, err := os.ReadFile(filepath.Join("..", "..", "module.json"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -109,6 +105,14 @@ func TestOpensWhatItsManifestNames(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// 1297/B2a: without a credential the program does not start, and says why.
|
||||
func TestNoCredential(t *testing.T) {
|
||||
err := run(context.Background(), func(string) string { return "" }, io.Discard)
|
||||
if err == nil || !strings.Contains(err.Error(), "MESH_BROKER_FILE") {
|
||||
t.Errorf("run without a credential = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// refusingJS opens every bucket but one, which it refuses with err.
|
||||
type refusingJS struct {
|
||||
jetstream.JetStream
|
||||
|
||||
@@ -43,14 +43,9 @@ type passer struct {
|
||||
// nodeFailed is, by node and module, the words of the condition last raised for a failure that is the node's own (ADR 0557
|
||||
// §1, issue 1496): raised once per words, cleared once when it ends.
|
||||
nodeFailed map[string]string
|
||||
// The seat mesh-placement's holder (seat.go, ADR 0567): its calls of the controller (nil: none is made), how it says
|
||||
// the seat's events (nil: none is said), the proposals it said, and what Intended computed by proposal and base.
|
||||
seat *seatActs
|
||||
publish publisher
|
||||
saidProposed map[string]bool
|
||||
// saidWaits is the last said of the seat's wait for its states (sayWaits).
|
||||
saidWaits string
|
||||
intended map[string]map[string]map[string]bool
|
||||
// wakeAt asks a pass at a time: the end of a placement candidate's answer bound (ADR 0559 §5), as a context's deadline
|
||||
// (ADR 0437 §8: no timer). Nil asks none.
|
||||
wakeAt func(at time.Time)
|
||||
}
|
||||
|
||||
func (p *passer) clock() time.Time {
|
||||
@@ -61,7 +56,6 @@ func (p *passer) clock() time.Time {
|
||||
}
|
||||
|
||||
func (p *passer) pass(ctx context.Context) {
|
||||
p.placementSeat(ctx)
|
||||
p.compositions(ctx)
|
||||
p.wants(ctx)
|
||||
p.puts(ctx)
|
||||
@@ -126,7 +120,7 @@ func (p *passer) wants(ctx context.Context) {
|
||||
}
|
||||
}
|
||||
wanted := p.w.wantsOf()
|
||||
frozen := p.sayPlacement(p.withCalls(p.w.placementDiffers()))
|
||||
frozen := p.sayPlacement(p.w.placementDiffers())
|
||||
placed := p.w.placedNodes()
|
||||
puts := p.putsByModule()
|
||||
p.staleAsks(ctx, trunk)
|
||||
@@ -488,6 +482,9 @@ func (p *passer) compositions(ctx context.Context) {
|
||||
} else {
|
||||
v = assign.Compose(*mesh, ch, p.sdk, time.Since(began))
|
||||
}
|
||||
if ch.Placement != nil && v.State != delivery.CompositionWaits {
|
||||
v = p.answers(*mesh, ch, v)
|
||||
}
|
||||
}
|
||||
if before, err := delivery.DecodeCompositionVerdict(written[out]); err == nil {
|
||||
before.Took, before.V = v.Took, 0
|
||||
@@ -499,6 +496,40 @@ func (p *passer) compositions(ctx context.Context) {
|
||||
}
|
||||
}
|
||||
|
||||
// answers judges every placed node's answer to the change's placement into v (ADR 0559 §5). While a node has not answered
|
||||
// within the bound the verdict waits, and the pass at the bound's end is asked (wakeAt), so a silent node is failed then,
|
||||
// named, and a later answer of the same head is judged on its own have's change.
|
||||
func (p *passer) answers(mesh assign.Mesh, ch assign.Change, v delivery.CompositionVerdict) delivery.CompositionVerdict {
|
||||
trunkPlaced := map[string]delivery.Placement{}
|
||||
for key, value := range p.w.read(delivery.PlacementBucket) {
|
||||
node, err := delivery.ParsePlacementKey(key)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if pl, err := delivery.DecodePlacement(value); err == nil && pl.Node == node {
|
||||
trunkPlaced[node] = pl
|
||||
}
|
||||
}
|
||||
fps := map[string]string{}
|
||||
for name, m := range p.w.trunk() {
|
||||
if m.FP2 != "" {
|
||||
fps[name] = m.FP2
|
||||
} else if m.FP != "" {
|
||||
fps[name] = m.FP
|
||||
}
|
||||
}
|
||||
intended, notes := assign.Intended(mesh.Input, trunkPlaced, ch.Placement, fps)
|
||||
a := assign.JudgeAnswers(ch.Placement, assign.PlacedNodes(trunkPlaced, ch.Placement), p.w.candidateAnswers(ch.Group), intended, p.w.online(), p.clock())
|
||||
v = assign.WithAnswers(v, a)
|
||||
if v.State != delivery.CompositionWaits {
|
||||
v.Warnings = append(v.Warnings, notes...)
|
||||
}
|
||||
if a.Waits && p.wakeAt != nil {
|
||||
p.wakeAt(a.Until)
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
// put writes one key of the assigner's own bucket; a write that fails is said and made again at the next pass.
|
||||
func (p *passer) put(ctx context.Context, bucket, key string, encode func() ([]byte, error)) {
|
||||
value, err := encode()
|
||||
|
||||
@@ -853,7 +853,7 @@ func aPairOneSourceNamesAloneGetsNoWantAndOneCondition(t *testing.T) {
|
||||
p.sayPlacement(p.w.placementDiffers())
|
||||
if !slices.Equal(got["g14"], []string{"a"}) || !frozen["g14.b"] || !frozen["g14.c"] || frozen["g14.a"] || len(said) != 2 ||
|
||||
said[0].State != asks.StateRaised || !strings.Contains(said[0].Explanation, "the controller's assignments names b on g14") ||
|
||||
!strings.Contains(said[1].Explanation, "the seat mesh-placement's placement names c on g14") {
|
||||
!strings.Contains(said[1].Explanation, "the catalogue's placement names c on g14") {
|
||||
t.Fatalf("one source alone: assigned %v, frozen %v, said %+v", got, frozen, said)
|
||||
}
|
||||
place("g14", "a", "b", "c")
|
||||
|
||||
@@ -1,810 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"maps"
|
||||
"reflect"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/compose"
|
||||
"git.novox.be/novox/mesh-sdk/go/compose/frombus"
|
||||
"git.novox.be/novox/mesh-sdk/go/compose/judge"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
"mesh-delivery-assigner/internal/assign"
|
||||
)
|
||||
|
||||
// The seat mesh-placement's holder in the assigner's process (novox/hq ADR 0567 §2-§8): each pass follows every open
|
||||
// proposal one step, from what the world holds. A proposal the verbs wrote is said (placement-proposed); the nodes it
|
||||
// must hear from are found by composing every node with and without it, and those its key does not change get a reached
|
||||
// candidate; their answers are judged (placement-judged) and, on a passing verdict, the operator is asked to approve it;
|
||||
// on the warrant each changed key is written by compare-and-set at the revision it was laid over (placement-changed),
|
||||
// and the controller's assignments are kept in step through its verbs until the lease write. A key moved meanwhile
|
||||
// lays the proposal again; a decline, an expiry or a withdrawal ends it (placement-withdrawn). Every event is said on the
|
||||
// assigner's own stream, so only this process says them. Nothing polls: the wake at an answer's bound is the one wake.
|
||||
|
||||
// publisher says one of the seat's events on the assigner's stream, under a message id the bus keeps one of.
|
||||
type publisher func(event string, body any, id string) error
|
||||
|
||||
// streamPublisher publishes on mesh.stream.mesh-delivery-assigner.event.<event>, with the headers an emit carries.
|
||||
func streamPublisher(ctx context.Context, js jetstream.JetStream, node string) publisher {
|
||||
return func(event string, body any, id string) error {
|
||||
data, err := json.Marshal(body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
msg := nats.NewMsg(process.ModuleStreamSubject(delivery.AssignerModule, event))
|
||||
msg.Header.Set("x-event-id", id)
|
||||
msg.Header.Set("x-source", delivery.AssignerModule)
|
||||
msg.Header.Set("x-node", node)
|
||||
msg.Header.Set("x-time", time.Now().UTC().Format(time.RFC3339Nano))
|
||||
msg.Header.Set("content-type", "application/json")
|
||||
msg.Data = data
|
||||
_, err = js.PublishMsg(ctx, msg, jetstream.WithMsgID(id))
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// seatActs keeps what the seat's calls of the controller came to while this process runs: the export read again at a
|
||||
// seed's warrant, and each call that keeps the controller's assignments in step (§8).
|
||||
type seatActs struct {
|
||||
mu sync.Mutex
|
||||
bus caller
|
||||
wake func()
|
||||
logf func(string, ...any)
|
||||
export map[string]exportCheck
|
||||
// calls is, by pair (node and module, or node and provision for a pin), what the last call for it came to.
|
||||
calls map[string]string
|
||||
}
|
||||
|
||||
// exportCheck is the export read again for one proposal: running, its answer, or why it was not read.
|
||||
type exportCheck struct {
|
||||
running bool
|
||||
export map[string]delivery.Placement
|
||||
err string
|
||||
}
|
||||
|
||||
func newSeatActs(bus caller, wake func(), logf func(string, ...any)) *seatActs {
|
||||
return &seatActs{bus: bus, wake: wake, logf: logf, export: map[string]exportCheck{}, calls: map[string]string{}}
|
||||
}
|
||||
|
||||
// exportOf is the export read again for a proposal, asked once in the background.
|
||||
func (a *seatActs) exportOf(id string) exportCheck {
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
if c, ok := a.export[id]; ok {
|
||||
return c
|
||||
}
|
||||
a.export[id] = exportCheck{running: true}
|
||||
go func() {
|
||||
defer a.wake()
|
||||
raw, err := a.bus.Call(exportCall, map[string]any{})
|
||||
c := exportCheck{}
|
||||
if err != nil {
|
||||
c.err = "the controller's placement-export did not answer: " + err.Error()
|
||||
} else if c.export, err = assign.ExportOf(raw); err != nil {
|
||||
c.err = err.Error()
|
||||
}
|
||||
a.mu.Lock()
|
||||
a.export[id] = c
|
||||
a.mu.Unlock()
|
||||
}()
|
||||
return exportCheck{running: true}
|
||||
}
|
||||
|
||||
// forget drops what is kept of a proposal's export, so a later warrant reads it again.
|
||||
func (a *seatActs) forget(id string) {
|
||||
if a == nil {
|
||||
return
|
||||
}
|
||||
a.mu.Lock()
|
||||
delete(a.export, id)
|
||||
a.mu.Unlock()
|
||||
}
|
||||
|
||||
// record records once, in the controller's hand-act log, that placement was written on the operator's warrant (§5): the
|
||||
// router's record of the teller's ask says who chose and how (ADR 0274).
|
||||
func (a *seatActs) record(warrant string) {
|
||||
if a == nil || warrant == "" {
|
||||
return
|
||||
}
|
||||
ask, _, _ := strings.Cut(warrant, "@")
|
||||
go func() {
|
||||
if _, err := a.bus.Call("seat:"+catalogue.ControllerSeatName+".warranted", map[string]any{"asker": delivery.TellerModule, "ask": ask}); err != nil {
|
||||
a.logf("the placement's warrant %s is not recorded in the hand-act log: %v", ask, err)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// handAct records a hand-back in the controller's hand-act log (§7).
|
||||
func (a *seatActs) handAct(prop delivery.PlacementProposal) {
|
||||
go func() {
|
||||
if _, err := a.bus.Call(handActCall, map[string]any{"what": "mesh-placement hand-back of " + strings.Join(prop.Nodes(), ", "),
|
||||
"why": prop.Why, "cause": "asked by " + prop.By + ", approved by the operator"}); err != nil {
|
||||
a.logf("the hand-back %s is not recorded in the hand-act log: %v", prop.ID, err)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// callAll makes the controller's calls in order in the background, each said in the journal and kept by its pair.
|
||||
func (a *seatActs) callAll(calls []assign.ControllerCall) {
|
||||
if len(calls) == 0 {
|
||||
return
|
||||
}
|
||||
a.mu.Lock()
|
||||
for _, c := range calls {
|
||||
a.calls[c.Node+"."+c.Module] = c.Key() + " is asked"
|
||||
}
|
||||
a.mu.Unlock()
|
||||
go func() {
|
||||
defer a.wake()
|
||||
for _, c := range calls {
|
||||
words := c.Key() + " answered"
|
||||
if _, err := a.bus.Call("seat:"+catalogue.ControllerSeatName+"."+c.Verb, c.Args); err != nil {
|
||||
words = c.Key() + " refused: " + err.Error()
|
||||
}
|
||||
a.logf("keeping the controller in step with placement: %s", words)
|
||||
a.mu.Lock()
|
||||
a.calls[c.Node+"."+c.Module] = words
|
||||
a.mu.Unlock()
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// callOf is what the last call for a pair came to, "" when none was made.
|
||||
func (a *seatActs) callOf(node, module string) string {
|
||||
if a == nil {
|
||||
return ""
|
||||
}
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
return a.calls[node+"."+module]
|
||||
}
|
||||
|
||||
// withCalls names, on each pair the two sources do not both name, the call the assigner made for it (§8).
|
||||
func (p *passer) withCalls(pairs []assign.Pair) []assign.Pair {
|
||||
for i := range pairs {
|
||||
pairs[i].Call = p.seat.callOf(pairs[i].Node, pairs[i].Module)
|
||||
}
|
||||
return pairs
|
||||
}
|
||||
|
||||
// sayWaits raises, once, that the seat waits for the states not yet bound, and clears it once, when all are.
|
||||
func (p *passer) sayWaits(unbound []string) {
|
||||
c := assign.SeatWaitsCondition(unbound)
|
||||
state := string(c.State) + c.Summary
|
||||
if p.say == nil || p.said[c.Key] && p.saidWaits == state {
|
||||
return
|
||||
}
|
||||
if c.State == asks.StateCleared && p.saidWaits == "" {
|
||||
return // nothing was raised by this process
|
||||
}
|
||||
if err := p.say(c); err != nil {
|
||||
p.logf("that the seat waits for its states could not be said, and is said at the next pass: %v", err)
|
||||
return
|
||||
}
|
||||
p.said[c.Key], p.saidWaits = true, state
|
||||
}
|
||||
|
||||
// seatPlacements is every node's placement key and its revision, as the watch holds them.
|
||||
func (p *passer) seatPlacements() (map[string]delivery.Placement, map[string]uint64) {
|
||||
out, revs := map[string]delivery.Placement{}, map[string]uint64{}
|
||||
for key, value := range p.w.read(delivery.PlacementBucket) {
|
||||
node, err := delivery.ParsePlacementKey(key)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if pl, err := delivery.DecodePlacement(value); err == nil && pl.Node == node {
|
||||
out[node], revs[node] = pl, p.w.metaOf(delivery.PlacementBucket, key).revision
|
||||
}
|
||||
}
|
||||
return out, revs
|
||||
}
|
||||
|
||||
// candidatesOf is a proposal's candidates by node, with their keys' revisions and the newest write.
|
||||
func (p *passer) candidatesOf(id string) (map[string]delivery.PlacementCandidate, map[string]uint64, time.Time) {
|
||||
out, revs := map[string]delivery.PlacementCandidate{}, map[string]uint64{}
|
||||
var newest time.Time
|
||||
for key, value := range p.w.read(delivery.PlacementCandidatesBucket) {
|
||||
pid, node, err := delivery.ParsePlacementCandidateKey(key)
|
||||
if err != nil || pid != id {
|
||||
continue
|
||||
}
|
||||
c, err := delivery.DecodePlacementCandidate(value)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
meta := p.w.metaOf(delivery.PlacementCandidatesBucket, key)
|
||||
out[node], revs[node] = c, meta.revision
|
||||
if meta.at.After(newest) {
|
||||
newest = meta.at
|
||||
}
|
||||
}
|
||||
return out, revs, newest
|
||||
}
|
||||
|
||||
// answersTo is, by node, each node's answer to the proposal in its `<node>.node` have.
|
||||
func (p *passer) answersTo(id string) map[string]*delivery.CandidateAnswer {
|
||||
out := map[string]*delivery.CandidateAnswer{}
|
||||
for key, value := range p.w.read(delivery.HaveBucket) {
|
||||
node, _, isNode, err := delivery.ParseHaveKey(key)
|
||||
if err != nil || !isNode {
|
||||
continue
|
||||
}
|
||||
h, err := delivery.DecodeHaveNode(value)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if a, ok := h.Candidates[id]; ok {
|
||||
out[node] = &a
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// placementSeat takes every open proposal one step further.
|
||||
func (p *passer) placementSeat(ctx context.Context) {
|
||||
for _, b := range seatBuckets {
|
||||
if p.kvs[b] == nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
if p.saidProposed == nil {
|
||||
p.saidProposed, p.intended = map[string]bool{}, map[string]map[string]map[string]bool{}
|
||||
}
|
||||
current, revs := p.seatPlacements()
|
||||
ids := make([]string, 0)
|
||||
raws := p.w.read(delivery.PlacementProposalsBucket)
|
||||
for id := range raws {
|
||||
ids = append(ids, id)
|
||||
}
|
||||
sort.Strings(ids)
|
||||
for _, id := range ids {
|
||||
prop, err := delivery.DecodePlacementProposal(raws[id])
|
||||
if err != nil || prop.ID != id {
|
||||
p.logf("the proposal %s cannot be read, so it is left: %v", id, err)
|
||||
continue
|
||||
}
|
||||
p.stepProposal(ctx, prop, p.w.metaOf(delivery.PlacementProposalsBucket, id).revision, current, revs)
|
||||
}
|
||||
}
|
||||
|
||||
// pulse writes a bound's key with its time to live (§4): its expiry, heard on the watch as the server's delete marker,
|
||||
// is the event that ends the wait. Written again over one that stands, so its time starts anew.
|
||||
func (p *passer) pulse(ctx context.Context, key string, ttl time.Duration) {
|
||||
kv := p.kvs[delivery.PlacementPulsesBucket]
|
||||
_ = kv.Delete(ctx, key)
|
||||
rev, err := kv.Create(ctx, key, []byte(`{}`), jetstream.KeyTTL(ttl))
|
||||
if err != nil {
|
||||
p.logf("the bound %s is not written, so its wait ends at once: %v", key, err)
|
||||
return
|
||||
}
|
||||
p.w.putEntry(delivery.PlacementPulsesBucket, key, []byte(`{}`), rev, p.clock())
|
||||
}
|
||||
|
||||
// pulsing says whether a bound's key stands.
|
||||
func (p *passer) pulsing(key string) bool { return p.w.has(delivery.PlacementPulsesBucket, key) }
|
||||
|
||||
// deletePulses deletes every bound of a proposal.
|
||||
func (p *passer) deletePulses(ctx context.Context, id string) {
|
||||
for key := range p.w.read(delivery.PlacementPulsesBucket) {
|
||||
if strings.HasSuffix(key, "."+id) || strings.Contains(key, "."+id+".") {
|
||||
if err := p.kvs[delivery.PlacementPulsesBucket].Delete(ctx, key); err == nil {
|
||||
p.w.put(delivery.PlacementPulsesBucket, key, nil)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ask raises (or raises again) a proposal's authorising ask and its bound, and answers the proposal asked.
|
||||
func (p *passer) ask(ctx context.Context, prop delivery.PlacementProposal, akey, brief string) (delivery.PlacementProposal, bool) {
|
||||
a, err := assign.PlacementAsk(akey, prop, brief, assign.PlacementAsking(prop, p.clock()), p.clock())
|
||||
if err != nil {
|
||||
p.logf("the ask about the proposal %s is not written: %v", prop.ID, err)
|
||||
return prop, false
|
||||
}
|
||||
p.put(ctx, delivery.AssignerAsksBucket, akey, func() ([]byte, error) { return delivery.EncodeAssignerAsk(akey, a) })
|
||||
if pk, err := delivery.PulseAskKey(prop.ID); err == nil {
|
||||
p.pulse(ctx, pk, a.Ask.Expires.Sub(p.clock()))
|
||||
}
|
||||
prop.State, prop.Ask = delivery.ProposalAsked, a.Ask.ID
|
||||
return prop, true
|
||||
}
|
||||
|
||||
// stepProposal takes one proposal one step further.
|
||||
func (p *passer) stepProposal(ctx context.Context, prop delivery.PlacementProposal, rev uint64, current map[string]delivery.Placement, revs map[string]uint64) {
|
||||
if !p.saidProposed[prop.ID] && prop.State == delivery.ProposalAnswering && prop.Verdict == nil {
|
||||
p.sayEvent(delivery.PlacementProposedEvent, delivery.PlacementEvent{Proposal: prop}, prop.ID)
|
||||
p.saidProposed[prop.ID] = true
|
||||
}
|
||||
if prop.Withdrawn != "" {
|
||||
p.endProposal(ctx, prop, prop.Withdrawn)
|
||||
return
|
||||
}
|
||||
akey, _ := delivery.PlacementAskKey(prop.ID)
|
||||
again := !prop.Again.IsZero() && (prop.Verdict == nil || prop.Again.After(prop.Verdict.At))
|
||||
if prop.State == delivery.ProposalAsked {
|
||||
askPulse, _ := delivery.PulseAskKey(prop.ID)
|
||||
switch {
|
||||
case p.answeredWith(akey, assign.OptionYes):
|
||||
if prop.HandBack {
|
||||
p.handBack(ctx, prop, akey, revs)
|
||||
} else {
|
||||
p.warranted(ctx, prop, rev, akey, current, revs)
|
||||
}
|
||||
return
|
||||
case p.answeredWith(akey, assign.OptionNo):
|
||||
p.endProposal(ctx, prop, "declined by the operator")
|
||||
return
|
||||
case !p.pulsing(askPulse) && !again:
|
||||
p.endProposal(ctx, prop, "its ask expired unanswered")
|
||||
return
|
||||
}
|
||||
}
|
||||
if prop.HandBack {
|
||||
// The operator's alone (§7): no node answers it; its ask is raised at once, and again on again.
|
||||
if prop.State != delivery.ProposalAsked || again {
|
||||
brief := fmt.Sprintf("%s (asked by %s): every node's placement key is deleted, %s, and each node returns to the "+
|
||||
"controller's compositions; the history is kept", prop.Why, prop.By, strings.Join(prop.Nodes(), ", "))
|
||||
now := p.clock().UTC()
|
||||
prop.Verdict = &delivery.PlacementVerdict{State: delivery.VerdictPass, At: now}
|
||||
if prop, ok := p.ask(ctx, prop, akey, brief); ok {
|
||||
p.writeProposal(ctx, prop, rev)
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
if moved(prop, revs) {
|
||||
p.layAgain(ctx, prop, rev, current, revs, "a key it was laid over moved since")
|
||||
return
|
||||
}
|
||||
cands, candRevs, written := p.candidatesOf(prop.ID)
|
||||
if len(prop.Must) == 0 {
|
||||
p.mustAnswer(ctx, prop, rev, current, cands)
|
||||
return
|
||||
}
|
||||
for _, node := range prop.Must {
|
||||
if _, ok := cands[node]; !ok {
|
||||
return // a candidate just written is not heard yet: its watch wakes the next pass
|
||||
}
|
||||
}
|
||||
if written.After(prop.Written) {
|
||||
prop.Written = written
|
||||
}
|
||||
pulsing := map[string]bool{}
|
||||
for _, node := range prop.Must {
|
||||
if k, err := delivery.PulseAnswerKey(prop.ID, node); err == nil {
|
||||
pulsing[node] = p.pulsing(k)
|
||||
}
|
||||
}
|
||||
v := judge.JudgeAnswers(prop, p.intendedOf(prop, current, cands), p.answersTo(prop.ID), candRevs, p.w.online(), pulsing, p.clock())
|
||||
if prop.Verdict != nil && sameVerdict(*prop.Verdict, v) && !again {
|
||||
return
|
||||
}
|
||||
prop.Verdict = &v
|
||||
switch v.State {
|
||||
case delivery.VerdictFail:
|
||||
prop.State, prop.Ask = delivery.ProposalFailed, ""
|
||||
case delivery.VerdictWaits:
|
||||
prop.State, prop.Ask = delivery.ProposalAnswering, ""
|
||||
case delivery.VerdictPass:
|
||||
if prop.State != delivery.ProposalAsked || again {
|
||||
var ok bool
|
||||
if prop, ok = p.ask(ctx, prop, akey, judge.Brief(prop, v)); !ok {
|
||||
return
|
||||
}
|
||||
if again {
|
||||
prop.Words = "judged and asked again (again)"
|
||||
}
|
||||
}
|
||||
}
|
||||
if p.writeProposal(ctx, prop, rev) {
|
||||
p.sayEvent(delivery.PlacementJudgedEvent, delivery.PlacementEvent{Proposal: prop},
|
||||
fmt.Sprintf("%s.judged.%d", prop.ID, v.At.UnixNano()))
|
||||
}
|
||||
}
|
||||
|
||||
// sameVerdict says two verdicts say the same, their time aside.
|
||||
func sameVerdict(a, b delivery.PlacementVerdict) bool {
|
||||
a.At, b.At = time.Time{}, time.Time{}
|
||||
return reflect.DeepEqual(a, b)
|
||||
}
|
||||
|
||||
// moved says whether a key the proposal changes is no longer at the revision it was laid over.
|
||||
func moved(prop delivery.PlacementProposal, revs map[string]uint64) bool {
|
||||
for node, base := range prop.Base {
|
||||
if revs[node] != base {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// intended is, by proposal and the revisions it was laid over, what Intended computed, kept while this process runs.
|
||||
func (p *passer) intendedOf(prop delivery.PlacementProposal, current map[string]delivery.Placement, cands map[string]delivery.PlacementCandidate) map[string]map[string]bool {
|
||||
key := fmt.Sprint(prop.ID, prop.Base)
|
||||
if got, ok := p.intended[key]; ok {
|
||||
return got
|
||||
}
|
||||
got, _, _ := p.compute(prop, current, cands)
|
||||
p.intended[key] = got
|
||||
return got
|
||||
}
|
||||
|
||||
// compute composes every node with the current placement and with the proposal laid over it (judge.Intended).
|
||||
func (p *passer) compute(prop delivery.PlacementProposal, current map[string]delivery.Placement, cands map[string]delivery.PlacementCandidate) (map[string]map[string]bool, []string, []string) {
|
||||
laid := map[string]*delivery.Placement{}
|
||||
for _, node := range prop.Nodes() {
|
||||
if c, ok := cands[node]; ok && !c.Reached {
|
||||
laid[node] = c.Placement
|
||||
}
|
||||
}
|
||||
get := kvGetter(p.kvs[delivery.ManifestsBucket])
|
||||
r := worldReader{w: p.w, manifests: get}
|
||||
var now, with *compose.Input
|
||||
if in, err := frombus.Input(context.Background(), r, frombus.Mesh{}, frombus.Scope{Every: true}); err == nil {
|
||||
now = &in
|
||||
}
|
||||
if in, err := frombus.Input(context.Background(), r, frombus.Mesh{}, frombus.Scope{Every: true, Placements: laid}); err == nil {
|
||||
with = &in
|
||||
}
|
||||
nodes := map[string]bool{}
|
||||
for n := range current {
|
||||
nodes[n] = true
|
||||
}
|
||||
if records, err := r.Records(context.Background()); err == nil {
|
||||
for n := range records {
|
||||
nodes[n] = true
|
||||
}
|
||||
}
|
||||
names := make([]string, 0, len(nodes))
|
||||
for n := range nodes {
|
||||
names = append(names, n)
|
||||
}
|
||||
fps := map[string]string{}
|
||||
for name, m := range p.w.trunk() {
|
||||
if m.FP2 != "" {
|
||||
fps[name] = m.FP2
|
||||
}
|
||||
}
|
||||
return judge.Intended(now, with, current, laid, names, fps)
|
||||
}
|
||||
|
||||
// mustAnswer finds the nodes the proposal must hear from (§3), writes a reached candidate for each whose key it does not
|
||||
// change, and writes them into the proposal.
|
||||
func (p *passer) mustAnswer(ctx context.Context, prop delivery.PlacementProposal, rev uint64, current map[string]delivery.Placement, cands map[string]delivery.PlacementCandidate) {
|
||||
for _, node := range prop.Nodes() {
|
||||
if _, ok := cands[node]; !ok {
|
||||
return // the verbs write the candidates before the proposal: wait to hear them
|
||||
}
|
||||
}
|
||||
intended, must, notes := p.compute(prop, current, cands)
|
||||
p.intended[fmt.Sprint(prop.ID, prop.Base)] = intended
|
||||
for _, node := range must {
|
||||
if _, ok := cands[node]; ok {
|
||||
continue
|
||||
}
|
||||
key, err := delivery.PlacementCandidateKey(prop.ID, node)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
c := delivery.PlacementCandidate{Proposal: prop.ID, Reached: true}
|
||||
if pl, ok := current[node]; ok {
|
||||
c.Placement = &pl
|
||||
}
|
||||
p.put(ctx, delivery.PlacementCandidatesBucket, key, func() ([]byte, error) { return delivery.EncodePlacementCandidate(c) })
|
||||
}
|
||||
for _, node := range must {
|
||||
if k, err := delivery.PulseAnswerKey(prop.ID, node); err == nil {
|
||||
p.pulse(ctx, k, judge.AnswerBound)
|
||||
}
|
||||
}
|
||||
prop.Must = must
|
||||
if len(notes) > 0 {
|
||||
prop.Words = strings.TrimPrefix(prop.Words+"; "+strings.Join(notes, "; "), "; ")
|
||||
}
|
||||
p.writeProposal(ctx, prop, rev)
|
||||
}
|
||||
|
||||
// layAgain lays a proposal over the keys as they stand now (§5: a key moved): its candidates are written again, the
|
||||
// nodes that must answer found again, and it is judged and asked again; a proposal that no longer lays ends.
|
||||
func (p *passer) layAgain(ctx context.Context, prop delivery.PlacementProposal, rev uint64, current map[string]delivery.Placement, revs map[string]uint64, why string) {
|
||||
held, recorded := p.lays(ctx)
|
||||
again, laid, findings := assign.Propose(assign.ProposeIn{ID: prop.ID, By: prop.By, Why: prop.Why, At: prop.At, Changes: prop.Changes,
|
||||
Current: current, Revisions: revs, Held: held, Recorded: recorded,
|
||||
Seed: prop.Seed, ExportDigest: prop.ExportDigest})
|
||||
if len(findings) > 0 {
|
||||
p.endProposal(ctx, prop, why+", and laid over the keys now it is refused: "+strings.Join(findings, "; "))
|
||||
return
|
||||
}
|
||||
p.deleteCandidates(ctx, prop.ID)
|
||||
p.deletePulses(ctx, prop.ID)
|
||||
for _, node := range again.Nodes() {
|
||||
key, _ := delivery.PlacementCandidateKey(prop.ID, node)
|
||||
c := delivery.PlacementCandidate{Proposal: prop.ID, Placement: laid[node]}
|
||||
p.put(ctx, delivery.PlacementCandidatesBucket, key, func() ([]byte, error) { return delivery.EncodePlacementCandidate(c) })
|
||||
}
|
||||
again.Written, again.Words = p.clock().UTC(), why+": laid again over the keys as they stand, to be judged and asked again"
|
||||
p.seat.forget(prop.ID)
|
||||
p.writeProposal(ctx, again, rev)
|
||||
}
|
||||
|
||||
// lays is what a proposal is checked against, as the verbs check it: the modules the indexer holds, by key, and the
|
||||
// nodes with a record.
|
||||
func (p *passer) lays(ctx context.Context) (func(string) bool, func(string) bool) {
|
||||
held := map[string]bool{}
|
||||
for key := range p.w.read(delivery.ModulesBucket) {
|
||||
if name, err := delivery.ParseModulesKey(key); err == nil {
|
||||
held[name] = true
|
||||
}
|
||||
}
|
||||
records, _ := worldReader{w: p.w}.Records(ctx)
|
||||
return func(m string) bool { return held[m] }, func(n string) bool { _, ok := records[n]; return ok }
|
||||
}
|
||||
|
||||
// warranted writes an approved proposal (§5): for the seed, after the export is read again and found unchanged (§7);
|
||||
// each changed key by compare-and-set at the revision it was laid over; then the warrant recorded, the candidates and the
|
||||
// proposal deleted, placement-changed said, and the controller kept in step (§8).
|
||||
func (p *passer) warranted(ctx context.Context, prop delivery.PlacementProposal, rev uint64, akey string, current map[string]delivery.Placement, revs map[string]uint64) {
|
||||
if err := p.performs(akey, assign.PlacementAct(prop)); err != nil {
|
||||
p.endProposal(ctx, prop, "its warrant does not bind these changes, so nothing is written: "+err.Error())
|
||||
return
|
||||
}
|
||||
answer, _ := p.w.answer(akey)
|
||||
if prop.Seed {
|
||||
if p.seat == nil {
|
||||
return
|
||||
}
|
||||
c := p.seat.exportOf(prop.ID)
|
||||
switch {
|
||||
case c.running:
|
||||
return // its answer wakes the next pass
|
||||
case c.err != "":
|
||||
if words := "the export cannot be read again at the warrant: " + c.err; prop.Words != words {
|
||||
prop.Words = words
|
||||
p.writeProposal(ctx, prop, rev)
|
||||
}
|
||||
return
|
||||
case delivery.ExportDigest(c.export) != prop.ExportDigest:
|
||||
p.endProposal(ctx, prop, "the controller's export changed since the seed was proposed: the new export is proposed as the seed")
|
||||
p.proposeSeed(ctx, prop, c.export)
|
||||
return
|
||||
}
|
||||
}
|
||||
if moved(prop, revs) {
|
||||
p.layAgain(ctx, prop, rev, current, revs, "a key moved since it was approved, so its warrant is spent")
|
||||
return
|
||||
}
|
||||
cands, _, _ := p.candidatesOf(prop.ID)
|
||||
laid := map[string]*delivery.Placement{}
|
||||
for _, node := range prop.Nodes() {
|
||||
laid[node] = cands[node].Placement
|
||||
}
|
||||
writes, err := assign.Writes(prop, laid)
|
||||
if err != nil {
|
||||
p.logf("the proposal %s cannot be written: %v", prop.ID, err)
|
||||
return
|
||||
}
|
||||
kv := p.kvs[delivery.PlacementBucket]
|
||||
var changes []delivery.PlacementChange
|
||||
writtenLaid := map[string]*delivery.Placement{}
|
||||
after := maps.Clone(current)
|
||||
afterRevs := maps.Clone(revs)
|
||||
for _, w := range writes {
|
||||
var rev2 uint64
|
||||
var err error
|
||||
switch {
|
||||
case w.Value == nil:
|
||||
err = kv.Delete(ctx, w.Key, jetstream.LastRevision(w.Base))
|
||||
default:
|
||||
raw, encErr := delivery.EncodePlacement(*w.Value)
|
||||
if encErr != nil {
|
||||
err = encErr
|
||||
} else if w.Base == 0 {
|
||||
rev2, err = kv.Create(ctx, w.Key, raw)
|
||||
} else {
|
||||
rev2, err = kv.Update(ctx, w.Key, raw, w.Base)
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
// Another writer came first (a hand-back): what is written stands and is followed through; the rest is laid
|
||||
// again over the keys as this write left them.
|
||||
p.logf("placement %s is not written at revision %d (%v): the rest of the proposal %s is laid again", w.Key, w.Base, err, prop.ID)
|
||||
p.followWrite(prop, answer, changes, current, writtenLaid)
|
||||
if rest, ok := assign.Rest(prop, writtenLaid); ok {
|
||||
p.layAgain(ctx, rest, rev, after, afterRevs, "a key moved while it was written, after "+strings.Join(sortedNodes(writtenLaid), ", ")+" were")
|
||||
} else {
|
||||
p.endProposal(ctx, prop, "a key moved while it was written: "+err.Error())
|
||||
}
|
||||
return
|
||||
}
|
||||
before := current[w.Node]
|
||||
now := delivery.Placement{Node: w.Node}
|
||||
if w.Value != nil {
|
||||
now = *w.Value
|
||||
after[w.Node] = *w.Value
|
||||
} else {
|
||||
delete(after, w.Node)
|
||||
}
|
||||
afterRevs[w.Node] = rev2
|
||||
c := delivery.ChangeOf(before, now)
|
||||
c.Before, c.After = w.Base, rev2
|
||||
changes = append(changes, c)
|
||||
writtenLaid[w.Node] = w.Value
|
||||
}
|
||||
p.deleteCandidates(ctx, prop.ID)
|
||||
p.deletePulses(ctx, prop.ID)
|
||||
_ = p.kvs[delivery.PlacementProposalsBucket].Delete(ctx, prop.ID)
|
||||
p.w.put(delivery.PlacementProposalsBucket, prop.ID, nil)
|
||||
p.followWrite(prop, answer, changes, current, writtenLaid)
|
||||
}
|
||||
|
||||
// followWrite follows the keys a warrant wrote: the warrant recorded, placement-changed said, and, until the lease
|
||||
// write, the controller's assignments kept in step for them (§8). The seed calls no controller verb: it is what the
|
||||
// controller holds already (§9).
|
||||
func (p *passer) followWrite(prop delivery.PlacementProposal, answer delivery.Answer, changes []delivery.PlacementChange,
|
||||
before map[string]delivery.Placement, written map[string]*delivery.Placement) {
|
||||
if len(changes) == 0 {
|
||||
return
|
||||
}
|
||||
p.seat.record(answer.Warrant)
|
||||
p.sayChanged(prop, answer, changes)
|
||||
if p.seat != nil && !assign.Leased && !prop.Seed {
|
||||
p.seat.callAll(assign.ControllerCalls(before, written, prop.ID, answer.Warrant))
|
||||
}
|
||||
}
|
||||
|
||||
func sortedNodes(m map[string]*delivery.Placement) []string {
|
||||
out := make([]string, 0, len(m))
|
||||
for n := range m {
|
||||
out = append(out, n)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// handBack deletes every key on the operator's warrant (§7) at the revision the hand-back read, records it in the
|
||||
// controller's hand-act log and says placement-changed; every node returns to the controller's compositions. A key moved
|
||||
// since ends it, and the operator hands back again.
|
||||
func (p *passer) handBack(ctx context.Context, prop delivery.PlacementProposal, akey string, revs map[string]uint64) {
|
||||
if err := p.performs(akey, assign.PlacementAct(prop)); err != nil {
|
||||
p.endProposal(ctx, prop, "its warrant does not bind this hand-back, so nothing is deleted: "+err.Error())
|
||||
return
|
||||
}
|
||||
answer, _ := p.w.answer(akey)
|
||||
current, _ := p.seatPlacements()
|
||||
var changes []delivery.PlacementChange
|
||||
for _, node := range prop.Nodes() {
|
||||
key, _ := delivery.PlacementKey(node)
|
||||
if err := p.kvs[delivery.PlacementBucket].Delete(ctx, key, jetstream.LastRevision(prop.Base[node])); err != nil {
|
||||
p.sayChanged(prop, answer, changes)
|
||||
p.endProposal(ctx, prop, fmt.Sprintf("%s moved since the hand-back was asked (%v): hand back again", key, err))
|
||||
return
|
||||
}
|
||||
c := delivery.ChangeOf(current[node], delivery.Placement{Node: node})
|
||||
c.Before = prop.Base[node]
|
||||
changes = append(changes, c)
|
||||
}
|
||||
if p.seat != nil {
|
||||
p.seat.handAct(prop)
|
||||
}
|
||||
p.seat.record(answer.Warrant)
|
||||
_ = p.kvs[delivery.PlacementProposalsBucket].Delete(ctx, prop.ID)
|
||||
p.w.put(delivery.PlacementProposalsBucket, prop.ID, nil)
|
||||
p.deletePulses(ctx, prop.ID)
|
||||
p.sayChanged(prop, answer, changes)
|
||||
}
|
||||
|
||||
// sayChanged says placement-changed for the keys written.
|
||||
func (p *passer) sayChanged(prop delivery.PlacementProposal, answer delivery.Answer, changes []delivery.PlacementChange) {
|
||||
if len(changes) == 0 {
|
||||
return
|
||||
}
|
||||
at := p.clock().UTC()
|
||||
if _, when, ok := strings.Cut(answer.Warrant, "@"); ok {
|
||||
if t, err := time.Parse(time.RFC3339Nano, when); err == nil {
|
||||
at = t
|
||||
}
|
||||
}
|
||||
delivery.SortChanges(changes)
|
||||
p.sayEvent(delivery.PlacementChangedEvent, delivery.PlacementEvent{Proposal: prop, Nodes: changes,
|
||||
Warrant: &delivery.PlacementWarrant{Warrant: answer.Warrant, Words: answer.Words, At: at}}, prop.ID+".changed")
|
||||
}
|
||||
|
||||
// proposeSeed proposes an export as the seed again, by the same proposer and why.
|
||||
func (p *passer) proposeSeed(ctx context.Context, was delivery.PlacementProposal, export map[string]delivery.Placement) {
|
||||
held, recorded := p.lays(ctx)
|
||||
now := p.clock()
|
||||
seed, laid, findings := assign.Propose(assign.ProposeIn{ID: delivery.NewProposalID(now), By: was.By, Why: was.Why, At: now,
|
||||
Changes: assign.SeedChanges(export), Current: map[string]delivery.Placement{}, Revisions: map[string]uint64{},
|
||||
Held: held, Recorded: recorded, Seed: true, ExportDigest: delivery.ExportDigest(export)})
|
||||
if len(findings) > 0 {
|
||||
p.logf("the new export is not proposed as the seed: %s", strings.Join(findings, "; "))
|
||||
return
|
||||
}
|
||||
for _, node := range seed.Nodes() {
|
||||
key, _ := delivery.PlacementCandidateKey(seed.ID, node)
|
||||
c := delivery.PlacementCandidate{Proposal: seed.ID, Placement: laid[node]}
|
||||
p.put(ctx, delivery.PlacementCandidatesBucket, key, func() ([]byte, error) { return delivery.EncodePlacementCandidate(c) })
|
||||
}
|
||||
seed.Written = now.UTC()
|
||||
p.writeProposal(ctx, seed, 0)
|
||||
}
|
||||
|
||||
// endProposal ends a proposal: its ask, its candidates and its key deleted, and placement-withdrawn said carrying it
|
||||
// whole with why it ended.
|
||||
func (p *passer) endProposal(ctx context.Context, prop delivery.PlacementProposal, why string) {
|
||||
if akey, err := delivery.PlacementAskKey(prop.ID); err == nil && p.w.has(delivery.AssignerAsksBucket, akey) {
|
||||
if err := p.kvs[delivery.AssignerAsksBucket].Delete(ctx, akey); err == nil {
|
||||
p.w.put(delivery.AssignerAsksBucket, akey, nil)
|
||||
}
|
||||
}
|
||||
p.deleteCandidates(ctx, prop.ID)
|
||||
p.deletePulses(ctx, prop.ID)
|
||||
if err := p.kvs[delivery.PlacementProposalsBucket].Delete(ctx, prop.ID); err != nil && !errors.Is(err, context.Canceled) {
|
||||
p.logf("the proposal %s is not deleted, and is ended at the next pass: %v", prop.ID, err)
|
||||
return
|
||||
}
|
||||
p.w.put(delivery.PlacementProposalsBucket, prop.ID, nil)
|
||||
p.seat.forget(prop.ID)
|
||||
p.logf("the proposal %s ends: %s", prop.ID, why)
|
||||
p.sayEvent(delivery.PlacementWithdrawnEvent, delivery.PlacementEvent{Proposal: prop, Ended: why}, prop.ID+".withdrawn")
|
||||
}
|
||||
|
||||
// deleteCandidates deletes every candidate of a proposal.
|
||||
func (p *passer) deleteCandidates(ctx context.Context, id string) {
|
||||
for key := range p.w.read(delivery.PlacementCandidatesBucket) {
|
||||
if pid, _, err := delivery.ParsePlacementCandidateKey(key); err == nil && pid == id {
|
||||
if err := p.kvs[delivery.PlacementCandidatesBucket].Delete(ctx, key); err == nil {
|
||||
p.w.put(delivery.PlacementCandidatesBucket, key, nil)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// writeProposal writes a proposal over the revision the pass read, by compare-and-set: a verb that marked it meanwhile
|
||||
// (withdraw, again) wins, and the next pass reads that. It says whether it wrote.
|
||||
func (p *passer) writeProposal(ctx context.Context, prop delivery.PlacementProposal, rev uint64) bool {
|
||||
raw, err := delivery.EncodePlacementProposal(prop)
|
||||
if err != nil {
|
||||
p.logf("the proposal %s is not written: %v", prop.ID, err)
|
||||
return false
|
||||
}
|
||||
kv := p.kvs[delivery.PlacementProposalsBucket]
|
||||
var at uint64
|
||||
if rev == 0 {
|
||||
at, err = kv.Create(ctx, prop.ID, raw)
|
||||
} else {
|
||||
at, err = kv.Update(ctx, prop.ID, raw, rev)
|
||||
}
|
||||
if err != nil {
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
p.logf("the proposal %s is not written, and is taken further at the next pass: %v", prop.ID, err)
|
||||
}
|
||||
return false
|
||||
}
|
||||
p.w.putEntry(delivery.PlacementProposalsBucket, prop.ID, raw, at, p.clock())
|
||||
return true
|
||||
}
|
||||
|
||||
// say says one of the seat's events, said in the journal when the bus refuses it.
|
||||
func (p *passer) sayEvent(event string, body delivery.PlacementEvent, id string) {
|
||||
if p.publish == nil {
|
||||
return
|
||||
}
|
||||
if err := p.publish(event, body, id); err != nil {
|
||||
p.logf("%s of %s is not said: %v", event, body.Proposal.ID, err)
|
||||
}
|
||||
}
|
||||
@@ -1,533 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
"mesh-delivery-assigner/internal/assign"
|
||||
)
|
||||
|
||||
// The seat mesh-placement's seven verbs (novox/hq ADR 0567 §10), served through the tool runner beside the put tools: a
|
||||
// process never serves (ADR 0436). They read every state the seat keeps and write what a person asks of it: a proposal
|
||||
// and its candidates (propose, adopt), a proposal marked to end (withdraw) or to be judged again (again), and the keys
|
||||
// deleted (hand-back). Everything that follows a proposal — composing it, judging the answers, the ask, the write at the
|
||||
// warrant and the seat's events on its stream — is the assigner's process's (seat.go), started by these writes.
|
||||
|
||||
// seatStore is what the seat's verbs need of the states: the runtime's, or a test's.
|
||||
type seatStore interface {
|
||||
store
|
||||
del(state, key string) error
|
||||
history(state, key string) ([]stdio.StateRevision, error)
|
||||
}
|
||||
|
||||
func (runtimeStore) del(state, key string) error { return stdio.State(state).Delete(key) }
|
||||
|
||||
func (runtimeStore) history(state, key string) ([]stdio.StateRevision, error) {
|
||||
return stdio.State(state).History(key)
|
||||
}
|
||||
|
||||
// The states the verbs read, as the runtime names them: the seat's own by their local names, the others'
|
||||
// `<module>.<state>`.
|
||||
const (
|
||||
modulesState = delivery.IndexerModule + ".modules"
|
||||
haveState = "node-engine.have"
|
||||
)
|
||||
|
||||
// seatTools serves the seat's verbs over a store, asking the controller through ask (stdio.Ask, or a test's).
|
||||
type seatTools struct {
|
||||
st seatStore
|
||||
ask func(key string, body any) (json.RawMessage, error)
|
||||
now func() time.Time
|
||||
}
|
||||
|
||||
func newSeatTools(st seatStore, ask func(string, any) (json.RawMessage, error)) *seatTools {
|
||||
return &seatTools{st: st, ask: ask, now: time.Now}
|
||||
}
|
||||
|
||||
// keyed is one placement key as a verb answers it.
|
||||
type keyed struct {
|
||||
Placement delivery.Placement `json:"placement"`
|
||||
Revision uint64 `json:"revision"`
|
||||
}
|
||||
|
||||
// placements reads every node's placement and its key's revision.
|
||||
func (s *seatTools) placements() (map[string]delivery.Placement, map[string]uint64, error) {
|
||||
keys, err := s.st.keys(delivery.PlacementState)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("placement cannot be listed: %w", err)
|
||||
}
|
||||
out, revs := map[string]delivery.Placement{}, map[string]uint64{}
|
||||
for _, key := range keys {
|
||||
node, err := delivery.ParsePlacementKey(key)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
raw, rev, found, err := s.st.get(delivery.PlacementState, key)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("placement of %s cannot be read: %w", node, err)
|
||||
}
|
||||
if !found {
|
||||
continue
|
||||
}
|
||||
p, err := delivery.DecodePlacement(raw)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("placement of %s cannot be read: %w", node, err)
|
||||
}
|
||||
out[node], revs[node] = p, rev
|
||||
}
|
||||
return out, revs, nil
|
||||
}
|
||||
|
||||
// lays is what a proposal is checked against: the modules the indexer holds and the nodes with a record.
|
||||
func (s *seatTools) lays() (func(string) bool, func(string) bool, error) {
|
||||
mods, err := s.st.keys(modulesState)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("the indexer's modules cannot be listed: %w", err)
|
||||
}
|
||||
held := map[string]bool{}
|
||||
for _, k := range mods {
|
||||
if name, err := delivery.ParseModulesKey(k); err == nil {
|
||||
held[name] = true
|
||||
}
|
||||
}
|
||||
haves, err := s.st.keys(haveState)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("the nodes' records cannot be listed: %w", err)
|
||||
}
|
||||
recorded := map[string]bool{}
|
||||
for _, k := range haves {
|
||||
node, _, isNode, err := delivery.ParseHaveKey(k)
|
||||
if err != nil || !isNode {
|
||||
continue
|
||||
}
|
||||
if raw, _, found, err := s.st.get(haveState, k); err == nil && found {
|
||||
if h, err := delivery.DecodeHaveNode(raw); err == nil && h.Record != nil {
|
||||
recorded[node] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
return func(m string) bool { return held[m] }, func(n string) bool { return recorded[n] }, nil
|
||||
}
|
||||
|
||||
// proposal reads one open proposal.
|
||||
func (s *seatTools) proposal(id string) (delivery.PlacementProposal, error) {
|
||||
if err := delivery.CheckProposalID(id); err != nil {
|
||||
return delivery.PlacementProposal{}, err
|
||||
}
|
||||
raw, _, found, err := s.st.get(delivery.PlacementProposalsState, id)
|
||||
if err != nil {
|
||||
return delivery.PlacementProposal{}, fmt.Errorf("the proposal %s cannot be read: %w", id, err)
|
||||
}
|
||||
if !found {
|
||||
return delivery.PlacementProposal{}, fmt.Errorf("no proposal %s is open: mesh-placement.proposals lists those that are", id)
|
||||
}
|
||||
return delivery.DecodePlacementProposal(raw)
|
||||
}
|
||||
|
||||
// write writes a proposal laid by assign.Propose: each candidate, then the proposal, whose write the assigner's process
|
||||
// takes as the event to compose and judge it.
|
||||
func (s *seatTools) write(p delivery.PlacementProposal, laid map[string]*delivery.Placement) error {
|
||||
p.Written = s.now().UTC()
|
||||
for _, node := range p.Nodes() {
|
||||
key, err := delivery.PlacementCandidateKey(p.ID, node)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
raw, err := delivery.EncodePlacementCandidate(delivery.PlacementCandidate{Proposal: p.ID, Placement: laid[node]})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := s.st.put(delivery.PlacementCandidatesState, key, raw); err != nil {
|
||||
return fmt.Errorf("the candidate %s is not written: %w", key, err)
|
||||
}
|
||||
}
|
||||
raw, err := delivery.EncodePlacementProposal(p)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := s.st.put(delivery.PlacementProposalsState, p.ID, raw); err != nil {
|
||||
return fmt.Errorf("the proposal %s is not written: %w", p.ID, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// proposed is what propose and adopt answer.
|
||||
type proposed struct {
|
||||
Proposal delivery.PlacementProposal `json:"proposal"`
|
||||
Follow string `json:"follow"`
|
||||
}
|
||||
|
||||
func followProposal(id string) string {
|
||||
return fmt.Sprintf("the nodes it touches answer it; mesh-placement.proposals with id %q says where it stands, and the "+
|
||||
"operator is asked to approve it once its verdict passes", id)
|
||||
}
|
||||
|
||||
// changesOf reads propose's changes: a JSON array, given as one or as its text.
|
||||
func changesOf(v any) ([]delivery.PlacementProposalChange, error) {
|
||||
var raw []byte
|
||||
switch c := v.(type) {
|
||||
case string:
|
||||
raw = []byte(c)
|
||||
case nil:
|
||||
return nil, errors.New("propose names its changes")
|
||||
default:
|
||||
var err error
|
||||
if raw, err = json.Marshal(c); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
var out []delivery.PlacementProposalChange
|
||||
dec := json.NewDecoder(strings.NewReader(string(raw)))
|
||||
dec.DisallowUnknownFields()
|
||||
if err := dec.Decode(&out); err != nil {
|
||||
return nil, fmt.Errorf("the changes are no list of changes: %w", err)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// doPropose is the verb propose (§2).
|
||||
func (s *seatTools) doPropose(args map[string]any, by string) (any, error) {
|
||||
changes, err := changesOf(args["changes"])
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, c := range changes {
|
||||
if c.Op == delivery.OpSeed {
|
||||
return nil, errors.New("the seed is adopt's alone: propose takes place, take-off, pin, unpin, hold, unhold, decide and config")
|
||||
}
|
||||
}
|
||||
current, revs, err := s.placements()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
held, recorded, err := s.lays()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
now := s.now()
|
||||
p, laid, findings := assign.Propose(assign.ProposeIn{ID: delivery.NewProposalID(now), By: by, Why: arg(args, "why"), At: now,
|
||||
Changes: changes, Current: current, Revisions: revs, Held: held, Recorded: recorded})
|
||||
if len(findings) > 0 {
|
||||
return nil, fmt.Errorf("the proposal is refused: %s", strings.Join(findings, "; "))
|
||||
}
|
||||
if err := s.write(p, laid); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return proposed{Proposal: p, Follow: followProposal(p.ID)}, nil
|
||||
}
|
||||
|
||||
// doWithdraw is the verb withdraw (§2): the proposal is marked, and the assigner deletes it and its candidates and says
|
||||
// placement-withdrawn carrying it whole.
|
||||
func (s *seatTools) doWithdraw(args map[string]any, by string) (any, error) {
|
||||
why := arg(args, "why")
|
||||
if why == "" {
|
||||
return nil, errors.New("withdraw says why: it is kept in the event that ends the proposal")
|
||||
}
|
||||
p, err := s.proposal(arg(args, "id"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
p.Withdrawn = fmt.Sprintf("withdrawn by %s: %s", by, why)
|
||||
raw, err := delivery.EncodePlacementProposal(p)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := s.st.put(delivery.PlacementProposalsState, p.ID, raw); err != nil {
|
||||
return nil, fmt.Errorf("the proposal %s is not withdrawn: %w", p.ID, err)
|
||||
}
|
||||
return map[string]any{"proposal": p.ID, "withdrawn": p.Withdrawn}, nil
|
||||
}
|
||||
|
||||
// doAgain is the verb again (§10): the proposal is marked, and the assigner judges it now and raises its ask again
|
||||
// when its verdict passes.
|
||||
func (s *seatTools) doAgain(args map[string]any) (any, error) {
|
||||
p, err := s.proposal(arg(args, "id"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
p.Again = s.now().UTC()
|
||||
raw, err := delivery.EncodePlacementProposal(p)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := s.st.put(delivery.PlacementProposalsState, p.ID, raw); err != nil {
|
||||
return nil, fmt.Errorf("the proposal %s is not asked again: %w", p.ID, err)
|
||||
}
|
||||
return map[string]any{"proposal": p.ID, "again": p.Again, "follow": followProposal(p.ID)}, nil
|
||||
}
|
||||
|
||||
// exportCall is the controller's verb the seed is read from, by request and reply on its seat (§7).
|
||||
const exportCall = "seat:" + catalogue.ControllerSeatName + ".placement-export"
|
||||
|
||||
// doAdopt is the verb adopt (§7): the controller's export proposed whole as the seed.
|
||||
func (s *seatTools) doAdopt(args map[string]any, by string) (any, error) {
|
||||
current, _, err := s.placements()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(current) > 0 {
|
||||
return nil, fmt.Errorf("placement is taken over already (%d keys): propose a change instead, or hand it back first", len(current))
|
||||
}
|
||||
if ids, err := s.st.keys(delivery.PlacementProposalsState); err == nil {
|
||||
for _, id := range ids {
|
||||
if p, err := s.proposal(id); err == nil && p.Seed {
|
||||
return nil, fmt.Errorf("the seed %s is open: withdraw it first, or answer its ask", id)
|
||||
}
|
||||
}
|
||||
}
|
||||
raw, err := s.ask(exportCall, map[string]any{})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the controller's placement-export did not answer: %w", err)
|
||||
}
|
||||
export, err := assign.ExportOf(raw)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
held, recorded, err := s.lays()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
now := s.now()
|
||||
p, laid, findings := assign.Propose(assign.ProposeIn{ID: delivery.NewProposalID(now), By: by, Why: arg(args, "why"), At: now,
|
||||
Changes: assign.SeedChanges(export), Current: map[string]delivery.Placement{}, Revisions: map[string]uint64{},
|
||||
Held: held, Recorded: recorded, Seed: true, ExportDigest: delivery.ExportDigest(export)})
|
||||
if len(findings) > 0 {
|
||||
return nil, fmt.Errorf("the seed is refused: %s", strings.Join(findings, "; "))
|
||||
}
|
||||
if err := s.write(p, laid); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return proposed{Proposal: p, Follow: followProposal(p.ID)}, nil
|
||||
}
|
||||
|
||||
// handActCall records a hand act in the controller's hand-act log.
|
||||
const handActCall = "seat:" + catalogue.ControllerSeatName + ".hand-act"
|
||||
|
||||
// doHandBack is the verb hand-back (§7): the operator's alone, so it is asked as a proposal is. It writes a hand-back
|
||||
// proposal naming every key at its revision; the assigner raises its authorising ask and, on its warrant, deletes the keys,
|
||||
// records the hand act and says placement-changed; every node returns to the controller's compositions.
|
||||
func (s *seatTools) doHandBack(args map[string]any, by string) (any, error) {
|
||||
if assign.Leased {
|
||||
return nil, errors.New("placement is the mesh's alone since the lease write: there is no controller to hand it back to")
|
||||
}
|
||||
why := arg(args, "why")
|
||||
if why == "" {
|
||||
return nil, errors.New("hand-back says why: it is recorded with the hand act")
|
||||
}
|
||||
if by == "" {
|
||||
return nil, errors.New("the runtime names no caller: a hand-back is made by someone")
|
||||
}
|
||||
_, revs, err := s.placements()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(revs) == 0 {
|
||||
return nil, errors.New("there is no placement key to hand back")
|
||||
}
|
||||
now := s.now()
|
||||
p := delivery.PlacementProposal{ID: delivery.NewProposalID(now), By: by, Why: why, At: now.UTC(), Written: now.UTC(),
|
||||
HandBack: true, Base: revs, State: delivery.ProposalAnswering}
|
||||
raw, err := delivery.EncodePlacementProposal(p)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := s.st.put(delivery.PlacementProposalsState, p.ID, raw); err != nil {
|
||||
return nil, fmt.Errorf("the hand-back is not written: %w", err)
|
||||
}
|
||||
return proposed{Proposal: p, Follow: "the operator is asked to approve it; on the warrant every key is deleted and each " +
|
||||
"node returns to the controller's compositions"}, nil
|
||||
}
|
||||
|
||||
// placementAnswer is what the verb placement answers.
|
||||
type placementAnswer struct {
|
||||
Keys map[string]keyed `json:"keys,omitempty"`
|
||||
Nodes []string `json:"nodes,omitempty"`
|
||||
Count int `json:"count"`
|
||||
History int `json:"history-per-key"`
|
||||
Changes []historyEntry `json:"changes,omitempty"`
|
||||
Older string `json:"older,omitempty"`
|
||||
}
|
||||
|
||||
// historyEntry is one change of a key in the state's revisions.
|
||||
type historyEntry struct {
|
||||
Node string `json:"node"`
|
||||
Revision uint64 `json:"revision"`
|
||||
At time.Time `json:"at"`
|
||||
Change delivery.PlacementChange `json:"change"`
|
||||
Deleted bool `json:"deleted,omitempty"`
|
||||
}
|
||||
|
||||
// doPlacement is the verb placement (§10, §6): one node's key, one module's nodes, or every key; with history, the
|
||||
// changes in order from the revisions the state keeps.
|
||||
func (s *seatTools) doPlacement(args map[string]any) (any, error) {
|
||||
all, revs, err := s.placements()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
node, module := arg(args, "node"), arg(args, "module")
|
||||
out := placementAnswer{Keys: map[string]keyed{}, Count: len(all), History: delivery.PlacementHistory}
|
||||
for n, p := range all {
|
||||
if node != "" && n != node {
|
||||
continue
|
||||
}
|
||||
if module != "" && !strings.Contains(" "+strings.Join(p.Assigned(), " ")+" ", " "+module+" ") {
|
||||
continue
|
||||
}
|
||||
out.Keys[n] = keyed{Placement: p, Revision: revs[n]}
|
||||
out.Nodes = append(out.Nodes, n)
|
||||
}
|
||||
sort.Strings(out.Nodes)
|
||||
if node != "" && len(out.Keys) == 0 && arg(args, "history") != "true" {
|
||||
return nil, fmt.Errorf("%s has no placement key", node)
|
||||
}
|
||||
if arg(args, "history") != "true" {
|
||||
return out, nil
|
||||
}
|
||||
nodes := out.Nodes
|
||||
if node != "" {
|
||||
nodes = []string{node}
|
||||
} else if module == "" {
|
||||
nodes = nil
|
||||
for n := range all {
|
||||
nodes = append(nodes, n)
|
||||
}
|
||||
sort.Strings(nodes)
|
||||
}
|
||||
for _, n := range nodes {
|
||||
key, err := delivery.PlacementKey(n)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
revisions, err := s.st.history(delivery.PlacementState, key)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the history of %s cannot be read: %w", key, err)
|
||||
}
|
||||
var before delivery.Placement
|
||||
for _, r := range revisions {
|
||||
e := historyEntry{Node: n, Revision: r.Revision, At: r.At}
|
||||
after := delivery.Placement{}
|
||||
if r.Op == "" || r.Op == "put" || r.Op == "PUT" {
|
||||
if p, err := delivery.DecodePlacement(r.Value); err == nil {
|
||||
after = p
|
||||
}
|
||||
} else {
|
||||
e.Deleted = true
|
||||
}
|
||||
e.Change = delivery.ChangeOf(before, after)
|
||||
e.Change.Before, e.Change.After = 0, r.Revision
|
||||
before = after
|
||||
if module != "" && !strings.Contains(" "+strings.Join(append(append(e.Change.Added, e.Change.Removed...), e.Change.Changed...), " ")+" ", " "+module+" ") {
|
||||
continue
|
||||
}
|
||||
out.Changes = append(out.Changes, e)
|
||||
}
|
||||
}
|
||||
out.Older = fmt.Sprintf("the state keeps %d revisions of each key; an older change is in its placement-changed event on "+
|
||||
"the assigner's stream %s, kept %d days", delivery.PlacementHistory, delivery.PlacementStream, delivery.PlacementStreamDays)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// openProposal is one open proposal as the verb proposals answers it: the proposal (its changes, base revisions,
|
||||
// verdict with the differences and the answers it judged, its ask), its candidates by node with their revisions, and
|
||||
// every node's answer now.
|
||||
type openProposal struct {
|
||||
Proposal delivery.PlacementProposal `json:"proposal"`
|
||||
Candidates map[string]uint64 `json:"candidates"`
|
||||
Answers map[string]delivery.CandidateAnswer `json:"answers,omitempty"`
|
||||
}
|
||||
|
||||
// doProposals is the verb proposals (§10).
|
||||
func (s *seatTools) doProposals(args map[string]any) (any, error) {
|
||||
ids, err := s.st.keys(delivery.PlacementProposalsState)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the proposals cannot be listed: %w", err)
|
||||
}
|
||||
cands, err := s.st.keys(delivery.PlacementCandidatesState)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the candidates cannot be listed: %w", err)
|
||||
}
|
||||
haves, err := s.st.keys(haveState)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the nodes' haves cannot be listed: %w", err)
|
||||
}
|
||||
answers := map[string]map[string]delivery.CandidateAnswer{} // by proposal, then node
|
||||
for _, k := range haves {
|
||||
node, _, isNode, err := delivery.ParseHaveKey(k)
|
||||
if err != nil || !isNode {
|
||||
continue
|
||||
}
|
||||
raw, _, found, err := s.st.get(haveState, k)
|
||||
if err != nil || !found {
|
||||
continue
|
||||
}
|
||||
h, err := delivery.DecodeHaveNode(raw)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
for id, a := range h.Candidates {
|
||||
if answers[id] == nil {
|
||||
answers[id] = map[string]delivery.CandidateAnswer{}
|
||||
}
|
||||
answers[id][node] = a
|
||||
}
|
||||
}
|
||||
sort.Strings(ids)
|
||||
out := []openProposal{}
|
||||
for _, id := range ids {
|
||||
if want := arg(args, "id"); want != "" && want != id {
|
||||
continue
|
||||
}
|
||||
p, err := s.proposal(id)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
o := openProposal{Proposal: p, Candidates: map[string]uint64{}, Answers: answers[id]}
|
||||
for _, k := range cands {
|
||||
pid, node, err := delivery.ParsePlacementCandidateKey(k)
|
||||
if err != nil || pid != id {
|
||||
continue
|
||||
}
|
||||
if _, rev, found, err := s.st.get(delivery.PlacementCandidatesState, k); err == nil && found {
|
||||
o.Candidates[node] = rev
|
||||
}
|
||||
}
|
||||
out = append(out, o)
|
||||
}
|
||||
if arg(args, "id") != "" && len(out) == 0 {
|
||||
return nil, fmt.Errorf("no proposal %s is open", arg(args, "id"))
|
||||
}
|
||||
return map[string]any{"proposals": out}, nil
|
||||
}
|
||||
|
||||
// tools are the seat's verbs as the runtime serves them, each named <seat>.<verb> (the tool runner serves a seat's verb
|
||||
// only from a tool of that name), with the seat's own schemas.
|
||||
func (s *seatTools) tools() []stdio.Tool {
|
||||
seat, _ := catalogue.SeatNamed(catalogue.PlacementSeat)
|
||||
input := map[string]map[string]any{}
|
||||
description := map[string]string{}
|
||||
for _, v := range seat.Serves {
|
||||
input[v.Name], description[v.Name] = v.Input, v.Description
|
||||
}
|
||||
run := map[string]func(map[string]any) (any, error){
|
||||
catalogue.PlacementVerbPlacement: s.doPlacement,
|
||||
catalogue.PlacementVerbProposals: s.doProposals,
|
||||
catalogue.PlacementVerbPropose: func(a map[string]any) (any, error) { return s.doPropose(a, stdio.CallerOf(a)) },
|
||||
catalogue.PlacementVerbWithdraw: func(a map[string]any) (any, error) { return s.doWithdraw(a, stdio.CallerOf(a)) },
|
||||
catalogue.PlacementVerbAgain: s.doAgain,
|
||||
catalogue.PlacementVerbAdopt: func(a map[string]any) (any, error) { return s.doAdopt(a, stdio.CallerOf(a)) },
|
||||
catalogue.PlacementVerbHandBack: func(a map[string]any) (any, error) { return s.doHandBack(a, stdio.CallerOf(a)) },
|
||||
}
|
||||
var out []stdio.Tool
|
||||
for _, v := range seat.Serves {
|
||||
out = append(out, stdio.Tool{Name: catalogue.PlacementSeat + "." + v.Name, Description: description[v.Name], Input: input[v.Name], Run: run[v.Name]})
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -1,157 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
func (s memStore) del(state, key string) error {
|
||||
delete(s[state].values, key)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s memStore) history(state, key string) ([]stdio.StateRevision, error) {
|
||||
v, ok := s[state].values[key]
|
||||
if !ok {
|
||||
return nil, nil
|
||||
}
|
||||
return []stdio.StateRevision{{Key: key, Value: v, Revision: uint64(len(v)), Op: "put"}}, nil
|
||||
}
|
||||
|
||||
// ADR 0567 §2, §7, §10: the seat's seven verbs over the states. propose lays a proposal and writes it with its candidates,
|
||||
// or refuses naming every finding and writes nothing; withdraw and again mark it for the holder; adopt proposes the
|
||||
// controller's export whole as the seed, once; hand-back records the hand act and deletes every key; placement and
|
||||
// proposals read every state the seat keeps, and history names the stream for what the state no longer keeps.
|
||||
func TestTheSeatsVerbs(t *testing.T) {
|
||||
anchor := delivery.Placement{V: delivery.Version, Node: "anchor", Modules: []delivery.PlacedModule{{Module: "web"}}}
|
||||
st := memStore{}
|
||||
for _, state := range []string{delivery.PlacementState, delivery.PlacementProposalsState, delivery.PlacementCandidatesState, modulesState, haveState} {
|
||||
st[state] = &memKV{values: map[string][]byte{}}
|
||||
}
|
||||
for _, m := range []string{"web", "shell"} {
|
||||
st[modulesState].values[m] = []byte("{}")
|
||||
}
|
||||
for _, n := range []string{"anchor", "laptop"} {
|
||||
raw, _ := delivery.EncodeHaveNode(delivery.HaveNode{Record: &delivery.NodeRecord{Node: n}})
|
||||
st[haveState].values[n+".node"] = raw
|
||||
}
|
||||
export, _ := json.Marshal(map[string]any{"anchor": anchor})
|
||||
var asked []string
|
||||
s := newSeatTools(st, func(key string, body any) (json.RawMessage, error) {
|
||||
asked = append(asked, key)
|
||||
if key == exportCall {
|
||||
out, _ := json.Marshal(map[string]any{"ok": true, "output": string(export)})
|
||||
return out, nil
|
||||
}
|
||||
return json.RawMessage(`{"ok":true}`), nil
|
||||
})
|
||||
s.now = func() time.Time { return time.Date(2026, 10, 28, 20, 0, 0, 0, time.UTC) }
|
||||
|
||||
// adopt: the export proposed whole as the seed, once.
|
||||
got, err := s.doAdopt(map[string]any{"why": "take over"}, "operator@laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
seed := got.(proposed).Proposal
|
||||
if !seed.Seed || seed.ExportDigest != delivery.ExportDigest(map[string]delivery.Placement{"anchor": anchor}) || !slices.Equal(asked, []string{exportCall}) ||
|
||||
len(st[delivery.PlacementCandidatesState].values) != 1 || st[delivery.PlacementProposalsState].values[seed.ID] == nil {
|
||||
t.Fatalf("adopt = %+v, asked %v", seed, asked)
|
||||
}
|
||||
if _, err := s.doAdopt(map[string]any{"why": "again"}, "op"); err == nil || !strings.Contains(err.Error(), "is open") {
|
||||
t.Errorf("a second adopt with a seed open: %v", err)
|
||||
}
|
||||
// withdraw and again mark the proposal; the holder ends or judges it.
|
||||
if _, err := s.doWithdraw(map[string]any{"id": seed.ID, "why": "not yet"}, "op"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := s.doAgain(map[string]any{"id": seed.ID}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if p, err := s.proposal(seed.ID); err != nil || !strings.Contains(p.Withdrawn, "not yet") || p.Again.IsZero() {
|
||||
t.Errorf("marked = %+v, %v", p, err)
|
||||
}
|
||||
if _, err := s.doWithdraw(map[string]any{"id": "p-none", "why": "x"}, "op"); err == nil {
|
||||
t.Error("an unknown proposal was withdrawn")
|
||||
}
|
||||
delete(st[delivery.PlacementProposalsState].values, seed.ID) // the holder ended it
|
||||
st[delivery.PlacementCandidatesState].values = map[string][]byte{}
|
||||
|
||||
// propose: refused whole, naming every finding; a valid move laid over the keys at their revisions.
|
||||
raw, _ := delivery.EncodePlacement(anchor)
|
||||
st[delivery.PlacementState].values["placement.anchor"] = raw
|
||||
if _, err := s.doPropose(map[string]any{"why": "x", "changes": `[{"op":"place","node":"anchor","module":"ghost"},{"op":"hold","node":"anchor","module":"web","claim":"x"}]`}, "op"); err == nil ||
|
||||
!strings.Contains(err.Error(), "ghost") || !strings.Contains(err.Error(), "lease write") || len(st[delivery.PlacementProposalsState].values) != 0 {
|
||||
t.Errorf("a refused proposal: %v", err)
|
||||
}
|
||||
if _, err := s.doPropose(map[string]any{"why": "x", "changes": `[{"op":"swap","node":"anchor"}]`}, "op"); err == nil {
|
||||
t.Error("an unknown change was taken")
|
||||
}
|
||||
got, err = s.doPropose(map[string]any{"why": "move web", "changes": []any{
|
||||
map[string]any{"op": "take-off", "node": "anchor", "module": "web"}, map[string]any{"op": "place", "node": "laptop", "module": "web"}}}, "operator@laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
move := got.(proposed).Proposal
|
||||
if move.By != "operator@laptop" || move.Base["anchor"] != uint64(len(raw)) || move.Base["laptop"] != 0 || len(st[delivery.PlacementCandidatesState].values) != 2 {
|
||||
t.Fatalf("propose = %+v", move)
|
||||
}
|
||||
|
||||
// placement and proposals read every state the seat keeps (§10); history names the stream for what is older.
|
||||
reads := map[string]bool{}
|
||||
pa, err := s.doPlacement(map[string]any{"module": "web", "history": "true"})
|
||||
if a := pa.(placementAnswer); err != nil || !slices.Equal(a.Nodes, []string{"anchor"}) || len(a.Changes) != 1 || !strings.Contains(a.Older, delivery.PlacementStream) {
|
||||
t.Errorf("placement of web = %+v, %v", pa, err)
|
||||
} else {
|
||||
reads[delivery.PlacementState], reads["stream "+delivery.PlacementStream] = true, true
|
||||
}
|
||||
if _, err := s.doPlacement(map[string]any{"node": "laptop"}); err == nil {
|
||||
t.Error("a node with no key was answered")
|
||||
}
|
||||
ps, err := s.doProposals(map[string]any{"id": move.ID})
|
||||
if list := ps.(map[string]any)["proposals"].([]openProposal); err != nil || len(list) != 1 || len(list[0].Candidates) != 2 {
|
||||
t.Errorf("proposals = %+v, %v", ps, err)
|
||||
} else {
|
||||
reads[delivery.PlacementProposalsState], reads[delivery.PlacementCandidatesState] = true, true
|
||||
}
|
||||
raw0, err := readManifest()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, state := range raw0.State {
|
||||
if slices.Contains([]string{delivery.PlacementState, delivery.PlacementProposalsState, delivery.PlacementCandidatesState}, state.Name) && !reads[state.Name] {
|
||||
t.Errorf("no verb reads the seat's state %s", state.Name)
|
||||
}
|
||||
}
|
||||
for _, stream := range raw0.Streams {
|
||||
if !reads["stream "+stream.Name] {
|
||||
t.Errorf("no verb reads the seat's stream %s", stream.Name)
|
||||
}
|
||||
}
|
||||
|
||||
// hand-back: the operator's alone, written as a hand-back proposal naming every key at its revision, for the assigner
|
||||
// to ask; nothing is deleted by the verb.
|
||||
got, err = s.doHandBack(map[string]any{"why": "back"}, "op")
|
||||
if hb := got.(proposed).Proposal; err != nil || !hb.HandBack || hb.Base["anchor"] != uint64(len(raw)) || len(st[delivery.PlacementState].values) != 1 {
|
||||
t.Errorf("hand-back = %+v, %v", got, err)
|
||||
}
|
||||
if tools := s.tools(); len(tools) != 7 || tools[2].Name != "mesh-placement.propose" || tools[2].Run == nil || tools[2].Input == nil {
|
||||
t.Errorf("the seat's verbs as served: %+v", tools)
|
||||
}
|
||||
}
|
||||
|
||||
// readManifest is the module's manifest, read as the controller reads it.
|
||||
func readManifest() (catalogue.Manifest, error) {
|
||||
raw, err := os.ReadFile("../../module.json")
|
||||
if err != nil {
|
||||
return catalogue.Manifest{}, err
|
||||
}
|
||||
return catalogue.ParseManifest(raw)
|
||||
}
|
||||
@@ -18,8 +18,8 @@ import (
|
||||
|
||||
// The assigner's tools (novox/hq ADR 0546), served through the tool runner from its bundle: a process never serves (ADR
|
||||
// 0436, mesh-sdk process). They write and read the puts; the assigner's process takes each put further (puts.go) and says
|
||||
// every step as the event put. They are the module's own tools; the seat mesh-placement's verbs are served beside them
|
||||
// (seat_tools.go, ADR 0567 §10).
|
||||
// every step as the event put. They are the module's own tools, not a seat's verbs: a new seat needs a change of the
|
||||
// controller, which takes none; they become the verbs of a seat once composing leaves the controller.
|
||||
|
||||
// putsState is the assigner's state of puts, as the runtime names a module's own state.
|
||||
const putsState = "puts"
|
||||
@@ -351,7 +351,7 @@ func serveTools() error {
|
||||
time.Sleep(wait)
|
||||
}
|
||||
}()
|
||||
return stdio.Serve("", append(s.tools(), newSeatTools(runtimeStore{}, stdio.Ask).tools()...))
|
||||
return stdio.Serve("", s.tools())
|
||||
}
|
||||
|
||||
// shortFP is a fingerprint's first twelve hex digits.
|
||||
|
||||
@@ -13,7 +13,6 @@ import (
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buscompose"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/compose"
|
||||
"git.novox.be/novox/mesh-sdk/go/compose/frombus"
|
||||
@@ -193,7 +192,7 @@ func (w *world) manifest(ctx context.Context, get getter, digest string) parsed
|
||||
return p
|
||||
}
|
||||
|
||||
// assigned is each node's modules both the controller's assignments and the seat mesh-placement's placement name, in name order
|
||||
// assigned is each node's modules both the controller's assignments and the indexer's placement name, in name order
|
||||
// (ADR 0549 §5; the assignments alone before any placement was read), with every open assignment by node and module.
|
||||
func (w *world) assigned() (map[string][]string, map[string]assignment) {
|
||||
byAssignment, all := w.assignedByController()
|
||||
@@ -202,7 +201,7 @@ func (w *world) assigned() (map[string][]string, map[string]assignment) {
|
||||
return both, all
|
||||
}
|
||||
|
||||
// placementDiffers is every pair the controller's assignments and the seat's placement do not both name.
|
||||
// placementDiffers is every pair the controller's assignments and the indexer's placement do not both name.
|
||||
func (w *world) placementDiffers() []assign.Pair {
|
||||
byAssignment, _ := w.assignedByController()
|
||||
placed, seeded := w.placed()
|
||||
@@ -210,7 +209,7 @@ func (w *world) placementDiffers() []assign.Pair {
|
||||
return differ
|
||||
}
|
||||
|
||||
// placed is each node's modules in the seat's placement (ADR 0567 §1), and whether any placement key was read: a key
|
||||
// placed is each node's modules in the indexer's placement (ADR 0549 §2), and whether any placement key was read: a key
|
||||
// that does not read places nothing on its node and is said by the indexer's own read tool.
|
||||
func (w *world) placed() (map[string][]string, bool) {
|
||||
out := map[string][]string{}
|
||||
@@ -229,7 +228,7 @@ func (w *world) placed() (map[string][]string, bool) {
|
||||
return out, len(values) > 0
|
||||
}
|
||||
|
||||
// placedNodes is every node the seat's placement names (ADR 0567 §1): those compose their own modules (ADR 0547 §2),
|
||||
// placedNodes is every node the indexer's placement names (ADR 0549 §2): those compose their own modules (ADR 0547 §2),
|
||||
// so their wants name the build alone and the seventh rule does not hold them.
|
||||
func (w *world) placedNodes() map[string]bool {
|
||||
placed, _ := w.placed()
|
||||
@@ -303,7 +302,7 @@ func (w *world) mesh(ctx context.Context, manifests getter) assign.Mesh {
|
||||
return m
|
||||
}
|
||||
|
||||
// worldReader is what the assigner watched as frombus reads the bus: the seat's placement, the indexer's modules, each node's
|
||||
// worldReader is what the assigner watched as frombus reads the bus: the indexer's placement and modules, each node's
|
||||
// record in its own have key, the manifests by digest; the check reads no config value, own secret or credential (the
|
||||
// composition it asks resolves, and stands in for what it does not carry).
|
||||
type worldReader struct {
|
||||
@@ -359,25 +358,7 @@ func (r worldReader) Manifest(ctx context.Context, digest string) ([]byte, error
|
||||
return []byte(m.JSON), nil
|
||||
}
|
||||
|
||||
// Values is one module's config values on one node as mesh-config resolved them, each a value its source lays
|
||||
// (catalogue.LaidValue): the holder composes every node a proposal touches with them (ADR 0567 §9).
|
||||
func (r worldReader) Values(_ context.Context, node, module string) (map[string]any, error) {
|
||||
var out map[string]any
|
||||
prefix := node + "." + module + "."
|
||||
for key, raw := range r.w.read(buscompose.ResolvedBucket) {
|
||||
name, ok := strings.CutPrefix(key, prefix)
|
||||
if !ok || strings.Contains(name, ".") {
|
||||
continue
|
||||
}
|
||||
if v, laid := catalogue.LaidValue(raw); laid {
|
||||
if out == nil {
|
||||
out = map[string]any{}
|
||||
}
|
||||
out[name] = v
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
func (worldReader) Values(context.Context, string, string) (map[string]any, error) { return nil, nil }
|
||||
|
||||
func (worldReader) OwnSecrets(context.Context, string) ([]compose.OwnSecretRow, error) {
|
||||
return nil, nil
|
||||
@@ -387,8 +368,8 @@ func (worldReader) Credentials(context.Context, string, []compose.CredentialKey,
|
||||
return nil, nil, nil
|
||||
}
|
||||
|
||||
// Build and Archive are read only for a node composing itself (frombus.Scope without Every): the check of every node
|
||||
// composes from the manifests the indexer read, and reads no build and no archive.
|
||||
// Build, Archive and Served are read only for a node composing itself (frombus.Scope without Every): the check of every
|
||||
// node composes from the manifests the indexer read, reads no build and no archive, and reads no config value.
|
||||
func (worldReader) Build(context.Context, string, string) (*delivery.BuilderBuild, error) {
|
||||
return nil, nil
|
||||
}
|
||||
@@ -397,18 +378,8 @@ func (worldReader) Archive(_ context.Context, url, _ string) ([]byte, error) {
|
||||
return nil, fmt.Errorf("the composition check reads no archive (%s)", url)
|
||||
}
|
||||
|
||||
func (r worldReader) Served(ctx context.Context, node, module string, keys []string) (map[string]any, error) {
|
||||
all, err := r.Values(ctx, node, module)
|
||||
var out map[string]any
|
||||
for _, k := range keys {
|
||||
if v, ok := all[k]; ok {
|
||||
if out == nil {
|
||||
out = map[string]any{}
|
||||
}
|
||||
out[k] = v
|
||||
}
|
||||
}
|
||||
return out, err
|
||||
func (worldReader) Served(context.Context, string, string, []string) (map[string]any, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// change is a candidate's change for the composition check, or why the composition waits for it.
|
||||
@@ -460,9 +431,77 @@ func (w *world) change(ctx context.Context, manifests getter, c delivery.Candida
|
||||
sort.Strings(ch.Twice[name])
|
||||
}
|
||||
}
|
||||
cp, waits := w.placementOf(ctx, manifests, c)
|
||||
if waits != "" {
|
||||
return ch, waits
|
||||
}
|
||||
ch.Placement = cp
|
||||
return ch, ""
|
||||
}
|
||||
|
||||
// placementOf is the change's own placement (ADR 0559 §5): the indexer's placement candidates of the group's catalogue
|
||||
// pull request, with the input the nodes compose from with them laid over the trunk's placement. None when the group has
|
||||
// no catalogue pull request or it changes no placement file. It waits while a key names another head than the group's.
|
||||
func (w *world) placementOf(ctx context.Context, manifests getter, c delivery.Candidate) (*assign.ChangePlacement, string) {
|
||||
var pull *delivery.CandidatePull
|
||||
for i := range c.Pulls {
|
||||
if delivery.SameRepository(c.Pulls[i].Repository, delivery.CatalogueRepository) {
|
||||
pull = &c.Pulls[i]
|
||||
}
|
||||
}
|
||||
if pull == nil {
|
||||
return nil, ""
|
||||
}
|
||||
cp := &assign.ChangePlacement{Placements: map[string]*delivery.Placement{}, Revisions: map[string]uint64{}, Head: pull.Head.Head}
|
||||
for key, value := range w.read(delivery.PlacementCandidatesBucket) {
|
||||
group, node, err := delivery.ParsePlacementCandidateKey(key)
|
||||
if err != nil || group != c.Group {
|
||||
continue
|
||||
}
|
||||
pc, err := delivery.DecodePlacementCandidate(value)
|
||||
if err != nil || pc.Pull.Number != pull.Number || !delivery.SameRepository(pc.Pull.Repository, pull.Repository) {
|
||||
continue
|
||||
}
|
||||
if pc.Pull.Head != pull.Head.Head {
|
||||
return nil, fmt.Sprintf("the indexer holds the placement candidates of %s#%d at %s, not yet at its head %s", pull.Repository, pull.Number, pc.Pull.Head, pull.Head.Head)
|
||||
}
|
||||
meta := w.metaOf(delivery.PlacementCandidatesBucket, key)
|
||||
cp.Placements[node], cp.Revisions[node] = pc.Placement, meta.revision
|
||||
if meta.at.After(cp.Written) {
|
||||
cp.Written = meta.at
|
||||
}
|
||||
}
|
||||
if len(cp.Placements) == 0 {
|
||||
return nil, ""
|
||||
}
|
||||
in, err := frombus.Input(ctx, worldReader{w: w, manifests: manifests}, frombus.Mesh{}, frombus.Scope{Every: true, Placements: cp.Placements})
|
||||
if err != nil {
|
||||
cp.Unread = err.Error()
|
||||
} else {
|
||||
cp.Input = &in
|
||||
}
|
||||
return cp, ""
|
||||
}
|
||||
|
||||
// candidateAnswers is, by node, each node's answer to the group's placement candidates in its `<node>.node` have.
|
||||
func (w *world) candidateAnswers(group string) map[string]*delivery.CandidateAnswer {
|
||||
out := map[string]*delivery.CandidateAnswer{}
|
||||
for key, value := range w.read(delivery.HaveBucket) {
|
||||
node, _, isNode, err := delivery.ParseHaveKey(key)
|
||||
if err != nil || !isNode {
|
||||
continue
|
||||
}
|
||||
h, err := delivery.DecodeHaveNode(value)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if a, ok := h.Candidates[group]; ok {
|
||||
out[node] = &a
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// online is, by node, whether the watchman holds the node online.
|
||||
func (w *world) online() map[string]bool {
|
||||
out := map[string]bool{}
|
||||
|
||||
@@ -48,7 +48,6 @@ func pull(repo string, number int, modules map[string]delivery.CandidateModule)
|
||||
// module with no manifest in the preview as removed, a mesh-sdk member's head, and a module the trunk holds from
|
||||
// another repository as defined twice.
|
||||
func TestTheChangeOfACandidate(t *testing.T) {
|
||||
theWorldCountsTheValuesAKeyTakes(t)
|
||||
f := fakeManifests{}
|
||||
web := f.add(t, "web", `{"module":"web","version":"2"}`)
|
||||
mail := f.add(t, "mail", `{"module":"mail","version":"1"}`)
|
||||
@@ -114,7 +113,7 @@ func TestAManifestFallingBackToAPredefinedKeyIsReadable(t *testing.T) {
|
||||
}
|
||||
|
||||
// Review round 1 (items 3 and 4): the world counts the values a key takes, so a have written again after an answer is seen.
|
||||
func theWorldCountsTheValuesAKeyTakes(t *testing.T) {
|
||||
func TestTheWorldCountsTheValuesAKeyTakes(t *testing.T) {
|
||||
w := newWorld()
|
||||
w.put(delivery.HaveBucket, "ace.web", []byte(`{"v":1}`))
|
||||
w.put(delivery.HaveBucket, "ace.web", []byte(`{"v":1}`))
|
||||
|
||||
@@ -3,7 +3,7 @@ module mesh-delivery-assigner
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028212605-77b48a4f1aff
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027201213-7a8e9f970648
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028212605-77b48a4f1aff h1:EHuM+ed+/M9woMZsMou6/2xm6hIdb1YCFINprrP54Ys=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028212605-77b48a4f1aff/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027193820-8ad9372b1952 h1:TmveT3Bija643RGWkilvkzP/h8UWlVvHFZZtQfJK5oU=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027193820-8ad9372b1952/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027201213-7a8e9f970648 h1:XoMsgg9kSgjQWa86NZM8h8OpxJ2kovxcMS2C0i6WSVU=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027201213-7a8e9f970648/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
|
||||
@@ -0,0 +1,242 @@
|
||||
package assign
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"reflect"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/compose"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// A change's own placement (novox/hq ADR 0559 §5, issue 1126): a catalogue pull request that changes placement files has
|
||||
// the indexer's placement candidates, one key `<group>.<node>` for each node whose file at its head differs from the
|
||||
// trunk's. The check composes the change with those placements laid over the trunk's (frombus, Scope.Placements), counts
|
||||
// every such node among the machines it touches, and then reads each placed node's own answer in its `<node>.node`
|
||||
// have: it fails on what a node composes differently that the change does not intend, on a refusal, and on a node
|
||||
// placed at the head or on the trunk that has not answered at the head within AnswerBound of the keys being written.
|
||||
|
||||
// AnswerBound is how long the check waits for every placed node's answer after the candidate keys were written (ADR 0559
|
||||
// §5): a node asleep or away is named past it, never passed. A variable so a test on the bus can shorten it.
|
||||
var AnswerBound = 10 * time.Minute
|
||||
|
||||
// ChangePlacement is the change's own placement, read from the indexer's placement candidates of its group.
|
||||
type ChangePlacement struct {
|
||||
// Placements is by node the candidate placement; nil when the head removes the node's file.
|
||||
Placements map[string]*delivery.Placement
|
||||
// Revisions is by node the revision of its candidate key: the answer to it names that revision.
|
||||
Revisions map[string]uint64
|
||||
// Head is the catalogue pull request's head the candidates were read at, and Written when the newest key was written.
|
||||
Head string
|
||||
Written time.Time
|
||||
// Input is the trunk's composition input with Placements laid over it (frombus.Input, Scope.Placements), its catalogue
|
||||
// the trunk's; nil with Unread saying why when it could not be read.
|
||||
Input *compose.Input
|
||||
Unread string
|
||||
}
|
||||
|
||||
// Nodes is every node the change's placement names, sorted.
|
||||
func (p *ChangePlacement) Nodes() []string {
|
||||
if p == nil {
|
||||
return nil
|
||||
}
|
||||
out := make([]string, 0, len(p.Placements))
|
||||
for node := range p.Placements {
|
||||
out = append(out, node)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// Intended is, by node the trunk places, the modules whose composition the change's placement changes: those it adds,
|
||||
// removes or whose placed entry differs, and those whose composition (with the trunk's catalogue, the same stand-ins
|
||||
// either side) differs between the trunk's input and the change's (delivery.CompareComposed), the node's own named
|
||||
// "node". A node the trunk does not place yet has none (ADR 0559 §5: at the seed every node must answer the same). fps
|
||||
// gives the build each module is composed at, the trunk's. A node whose compositions cannot be made either side keeps
|
||||
// the modules its placement entries name, and is said in notes.
|
||||
func Intended(trunk *compose.Input, trunkPlaced map[string]delivery.Placement, cp *ChangePlacement, fps map[string]string) (intended map[string]map[string]bool, notes []string) {
|
||||
intended = map[string]map[string]bool{}
|
||||
if cp == nil {
|
||||
return intended, nil
|
||||
}
|
||||
for _, node := range cp.Nodes() {
|
||||
before, placed := trunkPlaced[node]
|
||||
if !placed {
|
||||
continue
|
||||
}
|
||||
set := map[string]bool{}
|
||||
after := cp.Placements[node]
|
||||
var afterModules []delivery.PlacedModule
|
||||
if after != nil {
|
||||
afterModules = after.Modules
|
||||
}
|
||||
entries := map[string]delivery.PlacedModule{}
|
||||
for _, m := range before.Modules {
|
||||
entries[m.Module] = m
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
for _, m := range afterModules {
|
||||
seen[m.Module] = true
|
||||
if e, ok := entries[m.Module]; !ok || !reflect.DeepEqual(e, m) {
|
||||
set[m.Module] = true
|
||||
}
|
||||
}
|
||||
for name := range entries {
|
||||
if !seen[name] {
|
||||
set[name] = true
|
||||
}
|
||||
}
|
||||
if trunk != nil && cp.Input != nil {
|
||||
was, _, _, errB := compose.CompositionsShadow(*trunk, node, fps)
|
||||
now, _, _, errA := compose.CompositionsShadow(*cp.Input, node, fps)
|
||||
if errA == nil && errB == nil {
|
||||
running := make([]delivery.Composed, 0, len(was))
|
||||
for _, c := range was {
|
||||
running = append(running, c)
|
||||
}
|
||||
cmp := delivery.CompareComposed(now, running)
|
||||
for _, d := range cmp.Differs {
|
||||
set[d.Module] = true
|
||||
}
|
||||
for _, d := range cmp.Added {
|
||||
set[d.Module] = true
|
||||
}
|
||||
for _, m := range cmp.Removed {
|
||||
set[m] = true
|
||||
}
|
||||
} else {
|
||||
notes = append(notes, fmt.Sprintf("what the change intends on %s is read from its placement file alone: its compositions cannot be made (trunk: %v; change: %v)", node, errB, errA))
|
||||
}
|
||||
}
|
||||
intended[node] = set
|
||||
}
|
||||
return intended, notes
|
||||
}
|
||||
|
||||
// Answered is what the placed nodes answered: the failures to name, the nodes that have not answered, and the notes.
|
||||
type Answered struct {
|
||||
Fails []string
|
||||
Silent []string
|
||||
Machine string
|
||||
Waits bool
|
||||
Until time.Time
|
||||
Warnings []string
|
||||
}
|
||||
|
||||
// JudgeAnswers reads each placed node's answer to the change's placement (ADR 0559 §5). placed is every node placed at
|
||||
// the head or on the trunk; answers is by node its HaveNode.Candidates entry for the group, nil when it has none; online
|
||||
// says whether the watchman holds the node online. An answer counts when it names the candidates' head and, for a node
|
||||
// with a key of its own, that key's revision. It fails on a refusal, on a module that differs, is added or is removed
|
||||
// that intended does not name, and, from Written plus AnswerBound on, on every node with no answer that counts, named,
|
||||
// asleep or not; before that the check waits for them.
|
||||
func JudgeAnswers(cp *ChangePlacement, placed []string, answers map[string]*delivery.CandidateAnswer, intended map[string]map[string]bool, online map[string]bool, now time.Time) Answered {
|
||||
var out Answered
|
||||
if cp == nil {
|
||||
return out
|
||||
}
|
||||
failAt := func(node, words string) {
|
||||
if out.Machine == "" {
|
||||
out.Machine = node
|
||||
}
|
||||
out.Fails = append(out.Fails, words)
|
||||
}
|
||||
for _, node := range placed {
|
||||
a := answers[node]
|
||||
rev, keyed := cp.Revisions[node]
|
||||
if a == nil || a.Head != cp.Head || (keyed && a.Revision != rev) {
|
||||
out.Silent = append(out.Silent, node)
|
||||
continue
|
||||
}
|
||||
if a.Refused != "" {
|
||||
failAt(node, fmt.Sprintf("%s cannot compose the candidate placement: %s", node, a.Refused))
|
||||
continue
|
||||
}
|
||||
for _, r := range a.Refusals {
|
||||
failAt(node, fmt.Sprintf("%s refuses the candidate placement for %s: %s", node, r.Module, r.Why))
|
||||
}
|
||||
want := intended[node]
|
||||
for _, d := range a.Differs {
|
||||
if !want[d.Module] {
|
||||
failAt(node, fmt.Sprintf("%s composes %s differently with the candidate placement, which the change does not intend: %s", node, d.Module, d.Why))
|
||||
}
|
||||
}
|
||||
for _, d := range a.Added {
|
||||
if !want[d.Module] {
|
||||
failAt(node, fmt.Sprintf("%s would run %s with the candidate placement, which the change does not intend", node, d.Module))
|
||||
}
|
||||
}
|
||||
for _, m := range a.Removed {
|
||||
if !want[m] {
|
||||
failAt(node, fmt.Sprintf("%s would stop running %s with the candidate placement, which the change does not intend", node, m))
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(out.Silent) == 0 {
|
||||
return out
|
||||
}
|
||||
out.Until = cp.Written.Add(AnswerBound)
|
||||
named := make([]string, len(out.Silent))
|
||||
for i, node := range out.Silent {
|
||||
named[i] = node
|
||||
if !online[node] {
|
||||
named[i] = node + " (offline)"
|
||||
}
|
||||
}
|
||||
if now.Before(out.Until) {
|
||||
out.Waits = true
|
||||
out.Warnings = append(out.Warnings, fmt.Sprintf("waits for the answers of %s to the candidate placement at %s, until %s",
|
||||
strings.Join(named, ", "), short(cp.Head), out.Until.UTC().Format(time.RFC3339)))
|
||||
return out
|
||||
}
|
||||
for _, node := range out.Silent {
|
||||
if out.Machine == "" {
|
||||
out.Machine = node
|
||||
}
|
||||
}
|
||||
out.Fails = append(out.Fails, fmt.Sprintf("%s %s (%s after the candidate keys were written at %s)",
|
||||
strings.Join(named, ", "), delivery.CompositionUnanswered, AnswerBound, cp.Written.UTC().Format(time.RFC3339)))
|
||||
return out
|
||||
}
|
||||
|
||||
// PlacedNodes is every node placed at the head or on the trunk: the trunk's placed nodes with the change's placement laid
|
||||
// over them (a node whose file the head removes stays, as the trunk places it), sorted.
|
||||
func PlacedNodes(trunkPlaced map[string]delivery.Placement, cp *ChangePlacement) []string {
|
||||
set := map[string]bool{}
|
||||
for node := range trunkPlaced {
|
||||
set[node] = true
|
||||
}
|
||||
if cp != nil {
|
||||
for node, p := range cp.Placements {
|
||||
if p != nil {
|
||||
set[node] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
out := make([]string, 0, len(set))
|
||||
for node := range set {
|
||||
out = append(out, node)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// WithAnswers is v with the placed nodes' answers judged in: a fail adds its failures (the words of nodes not heard end
|
||||
// with delivery.CompositionUnanswered, so the checker takes a later composition of the same heads in when they answer), a
|
||||
// wait makes a passing verdict wait (a composition that waits composed no machine), and a failing one stands.
|
||||
func WithAnswers(v delivery.CompositionVerdict, a Answered) delivery.CompositionVerdict {
|
||||
if len(a.Fails) > 0 {
|
||||
if v.State == delivery.CompositionFail {
|
||||
v.Words += "; " + strings.Join(a.Fails, "; ")
|
||||
} else {
|
||||
v.State, v.Words, v.Machine = delivery.CompositionFail, strings.Join(a.Fails, "; "), a.Machine
|
||||
}
|
||||
return v
|
||||
}
|
||||
if a.Waits && v.State == delivery.CompositionPass {
|
||||
return delivery.CompositionVerdict{Group: v.Group, Heads: v.Heads, State: delivery.CompositionWaits, SDK: v.SDK, Took: v.Took,
|
||||
Words: strings.Join(a.Warnings, "; ")}
|
||||
}
|
||||
return v
|
||||
}
|
||||
@@ -59,13 +59,15 @@ type Change struct {
|
||||
// Members the head of each member by repository: what a composer at the group's mesh-sdk reads (issues 1622, 1623).
|
||||
Digests map[string]string
|
||||
Members map[string]string
|
||||
// Placement is the change's own placement, read from the indexer's placement candidates of the group (ADR 0559 §5);
|
||||
// nil when the group changes no placement file.
|
||||
Placement *ChangePlacement
|
||||
}
|
||||
|
||||
// Touched is the machines a change touches (ADR 0446 clause 3): those running a module the change moves, adds or
|
||||
// removes, those running a consumer of a provision such a module provides (before or after), and those running a
|
||||
// provider of a provision such a module requires after the change. A module the change adds that is assigned nowhere
|
||||
// touches no machine. A change of placement is no change of a repository (ADR 0567 §4): the seat
|
||||
// mesh-placement judges it.
|
||||
// touches no machine. Every node whose placement the change changes is touched too (ADR 0559 §5, changing ADR 0446 §3).
|
||||
func Touched(m Mesh, c Change) []string {
|
||||
moved := map[string]bool{}
|
||||
for name := range c.Manifests {
|
||||
@@ -99,6 +101,11 @@ func Touched(m Mesh, c Change) []string {
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, node := range c.Placement.Nodes() {
|
||||
if !slices.Contains(out, node) {
|
||||
out = append(out, node)
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
@@ -171,6 +178,25 @@ func Compose(m Mesh, c Change, sdk string, took time.Duration) delivery.Composit
|
||||
delete(changed, name)
|
||||
}
|
||||
after := input(changed, m.Assigned)
|
||||
if c.Placement != nil {
|
||||
// A change of placement is composed over the input the nodes compose from, its placement laid over the trunk's
|
||||
// (frombus, ADR 0559 §5): the trunk as the nodes compose it where the catalogue places them, the change's
|
||||
// manifests over the trunk's catalogue.
|
||||
if m.Input != nil {
|
||||
trunk = *m.Input
|
||||
trunk.Catalogue = withOverlay(m.Input.Catalogue, m.Manifests)
|
||||
}
|
||||
if c.Placement.Input == nil {
|
||||
failAt("", "the change's placement cannot be composed: the input with it cannot be read from the bus: "+c.Placement.Unread)
|
||||
} else {
|
||||
after = *c.Placement.Input
|
||||
cat := withOverlay(c.Placement.Input.Catalogue, changed)
|
||||
for _, name := range c.Removed {
|
||||
delete(cat, name)
|
||||
}
|
||||
after.Catalogue = cat
|
||||
}
|
||||
}
|
||||
for _, node := range machines {
|
||||
_, before := compose.Resolve(trunk, node)
|
||||
_, now := compose.Resolve(after, node)
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
@@ -57,6 +58,79 @@ func TestAMachineTheChangeBreaksFails(t *testing.T) {
|
||||
if v.State != delivery.CompositionFail || v.Machine != "ace" || !strings.Contains(v.Words, "database") {
|
||||
t.Fatalf("composition = %+v, want a fail on ace naming the provision nothing provides", v)
|
||||
}
|
||||
|
||||
// ADR 0559 §5 (issue 1126): a change of placement composes its placement laid over the trunk's input and fails naming
|
||||
// the node it breaks: a module no manifest names placed on g14; ace,
|
||||
// is no longer composed with it; a placement whose input cannot be read fails; a placement that breaks nothing passes, g14 touched.
|
||||
m := meshOf()
|
||||
placed := func(assigned map[string][]string) *ChangePlacement {
|
||||
in := input(m.Manifests, assigned)
|
||||
return &ChangePlacement{Placements: map[string]*delivery.Placement{"g14": {Node: "g14"}}, Input: &in}
|
||||
}
|
||||
for _, row := range []struct {
|
||||
name string
|
||||
cp *ChangePlacement
|
||||
state string
|
||||
machine string
|
||||
words string
|
||||
}{
|
||||
{"a placement that breaks g14", placed(map[string][]string{"ace": {"mail", "store"}, "g14": {"web", "ghost"}}), delivery.CompositionFail, "g14", "ghost"},
|
||||
{"a placement whose input is not read", &ChangePlacement{Placements: map[string]*delivery.Placement{"g14": nil}, Unread: "no record of g14"}, delivery.CompositionFail, "", "no record of g14"},
|
||||
{"a placement that breaks nothing", placed(map[string][]string{"ace": {"mail", "store"}, "g14": {"web"}}), delivery.CompositionPass, "", ""},
|
||||
} {
|
||||
v := Compose(m, Change{Group: "g", Heads: "0123456789ab", Placement: row.cp}, "sdk", 0)
|
||||
if v.State != row.state || v.Machine != row.machine || !strings.Contains(v.Words, row.words) || !slices.Contains(v.Machines, "g14") {
|
||||
t.Errorf("%s: %+v", row.name, v)
|
||||
}
|
||||
}
|
||||
|
||||
// ADR 0559 §5: the placed nodes' answers. ace is placed on the trunk and its candidate moves web there (intended);
|
||||
// g14 is not placed on the trunk (the seed), so nothing is intended there; novox is placed and unchanged.
|
||||
at := time.Date(2026, 10, 27, 2, 0, 0, 0, time.UTC)
|
||||
cp := &ChangePlacement{Head: "h2", Written: at, Revisions: map[string]uint64{"ace": 7, "g14": 8},
|
||||
Placements: map[string]*delivery.Placement{"ace": {Node: "ace"}, "g14": {Node: "g14"}}}
|
||||
intended := map[string]map[string]bool{"ace": {"web": true}}
|
||||
same := func(rev uint64) *delivery.CandidateAnswer {
|
||||
return &delivery.CandidateAnswer{Head: "h2", Revision: rev, Verdict: delivery.CandidateSame, Same: []string{"mail"}}
|
||||
}
|
||||
placedNodes := []string{"ace", "g14", "novox"}
|
||||
for _, row := range []struct {
|
||||
name string
|
||||
answers map[string]*delivery.CandidateAnswer
|
||||
now time.Time
|
||||
state string
|
||||
words string
|
||||
}{
|
||||
{"all the same", map[string]*delivery.CandidateAnswer{"ace": same(7), "g14": same(8), "novox": same(0)}, at, delivery.CompositionPass, ""},
|
||||
{"a module added where intended", map[string]*delivery.CandidateAnswer{"ace": {Head: "h2", Revision: 7, Verdict: delivery.CandidateDiffers,
|
||||
Added: []delivery.ModuleAdded{{Module: "web", Unissued: []string{"its own secret"}}}}, "g14": same(8), "novox": same(0)}, at, delivery.CompositionPass, ""},
|
||||
{"a module differing where not intended", map[string]*delivery.CandidateAnswer{"ace": {Head: "h2", Revision: 7, Verdict: delivery.CandidateDiffers,
|
||||
Differs: []delivery.ModuleDiffers{{Module: "mail", Why: "the port differs"}}}, "g14": same(8), "novox": same(0)}, at, delivery.CompositionFail, "ace composes mail differently"},
|
||||
{"a node's first placement file with a module differing", map[string]*delivery.CandidateAnswer{"ace": same(7), "g14": {Head: "h2", Revision: 8, Verdict: delivery.CandidateDiffers,
|
||||
Differs: []delivery.ModuleDiffers{{Module: "web", Why: "a binding differs"}}}, "novox": same(0)}, at, delivery.CompositionFail, "g14 composes web differently"},
|
||||
{"a refusal", map[string]*delivery.CandidateAnswer{"ace": same(7), "g14": {Head: "h2", Revision: 8, Verdict: delivery.CandidateRefused,
|
||||
Refusals: []delivery.ModuleRefused{{Module: "web", Why: "no own secret"}}}, "novox": same(0)}, at, delivery.CompositionFail, "g14 refuses"},
|
||||
{"a node not answering, within its bound", map[string]*delivery.CandidateAnswer{"ace": same(7), "novox": same(0)}, at.Add(AnswerBound - time.Second), delivery.CompositionWaits, "g14 (offline)"},
|
||||
{"an answer of an earlier revision, past its bound", map[string]*delivery.CandidateAnswer{"ace": same(6), "g14": same(8), "novox": same(0)}, at.Add(AnswerBound), delivery.CompositionFail, "ace " + delivery.CompositionUnanswered},
|
||||
{"its later answer", map[string]*delivery.CandidateAnswer{"ace": same(7), "g14": same(8), "novox": same(0)}, at.Add(time.Hour), delivery.CompositionPass, ""},
|
||||
} {
|
||||
pass := delivery.CompositionVerdict{Group: "g", Heads: "0123456789ab", State: delivery.CompositionPass, SDK: "sdk", Machines: []string{"ace", "g14"}}
|
||||
v := WithAnswers(pass, JudgeAnswers(cp, placedNodes, row.answers, intended, map[string]bool{"ace": true, "novox": true}, row.now))
|
||||
if v.State != row.state || !strings.Contains(v.Words, row.words) {
|
||||
t.Errorf("%s: %+v", row.name, v)
|
||||
}
|
||||
if _, err := delivery.EncodeCompositionVerdict(v); err != nil {
|
||||
t.Errorf("%s: the verdict cannot be written: %v", row.name, err)
|
||||
}
|
||||
}
|
||||
trunk := map[string]delivery.Placement{"ace": {Node: "ace", Modules: []delivery.PlacedModule{{Module: "mail"}}}, "novox": {Node: "novox"}}
|
||||
if got := PlacedNodes(trunk, &ChangePlacement{Placements: map[string]*delivery.Placement{"novox": nil, "g14": {Node: "g14"}}}); !slices.Equal(got, []string{"ace", "g14", "novox"}) {
|
||||
t.Errorf("placed at the head or on the trunk = %v", got)
|
||||
}
|
||||
got, _ := Intended(nil, trunk, &ChangePlacement{Placements: map[string]*delivery.Placement{"ace": {Node: "ace", Modules: []delivery.PlacedModule{{Module: "mail", Ports: map[string]int{"25": 2525}}, {Module: "web"}}}, "g14": {Node: "g14", Modules: []delivery.PlacedModule{{Module: "web"}}}}}, nil)
|
||||
if !got["ace"]["mail"] || !got["ace"]["web"] || len(got["g14"]) != 0 {
|
||||
t.Errorf("intended = %v, want ace's mail and web, nothing on g14 the trunk does not place", got)
|
||||
}
|
||||
}
|
||||
|
||||
// 1297/B3c: what was already broken on the trunk is said and fails nothing.
|
||||
@@ -109,6 +183,10 @@ func TestTouched(t *testing.T) {
|
||||
if got := Touched(m, Change{Manifests: map[string]catalogue.Manifest{"store": m.Manifests["store"]}}); !slices.Equal(got, []string{"ace", "g14"}) {
|
||||
t.Errorf("touched by store with a consumer on g14 = %v", got)
|
||||
}
|
||||
// ADR 0559 §5: every node whose placement the change changes is touched, its file removed or not.
|
||||
if got := Touched(m, Change{Placement: &ChangePlacement{Placements: map[string]*delivery.Placement{"novox": nil, "shanks": {Node: "shanks"}}}}); !slices.Equal(got, []string{"novox", "shanks"}) {
|
||||
t.Errorf("touched by a placement = %v", got)
|
||||
}
|
||||
c := Change{Group: "g", Heads: "0123456789ab", Manifests: map[string]catalogue.Manifest{"new": {Module: "new", Version: "1"}}}
|
||||
if v := Compose(m, c, "sdk", 0); len(v.Machines) != 0 || !hasWarning(v, "touches no machine") || v.State != delivery.CompositionPass {
|
||||
t.Errorf("an added module assigned nowhere: %+v", v)
|
||||
|
||||
@@ -50,6 +50,17 @@ func HeadJob(m Mesh, c Change) (delivery.ComposeJob, error) {
|
||||
fromClaims = len(in.Holdings) == 0
|
||||
}
|
||||
changeCase := composer.Case{Name: CaseChange, Manifests: change}
|
||||
if c.Placement != nil {
|
||||
// The change's own placement, laid over the input's in the change case (ADR 0559 §5; composer.Case's Placements):
|
||||
// a node the input does not hold yet is added bare, so the composer places it.
|
||||
changeCase.Placements = c.Placement.Placements
|
||||
in.Nodes = slices.Clone(in.Nodes)
|
||||
for _, node := range c.Placement.Nodes() {
|
||||
if !slices.ContainsFunc(in.Nodes, func(n compose.Node) bool { return n.Name == node }) {
|
||||
in.Nodes = append(in.Nodes, compose.Node{Name: node})
|
||||
}
|
||||
}
|
||||
}
|
||||
req := composer.Request{V: composer.Version, Input: in, Depth: composer.DepthResolve,
|
||||
HoldingsFromClaims: fromClaims, Machines: Touched(m, c),
|
||||
Cases: []composer.Case{{Name: CaseTrunk, Manifests: trunk}, changeCase}}
|
||||
|
||||
@@ -120,6 +120,22 @@ func TestAGroupWithAnSDKMemberAsksTheComposerAtItsHead(t *testing.T) {
|
||||
len(placed.Input.Holdings) != 1 || placed.HoldingsFromClaims {
|
||||
t.Errorf("the placed check's request = %+v, holdings from claims %v", placed.Input, placed.HoldingsFromClaims)
|
||||
}
|
||||
|
||||
// ADR 0559 §5: a change of placement composes its own placement in the change case alone (composer.Case's Placements),
|
||||
// a node the input does not hold yet added bare so the composer places it, and the node touched.
|
||||
c.Placement = &ChangePlacement{Placements: map[string]*delivery.Placement{"novox": {Node: "novox"}}}
|
||||
candJob, err := HeadJob(m, c)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cand, err := composer.Decode(candJob.Request)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if cand.Cases[0].Placements != nil || cand.Cases[1].Placements["novox"] == nil || len(cand.Input.Nodes) != 2 || cand.Input.Nodes[1].Name != "novox" ||
|
||||
!slices.Equal(cand.Machines, []string{"g14", "novox"}) || len(m.Input.Nodes) != 1 {
|
||||
t.Errorf("the placement's request = cases %+v, nodes %+v, machines %v", cand.Cases, cand.Input.Nodes, cand.Machines)
|
||||
}
|
||||
}
|
||||
|
||||
// 1622/AC2, AC3: a catalogue member using a field only the SDK head reads passes with the head's composer, and the
|
||||
|
||||
@@ -3,32 +3,27 @@ package assign
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// Until ADR 0547's third milestone two sources name which module runs on which node: the controller's assignments and
|
||||
// the seat mesh-placement's placement (ADR 0549 §5, ADR 0567 §8). The assigner writes wants only for a pair both name:
|
||||
// the indexer's placement, read from the catalogue (ADR 0549 §5). The assigner writes wants only for a pair both name:
|
||||
// for a pair one source names alone it writes no want and touches none it wrote before, so nothing moves and nothing is
|
||||
// removed on one source's word, and it raises a condition naming the pair until both agree. After an approved proposal
|
||||
// the assigner keeps the controller in step itself, calling its assign, unassign, pin and unpin; a call refused or not
|
||||
// yet answered leaves the pair differing, and the condition names the call (Pair.Call). Before the first placement key
|
||||
// is written the controller's assignments are the one source, as before.
|
||||
// removed on one source's word, and it raises a condition naming the pair until both agree. Before the first placement
|
||||
// file reaches the indexer (no placement key at all) the controller's assignments are the one source, as before.
|
||||
|
||||
// Pair is a module on a node that one source names and the other does not; Only says which: OnlyAssignments or
|
||||
// OnlyPlacement.
|
||||
type Pair struct {
|
||||
Node, Module, Only string
|
||||
// Call is what became of the assigner's call of the controller's verb for the pair, in words, when it made one.
|
||||
Call string
|
||||
}
|
||||
|
||||
// The source that names a pair alone.
|
||||
const (
|
||||
OnlyAssignments = "the controller's assignments"
|
||||
OnlyPlacement = "the seat mesh-placement's placement"
|
||||
OnlyPlacement = "the catalogue's placement"
|
||||
)
|
||||
|
||||
// Placed is, by node, the modules both sources name, in name order, and every pair one names alone, by node then
|
||||
@@ -84,7 +79,7 @@ func PlacementCondition(p Pair, raised bool) asks.Condition {
|
||||
subject := asks.Scope{Scope: "module", ID: p.Module, Machine: p.Node}
|
||||
if !raised {
|
||||
return asks.Condition{Key: key, Subject: subject, State: asks.StateCleared,
|
||||
Resolved: fmt.Sprintf("the controller's assignments and the seat's placement agree on %s on %s.", p.Module, p.Node)}
|
||||
Resolved: fmt.Sprintf("the controller's assignments and the catalogue's placement agree on %s on %s.", p.Module, p.Node)}
|
||||
}
|
||||
headline := fmt.Sprintf("%s on %s is placed by one source only", p.Module, p.Node)
|
||||
if n := len([]rune(headline)); n > asks.HeadlineLength {
|
||||
@@ -92,12 +87,9 @@ func PlacementCondition(p Pair, raised bool) asks.Condition {
|
||||
}
|
||||
var needs string
|
||||
if p.Only == OnlyAssignments {
|
||||
needs = fmt.Sprintf("propose placing %s on %s (mesh-placement.propose), or unassign it at the controller", p.Module, p.Node)
|
||||
needs = fmt.Sprintf("add %s to the catalogue's placement file of %s, or unassign it at the controller", p.Module, p.Node)
|
||||
} else {
|
||||
needs = fmt.Sprintf("assign %s to %s at the controller, as the approved placement says, or propose taking it off", p.Module, p.Node)
|
||||
}
|
||||
if p.Call != "" {
|
||||
needs += "; the assigner's call: " + p.Call
|
||||
needs = fmt.Sprintf("assign %s to %s at the controller, as the merged placement file says", p.Module, p.Node)
|
||||
}
|
||||
return asks.Condition{Key: key, Severity: asks.SeverityUrgent, Subject: subject, State: asks.StateRaised,
|
||||
Summary: headline, Headline: headline, Needs: needs,
|
||||
@@ -105,17 +97,3 @@ func PlacementCondition(p Pair, raised bool) asks.Condition {
|
||||
"name a module on a node (ADR 0549 §5): the assigner writes no want for it there and leaves the one it wrote before, so "+
|
||||
"nothing moves or is removed. This clears once both name it or neither does.", asks.NeedsYou, needs, p.Only, p.Module, p.Node)}
|
||||
}
|
||||
|
||||
// SeatWaitsCondition is the assigner's word on the seat mesh-placement while a state of the seat is not bound (ADR 0567
|
||||
// §1): raised naming each, cleared when none is left.
|
||||
func SeatWaitsCondition(unbound []string) asks.Condition {
|
||||
key := delivery.AssignerModule + ".seat-states"
|
||||
subject := asks.Scope{Scope: "module", ID: delivery.AssignerModule}
|
||||
if len(unbound) == 0 {
|
||||
return asks.Condition{Key: key, Subject: subject, State: asks.StateCleared, Resolved: "the seat mesh-placement holds all its states."}
|
||||
}
|
||||
why := "the seat waits for its states: " + strings.Join(unbound, ", ") + " are not on the bus or not granted yet"
|
||||
return asks.Condition{Key: key, Severity: asks.SeverityWarning, Subject: subject, State: asks.StateRaised,
|
||||
Summary: "the seat waits for its states", Headline: "the seat waits for its states",
|
||||
Explanation: asks.NothingToDo + " " + why + "; each is bound at the next pass once it is, with no restart."}
|
||||
}
|
||||
|
||||
@@ -1,296 +0,0 @@
|
||||
package assign
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// The seat mesh-placement (novox/hq ADR 0567), held by the assigner beside its wants: placement is its state
|
||||
// `placement`, changed only by a proposal (§2) the nodes it touches answer (§3), judged (§4) and written on the
|
||||
// operator's Approve at the revision it was laid over (§5). What here decides; the tools and the passes do.
|
||||
|
||||
// Leased says whether the lease write of ADR 0547 has happened, after which the controller's assignments are read no
|
||||
// more (§8). Not before L3: until then the holder keeps the controller in step and refuses what its verbs cannot take.
|
||||
var Leased = false
|
||||
|
||||
// ProposeIn is what a proposal is laid over: its id, who and why, its changes; every node's current placement and the
|
||||
// revision of its key; which modules the indexer holds and which nodes have a record; and, for the seed, the export's
|
||||
// digest.
|
||||
type ProposeIn struct {
|
||||
ID, By, Why string
|
||||
At time.Time
|
||||
Changes []delivery.PlacementProposalChange
|
||||
Current map[string]delivery.Placement
|
||||
Revisions map[string]uint64
|
||||
Held func(module string) bool
|
||||
Recorded func(node string) bool
|
||||
Seed bool
|
||||
ExportDigest string
|
||||
}
|
||||
|
||||
// Propose lays a proposal over the current keys (§2): the proposal (state answering, each changed node's base revision),
|
||||
// the laid placement of every node whose key it changes, and every finding, which refuses it whole: a change the
|
||||
// controller's verbs cannot take before the lease write (§8), a change that does not fit its node, and what
|
||||
// CheckPlacements refuses of the result laid over every placement (two modules on one host port, a provider not placed,
|
||||
// a seat held once held twice, two nodes with one address, two hubs, a module the indexer does not hold, a node with no
|
||||
// record). A key whose value the proposal leaves as it is changes nothing and is not laid.
|
||||
func Propose(in ProposeIn) (delivery.PlacementProposal, map[string]*delivery.Placement, []string) {
|
||||
p := delivery.PlacementProposal{ID: in.ID, By: strings.TrimSpace(in.By), Why: strings.TrimSpace(in.Why), At: in.At.UTC(),
|
||||
Changes: in.Changes, Seed: in.Seed, ExportDigest: in.ExportDigest, Base: map[string]uint64{}, State: delivery.ProposalAnswering}
|
||||
var findings []string
|
||||
if p.By == "" {
|
||||
findings = append(findings, "the runtime names no caller: a proposal is made by someone")
|
||||
}
|
||||
if p.Why == "" {
|
||||
findings = append(findings, "a proposal says why: it is shown to the operator")
|
||||
}
|
||||
if len(in.Changes) == 0 {
|
||||
findings = append(findings, "a proposal changes something: it names no change")
|
||||
}
|
||||
for i, c := range in.Changes {
|
||||
if (c.Op == delivery.OpSeed) != in.Seed {
|
||||
findings = append(findings, fmt.Sprintf("change %d: the seed is adopt's alone, and never mixed with other changes", i+1))
|
||||
continue
|
||||
}
|
||||
if !Leased {
|
||||
if why := c.BeforeLease(); why != "" {
|
||||
findings = append(findings, fmt.Sprintf("change %d: %s", i+1, why))
|
||||
}
|
||||
}
|
||||
}
|
||||
laid, lay := delivery.LayChanges(in.Current, in.Changes)
|
||||
findings = append(findings, lay...)
|
||||
all := map[string]delivery.Placement{}
|
||||
for node, pl := range in.Current {
|
||||
all[node] = pl
|
||||
}
|
||||
out := map[string]*delivery.Placement{}
|
||||
for node, pl := range laid {
|
||||
if before, ok := in.Current[node]; ok && delivery.SamePlacement(before, pl) {
|
||||
continue
|
||||
}
|
||||
pl := pl
|
||||
all[node] = pl
|
||||
out[node] = &pl
|
||||
p.Base[node] = in.Revisions[node]
|
||||
}
|
||||
if len(findings) == 0 && len(out) == 0 {
|
||||
findings = append(findings, "the changes leave every placement as it is")
|
||||
}
|
||||
held, recorded := in.Held, in.Recorded
|
||||
if held == nil {
|
||||
held = func(string) bool { return false }
|
||||
}
|
||||
if recorded == nil {
|
||||
recorded = func(string) bool { return false }
|
||||
}
|
||||
for _, r := range delivery.CheckPlacements(all, held, recorded) {
|
||||
findings = append(findings, r)
|
||||
}
|
||||
return p, out, findings
|
||||
}
|
||||
|
||||
// Write is one key a warrant writes (§5): the node, its key, the revision it is written at (0: it has none, so it is
|
||||
// created) and its value (nil: it is deleted).
|
||||
type Write struct {
|
||||
Node, Key string
|
||||
Base uint64
|
||||
Value *delivery.Placement
|
||||
}
|
||||
|
||||
// Writes is every key a proposal writes, by node order.
|
||||
func Writes(p delivery.PlacementProposal, laid map[string]*delivery.Placement) ([]Write, error) {
|
||||
var out []Write
|
||||
for _, node := range p.Nodes() {
|
||||
key, err := delivery.PlacementKey(node)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, Write{Node: node, Key: key, Base: p.Base[node], Value: laid[node]})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// ControllerCall is one call of the controller's seat verbs the holder makes after writing an approved proposal, until
|
||||
// the lease write (§8): assign, unassign, pin or unpin, with its arguments.
|
||||
type ControllerCall struct {
|
||||
Verb string
|
||||
Args map[string]any
|
||||
// Node and Module name the pair the call keeps in step (a pin names no module).
|
||||
Node, Module string
|
||||
}
|
||||
|
||||
// Key names the call, as the condition of its pair says it.
|
||||
func (c ControllerCall) Key() string {
|
||||
return fmt.Sprintf("mesh-controller.%s %s %s", c.Verb, c.Node, c.Module)
|
||||
}
|
||||
|
||||
// ControllerCalls is every call that brings the controller's assignments and pins to the placement after: assign for a
|
||||
// module placed on a node before not, unassign for one taken off, pin for a pin set or changed, unpin for one removed,
|
||||
// each naming the proposal and its warrant in its why. In node order, unassigns before assigns, so a module moved
|
||||
// between nodes is first taken off.
|
||||
func ControllerCalls(before map[string]delivery.Placement, after map[string]*delivery.Placement, proposal, warrant string) []ControllerCall {
|
||||
why := fmt.Sprintf("placement proposal %s, approved (%s)", proposal, warrant)
|
||||
nodes := make([]string, 0, len(after))
|
||||
for node := range after {
|
||||
nodes = append(nodes, node)
|
||||
}
|
||||
sort.Strings(nodes)
|
||||
var off, on, pins []ControllerCall
|
||||
for _, node := range nodes {
|
||||
was, now := before[node], delivery.Placement{}
|
||||
if a := after[node]; a != nil {
|
||||
now = *a
|
||||
}
|
||||
had, has := map[string]bool{}, map[string]bool{}
|
||||
for _, m := range was.Modules {
|
||||
had[m.Module] = true
|
||||
}
|
||||
for _, m := range now.Modules {
|
||||
has[m.Module] = true
|
||||
if !had[m.Module] {
|
||||
on = append(on, ControllerCall{Verb: "assign", Node: node, Module: m.Module, Args: map[string]any{"node": node, "module": m.Module}})
|
||||
}
|
||||
}
|
||||
for _, m := range was.Modules {
|
||||
if !has[m.Module] {
|
||||
off = append(off, ControllerCall{Verb: "unassign", Node: node, Module: m.Module, Args: map[string]any{"node": node, "module": m.Module}})
|
||||
}
|
||||
}
|
||||
for _, provision := range sortedKeys(now.Pins) {
|
||||
to := now.Pins[provision]
|
||||
if old, ok := was.Pins[provision]; !ok || old != to {
|
||||
pins = append(pins, ControllerCall{Verb: "pin", Node: node, Module: provision,
|
||||
Args: map[string]any{"node": node, "provision": provision, "from": to.Node, "module": to.Module}})
|
||||
}
|
||||
}
|
||||
for _, provision := range sortedKeys(was.Pins) {
|
||||
if _, ok := now.Pins[provision]; !ok {
|
||||
pins = append(pins, ControllerCall{Verb: "unpin", Node: node, Module: provision, Args: map[string]any{"node": node, "provision": provision}})
|
||||
}
|
||||
}
|
||||
}
|
||||
out := append(append(off, on...), pins...)
|
||||
for i := range out {
|
||||
out[i].Args["why"] = why
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// PlacementAct is the act an approved proposal's yes binds: writing exactly its changes at its base revisions.
|
||||
func PlacementAct(p delivery.PlacementProposal) asks.Act {
|
||||
return asks.Act{"placement": "write", "proposal": p.ID, "changes": delivery.ChangesDigest(p)}
|
||||
}
|
||||
|
||||
// PlacementAsking is a proposal's asking number: each judging that raises its ask again (again, a key moved) is
|
||||
// another asking, under an id of its own.
|
||||
func PlacementAsking(p delivery.PlacementProposal, at time.Time) int {
|
||||
return int(at.Unix())
|
||||
}
|
||||
|
||||
// PlacementAsk is the authorising ask about a proposal (§5): an Approve that binds its act, so only a channel that
|
||||
// proves who answered takes it, with the brief of the verdict by node (judge.Brief) as its explanation.
|
||||
func PlacementAsk(key string, p delivery.PlacementProposal, brief string, asking int, now time.Time) (delivery.AssignerAsk, error) {
|
||||
binds, err := asks.ActDigest(PlacementAct(p))
|
||||
if err != nil {
|
||||
return delivery.AssignerAsk{}, err
|
||||
}
|
||||
nodes := p.Nodes()
|
||||
headline := fmt.Sprintf("Change placement on %s?", strings.Join(nodes, ", "))
|
||||
if p.Seed {
|
||||
headline = "Take placement over from the controller?"
|
||||
}
|
||||
if n := len([]rune(headline)); n > asks.HeadlineLength {
|
||||
headline = fmt.Sprintf("Change placement on %d machines?", len(nodes))
|
||||
}
|
||||
return delivery.AssignerAsk{Ask: asks.Ask{
|
||||
ID: delivery.AskingID(key, asking),
|
||||
Headline: headline,
|
||||
Explanation: brief,
|
||||
Options: []asks.Option{
|
||||
{ID: OptionYes, Label: "Approve", Does: "the placement of " + strings.Join(nodes, ", ") + " is written as proposed",
|
||||
Level: asks.Approve, Binds: binds},
|
||||
{ID: OptionNo, Label: "Decline", Does: "nothing is written, and the proposal ends", Level: asks.Acknowledge},
|
||||
},
|
||||
Who: asks.Operator,
|
||||
Expires: now.Add(asks.ApproveLasts),
|
||||
OnExpiry: "nothing is written, and the proposal ends",
|
||||
About: key,
|
||||
}, Asking: asking}, nil
|
||||
}
|
||||
|
||||
// ExportOf reads the controller's answer to placement-export (§7): its verb answers what the command printed, the
|
||||
// JSON of every node's placement, in `output` (the controller's verbAnswer; `answer` when it carries it as data). A
|
||||
// refused or unreadable answer is an error naming it.
|
||||
func ExportOf(raw []byte) (map[string]delivery.Placement, error) {
|
||||
var a struct {
|
||||
Output string `json:"output"`
|
||||
OK *bool `json:"ok"`
|
||||
Answer json.RawMessage `json:"answer"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &a); err != nil {
|
||||
return nil, fmt.Errorf("the controller's placement-export answered no verb answer: %w", err)
|
||||
}
|
||||
if a.OK != nil && !*a.OK {
|
||||
return nil, fmt.Errorf("the controller's placement-export refused: %s", strings.TrimSpace(a.Output))
|
||||
}
|
||||
var byNode map[string]json.RawMessage
|
||||
if len(a.Answer) > 0 && string(a.Answer) != "null" {
|
||||
if err := json.Unmarshal(a.Answer, &byNode); err != nil {
|
||||
return nil, fmt.Errorf("the controller's placement-export is no placement by node: %w", err)
|
||||
}
|
||||
} else if err := json.NewDecoder(strings.NewReader(a.Output)).Decode(&byNode); err != nil {
|
||||
// The command prints its JSON first; what its standard error said after it is left.
|
||||
return nil, fmt.Errorf("the controller's placement-export printed no placement by node: %w", err)
|
||||
}
|
||||
out := map[string]delivery.Placement{}
|
||||
for node, rawP := range byNode {
|
||||
p, err := delivery.DecodePlacement(rawP)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the controller's placement-export of %s cannot be read: %w", node, err)
|
||||
}
|
||||
if p.Node != node {
|
||||
return nil, fmt.Errorf("the controller's placement-export names %s under %s", p.Node, node)
|
||||
}
|
||||
out[node] = p
|
||||
}
|
||||
if len(out) == 0 {
|
||||
return nil, fmt.Errorf("the controller's placement-export names no node")
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// SeedChanges is the seed's changes: every node's placement of the export, whole, in node order.
|
||||
func SeedChanges(export map[string]delivery.Placement) []delivery.PlacementProposalChange {
|
||||
var out []delivery.PlacementProposalChange
|
||||
for _, node := range sortedKeys(export) {
|
||||
p := export[node]
|
||||
out = append(out, delivery.PlacementProposalChange{Op: delivery.OpSeed, Node: node, Whole: &p})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Rest is what a proposal leaves to write once the keys of written are: its changes of the other nodes, to be laid again
|
||||
// over the keys as the write left them (§5: a key moved while it was written). False when nothing is left.
|
||||
func Rest(p delivery.PlacementProposal, written map[string]*delivery.Placement) (delivery.PlacementProposal, bool) {
|
||||
rest := p
|
||||
rest.Changes, rest.Base = nil, map[string]uint64{}
|
||||
for _, c := range p.Changes {
|
||||
if _, done := written[c.Node]; !done {
|
||||
rest.Changes = append(rest.Changes, c)
|
||||
}
|
||||
}
|
||||
for node, base := range p.Base {
|
||||
if _, done := written[node]; !done {
|
||||
rest.Base[node] = base
|
||||
}
|
||||
}
|
||||
rest.Must, rest.Verdict, rest.Ask, rest.State = nil, nil, "", delivery.ProposalAnswering
|
||||
return rest, len(rest.Changes) > 0
|
||||
}
|
||||
@@ -1,157 +0,0 @@
|
||||
package assign
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// ADR 0567 §2, §5, §7, §8: what the seat's holder decides of a proposal: every refusal named at once, a valid one laid
|
||||
// with the base revision of each key it changes, the writes at those revisions, the controller's calls that keep its
|
||||
// assignments in step, the authorising ask bound to the changes, and the export read as the controller answers it.
|
||||
func TestThePlacementSeatDecides(t *testing.T) {
|
||||
anchor := delivery.Placement{V: delivery.Version, Node: "anchor", Decided: delivery.NodeDecided{Hub: true, Address: "10.0.0.1"},
|
||||
Modules: []delivery.PlacedModule{{Module: "store", Ports: map[string]int{"5432": 5432}}, {Module: "web"}}}
|
||||
laptop := delivery.Placement{V: delivery.Version, Node: "laptop", Decided: delivery.NodeDecided{Address: "10.0.0.2"}, Modules: []delivery.PlacedModule{{Module: "shell"}}}
|
||||
in := func(changes ...delivery.PlacementProposalChange) ProposeIn {
|
||||
return ProposeIn{ID: "p-one", By: "operator@laptop", Why: "move web", At: time.Unix(1, 0), Changes: changes,
|
||||
Current: map[string]delivery.Placement{"anchor": anchor, "laptop": laptop}, Revisions: map[string]uint64{"anchor": 4, "laptop": 9},
|
||||
Held: func(m string) bool { return m != "ghost" },
|
||||
Recorded: func(n string) bool { return n == "anchor" || n == "laptop" }}
|
||||
}
|
||||
move := []delivery.PlacementProposalChange{{Op: delivery.OpTakeOff, Node: "anchor", Module: "web"}, {Op: delivery.OpPlace, Node: "laptop", Module: "web"}}
|
||||
p, laid, findings := Propose(in(move...))
|
||||
if len(findings) != 0 || p.State != delivery.ProposalAnswering || p.Base["anchor"] != 4 || p.Base["laptop"] != 9 ||
|
||||
!slices.Equal(laid["laptop"].Assigned(), []string{"shell", "web"}) || !slices.Equal(laid["anchor"].Assigned(), []string{"store"}) {
|
||||
t.Fatalf("a move: %+v, %+v, %v", p, laid, findings)
|
||||
}
|
||||
if _, err := delivery.EncodePlacementProposal(p); err != nil {
|
||||
t.Fatalf("the proposal cannot be written: %v", err)
|
||||
}
|
||||
for name, c := range map[string]struct {
|
||||
in ProposeIn
|
||||
want string
|
||||
}{
|
||||
"a port before the lease": {in(delivery.PlacementProposalChange{Op: delivery.OpPlace, Node: "laptop", Module: "web", Ports: map[string]int{"80": 8080}}), "lease write"},
|
||||
"a holding": {in(delivery.PlacementProposalChange{Op: delivery.OpHold, Node: "anchor", Module: "store", Claim: "mesh-store", Scope: "mesh"}), "lease write"},
|
||||
"a binding": {in(delivery.PlacementProposalChange{Op: delivery.OpPlace, Node: "laptop", Module: "web", Bindings: map[string]delivery.Provider{"db": {Node: "anchor", Module: "store"}}}), "lease write"},
|
||||
"a config place": {in(delivery.PlacementProposalChange{Op: delivery.OpConfig, Node: "anchor", Module: "web", Layer: map[string]any{"places": []any{}}}), "lease write"},
|
||||
"a config access": {in(delivery.PlacementProposalChange{Op: delivery.OpConfig, Node: "anchor", Module: "web", MeshWide: map[string]any{"accesses": []any{}}}), "lease write"},
|
||||
"a module not held": {in(delivery.PlacementProposalChange{Op: delivery.OpPlace, Node: "laptop", Module: "ghost"}), "the indexer holds no module ghost"},
|
||||
"a node with no record": {in(delivery.PlacementProposalChange{Op: delivery.OpPlace, Node: "stranger", Module: "web"}), "stranger is a node with no record"},
|
||||
"nothing changes": {in(delivery.PlacementProposalChange{Op: delivery.OpPlace, Node: "anchor", Module: "web"}), "leave every placement as it is"},
|
||||
"no why": {func() ProposeIn { i := in(move...); i.Why = " "; return i }(), "says why"},
|
||||
"no caller": {func() ProposeIn { i := in(move...); i.By = ""; return i }(), "names no caller"},
|
||||
"a seed by hand": {in(delivery.PlacementProposalChange{Op: delivery.OpSeed, Node: "laptop", Whole: &laptop}), "adopt's alone"},
|
||||
"a pin of nobody placed": {in(delivery.PlacementProposalChange{Op: delivery.OpPin, Node: "laptop", Provision: "db", Provider: &delivery.Provider{Node: "laptop", Module: "store"}}), "not placed there"},
|
||||
} {
|
||||
if _, _, f := Propose(c.in); !slices.ContainsFunc(f, func(s string) bool { return strings.Contains(s, c.want) }) {
|
||||
t.Errorf("%s: findings %v, want one naming %q", name, f, c.want)
|
||||
}
|
||||
}
|
||||
|
||||
// A module's config values may be proposed before the lease write (§13); once the lease is written every change is
|
||||
// the state's alone, and none waits.
|
||||
if _, _, f := Propose(in(delivery.PlacementProposalChange{Op: delivery.OpConfig, Node: "anchor", Module: "web", Values: map[string]any{"puid": 1}})); len(f) != 0 {
|
||||
t.Errorf("config values before the lease write: %v", f)
|
||||
}
|
||||
Leased = true
|
||||
for _, c := range []delivery.PlacementProposalChange{
|
||||
{Op: delivery.OpPlace, Node: "laptop", Module: "web", Ports: map[string]int{"80": 8080}, Bindings: map[string]delivery.Provider{"db": {Node: "anchor", Module: "store"}}},
|
||||
{Op: delivery.OpConfig, Node: "anchor", Module: "web", Layer: map[string]any{"places": []any{}}},
|
||||
{Op: delivery.OpDecide, Node: "laptop", Field: "site", Value: json.RawMessage(`"attic"`)},
|
||||
{Op: delivery.OpHold, Node: "anchor", Module: "store", Claim: "node-shell", Scope: "node"},
|
||||
} {
|
||||
if _, _, f := Propose(in(c)); len(f) != 0 {
|
||||
t.Errorf("after the lease write %s is refused: %v", c.Op, f)
|
||||
}
|
||||
}
|
||||
Leased = false
|
||||
|
||||
// A key moved while the warrant wrote: what is left to write is the other nodes' changes, to be laid and asked again.
|
||||
if rest, ok := Rest(p, map[string]*delivery.Placement{"anchor": laid["anchor"]}); !ok || len(rest.Changes) != 1 || rest.Changes[0].Node != "laptop" ||
|
||||
len(rest.Base) != 1 || rest.State != delivery.ProposalAnswering || rest.Must != nil {
|
||||
t.Errorf("the rest = %+v, %v", rest, ok)
|
||||
}
|
||||
if _, ok := Rest(p, laid); ok {
|
||||
t.Error("a proposal written whole leaves something to write")
|
||||
}
|
||||
|
||||
// The seat says it waits while one of its states is not bound, naming it, and clears once all are.
|
||||
for _, unbound := range [][]string{{delivery.PlacementPulsesBucket}, nil} {
|
||||
c := SeatWaitsCondition(unbound)
|
||||
if _, err := asks.EncodeCondition(c); err != nil || (len(unbound) > 0) != (c.State == asks.StateRaised) ||
|
||||
(len(unbound) > 0 && !strings.Contains(c.Explanation, "the seat waits for its states: "+delivery.PlacementPulsesBucket)) {
|
||||
t.Errorf("waiting for %v: %+v, %v", unbound, c, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Issue 1126: the seed places the private network, which no repository holds; it is the mesh's own, and passes.
|
||||
wired := laptop
|
||||
wired.Modules = append(slices.Clone(laptop.Modules), delivery.PlacedModule{Module: "mesh-wireguard"})
|
||||
seed := in(SeedChanges(map[string]delivery.Placement{"laptop": wired})...)
|
||||
seed.Seed, seed.ExportDigest, seed.Current, seed.Revisions = true, "sha256:x", map[string]delivery.Placement{}, map[string]uint64{}
|
||||
seed.Held = func(m string) bool { return m != "mesh-wireguard" } // the indexer holds every module but it
|
||||
if _, _, f := Propose(seed); len(f) != 0 {
|
||||
t.Errorf("a seed placing the private network is refused: %v", f)
|
||||
}
|
||||
|
||||
writes, err := Writes(p, laid)
|
||||
if err != nil || len(writes) != 2 || writes[0].Key != "placement.anchor" || writes[0].Base != 4 || writes[1].Base != 9 {
|
||||
t.Fatalf("writes = %+v, %v", writes, err)
|
||||
}
|
||||
pinned := *laid["laptop"]
|
||||
pinned.Pins = map[string]delivery.Provider{"db": {Node: "anchor", Module: "store"}}
|
||||
calls := ControllerCalls(map[string]delivery.Placement{"anchor": anchor, "laptop": laptop},
|
||||
map[string]*delivery.Placement{"anchor": laid["anchor"], "laptop": &pinned}, "p-one", "w1@t")
|
||||
var said []string
|
||||
for _, c := range calls {
|
||||
said = append(said, c.Key())
|
||||
if !strings.Contains(c.Args["why"].(string), "p-one") {
|
||||
t.Errorf("%s does not name the proposal: %v", c.Key(), c.Args)
|
||||
}
|
||||
}
|
||||
if !slices.Equal(said, []string{"mesh-controller.unassign anchor web", "mesh-controller.assign laptop web", "mesh-controller.pin laptop db"}) {
|
||||
t.Errorf("the controller's calls = %v", said)
|
||||
}
|
||||
|
||||
a, err := PlacementAsk("placement.p-one", p, "move web (proposed by operator@laptop)", 7, time.Unix(100, 0))
|
||||
if err != nil || a.Ask.Check(time.Unix(100, 0)) != nil || a.Ask.Options[0].Level != asks.Approve || a.Ask.Options[0].Performs(PlacementAct(p)) != nil {
|
||||
t.Fatalf("the ask = %+v, %v", a, err)
|
||||
}
|
||||
q := p
|
||||
q.Changes = append(slices.Clone(p.Changes), delivery.PlacementProposalChange{Op: delivery.OpUnpin, Node: "laptop", Provision: "x"})
|
||||
if a.Ask.Options[0].Performs(PlacementAct(q)) == nil {
|
||||
t.Error("the ask's yes binds changes it was not asked about")
|
||||
}
|
||||
|
||||
export, _ := json.Marshal(map[string]any{"anchor": anchor, "laptop": laptop})
|
||||
for name, c := range map[string]struct {
|
||||
raw string
|
||||
want int
|
||||
}{
|
||||
"in output": {`{"ok":true,"output":` + quote(string(export)+"\n") + `}`, 2},
|
||||
"as answer": {`{"ok":true,"output":"","answer":` + string(export) + `}`, 2},
|
||||
"refused": {`{"ok":false,"output":"no store"}`, 0},
|
||||
"no node": {`{"ok":true,"output":"{}"}`, 0},
|
||||
"no version": {`{"ok":true,"output":"{\"anchor\":{\"node\":\"anchor\",\"modules\":[]}}"}`, 0},
|
||||
} {
|
||||
got, err := ExportOf([]byte(c.raw))
|
||||
if len(got) != c.want || (c.want == 0) != (err != nil) {
|
||||
t.Errorf("%s: %d nodes, %v", name, len(got), err)
|
||||
}
|
||||
}
|
||||
if seed := SeedChanges(map[string]delivery.Placement{"laptop": laptop, "anchor": anchor}); len(seed) != 2 || seed[0].Node != "anchor" || seed[0].Op != delivery.OpSeed {
|
||||
t.Errorf("the seed's changes = %+v", seed)
|
||||
}
|
||||
}
|
||||
|
||||
func quote(s string) string {
|
||||
raw, _ := json.Marshal(s)
|
||||
return string(raw)
|
||||
}
|
||||
@@ -7,7 +7,6 @@ import (
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
func load(t *testing.T) catalogue.Manifest {
|
||||
@@ -33,9 +32,9 @@ func TestManifest(t *testing.T) {
|
||||
}
|
||||
reads := slices.Clone(m.Reads)
|
||||
slices.Sort(reads)
|
||||
want := []string{"mesh-build-go.steps", "mesh-config.resolved", "mesh-controller.assignments", "mesh-controller.builds", "mesh-controller.composed", "mesh-controller.grants",
|
||||
want := []string{"mesh-build-go.steps", "mesh-controller.assignments", "mesh-controller.builds", "mesh-controller.composed", "mesh-controller.grants",
|
||||
"mesh-delivery-builder.builds", "mesh-delivery-indexer.candidates", "mesh-delivery-indexer.manifests",
|
||||
"mesh-delivery-indexer.modules", "mesh-delivery-teller.answers", "mesh-delivery-watchman.presence", "node-engine.have"}
|
||||
"mesh-delivery-indexer.modules", "mesh-delivery-indexer.placement", "mesh-delivery-indexer.placement-candidates", "mesh-delivery-teller.answers", "mesh-delivery-watchman.presence", "node-engine.have"}
|
||||
if !reflect.DeepEqual(reads, want) {
|
||||
t.Errorf("reads = %v, want exactly %v", reads, want)
|
||||
}
|
||||
@@ -43,8 +42,7 @@ func TestManifest(t *testing.T) {
|
||||
for _, s := range m.State {
|
||||
states[s.Name] = s.History
|
||||
}
|
||||
if want := map[string]int{"asks": 5, "compositions": 5, "differences": 64, "puts": 20, "wants": 20,
|
||||
"placement": 64, "proposals": 5, "candidates": 5, "pulses": 0}; !reflect.DeepEqual(states, want) {
|
||||
if want := map[string]int{"asks": 5, "compositions": 5, "differences": 64, "puts": 20, "wants": 20}; !reflect.DeepEqual(states, want) {
|
||||
t.Errorf("state = %v, want %v", states, want)
|
||||
}
|
||||
if _, ok := m.OwnSecrets["broker"]; !ok || len(m.OwnSecrets) != 1 {
|
||||
@@ -59,27 +57,12 @@ func TestManifest(t *testing.T) {
|
||||
t.Errorf("uses = %v, want exactly node-build-go and operator-channel", m.Uses)
|
||||
}
|
||||
// ADR 0546: it serves its own tools (put, release, puts) from its bundle through the tool runner (the artifact loads the
|
||||
// module), says each put's step as the event put, and calls the controller's warranted and the bus node's backup holder.
|
||||
// ADR 0567 §1, §6-§10: it holds the seat mesh-placement, serving its seven verbs; keeps the seat's events on its stream
|
||||
// placement for 30 days; calls the controller's export, its four verbs that keep its assignments in step and its
|
||||
// hand-act log; and reads mesh-config's resolved values to compose every node a proposal touches.
|
||||
if len(m.Claims) != 1 || m.Claims[0].Name != catalogue.PlacementSeat || m.Claims[0].At() != catalogue.ScopeMesh ||
|
||||
!slices.Equal(m.Claims[0].Serves, []string{"placement", "proposals", "propose", "withdraw", "again", "adopt", "hand-back"}) {
|
||||
t.Errorf("the assigner claims %+v, want mesh-placement with its seven verbs", m.Claims)
|
||||
// module), claims no seat (a new seat needs a change of the controller), says each put's step as the event put, and
|
||||
// calls the controller's warranted and the bus node's backup holder.
|
||||
if len(m.Claims) != 0 || !slices.Equal(m.Tools, []string{"assigner_put", "assigner_release", "assigner_puts"}) {
|
||||
t.Errorf("the assigner claims %v and serves tools %v, want no claim and its three put tools", m.Claims, m.Tools)
|
||||
}
|
||||
if seat, _ := catalogue.SeatNamed(catalogue.PlacementSeat); catalogue.CanHold(m, seat) != nil {
|
||||
t.Errorf("the assigner cannot hold the seat: %v", catalogue.CanHold(m, seat))
|
||||
}
|
||||
if !slices.Equal(m.Tools, []string{"assigner_put", "assigner_release", "assigner_puts"}) {
|
||||
t.Errorf("the assigner serves tools %v, want its three put tools", m.Tools)
|
||||
}
|
||||
if len(m.Streams) != 1 || m.Streams[0].Name != delivery.PlacementStream || m.Streams[0].MaxAgeDays != delivery.PlacementStreamDays ||
|
||||
!slices.Equal(slices.Sorted(slices.Values(m.Streams[0].Events)), slices.Sorted(slices.Values(delivery.PlacementEvents))) {
|
||||
t.Errorf("streams = %+v, want placement carrying the seat's four events for %d days", m.Streams, delivery.PlacementStreamDays)
|
||||
}
|
||||
if want := []string{"seat:mesh-controller.assign", "seat:mesh-controller.hand-act", "seat:mesh-controller.pin",
|
||||
"seat:mesh-controller.placement-export", "seat:mesh-controller.unassign", "seat:mesh-controller.unpin",
|
||||
"seat:mesh-controller.warranted", "seat:node-backup.backed-up", "seat:node-backup.now"}; !slices.Equal(slices.Sorted(slices.Values(m.Invokes)), want) {
|
||||
if want := []string{"seat:mesh-controller.warranted", "seat:node-backup.backed-up", "seat:node-backup.now"}; !slices.Equal(slices.Sorted(slices.Values(m.Invokes)), want) {
|
||||
t.Errorf("invokes = %v, want %v", m.Invokes, want)
|
||||
}
|
||||
if !slices.Equal(m.Emits, []string{"put"}) {
|
||||
|
||||
@@ -2,24 +2,8 @@
|
||||
"module": "mesh-delivery-assigner",
|
||||
"version": "1",
|
||||
"slug": "assigner",
|
||||
"claims": [
|
||||
{
|
||||
"name": "mesh-placement",
|
||||
"scope": "mesh",
|
||||
"serves": [
|
||||
"placement",
|
||||
"proposals",
|
||||
"propose",
|
||||
"withdraw",
|
||||
"again",
|
||||
"adopt",
|
||||
"hand-back"
|
||||
]
|
||||
}
|
||||
],
|
||||
"reads": [
|
||||
"mesh-build-go.steps",
|
||||
"mesh-config.resolved",
|
||||
"mesh-controller.assignments",
|
||||
"mesh-controller.builds",
|
||||
"mesh-controller.composed",
|
||||
@@ -28,6 +12,8 @@
|
||||
"mesh-delivery-indexer.candidates",
|
||||
"mesh-delivery-indexer.manifests",
|
||||
"mesh-delivery-indexer.modules",
|
||||
"mesh-delivery-indexer.placement",
|
||||
"mesh-delivery-indexer.placement-candidates",
|
||||
"mesh-delivery-teller.answers",
|
||||
"mesh-delivery-watchman.presence",
|
||||
"node-engine.have"
|
||||
@@ -37,15 +23,9 @@
|
||||
"operator-channel"
|
||||
],
|
||||
"invokes": [
|
||||
"seat:mesh-controller.assign",
|
||||
"seat:mesh-controller.hand-act",
|
||||
"seat:mesh-controller.pin",
|
||||
"seat:mesh-controller.placement-export",
|
||||
"seat:mesh-controller.unassign",
|
||||
"seat:mesh-controller.unpin",
|
||||
"seat:mesh-controller.warranted",
|
||||
"seat:node-backup.backed-up",
|
||||
"seat:node-backup.now"
|
||||
"seat:node-backup.now",
|
||||
"seat:node-backup.backed-up"
|
||||
],
|
||||
"emits": [
|
||||
"put"
|
||||
@@ -56,28 +36,12 @@
|
||||
"assigner_puts"
|
||||
],
|
||||
"state": [
|
||||
{ "name": "placement", "history": 64 },
|
||||
{ "name": "proposals", "history": 5 },
|
||||
{ "name": "candidates", "history": 5 },
|
||||
{ "name": "pulses", "key-ttl": true },
|
||||
{ "name": "asks", "history": 5 },
|
||||
{ "name": "compositions", "history": 5 },
|
||||
{ "name": "differences", "history": 64 },
|
||||
{ "name": "puts", "history": 20 },
|
||||
{ "name": "wants", "history": 20 }
|
||||
],
|
||||
"streams": [
|
||||
{
|
||||
"name": "placement",
|
||||
"events": [
|
||||
"placement-proposed",
|
||||
"placement-judged",
|
||||
"placement-changed",
|
||||
"placement-withdrawn"
|
||||
],
|
||||
"max-age-days": 30
|
||||
}
|
||||
],
|
||||
"own-secrets": {
|
||||
"broker": "${dir:state}/broker"
|
||||
},
|
||||
|
||||
@@ -7,7 +7,6 @@ package main
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
@@ -25,8 +24,8 @@ import (
|
||||
type queueAsker func(key string, value []byte) (json.RawMessage, error)
|
||||
|
||||
// Tools are the builder's tools: stuck reads, askAgain writes the reasks value, queue writes a queues value and waits for
|
||||
// the builder's read, buildAgain writes a reasks value of mode build and waits for the builder's answer.
|
||||
func Tools(r build.StuckReader, askAgain func(key string, value []byte) error, queue, buildAgain queueAsker, now func() time.Time) []stdio.Tool {
|
||||
// the builder's read.
|
||||
func Tools(r build.StuckReader, askAgain func(key string, value []byte) error, queue queueAsker, now func() time.Time) []stdio.Tool {
|
||||
str := func(args map[string]any, k string) string { s, _ := args[k].(string); return strings.TrimSpace(s) }
|
||||
return []stdio.Tool{
|
||||
{Name: build.StuckTool,
|
||||
@@ -58,42 +57,6 @@ func Tools(r build.StuckReader, askAgain func(key string, value []byte) error, q
|
||||
return map[string]any{"key": key, "asked": true,
|
||||
"next": "the builder answers in " + build.ReAsksState + "." + key + "; " + build.StuckTool + " shows the answer and the new ask"}, nil
|
||||
}},
|
||||
{Name: build.BuildAgainTool,
|
||||
Description: "Build a failed key again (rebuild a module, build a failed key again): the builder writes the key of module and " +
|
||||
"fingerprint building as a new revision of the same key and asks the build seats, as it does by itself when it looks at a module " +
|
||||
"again (ADR 0429 clause 18). Answers the new revision and the ask id of each build module (go, node, image). Refuses a key that " +
|
||||
"is not failed, and says why: done (nothing to do), or waiting (building or queued: " + build.AskAgainTool + " asks its waiting asks " +
|
||||
"again); a key failed for a reason the builder never retries on the same fingerprint (a pin missing, a version taken, digests that " +
|
||||
"differ) and a fingerprint its module no longer has are refused too. Use it for a key builder_stuck does not show, which failed.",
|
||||
Input: map[string]any{"type": "object", "required": []string{"module", "fp", "why"}, "properties": map[string]any{
|
||||
"module": map[string]any{"type": "string", "description": "the module's name"},
|
||||
"fp": map[string]any{"type": "string", "description": "its new-form fingerprint, src2:<hex>"},
|
||||
"why": map[string]any{"type": "string", "description": "why it is built again, in plain words"}}},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
key, value, err := build.NewBuildAgain(str(args, "module"), str(args, "fp"), str(args, "why"), stdio.CallerOf(args), now())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
raw, err := buildAgain(key, value)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
r, err := build.DecodeReAsk(raw)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if r.Refused {
|
||||
return nil, errors.New(r.Answer)
|
||||
}
|
||||
if r.Failed != "" {
|
||||
return nil, errors.New(r.Answer)
|
||||
}
|
||||
if r.Revision == 0 && !r.Pending {
|
||||
return nil, fmt.Errorf("the builder answered neither a new revision nor a refusal (an older builder?): %s", r.Answer)
|
||||
}
|
||||
return map[string]any{"key": key, "revision": r.Revision, "asks": r.Asks, "answer": r.Answer,
|
||||
"next": build.StuckTool + " shows the key and its asks while they wait"}, nil
|
||||
}},
|
||||
{Name: build.SeatQueueTool,
|
||||
Description: "What waits on a build seat's stream (SEAT_NODE_BUILD_<LANGUAGE>), as the builder reads it: the stream's message count and " +
|
||||
"sequences, and per kind (delivery, check, recheck) its messages oldest first, at most limit each, with sequence, ask id, when asked " +
|
||||
@@ -124,17 +87,8 @@ func Tools(r build.StuckReader, askAgain func(key string, value []byte) error, q
|
||||
|
||||
// askQueue writes the queues value at key and waits, on a watch of the key, at most bound for the builder's answer.
|
||||
func askQueue(s stdio.KeptState, bound time.Duration) queueAsker {
|
||||
return askKept(s, build.QueuesState, build.QueuesBucket, "the read", bound, func(v []byte) bool {
|
||||
a, err := build.DecodeQueueAsk(v)
|
||||
return err == nil && !a.Answered.IsZero()
|
||||
})
|
||||
}
|
||||
|
||||
// askKept writes a value at key of a state the builder watches and answers there, and waits, on a watch of the key, at most
|
||||
// bound for a value written after it that answered says is the builder's answer.
|
||||
func askKept(s stdio.KeptState, state, bucket, what string, bound time.Duration, answered func([]byte) bool) queueAsker {
|
||||
return func(key string, value []byte) (json.RawMessage, error) {
|
||||
heard := make(chan json.RawMessage, 1)
|
||||
answered := make(chan json.RawMessage, 1)
|
||||
var rev uint64
|
||||
var mu sync.Mutex
|
||||
if err := s.Watch(key, func(c stdio.StateChange) error {
|
||||
@@ -144,34 +98,36 @@ func askKept(s stdio.KeptState, state, bucket, what string, bound time.Duration,
|
||||
if c.Op != "put" || mine == 0 || c.Revision <= mine {
|
||||
return nil
|
||||
}
|
||||
if answered(c.Value) {
|
||||
if a, err := build.DecodeQueueAsk(c.Value); err == nil && !a.Answered.IsZero() {
|
||||
select {
|
||||
case heard <- c.Value:
|
||||
case answered <- c.Value:
|
||||
default:
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}); err != nil {
|
||||
return nil, fmt.Errorf("watching %s.%s: %w", state, key, err)
|
||||
return nil, fmt.Errorf("watching %s.%s: %w", build.QueuesState, key, err)
|
||||
}
|
||||
r, err := s.Put(key, json.RawMessage(value))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s could not be written to %s: %w", what, bucket, err)
|
||||
return nil, fmt.Errorf("the read could not be written to %s: %w", build.QueuesBucket, err)
|
||||
}
|
||||
mu.Lock()
|
||||
rev = r
|
||||
mu.Unlock()
|
||||
// The watch may have heard the answer before the revision was known: read it once too.
|
||||
if e, err := s.Get(key); err == nil && e != nil && e.Revision > r && answered(e.Value) {
|
||||
return e.Value, nil
|
||||
if e, err := s.Get(key); err == nil && e != nil && e.Revision > r {
|
||||
if a, err := build.DecodeQueueAsk(e.Value); err == nil && !a.Answered.IsZero() {
|
||||
return e.Value, nil
|
||||
}
|
||||
}
|
||||
wait, cancel := context.WithTimeout(context.Background(), bound) // a context's bound, no timer (ADR 0437 §8)
|
||||
defer cancel()
|
||||
select {
|
||||
case v := <-heard:
|
||||
case v := <-answered:
|
||||
return v, nil
|
||||
case <-wait.Done():
|
||||
return nil, fmt.Errorf("the builder did not answer %s.%s within %s: it is not running, or cannot read %s", state, key, bound, bucket)
|
||||
return nil, fmt.Errorf("the builder did not answer %s.%s within %s: it is not running, or cannot read %s", build.QueuesState, key, bound, build.QueuesBucket)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -218,11 +174,7 @@ func main() {
|
||||
_, err := reasks.Put(key, json.RawMessage(value))
|
||||
return err
|
||||
}
|
||||
buildAgain := askKept(reasks, build.ReAsksState, build.ReAsksBucket, "the request", 25*time.Second, func(v []byte) bool {
|
||||
a, err := build.DecodeReAsk(v)
|
||||
return err == nil && !a.Answered.IsZero()
|
||||
})
|
||||
if err := stdio.Serve("", Tools(r, put, askQueue(stdio.State(build.QueuesState), 25*time.Second), buildAgain, time.Now)); err != nil {
|
||||
if err := stdio.Serve("", Tools(r, put, askQueue(stdio.State(build.QueuesState), 25*time.Second), time.Now)); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "[mesh-delivery-builder-tools] %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
@@ -92,16 +92,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
hearing := buildrun.NewDeadLetterHearing(ctx, buildrun.JetStreamDeadLetters{JS: js}, delivery.AskLanguages, dead, logger.Printf,
|
||||
buildrun.SayHearing(ctx, hearingKV, logger.Printf))
|
||||
defer hearing.Stop()
|
||||
// Each ask's step is heard by its own watch and, at every step end of its build module and every reconnect, read
|
||||
// (buildrun.HearEnds): a watch that goes deaf loses no end (the lost check of 2026-10-28).
|
||||
var buildModules []string
|
||||
for _, l := range delivery.AskLanguages {
|
||||
if m, err := delivery.BuildModuleOf(l); err == nil {
|
||||
buildModules = append(buildModules, m)
|
||||
}
|
||||
}
|
||||
heard := buildrun.HearEnds(ctx, asks, asks, buildModules, logger.Printf)
|
||||
conn.SetReconnectHandler(func(*nats.Conn) { hearing.Again(); heard.Nudge("") })
|
||||
conn.SetReconnectHandler(func(*nats.Conn) { hearing.Again() })
|
||||
// The builder asks the controller, as its own account, whether the artifact store still holds a done key's blobs
|
||||
// (the seat verb held, novox/hq issue 1937) before it leaves the key done.
|
||||
calls, err := process.Connect(cred, nats.Name(cred.Node+"."+cred.Module+".calls"), nats.RetryOnFailedConnect(true))
|
||||
@@ -111,7 +102,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
defer calls.Close()
|
||||
b := &build.Builder{
|
||||
Claimer: build.Claimer{Store: kvStore{builds}, By: cred.Node + "." + cred.Module},
|
||||
Asks: buildrun.Asker{Publisher: asks, Waiter: buildrun.LostAskWaiter{Inner: heard, Dead: dead, Hearing: hearing}, Bound: bound},
|
||||
Asks: buildrun.Asker{Publisher: asks, Waiter: buildrun.LostAskWaiter{Inner: asks, Dead: dead, Hearing: hearing}, Bound: bound},
|
||||
Held: heldOnTheBus{js},
|
||||
// An ask that can no longer be answered is asked again (issue 1945), read at the step bound: a context's bound.
|
||||
Gone: heldOnTheBus{js},
|
||||
|
||||
@@ -18,13 +18,8 @@ import (
|
||||
// askAgainer is the part of the builder a reasks value asks through.
|
||||
type askAgainer interface {
|
||||
AskAgain(key, why string) []string
|
||||
BuildAgain(ctx context.Context, module, fp, why, by string, bound time.Duration) build.BuildAgainResult
|
||||
}
|
||||
|
||||
// buildAgainBound is how long the builder waits for the claim and the asks of a key built again by hand; the tool waits
|
||||
// longer for the builder's answer.
|
||||
const buildAgainBound = 15 * time.Second
|
||||
|
||||
// reAsked answers one value of the reasks state (the tool builder_ask_again, issue 1945): a value not yet answered asks the
|
||||
// key's waiting lanes again and is written back, by compare-and-set at its revision, with what the builder did. A value
|
||||
// answered already (this builder's own write) or deleted is left.
|
||||
@@ -40,11 +35,6 @@ func reAsked(ctx context.Context, b askAgainer, kv jetstream.KeyValue, e jetstre
|
||||
if r.Answer != "" {
|
||||
return
|
||||
}
|
||||
if r.Mode == build.ModeBuild {
|
||||
// The answer comes when the claim and the asks are written, so it is made beside the watch, not in it.
|
||||
go builtAgain(ctx, b, kv, e, r, logf)
|
||||
return
|
||||
}
|
||||
asked := b.AskAgain(e.Key(), r.Why)
|
||||
if len(asked) == 0 {
|
||||
r.Answer = "no ask of this key waits in this builder now (it is not building, or its build has not reached its ask yet): nothing was asked; builder_stuck shows where it stands"
|
||||
@@ -84,33 +74,3 @@ func emitAskedAgain(ctx context.Context, js jetstream.JetStream, cred process.Ac
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// builtAgain answers a reasks value of mode build (the tool builder_build_again, issue 1126 note 72): the failed key is
|
||||
// built again as a new revision, and the value is written back with the revision and the asks put, or why nothing was done.
|
||||
func builtAgain(ctx context.Context, b askAgainer, kv jetstream.KeyValue, e jetstream.KeyValueEntry, r build.ReAsk, logf func(string, ...any)) {
|
||||
res := b.BuildAgain(ctx, r.Module, r.FP, r.Why, r.By, buildAgainBound)
|
||||
switch {
|
||||
case res.Refused != "":
|
||||
r.Answer, r.Refused = res.Refused, true
|
||||
case res.Failed != "":
|
||||
r.Answer, r.Revision, r.Failed = fmt.Sprintf("the key is written building as revision %d, and the build failed before any ask was put: %s", res.Revision, res.Failed), res.Revision, res.Failed
|
||||
default:
|
||||
r.Pending = res.Revision == 0
|
||||
r.Answer, r.Revision, r.Asks = "built again: the key is written building as revision "+fmt.Sprint(res.Revision)+" and its asks are put", res.Revision, res.Asks
|
||||
if res.Pending != "" {
|
||||
r.Answer = res.Pending
|
||||
if res.Revision != 0 {
|
||||
r.Answer = "built again: the key is written building as revision " + fmt.Sprint(res.Revision) + "; " + res.Pending
|
||||
}
|
||||
}
|
||||
}
|
||||
r.Answered = time.Now().UTC()
|
||||
logf("%s: build again asked by %s (%s): %s", e.Key(), r.By, r.Why, r.Answer)
|
||||
data, err := json.Marshal(r)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if _, err := kv.Update(ctx, e.Key(), data, e.Revision()); err != nil && !errors.Is(err, jetstream.ErrKeyExists) && !errors.Is(err, jetstream.ErrKeyRevisionMismatch) {
|
||||
logf("%s.%s: the answer could not be written: %v", build.ReAsksState, e.Key(), err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,7 +3,7 @@ module mesh-delivery-builder
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028193502-f91340fe82ef
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027193820-8ad9372b1952
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028193502-f91340fe82ef h1:ggwHXCvAOhBCKAY+ErehKEhu/IQTS4poJBRDokGYnDc=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028193502-f91340fe82ef/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027193820-8ad9372b1952 h1:TmveT3Bija643RGWkilvkzP/h8UWlVvHFZZtQfJK5oU=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027193820-8ad9372b1952/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
|
||||
@@ -1,183 +0,0 @@
|
||||
package build
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"maps"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||
)
|
||||
|
||||
// **A failed key is built again by hand** (novox/hq issue 1126 note 72; ADR 0429 clause 18: "a fingerprint is built again
|
||||
// only when its build key failed, as a new revision of the same key"). The builder does it by itself whenever it looks at a
|
||||
// module again (a modules value, a change of the toolchains, artifacts-gone, a moved trunk): Claimer.Claim writes a failed
|
||||
// key building as a new revision and the build asks the build seats. Nothing looks at an unchanged module on its own, so a key
|
||||
// that failed (a node refused the build, a toolchain was missing) stayed failed until the module changed. The tool
|
||||
// builder_build_again (BuildAgain) makes the builder look at one module now, which does exactly that and no more: it keeps
|
||||
// the claim's rules, so a key failed for a reason the builder never retries on the same fingerprint stays refused.
|
||||
|
||||
// BuildAgainResult is the builder's answer to a request to build a key again.
|
||||
type BuildAgainResult struct {
|
||||
// Refused is why nothing was done, "" when the key was written building as a new revision.
|
||||
Refused string
|
||||
// Revision is the revision the claim wrote; Asks are the ask ids put by language, as far as they were put within the bound.
|
||||
Revision uint64
|
||||
Asks map[string]string
|
||||
// Pending says the builder had not claimed the key, or not put every ask, when it answered: not a refusal.
|
||||
Pending string
|
||||
// Failed is the error of the key when the build ended failed before its asks were put (a plan that cannot be read, a
|
||||
// toolchain not agreed): the new revision was written, and nothing was asked.
|
||||
Failed string
|
||||
}
|
||||
|
||||
// handWatch follows one request to build a key again until its asks are put.
|
||||
type handWatch struct {
|
||||
rev uint64
|
||||
lanes int
|
||||
asks map[string]string
|
||||
done chan struct{}
|
||||
shut bool
|
||||
// ended is set when the look ended before the asks were all put.
|
||||
ended bool
|
||||
}
|
||||
|
||||
// settle closes done once the claim is written and every lane has put its ask. Call with b.mu held.
|
||||
func (w *handWatch) settle() {
|
||||
if !w.shut && w.rev > 0 && w.lanes >= 0 && len(w.asks) >= w.lanes {
|
||||
w.shut = true
|
||||
close(w.done)
|
||||
}
|
||||
}
|
||||
|
||||
func (b *Builder) hand(key string, f func(w *handWatch)) {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
if w := b.hands[key]; w != nil {
|
||||
f(w)
|
||||
}
|
||||
}
|
||||
|
||||
func (b *Builder) handClaimed(key string, rev uint64) {
|
||||
b.hand(key, func(w *handWatch) { w.rev = rev; w.settle() })
|
||||
}
|
||||
|
||||
func (b *Builder) handLanes(key string, n int) {
|
||||
b.hand(key, func(w *handWatch) { w.lanes = n; w.settle() })
|
||||
}
|
||||
|
||||
func (b *Builder) handAsk(key, language, id string) {
|
||||
b.hand(key, func(w *handWatch) { w.asks[language] = id; w.settle() })
|
||||
}
|
||||
|
||||
// handEnded ends the watch of a request whose look ended: it wrote no claim, or the build ended.
|
||||
func (b *Builder) handEnded(key string) {
|
||||
b.hand(key, func(w *handWatch) {
|
||||
if !w.shut {
|
||||
w.shut, w.ended = true, true
|
||||
close(w.done)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// BuildAgain builds the failed key of module at fp again, as the builder does by itself: the key is written building as a
|
||||
// new revision and the build module asks are put on the build seats. It answers once the asks are put, or at bound. A key
|
||||
// that is not failed is refused with why: done (nothing to do) or waiting (builder_ask_again asks its waiting asks again).
|
||||
// ctx is the builder's own context: the build it starts lives on it.
|
||||
func (b *Builder) BuildAgain(ctx context.Context, module, fp, why, by string, bound time.Duration) BuildAgainResult {
|
||||
b.init()
|
||||
key, err := delivery.BuilderBuildKey(module, fp)
|
||||
if err != nil {
|
||||
return BuildAgainResult{Refused: err.Error()}
|
||||
}
|
||||
value, _, found, err := b.Claimer.Store.Get(ctx, key)
|
||||
if err != nil {
|
||||
return BuildAgainResult{Refused: fmt.Sprintf("%s cannot be read: %v", key, err)}
|
||||
}
|
||||
if !found {
|
||||
return BuildAgainResult{Refused: fmt.Sprintf("%s has no build key: nothing failed. The builder builds a fingerprint by itself once the indexer writes it to its module", key)}
|
||||
}
|
||||
cur, err := delivery.DecodeBuilderBuild(value)
|
||||
if err != nil {
|
||||
return BuildAgainResult{Refused: fmt.Sprintf("%s holds what is not a build: %v", key, err)}
|
||||
}
|
||||
switch {
|
||||
case cur.State == delivery.BuildDone:
|
||||
return BuildAgainResult{Refused: fmt.Sprintf("%s is done: nothing to do", key)}
|
||||
case cur.State == delivery.BuildBuilding || cur.State == delivery.BuildQueued:
|
||||
return BuildAgainResult{Refused: fmt.Sprintf("%s is waiting (%s: %s): %s asks its waiting asks again", key, cur.State, cur.Step, AskAgainTool)}
|
||||
case cur.State != delivery.BuildFailed:
|
||||
return BuildAgainResult{Refused: fmt.Sprintf("%s is %q, not failed", key, cur.State)}
|
||||
case delivery.FailedUntilChanged(cur.Error) || strings.HasPrefix(cur.Error, buildrun.DigestsDifferPrefix):
|
||||
return BuildAgainResult{Refused: fmt.Sprintf("%s failed for a reason the builder does not retry on the same fingerprint, it waits for another fingerprint: %s", key, cur.Error)}
|
||||
}
|
||||
// One critical section: the key is refused when it is looked at or asked already, the module still has the fingerprint,
|
||||
// and the watch and the look are registered before anything else can take the key.
|
||||
w := &handWatch{lanes: -1, asks: map[string]string{}, done: make(chan struct{})}
|
||||
b.mu.Lock()
|
||||
m, known := b.known[module]
|
||||
switch {
|
||||
case !known || m.FP2 != fp:
|
||||
b.mu.Unlock()
|
||||
return BuildAgainResult{Refused: fmt.Sprintf("module %s does not have the fingerprint %s now (a newer one superseded it, or its modules value is not read yet): the builder builds the module's current fingerprint", module, fp)}
|
||||
case m.Manifest == "":
|
||||
b.mu.Unlock()
|
||||
return BuildAgainResult{Refused: fmt.Sprintf("module %s names no manifest at %s yet, so the builder looks at nothing", module, fp)}
|
||||
case b.inflight[key] || b.hands[key] != nil:
|
||||
b.mu.Unlock()
|
||||
return BuildAgainResult{Refused: fmt.Sprintf("%s is being looked at by this builder now (or asked again by hand): wait for it to end", key)}
|
||||
}
|
||||
b.hands[key] = w
|
||||
b.inflight[key] = true
|
||||
b.mu.Unlock()
|
||||
defer func() {
|
||||
b.mu.Lock()
|
||||
if b.hands[key] == w {
|
||||
delete(b.hands, key)
|
||||
}
|
||||
b.mu.Unlock()
|
||||
}()
|
||||
b.logf("%s: failed (%s); built again by hand, asked by %s: %s", key, cur.Error, by, why)
|
||||
b.startOnce(ctx, key, module, m)
|
||||
wait, stop := context.WithTimeout(ctx, bound) // a context's bound for a person's answer (ADR 0437 §8)
|
||||
defer stop()
|
||||
select {
|
||||
case <-w.done:
|
||||
case <-wait.Done():
|
||||
}
|
||||
b.mu.Lock()
|
||||
res := BuildAgainResult{Revision: w.rev, Asks: maps.Clone(w.asks)}
|
||||
ended, shut, lanes := w.ended, w.shut, w.lanes
|
||||
b.mu.Unlock()
|
||||
switch {
|
||||
case res.Revision == 0 && shut:
|
||||
res.Refused = fmt.Sprintf("the builder looked at %s and wrote nothing (a toolchain moved, the claim was lost or could not be written): its log says why", key)
|
||||
case res.Revision == 0:
|
||||
res.Pending = fmt.Sprintf("the builder has not claimed %s within %s: it waits behind other builds; builder_stuck shows where it stands", key, bound)
|
||||
case ended && len(res.Asks) < max(lanes, 0)+boolInt(lanes < 0):
|
||||
// The look ended before every ask was put: say how the key stands, not that it was asked.
|
||||
if v, _, found, err := b.Claimer.Store.Get(ctx, key); err == nil && found {
|
||||
if cur, err := delivery.DecodeBuilderBuild(v); err == nil {
|
||||
switch cur.State {
|
||||
case delivery.BuildFailed:
|
||||
res.Failed = cur.Error
|
||||
case delivery.BuildDone:
|
||||
default:
|
||||
res.Pending = fmt.Sprintf("the look ended with the key %s (%s) before every ask was put: its log says why", cur.State, cur.Step)
|
||||
}
|
||||
}
|
||||
}
|
||||
case !shut:
|
||||
res.Pending = fmt.Sprintf("the key is building at revision %d, and not every ask was put within %s; builder_stuck shows the asks", res.Revision, bound)
|
||||
}
|
||||
return res
|
||||
}
|
||||
|
||||
func boolInt(b bool) int {
|
||||
if b {
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
@@ -89,8 +89,6 @@ type Builder struct {
|
||||
held map[string]bool
|
||||
// waiting are the lanes that wait for an ask's step, by build key and language: AskAgain asks through them.
|
||||
waiting map[string]map[string]chan string
|
||||
// hands are the requests to build a key again by hand that wait for the claim and its asks (BuildAgain), by build key.
|
||||
hands map[string]*handWatch
|
||||
}
|
||||
|
||||
type planned struct {
|
||||
@@ -108,7 +106,6 @@ func (b *Builder) init() {
|
||||
b.cands, b.via, b.trunkFP = map[string]delivery.Candidate{}, map[string]via{}, map[string]bool{}
|
||||
b.known, b.inflight, b.again, b.plans, b.tried = map[string]delivery.Module{}, map[string]bool{}, map[string]delivery.Module{}, map[string]planned{}, map[string]bool{}
|
||||
b.published, b.held, b.waiting = map[string]bool{}, map[string]bool{}, map[string]map[string]chan string{}
|
||||
b.hands = map[string]*handWatch{}
|
||||
})
|
||||
}
|
||||
|
||||
@@ -375,35 +372,9 @@ func (b *Builder) consider(ctx context.Context, module string, m delivery.Module
|
||||
}()
|
||||
}
|
||||
|
||||
// startOnce looks at the module for a key the caller has set inflight already (BuildAgain): the by-hand look runs once, and
|
||||
// a look the builder queued for the key meanwhile (again) still runs after it, as consider's loop does.
|
||||
func (b *Builder) startOnce(ctx context.Context, key, module string, m delivery.Module) {
|
||||
b.wg.Add(1)
|
||||
go func() {
|
||||
defer b.wg.Done()
|
||||
b.sem <- struct{}{}
|
||||
defer func() { <-b.sem }()
|
||||
for {
|
||||
b.try(ctx, module, m)
|
||||
b.mu.Lock()
|
||||
next, more := b.again[key]
|
||||
delete(b.again, key)
|
||||
if !more || ctx.Err() != nil {
|
||||
delete(b.inflight, key)
|
||||
b.mu.Unlock()
|
||||
return
|
||||
}
|
||||
b.mu.Unlock()
|
||||
m = next // a look the builder queued meanwhile is not lost (consider's rule); it is not a second by-hand build
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// try claims the fingerprint and builds it, writing the key done or failed. A build that ends because ctx did is left
|
||||
// building by this builder: it takes it again when it starts (ADR 0478 clause 5).
|
||||
func (b *Builder) try(ctx context.Context, module string, m delivery.Module) {
|
||||
key, _ := delivery.BuilderBuildKey(module, m.FP2)
|
||||
defer b.handEnded(key)
|
||||
plan, planErr, read := b.planOf(ctx, m.Manifest)
|
||||
nothing := read && planErr == nil && len(plan.Tasks) == 0
|
||||
if planErr == nil && !nothing {
|
||||
@@ -414,6 +385,7 @@ func (b *Builder) try(ctx context.Context, module string, m delivery.Module) {
|
||||
return
|
||||
}
|
||||
}
|
||||
key, _ := delivery.BuilderBuildKey(module, m.FP2)
|
||||
b.mu.Lock()
|
||||
trunk := b.trunkFP[key]
|
||||
first := !b.tried[key]
|
||||
@@ -428,7 +400,6 @@ func (b *Builder) try(ctx context.Context, module string, m delivery.Module) {
|
||||
if cl == nil {
|
||||
return
|
||||
}
|
||||
b.handClaimed(key, cl.Rev)
|
||||
if cl.Unasked != nil {
|
||||
b.logf("%v", cl.Unasked)
|
||||
}
|
||||
@@ -564,9 +535,6 @@ func (b *Builder) build(ctx context.Context, module string, m delivery.Module, p
|
||||
// One ask builds each fingerprint's part of a build module, on whichever node pulls it (ADR 0525 §3).
|
||||
lanes = append(lanes, &lane{module: bm, tasks: tasks, toolchain: toolchain, digests: map[string]string{}, references: map[string]string{}})
|
||||
}
|
||||
if key, err := delivery.BuilderBuildKey(module, m.FP2); err == nil {
|
||||
b.handLanes(key, len(lanes))
|
||||
}
|
||||
var wg sync.WaitGroup
|
||||
for _, l := range lanes {
|
||||
wg.Add(1)
|
||||
@@ -749,11 +717,7 @@ func (n *askNotes) note(ctx context.Context, language, id, step string) error {
|
||||
v.Asks = map[string]string{}
|
||||
}
|
||||
v.Asks[language] = id
|
||||
if err := n.b.Claimer.Finish(ctx, n.cl, v); err != nil {
|
||||
return err
|
||||
}
|
||||
n.b.handAsk(n.cl.Key, language, id)
|
||||
return nil
|
||||
return n.b.Claimer.Finish(ctx, n.cl, v)
|
||||
}
|
||||
|
||||
// runLane puts the lane's part of the build as one ask on its build seat's stream (the delivery stream for a trunk key, the
|
||||
|
||||
@@ -691,129 +691,18 @@ func TestABuildAskWaitingInItsStreamIsNotBounded(t *testing.T) {
|
||||
byHand bool
|
||||
always bool // Gone says every ask cannot be answered, not only the first
|
||||
dead bool // DEAD_LETTERS keeps the first ask: the seat gave it up (issue 734)
|
||||
deaf bool // the ask's own watch hears nothing though its step ends (the lost check of 2026-10-28)
|
||||
puts int
|
||||
done bool
|
||||
asking int // the asking the event says, 0 when none is said
|
||||
// again, when set, is a request to build the key again by hand (BuildAgain, issue 1126 note 72) of the key as
|
||||
// it stands (key, absent when nil), the module known at known's fingerprint ("" for the rig's): refuse is a part of why it
|
||||
// is refused, "" when it is built again as a new revision and its ask id answered.
|
||||
again bool
|
||||
key *delivery.BuilderBuild
|
||||
known string
|
||||
refuse string
|
||||
// edit changes the module the builder has read, busy marks the key looked at (1) or asked by hand (2), failed is a part of
|
||||
// the error the key then holds when its build ends before its asks are put.
|
||||
edit func(m *delivery.Module)
|
||||
busy int
|
||||
failed string
|
||||
// during is a look of the builder's own at the module while the by-hand build runs: it still runs afterwards.
|
||||
during bool
|
||||
}{
|
||||
{name: "build again: a look the builder queued during the by-hand build still runs afterwards", again: true, during: true,
|
||||
key: &delivery.BuilderBuild{State: delivery.BuildFailed, Error: "ace: no tun"}},
|
||||
{name: "build again: a key looked at by the builder now is refused, not built twice", again: true, refuse: "is being looked at", busy: 1,
|
||||
key: &delivery.BuilderBuild{State: delivery.BuildFailed, Error: "ace: no tun"}},
|
||||
{name: "build again: a key already asked by hand is refused, its watch kept", again: true, refuse: "is being looked at", busy: 2,
|
||||
key: &delivery.BuilderBuild{State: delivery.BuildFailed, Error: "ace: no tun"}},
|
||||
{name: "build again: a module with no manifest is refused", again: true, refuse: "names no manifest", edit: func(m *delivery.Module) { m.Manifest = "" },
|
||||
key: &delivery.BuilderBuild{State: delivery.BuildFailed, Error: "ace: no tun"}},
|
||||
{name: "build again: a toolchain that moved writes nothing and is said at once", again: true, refuse: "wrote nothing",
|
||||
edit: func(m *delivery.Module) {
|
||||
m.Toolchains = map[string]delivery.ToolchainUsed{"go": {Version: "go1.0", Recipe: "1"}}
|
||||
},
|
||||
key: &delivery.BuilderBuild{State: delivery.BuildFailed, Error: "ace: no tun"}},
|
||||
{name: "build again: a build that fails before its asks says the key failed, not that it was asked", again: true, failed: "cannot be read",
|
||||
edit: func(m *delivery.Module) { m.Manifest = mdigest('b') },
|
||||
key: &delivery.BuilderBuild{State: delivery.BuildFailed, Error: "ace: no tun"}},
|
||||
{name: "build again: a failed key is written building as a new revision and asks the seat", again: true,
|
||||
key: &delivery.BuilderBuild{State: delivery.BuildFailed, Error: "ace: the step failed: no tun"}},
|
||||
{name: "build again: a done key is refused, nothing to do", again: true, refuse: "is done: nothing to do",
|
||||
key: &delivery.BuilderBuild{State: delivery.BuildDone, Step: "built"}},
|
||||
{name: "build again: a building key is refused, builder_ask_again is for it", again: true, refuse: "is waiting",
|
||||
key: &delivery.BuilderBuild{State: delivery.BuildBuilding, By: "novox.builder", Step: "asked mesh-build-go", Asks: map[string]string{delivery.AskGo: "mvl0frv18d1960"}}},
|
||||
{name: "build again: a key failed for a pin missing is refused, it waits for another fingerprint", again: true, refuse: "does not retry",
|
||||
key: &delivery.BuilderBuild{State: delivery.BuildFailed, Error: delivery.NotPinned + "web"}},
|
||||
{name: "build again: a key with no build is refused", again: true, refuse: "has no build key"},
|
||||
{name: "build again: a fingerprint its module no longer has is refused", again: true, refuse: "does not have the fingerprint", known: "src2:" + strings.Repeat("9", 64),
|
||||
key: &delivery.BuilderBuild{State: delivery.BuildFailed, Error: "ace: no tun"}},
|
||||
{name: "given up by its stream: asked again under a new id and done from it", puts: 2, done: true, asking: 2},
|
||||
{name: "asked again by hand: the same, with no check bound", byHand: true, puts: 2, done: true, asking: 2},
|
||||
{name: "given up into DEAD_LETTERS: its wait ends failed with the words and it is asked again at once", dead: true, puts: 2, done: true, asking: 2},
|
||||
{name: "its step ended while its watch was deaf: read at its build module's next step end, done with no new ask", deaf: true, puts: 1, done: true},
|
||||
{name: "never answerable: failed after its askings", askings: 2, always: true, puts: 2},
|
||||
{name: "taken over at its last asking: failed, not asked again", askings: 3, always: true, step: "asked mesh-build-go again (asking 3): x", puts: 0},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) { // 1945
|
||||
r := newRig(t, goBundle)
|
||||
if c.again {
|
||||
var rev uint64
|
||||
if c.key != nil {
|
||||
c.key.Manifest = manifestD
|
||||
rev = putBuild(t, r.st, r.key, *c.key)
|
||||
}
|
||||
m, err := delivery.DecodeModule(moduleValue(t, "modules/m"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if c.known != "" {
|
||||
m.FP2 = c.known
|
||||
}
|
||||
if c.edit != nil {
|
||||
c.edit(&m)
|
||||
}
|
||||
r.b.init()
|
||||
var stored atomic.Int64
|
||||
if c.during {
|
||||
r.asks.Answer = nil
|
||||
r.b.Stored = func(context.Context, []string) ([]string, error) { stored.Add(1); return nil, nil }
|
||||
}
|
||||
switch c.busy {
|
||||
case 1:
|
||||
r.b.inflight[r.key] = true
|
||||
case 2:
|
||||
r.b.hands[r.key] = &handWatch{}
|
||||
}
|
||||
r.b.known["m"] = m // the builder has read the modules value, and looks at nothing until asked
|
||||
log := &logged{}
|
||||
r.b.Logf = log.logf
|
||||
res := r.b.BuildAgain(context.Background(), "m", fp1, "no tun on ace", "shanks.lead", 2*time.Second)
|
||||
if c.during {
|
||||
// The trunk's modules value arrives while the by-hand build waits for its ask: it is queued, and run after it.
|
||||
r.b.Module(context.Background(), "m", moduleValue(t, "modules/m"))
|
||||
r.asks.End(r.answer(waitForAsks(t, r, 1)[0]))
|
||||
r.b.Wait()
|
||||
if v := valueOf(t, r.st, r.key); v.State != delivery.BuildDone || res.Revision <= rev || stored.Load() != 1 {
|
||||
t.Fatalf("key %+v, answer %+v, %d asks of the store; want built from the new revision and the queued look run once after it", v, res, stored.Load())
|
||||
}
|
||||
return
|
||||
}
|
||||
r.b.Wait()
|
||||
after := r.st.revs[r.key]
|
||||
if c.refuse != "" {
|
||||
if !strings.Contains(res.Refused, c.refuse) || res.Revision != 0 || len(r.asks.Asks()) != 0 || after != rev {
|
||||
t.Fatalf("refused %q (revision %d, %d asks, key at revision %d was %d); want refused with %q and nothing written or asked", res.Refused, res.Revision, len(r.asks.Asks()), after, rev, c.refuse)
|
||||
}
|
||||
return
|
||||
}
|
||||
if c.failed != "" {
|
||||
if !strings.Contains(res.Failed, c.failed) || res.Revision <= rev || len(res.Asks) != 0 || len(r.asks.Asks()) != 0 {
|
||||
t.Fatalf("answer %+v after %d asks; want a new revision (was %d), no ask and the key's failure %q", res, len(r.asks.Asks()), rev, c.failed)
|
||||
}
|
||||
return
|
||||
}
|
||||
asks := r.asks.Asks()
|
||||
if res.Refused != "" || res.Pending != "" || res.Revision <= rev || len(asks) != 1 || res.Asks[delivery.AskGo] != asks[0].ID {
|
||||
t.Fatalf("answer %+v after %d asks; want the new revision (was %d) and the ask id of the one ask", res, len(asks), rev)
|
||||
}
|
||||
if v := valueOf(t, r.st, r.key); v.State != delivery.BuildDone || after <= res.Revision {
|
||||
t.Errorf("the key = %+v at revision %d; want built from the new revision %d", v, after, res.Revision)
|
||||
}
|
||||
if !log.has(r.key, "built again by hand", "no tun on ace", "shanks.lead") {
|
||||
t.Errorf("the why is not logged: %v", log.lines)
|
||||
}
|
||||
return
|
||||
}
|
||||
r.asks.Answer = nil
|
||||
r.b.Askings = c.askings
|
||||
var said []AskedAgain
|
||||
@@ -831,12 +720,7 @@ func TestABuildAskWaitingInItsStreamIsNotBounded(t *testing.T) {
|
||||
if c.dead {
|
||||
r.b.Asks = buildrun.Asker{Publisher: r.asks, Waiter: buildrun.LostAskWaiter{Inner: r.asks, Dead: dead}, Bound: 2 * time.Second}
|
||||
}
|
||||
var heard *buildrun.HeardEnds
|
||||
if c.deaf {
|
||||
heard = &buildrun.HeardEnds{Inner: silentWatch{}, Read: r.asks}
|
||||
r.b.Asks = buildrun.Asker{Publisher: r.asks, Waiter: heard, Bound: 2 * time.Second}
|
||||
}
|
||||
if !c.byHand && !c.dead && !c.deaf {
|
||||
if !c.byHand && !c.dead {
|
||||
gone.why = func(a delivery.StepAsk) string {
|
||||
if c.always || a.ID == first || first == "" {
|
||||
return "it waits first in line while a node took a later ask"
|
||||
@@ -858,16 +742,7 @@ func TestABuildAskWaitingInItsStreamIsNotBounded(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
if c.deaf {
|
||||
r.asks.End(r.answer(waitForAsks(t, r, 1)[0]))
|
||||
for deadline := time.Now().Add(2 * time.Second); heard.Waiting() == 0; time.Sleep(time.Millisecond) {
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatal("no ask is waited on")
|
||||
}
|
||||
}
|
||||
heard.Nudge(buildrun.GoBuildModule) // a step of mesh-build-go ended
|
||||
}
|
||||
if c.done && !c.deaf {
|
||||
if c.done {
|
||||
asks := waitForAsks(t, r, c.puts)
|
||||
if first == "" && !c.byHand {
|
||||
first = asks[0].ID
|
||||
@@ -892,7 +767,7 @@ func TestABuildAskWaitingInItsStreamIsNotBounded(t *testing.T) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
// Note 62: the old message of an ask asked again because its stream gave it up is dropped from the stream.
|
||||
if gone.mu.Lock(); c.done && !c.byHand && !c.dead && !c.deaf && (len(gone.dropped) != 1 || gone.dropped[0] != first) {
|
||||
if gone.mu.Lock(); c.done && !c.byHand && !c.dead && (len(gone.dropped) != 1 || gone.dropped[0] != first) {
|
||||
t.Errorf("dropped %v from the stream, want the old ask %s", gone.dropped, first)
|
||||
}
|
||||
gone.mu.Unlock()
|
||||
@@ -1102,10 +977,3 @@ func TestATakeOverThatCannotReadTheStreamWaitsForTheAsk(t *testing.T) {
|
||||
t.Fatalf("the key = %+v after %d asks, want done from the ask's step, none put", v, len(r.asks.Asks()))
|
||||
}
|
||||
}
|
||||
|
||||
// silentWatch is a watch of an ask's step that stays open and hears nothing.
|
||||
type silentWatch struct{}
|
||||
|
||||
func (silentWatch) WatchAsk(context.Context, string, string) (<-chan delivery.Step, error) {
|
||||
return make(chan delivery.Step), nil
|
||||
}
|
||||
|
||||
@@ -22,8 +22,6 @@ import (
|
||||
const (
|
||||
StuckTool = "builder_stuck"
|
||||
AskAgainTool = "builder_ask_again"
|
||||
// BuildAgainTool builds a failed key again by hand (issue 1126 note 72): it writes a reasks value of Mode ModeBuild.
|
||||
BuildAgainTool = "builder_build_again"
|
||||
// ReAsksState is the builder's state of asks to ask again: the bucket mesh-delivery-builder_reasks, keyed as the build key.
|
||||
ReAsksState = "reasks"
|
||||
ReAsksBucket = delivery.BuilderModule + "_" + ReAsksState
|
||||
@@ -38,24 +36,8 @@ type ReAsk struct {
|
||||
At time.Time `json:"at"`
|
||||
Answer string `json:"answer,omitempty"`
|
||||
Answered time.Time `json:"answered,omitzero"`
|
||||
// Mode is ModeBuild for a request to build a failed key again (BuildAgainTool); empty for an ask asked again.
|
||||
Mode string `json:"mode,omitempty"`
|
||||
Module string `json:"module,omitempty"`
|
||||
FP string `json:"fp,omitempty"`
|
||||
// Refused, Revision and Asks are the builder's answer to a ModeBuild request: why it did nothing, or the key's new
|
||||
// revision (building) and the ask ids it put by language.
|
||||
Refused bool `json:"refused,omitempty"`
|
||||
// Pending says the builder answered before the key was claimed or all its asks were put: neither a refusal nor done.
|
||||
// Failed is the error of a key written building that failed before its asks were put.
|
||||
Pending bool `json:"pending,omitempty"`
|
||||
Failed string `json:"failed,omitempty"`
|
||||
Revision uint64 `json:"revision,omitempty"`
|
||||
Asks map[string]string `json:"asks,omitempty"`
|
||||
}
|
||||
|
||||
// ModeBuild is the Mode of a reasks value that asks the key to be built again, not its waiting asks to be asked again.
|
||||
const ModeBuild = "build"
|
||||
|
||||
// DecodeReAsk reads a reasks value.
|
||||
func DecodeReAsk(data []byte) (ReAsk, error) {
|
||||
var r ReAsk
|
||||
@@ -81,19 +63,6 @@ func NewReAsk(module, fp, why, by string, at time.Time) (string, []byte, error)
|
||||
return key, data, err
|
||||
}
|
||||
|
||||
// NewBuildAgain is the value builder_build_again writes for a module's fingerprint: its key and the value.
|
||||
func NewBuildAgain(module, fp, why, by string, at time.Time) (string, []byte, error) {
|
||||
key, err := delivery.BuilderBuildKey(module, fp)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
if strings.TrimSpace(why) == "" {
|
||||
return "", nil, errors.New("say why the key is built again (why)")
|
||||
}
|
||||
data, err := json.Marshal(ReAsk{V: 1, Why: strings.TrimSpace(why), By: by, At: at.UTC(), Mode: ModeBuild, Module: module, FP: fp})
|
||||
return key, data, err
|
||||
}
|
||||
|
||||
// StateRead is a state the tools read: its keys, and a key's value with when it was written (zero when not known).
|
||||
type StateRead struct {
|
||||
Keys func() ([]string, error)
|
||||
|
||||
@@ -116,7 +116,7 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1
|
||||
t.Errorf("secrets-owner = %q", m.SecretsOwner)
|
||||
}
|
||||
// 1945: the builder's tools (builder_seat_queue: note 62), served by its tools bundle, and the event it says when it asks an ask again.
|
||||
if want := []string{"builder_ask_again", "builder_build_again", "builder_seat_queue", "builder_stuck"}; !reflect.DeepEqual(sorted(m.Tools), want) {
|
||||
if want := []string{"builder_ask_again", "builder_seat_queue", "builder_stuck"}; !reflect.DeepEqual(sorted(m.Tools), want) {
|
||||
t.Errorf("tools = %v, want %v", m.Tools, want)
|
||||
}
|
||||
if want := []string{"build.asked-again"}; !reflect.DeepEqual(m.Emits, want) {
|
||||
|
||||
@@ -4,7 +4,6 @@
|
||||
"slug": "builder",
|
||||
"tools": [
|
||||
"builder_ask_again",
|
||||
"builder_build_again",
|
||||
"builder_seat_queue",
|
||||
"builder_stuck"
|
||||
],
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
// Command mesh-delivery-checker-tools serves the checker's read tools (novox/hq ADR 0484 clause 13, issue 1686): the runtime
|
||||
// launches it for a call, it reads the checker's verdicts and kept states and the indexer's candidates through the checker's
|
||||
// state and reads. It writes only the tries (the try's tools) and, asking a check again by hand, one verdict. The checker is the other program: it serves no tool (ADR 0437 §4).
|
||||
// state and reads, and writes nothing. The checker is the other program: it serves no tool (ADR 0437 §4).
|
||||
package main
|
||||
|
||||
import (
|
||||
@@ -170,6 +170,21 @@ func TryTools(tr Trying) []stdio.Tool {
|
||||
}
|
||||
}
|
||||
|
||||
// Seat is the seat the checker holds (novox/hq ADR 0547 §10, issue 774); its verbs are served as tools named <seat>.<verb>.
|
||||
const Seat = "mesh-delivery-checker"
|
||||
|
||||
// SeatTryTools are the try's tools as the seat's verbs: try, try-read and try-end, run by the same code as checker_try,
|
||||
// checker_try_read and checker_try_end, so the answer is the same. The module tools stay beside them.
|
||||
func SeatTryTools(tr Trying) []stdio.Tool {
|
||||
verbs := map[string]string{check.TryTool: "try", check.TryReadTool: "try-read", check.TryEndTool: "try-end"}
|
||||
var out []stdio.Tool
|
||||
for _, t := range TryTools(tr) {
|
||||
t.Name = Seat + "." + verbs[t.Name]
|
||||
out = append(out, t)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Tools are the read tools over the checker's states.
|
||||
func Tools(r check.Reader) []stdio.Tool {
|
||||
return []stdio.Tool{
|
||||
@@ -235,24 +250,6 @@ func Tools(r check.Reader) []stdio.Tool {
|
||||
}
|
||||
}
|
||||
|
||||
// RecoveryTools are the tools that steer a check by hand (tools-are-how-a-module-is-spoken-to): ask a check again whose
|
||||
// wait judged nothing (the lost check of 2026-10-28). They write the checker's verdicts through w.
|
||||
func RecoveryTools(r check.Reader, w check.VerdictEntry) []stdio.Tool {
|
||||
str := func(args map[string]any, k string) string { s, _ := args[k].(string); return s }
|
||||
return []stdio.Tool{{Name: check.AskAgainTool,
|
||||
Description: "Ask a candidate's check again by hand, whose verdict stays running while nothing runs for it (" + check.StatusReadTool + " names it " +
|
||||
"under lost): its verdict is ended as asked again, saying why, and the checker, which hears its verdicts, stops the wait that judged nothing " +
|
||||
"and asks the check anew, whole. Refused for a group with no candidate, a verdict that is not running (a pass or failure stands until a push or " +
|
||||
"a trunk move), and while a step of its pull requests can still run on a build module (its end judges it). The checker reads a step that " +
|
||||
"ended while its watch was deaf by itself at the next step end of its build module; this is for what that does not reach.",
|
||||
Input: map[string]any{"type": "object", "required": []string{"group", "why"}, "properties": map[string]any{
|
||||
"group": map[string]any{"type": "string", "description": "the delivery group's branch name"},
|
||||
"why": map[string]any{"type": "string", "description": "why it is asked again, in words; kept in the verdict it ends"}}},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
return r.AskAgain(w, str(args, "group"), str(args, "why"))
|
||||
}}}
|
||||
}
|
||||
|
||||
// stepBound is the checker's step bound as the checker reads it (MESH_CHECKER_STEP_BOUND), its default when unset or not a
|
||||
// duration.
|
||||
func stepBound() time.Duration {
|
||||
@@ -267,8 +264,7 @@ func main() {
|
||||
PreviewState: read(stdio.State("previews")),
|
||||
StepStates: map[string]check.StateRead{"mesh-build-go": read(stdio.State("mesh-build-go.steps")), "mesh-build-node": read(stdio.State("mesh-build-node.steps"))},
|
||||
HaveState: read(stdio.State("node-engine.have")), StepBound: stepBound(),
|
||||
TurnState: read(stdio.State("mesh-delivery-gate.turns")), HearingState: read(stdio.State(buildrun.HearingState)),
|
||||
AskState: read(stdio.State(check.AsksState))}
|
||||
TurnState: read(stdio.State("mesh-delivery-gate.turns")), HearingState: read(stdio.State(buildrun.HearingState))}
|
||||
r.Runs = runsOf{read(stdio.State("mesh-delivery-indexer.modules"))} // the node-engine at the commit the mesh runs, as the checker reads it
|
||||
tries := stdio.State(check.TriesState)
|
||||
source := stdio.State("mesh-delivery-gate.source")
|
||||
@@ -296,20 +292,7 @@ func main() {
|
||||
return err
|
||||
}}
|
||||
trying := Trying{Tries: store, Ender: store, Trunks: trunks}
|
||||
verdicts := stdio.State("verdicts")
|
||||
written := stateStore{entry: func(key string) (json.RawMessage, uint64, error) {
|
||||
reads.Add(1)
|
||||
e, err := verdicts.Get(key)
|
||||
if err != nil || e == nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
return e.Value, e.Revision, nil
|
||||
}, update: func(key string, value []byte, revision uint64) error {
|
||||
_, err := verdicts.Update(key, json.RawMessage(value), revision)
|
||||
return err
|
||||
}}
|
||||
tools := append(append(Tools(r), TryTools(trying)...), RecoveryTools(r, written)...)
|
||||
if err := stdio.Serve("", logged(tools)); err != nil {
|
||||
if err := stdio.Serve("", logged(append(append(Tools(r), TryTools(trying)...), SeatTryTools(trying)...))); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "[mesh-delivery-checker-tools] %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
@@ -23,14 +23,6 @@ func TestTheToolsServedAreTheFiveReadTools(t *testing.T) {
|
||||
if !slices.Equal(names, want) {
|
||||
t.Errorf("tools = %v, want %v", names, want)
|
||||
}
|
||||
// The recovery tool beside them asks a lost check again; with nothing to write it refuses.
|
||||
recovery := RecoveryTools(check.Reader{}, nil)
|
||||
if len(recovery) != 1 || recovery[0].Name != check.AskAgainTool || recovery[0].Description == "" {
|
||||
t.Errorf("recovery tools = %+v", recovery)
|
||||
}
|
||||
if _, err := recovery[0].Run(map[string]any{"group": "feat/x", "why": "lost"}); err == nil {
|
||||
t.Errorf("%s answered with no state to write", check.AskAgainTool)
|
||||
}
|
||||
// With nothing to read from a tool says so and changes nothing.
|
||||
for _, tl := range Tools(check.Reader{}) {
|
||||
if tl.Name == check.StatusReadTool {
|
||||
@@ -88,6 +80,24 @@ func TestTheTryToolsAskAndReadATry(t *testing.T) { // ADR 0501 §5, the agreed i
|
||||
if err != nil || !strings.Contains(string(raw.(json.RawMessage)), `"state":"queued"`) {
|
||||
t.Errorf("read = %s, %v", raw, err)
|
||||
}
|
||||
// The seat's verbs answer as the tools do (ADR 0547 §10, issue 774): the same ask is the same try, and the read is alike.
|
||||
verbs := SeatTryTools(Trying{Tries: store, Ender: store, Now: func() time.Time { return time.Unix(1, 5) },
|
||||
Trunks: func() (map[string]string, error) {
|
||||
return map[string]string{"novox/mesh-delivery": strings.Repeat("a", 40)}, nil
|
||||
}})
|
||||
if len(verbs) != 3 || verbs[0].Name != Seat+".try" || verbs[1].Name != Seat+".try-read" || verbs[2].Name != Seat+".try-end" {
|
||||
t.Fatalf("verbs = %+v", verbs)
|
||||
}
|
||||
again, err := verbs[0].Run(map[string]any{"repository": "novox/mesh-delivery", "head": strings.Repeat("1", 40), "trunk": "main"})
|
||||
if err != nil || again.(map[string]any)["id"] != id {
|
||||
t.Errorf("verb try = %v, %v, want the waiting try %s", again, err, id)
|
||||
}
|
||||
if viaVerb, err := verbs[1].Run(map[string]any{"id": id}); err != nil || string(viaVerb.(json.RawMessage)) != string(raw.(json.RawMessage)) {
|
||||
t.Errorf("verb try-read = %s, %v; tool = %s", viaVerb, err, raw)
|
||||
}
|
||||
if _, err := verbs[0].Run(map[string]any{"repository": "novox/hq", "head": strings.Repeat("1", 40)}); err == nil {
|
||||
t.Error("the verb tried a repository nothing checks")
|
||||
}
|
||||
// Issue 734: a try whose ask was lost is ended by hand with why, and its head is asked anew.
|
||||
ended, err := tools[2].Run(map[string]any{"id": id, "why": "its ask was lost"})
|
||||
if err != nil || ended.(map[string]any)["state"] != "error" {
|
||||
|
||||
@@ -9,7 +9,6 @@ import (
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
"mesh-delivery-checker/internal/check"
|
||||
)
|
||||
@@ -131,55 +130,3 @@ func (m manifestReader) Manifest(ctx context.Context, digest string) (catalogue.
|
||||
func parseManifest(raw string) (catalogue.Manifest, error) {
|
||||
return catalogue.ParseManifestForInputs([]byte(raw))
|
||||
}
|
||||
|
||||
// awaitedKV is a bucket that may appear after the start (process.AwaitBucket) as a check.KV: a call while it is not on the
|
||||
// bus fails saying so.
|
||||
type awaitedKV struct{ b *process.AwaitedBucket }
|
||||
|
||||
func (a awaitedKV) kv() (kvStore, error) {
|
||||
kv := a.b.KV()
|
||||
if kv == nil {
|
||||
return kvStore{}, errors.New("the bucket is not on the bus yet")
|
||||
}
|
||||
return kvStore{kv}, nil
|
||||
}
|
||||
|
||||
func (a awaitedKV) Get(ctx context.Context, key string) ([]byte, uint64, bool, error) {
|
||||
s, err := a.kv()
|
||||
if err != nil {
|
||||
return nil, 0, false, err
|
||||
}
|
||||
return s.Get(ctx, key)
|
||||
}
|
||||
|
||||
func (a awaitedKV) Create(ctx context.Context, key string, value []byte) (uint64, error) {
|
||||
s, err := a.kv()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return s.Create(ctx, key, value)
|
||||
}
|
||||
|
||||
func (a awaitedKV) Update(ctx context.Context, key string, value []byte, revision uint64) (uint64, error) {
|
||||
s, err := a.kv()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return s.Update(ctx, key, value, revision)
|
||||
}
|
||||
|
||||
func (a awaitedKV) Keys(ctx context.Context) ([]string, error) {
|
||||
s, err := a.kv()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return s.Keys(ctx)
|
||||
}
|
||||
|
||||
func (a awaitedKV) Delete(ctx context.Context, key string) error {
|
||||
s, err := a.kv()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return s.Delete(ctx, key)
|
||||
}
|
||||
|
||||
@@ -89,8 +89,6 @@ type fakeBuildModules struct {
|
||||
// beside is, by workspace, the siblings the fetch placed in it as `<owner/repo> <commit>`, sorted: a digest echoes them
|
||||
// (1929/AC2; mesh-sdk buildstep.besideOf), or the level runs whole and keeps nothing.
|
||||
beside map[string][]string
|
||||
// names are the nodes' names the last check ask carried, for the names check (ADR 0567 §12).
|
||||
names []string
|
||||
}
|
||||
|
||||
func (f *fakeBuildModules) count(tool string) int {
|
||||
@@ -193,14 +191,7 @@ func (f *fakeBuildModules) serve(t *testing.T, module string, kept jetstream.Key
|
||||
}
|
||||
p := &buildstep.Puller{Source: &buildstep.JetStreamAsks{JS: f.js, Language: language}, Language: language, Node: "n1", Put: put,
|
||||
Capacity: func() int { return 2 }, PullWait: 200 * time.Millisecond,
|
||||
Run: func(ctx context.Context, a delivery.StepAsk) (any, error) {
|
||||
if a.Check != nil {
|
||||
f.mu.Lock()
|
||||
f.names = a.Check.NodeNames
|
||||
f.mu.Unlock()
|
||||
}
|
||||
return buildstep.RunAsk(ctx, steps, a)
|
||||
}}
|
||||
Run: func(ctx context.Context, a delivery.StepAsk) (any, error) { return buildstep.RunAsk(ctx, steps, a) }}
|
||||
go func() { _ = p.Pull(f.ctx) }()
|
||||
}
|
||||
|
||||
@@ -343,7 +334,6 @@ func startBed(t *testing.T) *bed {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b.setOnline(t, "n1", true)
|
||||
b.setNode(t, "n1")
|
||||
for _, module := range []string{"mesh-build-go", "mesh-build-node"} {
|
||||
b.setHealth(t, "n1", module)
|
||||
}
|
||||
@@ -376,20 +366,6 @@ func (b *bed) setOnline(t *testing.T, node string, online bool) {
|
||||
}
|
||||
}
|
||||
|
||||
// setNode writes the node's record in the node-engine's have, as the node-engine does, so the checker names it in each
|
||||
// check ask (ADR 0567 §12); with no record the checker asks no check.
|
||||
func (b *bed) setNode(t *testing.T, node string) {
|
||||
t.Helper()
|
||||
data, err := delivery.EncodeNodeRecord(delivery.NodeRecord{V: 1, Node: node})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
key, _ := delivery.HaveNodeKey(node)
|
||||
if _, err := b.buckets[delivery.HaveBucket].Put(b.ctx, key, data); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func (b *bed) setHealth(t *testing.T, node, module string) {
|
||||
t.Helper()
|
||||
data, err := delivery.EncodeHave(delivery.Have{Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
|
||||
@@ -517,12 +493,6 @@ func TestCheckerJudgesACandidateAndOwesItsStatus(t *testing.T) { // N6/AC5, 1300
|
||||
if v.Composition == nil || v.Composition.State != delivery.CompositionNotRun || len(v.Machines) != 1 || v.Machines[0] != "n1" {
|
||||
t.Errorf("composition %+v machines %v", v.Composition, v.Machines)
|
||||
}
|
||||
b.fake.mu.Lock()
|
||||
names := b.fake.names
|
||||
b.fake.mu.Unlock()
|
||||
if len(names) != 1 || names[0] != "n1" {
|
||||
t.Errorf("the check ask named the nodes %v, want [n1] from the have's node record", names)
|
||||
}
|
||||
o := b.owed(t, 7, head1, "success")
|
||||
if o.Context != delivery.StatusContextCheck || !strings.HasPrefix(o.Text, "pass 111111111111 on aaaaaaaaaaaa") || !strings.Contains(o.Text, "unit 2 run") {
|
||||
t.Errorf("owed = %+v", o)
|
||||
@@ -827,8 +797,7 @@ func TestTheCheckerSaysOwedReadableOnTheBus(t *testing.T) { // R2-1
|
||||
break
|
||||
}
|
||||
if said.Module != "mesh-delivery-checker" || said.Node != "n1" || !said.Keeps("owed") || !said.Readable("owed") ||
|
||||
!said.Readable("previews") || !said.Readable("verdicts") || !said.Readable("kept") || !said.Readable("tries") || said.Readable("hearing") ||
|
||||
!said.Keeps("asks") || said.Readable("asks") || len(said.States) != 7 {
|
||||
t.Fatalf("the checker said %+v, want its seven states, asks and hearing its own, with kept, owed, previews, tries and verdicts readable, tries by the delivery view (1929/AC4; 1887 A5b; build_usage)", said)
|
||||
!said.Readable("previews") || !said.Readable("verdicts") || !said.Readable("kept") || !said.Readable("tries") || said.Readable("hearing") || len(said.States) != 6 {
|
||||
t.Fatalf("the checker said %+v, want its six states with kept, owed, previews, tries and verdicts readable, tries by the delivery view (1929/AC4; 1887 A5b; build_usage)", said)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,16 +2,12 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
"mesh-delivery-checker/internal/check"
|
||||
)
|
||||
@@ -29,46 +25,11 @@ type judgeDeps struct {
|
||||
builds check.BuildStates
|
||||
// compositions is the assigner's bucket of compositions; nil leaves every composition not run.
|
||||
compositions check.CompositionReader
|
||||
// nodes names every node with a record; nil hands no names.
|
||||
nodes *nodeNames
|
||||
logf func(format string, args ...any)
|
||||
}
|
||||
|
||||
// nodeNames reads every node's name from the node-engine's have, one key `<node>.node` per node with a record (novox/hq
|
||||
// ADR 0567 §12), at each ask: a node enrolled since is named in the next check.
|
||||
type nodeNames struct{ kv jetstream.KeyValue }
|
||||
|
||||
// openNodeNames opens the node-engine's have for the nodes' names alone: it is not watched, so no check waits on a node.
|
||||
func openNodeNames(ctx context.Context, js process.BucketOpener, logf func(string, ...any)) *nodeNames {
|
||||
return &nodeNames{kv: process.OptionalBucket(ctx, js, delivery.HaveBucket, "no check is asked until it is: the names check needs the nodes' names", logf)}
|
||||
}
|
||||
|
||||
func (n nodeNames) Names(ctx context.Context) ([]string, error) {
|
||||
if n.kv == nil {
|
||||
return nil, fmt.Errorf("the bucket %s could not be opened at the checker's start", delivery.HaveBucket)
|
||||
}
|
||||
lister, err := n.kv.ListKeysFiltered(ctx, "*."+delivery.HaveNodeWord)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out []string
|
||||
for key := range lister.Keys() {
|
||||
if node, _, isNode, err := delivery.ParseHaveKey(key); err == nil && isNode {
|
||||
out = append(out, node)
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out, nil
|
||||
logf func(format string, args ...any)
|
||||
}
|
||||
|
||||
func newJudge(d judgeDeps) check.Judge {
|
||||
var nodes interface {
|
||||
Names(ctx context.Context) ([]string, error)
|
||||
}
|
||||
if d.nodes != nil {
|
||||
nodes = d.nodes
|
||||
}
|
||||
return &check.Examiner{Nodes: nodes, Compositions: d.compositions, Asks: d.asks, Toolchains: d.tools,
|
||||
return &check.Examiner{Compositions: d.compositions, Asks: d.asks, Toolchains: d.tools,
|
||||
Kept: check.Kept{Store: d.kept}, Previews: d.previews, Modules: d.repos, Packages: d.repos, Forge: d.forge, Runs: d.repos, Builds: d.builds, Announce: d.announce, Trunks: d.trunks, Logf: d.logf}
|
||||
}
|
||||
|
||||
|
||||
@@ -69,8 +69,6 @@ var keptStates = []announce.State{
|
||||
// design limit 1); the checker alone writes them (ADR 0444 clause 5).
|
||||
// Whether DEAD_LETTERS is heard (issue 734): read by the checker's own status tool alone.
|
||||
{Name: "hearing"},
|
||||
// The asks each running check put (check.AsksState): read by the checker's own checker_ask_again alone.
|
||||
{Name: "asks"},
|
||||
{Name: "kept", Readable: true},
|
||||
{Name: "owed", Readable: true},
|
||||
{Name: "previews", Readable: true},
|
||||
@@ -146,14 +144,9 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
hearing := buildrun.NewDeadLetterHearing(ctx, buildrun.JetStreamDeadLetters{JS: js}, []string{delivery.AskGo, delivery.AskNode}, dead,
|
||||
logger.Printf, buildrun.SayHearing(ctx, hearingKV, logger.Printf))
|
||||
defer hearing.Stop()
|
||||
// Each ask's step is heard by its own watch and, at every step end of its build module and every reconnect, read
|
||||
// (buildrun.HearEnds): a watch that goes deaf loses no end (the lost check of 2026-10-28).
|
||||
heard := buildrun.HearEnds(ctx, asks, asks, []string{buildrun.GoBuildModule, buildrun.NodeBuildModule}, logger.Printf)
|
||||
conn.SetReconnectHandler(func(*nats.Conn) { hearing.Again(); heard.Nudge("") })
|
||||
waiter := buildrun.LostAskWaiter{Inner: heard, Dead: dead, Hearing: hearing}
|
||||
asksKV := process.AwaitBucket(ctx, js, conn, check.AsksBucket, "checker_ask_again refuses every check, as it cannot tell whether its ask waits in a stream", logger.Printf)
|
||||
conn.SetReconnectHandler(func(*nats.Conn) { hearing.Again() })
|
||||
waiter := buildrun.LostAskWaiter{Inner: asks, Dead: dead, Hearing: hearing}
|
||||
ch := &check.Checker{
|
||||
Asks: awaitedKV{asksKV},
|
||||
Verdicts: kvStore{buckets[delivery.CheckerVerdictsBucket]},
|
||||
Owed: kvStore{buckets[delivery.CheckerOwedBucket]},
|
||||
Trunks: trunks,
|
||||
@@ -201,7 +194,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
kept: kvStore{buckets[delivery.CheckerKeptBucket]}, previews: kvStore{buckets[delivery.PreviewsBucket]},
|
||||
repos: repos, builds: builds, forge: forgeReads{call: caller.Call}, logf: logger.Printf,
|
||||
announce: announce, compositions: compositions,
|
||||
nodes: openNodeNames(ctx, js, logger.Printf),
|
||||
})
|
||||
ch.Announce = announce
|
||||
tryer.Judge = ch.Judge
|
||||
@@ -338,21 +330,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
}
|
||||
})
|
||||
}()
|
||||
// The checker's own verdicts, updates only, in one watch: one checker_ask_again wrote asks its check again
|
||||
// (Checker.VerdictWritten); a deleted key of heads a candidate holds is judged again (Checker.VerdictDeleted, issue 1733).
|
||||
vw, err := buckets[delivery.CheckerVerdictsBucket].WatchAll(ctx, jetstream.UpdatesOnly())
|
||||
if err != nil {
|
||||
return fmt.Errorf("the checker's verdicts cannot be watched, so no check can be asked again by hand and a deleted verdict would never be written again: %w", err)
|
||||
}
|
||||
go func() {
|
||||
errs <- follow(ctx, vw, func(e jetstream.KeyValueEntry) {
|
||||
if opOf(e) {
|
||||
ch.VerdictWritten(ctx, e.Key(), e.Value())
|
||||
} else {
|
||||
ch.VerdictDeleted(ctx, e.Key())
|
||||
}
|
||||
})
|
||||
}()
|
||||
logger.Printf("started on %s", cred.Node)
|
||||
go func() { errs <- follow(ctx, w, each) }()
|
||||
select {
|
||||
|
||||
@@ -7,8 +7,7 @@ import (
|
||||
)
|
||||
|
||||
// 1912: with checks as asks on the build seats' streams, the stream redelivers a lost node's ask (ADR 0525 §6), so the
|
||||
// checker program watches no node-engine have to start again a check that failed for want of a node. It reads the have's
|
||||
// keys only for the nodes' names at each ask (judge.go, novox/hq ADR 0567 §12).
|
||||
// checker program opens no node-engine have to start again a check that failed for want of a node.
|
||||
func TestTheCheckerWatchesNoHave(t *testing.T) {
|
||||
raw, err := os.ReadFile("main.go")
|
||||
if err != nil {
|
||||
|
||||
@@ -3,7 +3,7 @@ module mesh-delivery-checker
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029000420-979f2cd71819
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027220144-3f7957e16972
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
@@ -1,9 +1,5 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028230718-261f41aec1ae h1:bwcj9NnfGk/DP0sbTRInPAA4aMUnvr8+CmQLhnDvV9w=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028230718-261f41aec1ae/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028235320-62a50ebcf822 h1:od9tMSw9yHRLe4c03/u9EVr7TpXZ+pEJa2XY5evx6Ho=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028235320-62a50ebcf822/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029000420-979f2cd71819 h1:ypKERyJAbbsydMKLlPmWKU++wU2YCPLzGC0nzdw6QvQ=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029000420-979f2cd71819/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027220144-3f7957e16972 h1:iYDYv4HpV1ic20Ban0MaOErryQfNT8ngjlh0bKW6sCU=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027220144-3f7957e16972/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
|
||||
@@ -1,198 +0,0 @@
|
||||
package check
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// **A check is asked again by hand** (the lost check of 2026-10-28; tools-are-how-a-module-is-spoken-to: every recovery
|
||||
// has a tool). checker_ask_again ends a candidate's running verdict, whose wait judged nothing, with AskedAgainWords and
|
||||
// the why; the checker hears it on its watch of its verdicts (Checker.VerdictWritten), ends the check of those heads
|
||||
// running in it, and claims and asks it anew, a whole check (IsEnded). It is refused while a step of the candidate's pull
|
||||
// requests can still run (checker_status's running_steps): that step's end judges it; and while an ask the check put is
|
||||
// still in its build seat's stream, taken by no node (it has no step key yet): asking again then would run the check twice
|
||||
// (issue 1791's kind). The checker keeps the asks each running check put in its state asks (AsksState, by verdict key).
|
||||
// An ask is not put again under its own id instead: the stream de-duplicates an id within its window, so the copy waiting
|
||||
// stays and the new one is dropped, and past the window both would run; nothing takes a put ask back (buildrun.Asker).
|
||||
|
||||
// AsksState is the checker's state of the asks each running check put, by verdict key (CheckAsks); AsksBucket its bucket.
|
||||
const (
|
||||
AsksState = "asks"
|
||||
AsksBucket = delivery.CheckerModule + "_" + AsksState
|
||||
)
|
||||
|
||||
// CheckAsks are the asks a check put, in order, as the state asks keeps them while it runs; its end deletes the key.
|
||||
type CheckAsks struct {
|
||||
Asks []string `json:"asks"`
|
||||
}
|
||||
|
||||
// AsksKV is the state asks: a KV whose key a check's end deletes, so the state holds only the checks that run.
|
||||
type AsksKV interface {
|
||||
KV
|
||||
Delete(ctx context.Context, key string) error
|
||||
}
|
||||
|
||||
// forgetAsks deletes the asks a check put at its end; one that cannot be deleted is said, and the next check of the same
|
||||
// heads writes the key anew.
|
||||
func (ch *Checker) forgetAsks(ctx context.Context, key string) {
|
||||
if err := ch.Asks.Delete(ctx, key); err != nil {
|
||||
ch.logf("%s: the asks it put could not be forgotten at its end: %v", key, err)
|
||||
}
|
||||
}
|
||||
|
||||
// recordAsks keeps the asks a check put under its verdict key; nil Asks keeps nothing. A record that cannot be written
|
||||
// is said: checker_ask_again then refuses the check, as it cannot tell whether its ask waits in a stream.
|
||||
func (ch *Checker) recordAsks(ctx context.Context, key string, asks []string) {
|
||||
if ch.Asks == nil {
|
||||
return
|
||||
}
|
||||
data, err := json.Marshal(CheckAsks{Asks: asks})
|
||||
if err == nil {
|
||||
_, err = putOnce(ctx, ch.Asks, key, data, func(cur []byte) bool { return string(cur) == string(data) })
|
||||
}
|
||||
if err != nil {
|
||||
ch.logf("%s: the asks it put cannot be kept (%v), so %s refuses it", key, err, AskAgainTool)
|
||||
}
|
||||
}
|
||||
|
||||
// AskAgainTool asks a candidate's check again by hand.
|
||||
const AskAgainTool = "checker_ask_again"
|
||||
|
||||
// AskedAgainWords open the words of a verdict AskAgainTool wrote; the why follows.
|
||||
const AskedAgainWords = EndedWords + "asked again by hand (" + AskAgainTool + "): "
|
||||
|
||||
// errAskedAgain is the cause a running check is ended with when its verdict was asked again by hand.
|
||||
var errAskedAgain = errors.New("its verdict was asked again by hand (" + AskAgainTool + ")")
|
||||
|
||||
// IsAskedAgain says whether a verdict was written by AskAgainTool.
|
||||
func IsAskedAgain(v delivery.Verdict) bool {
|
||||
return IsEnded(v) && strings.HasPrefix(v.Failed.Words, AskedAgainWords)
|
||||
}
|
||||
|
||||
// VerdictEntry is the verdicts state as the ask-again tool writes it: a key's value and revision (nil and 0 for none),
|
||||
// and a compare-and-set.
|
||||
type VerdictEntry interface {
|
||||
Entry(key string) (json.RawMessage, uint64, error)
|
||||
Update(key string, value []byte, revision uint64) error
|
||||
}
|
||||
|
||||
// AskedAgain is the answer of AskAgainTool.
|
||||
type AskedAgain struct {
|
||||
Key string `json:"key"`
|
||||
Was string `json:"was"`
|
||||
Said string `json:"said"`
|
||||
Note string `json:"note"`
|
||||
}
|
||||
|
||||
// AskAgain ends the running verdict of the group's candidate as asked again by hand, saying why. It refuses a group with
|
||||
// no candidate, a verdict that is not running (a pass or a failure stands until a push or a trunk move), one a step of
|
||||
// whose pull requests can still run, and one whose asks are not known or one of which no node took yet.
|
||||
func (r Reader) AskAgain(w VerdictEntry, group, why string) (AskedAgain, error) {
|
||||
why = strings.TrimSpace(why)
|
||||
if group == "" || why == "" {
|
||||
return AskedAgain{}, fmt.Errorf("give the group and why it is asked again")
|
||||
}
|
||||
if w == nil {
|
||||
return AskedAgain{}, fmt.Errorf("the verdicts state cannot be written here")
|
||||
}
|
||||
c, ok := r.candidate(group)
|
||||
if !ok {
|
||||
return AskedAgain{}, fmt.Errorf("no candidate is held for the group %q", group)
|
||||
}
|
||||
key, err := delivery.CheckerVerdictKey(c.Group, c.HeadsHash)
|
||||
if err != nil {
|
||||
return AskedAgain{}, err
|
||||
}
|
||||
raw, rev, err := w.Entry(key)
|
||||
if err != nil {
|
||||
return AskedAgain{}, fmt.Errorf("%s cannot be read: %w", key, err)
|
||||
}
|
||||
if len(raw) == 0 {
|
||||
return AskedAgain{}, fmt.Errorf("%s holds no verdict: the checker claims its check when it looks at the candidate (%s)", key, StatusReadTool)
|
||||
}
|
||||
v, err := delivery.DecodeVerdict(raw)
|
||||
if err != nil {
|
||||
return AskedAgain{}, fmt.Errorf("%s holds what is not a verdict: %w", key, err)
|
||||
}
|
||||
if v.State != delivery.VerdictRunning {
|
||||
return AskedAgain{}, fmt.Errorf("%s is %s, not running: a verdict stands until a push or a trunk move asks it again", key, v.State)
|
||||
}
|
||||
steps := r.readSteps()
|
||||
for _, rs := range steps.running {
|
||||
if asksOf(c, rs.What) {
|
||||
return AskedAgain{}, fmt.Errorf("a step of it runs: %s %s on %s since %s (%s); its end judges it", rs.Module, rs.ID, rs.Node, rs.Since, rs.What)
|
||||
}
|
||||
}
|
||||
if r.AskState.Get == nil {
|
||||
return AskedAgain{}, fmt.Errorf("the checker's asks are not readable here, so whether its ask waits in a build seat's stream cannot be told")
|
||||
}
|
||||
kept, err := r.AskState.Get(key)
|
||||
var put CheckAsks
|
||||
if err == nil && len(kept) > 0 {
|
||||
err = json.Unmarshal(kept, &put)
|
||||
}
|
||||
if err != nil || len(put.Asks) == 0 {
|
||||
return AskedAgain{}, fmt.Errorf("the checker names no ask of %s (%v), so whether one waits in a build seat's stream cannot be told; asking again could run it twice", key, err)
|
||||
}
|
||||
taken := map[string]bool{}
|
||||
for _, s := range steps.all {
|
||||
taken[s.step.ID] = true
|
||||
}
|
||||
for _, id := range put.Asks {
|
||||
if !taken[id] {
|
||||
return AskedAgain{}, fmt.Errorf("its ask %s waits in its build seat's stream, taken by no node yet: asking again would run it twice; "+
|
||||
"the seats' backlog is in mesh-delivery-view.build_usage", id)
|
||||
}
|
||||
}
|
||||
words := AskedAgainWords + why
|
||||
ended := delivery.Verdict{Group: v.Group, Heads: v.Heads, Trunk: v.Trunk, State: delivery.VerdictFail,
|
||||
Failed: delivery.FailedFirst{Machine: "checker", Words: words}}
|
||||
data, err := delivery.EncodeVerdict(ended)
|
||||
if err != nil {
|
||||
return AskedAgain{}, err
|
||||
}
|
||||
if err := w.Update(key, data, rev); err != nil {
|
||||
return AskedAgain{}, fmt.Errorf("%s changed while it was asked again, so it was not; read it again: %w", key, err)
|
||||
}
|
||||
return AskedAgain{Key: key, Was: v.State, Said: words,
|
||||
Note: "the checker hears it on its watch of its verdicts, ends the wait that judged nothing and asks the check anew, whole; " +
|
||||
VerdictsReadTool + " shows it running again"}, nil
|
||||
}
|
||||
|
||||
// VerdictWritten takes a change of the checker's verdicts in (its watch, updates only): a verdict AskAgainTool wrote ends
|
||||
// the check of those heads running in this process, which then considers them again; with none running, the candidate
|
||||
// that holds them is considered at once. Any other verdict is the checker's own, and left.
|
||||
func (ch *Checker) VerdictWritten(ctx context.Context, key string, value []byte) {
|
||||
v, err := delivery.DecodeVerdict(value)
|
||||
if err != nil || !IsAskedAgain(v) {
|
||||
return
|
||||
}
|
||||
ch.init()
|
||||
ch.mu.Lock()
|
||||
if l, ok := ch.live[key]; ok {
|
||||
l.end(errAskedAgain)
|
||||
ch.mu.Unlock()
|
||||
ch.logf("%s: %s; the check running for it is ended and asked again", key, v.Failed.Words)
|
||||
return
|
||||
}
|
||||
var group string
|
||||
var cand delivery.Candidate
|
||||
for k, c := range ch.known {
|
||||
if vk, err := delivery.CheckerVerdictKey(c.Group, c.HeadsHash); err == nil && vk == key {
|
||||
group, cand = k, c
|
||||
break
|
||||
}
|
||||
}
|
||||
ch.mu.Unlock()
|
||||
if group == "" {
|
||||
ch.logf("%s: %s, but no candidate holds these heads, so nothing is asked", key, v.Failed.Words)
|
||||
return
|
||||
}
|
||||
ch.logf("%s: %s; asked again", key, v.Failed.Words)
|
||||
ch.consider(ctx, group, cand)
|
||||
}
|
||||
@@ -115,7 +115,7 @@ func CaseOf(c delivery.Candidate, trunk map[string]string) *Case {
|
||||
for _, p := range c.Pulls {
|
||||
switch head, known := trunk[p.Repository]; {
|
||||
case !known || head == "":
|
||||
cs.Left = append(cs.Left, fmt.Sprintf("%s#%d: %s", p.Repository, p.Number, UntrunkedWordsOf([]string{p.Repository})))
|
||||
cs.Left = append(cs.Left, fmt.Sprintf("%s#%d: the trunk head of %s is not known", p.Repository, p.Number, p.Repository))
|
||||
default:
|
||||
cs.Pulls = append(cs.Pulls, Pull{Repository: p.Repository, Number: p.Number, Head: p.Head.Head, Trunk: head, Modules: p.Modules, Fps: p.Fps})
|
||||
cs.Trunk[p.Repository] = head
|
||||
@@ -133,42 +133,6 @@ func CaseOf(c delivery.Candidate, trunk map[string]string) *Case {
|
||||
return cs
|
||||
}
|
||||
|
||||
// UntrunkedWords open the words of a verdict, and of a status, of a pull request that cannot be judged because the gate's
|
||||
// source.main names no trunk head of its repository (issue 1422): never a failure of the change, and judged whole once the
|
||||
// trunk head is known. It is said loudly, never left silently "new".
|
||||
const UntrunkedWords = "cannot be judged: the trunk head of "
|
||||
|
||||
// UntrunkedWordsOf are those words for the repositories whose trunk head is not known, in order, each named once.
|
||||
func UntrunkedWordsOf(repos []string) string {
|
||||
rs := slices.Clone(repos)
|
||||
slices.Sort(rs)
|
||||
rs = slices.Compact(rs)
|
||||
return UntrunkedWords + strings.Join(rs, ", ") + " is not known: the gate's source.main names none, so there is no commit to merge it onto"
|
||||
}
|
||||
|
||||
// IsUntrunked says whether a verdict says its heads cannot be judged for want of a trunk head (UntrunkedWords).
|
||||
func IsUntrunked(v delivery.Verdict) bool {
|
||||
return v.State == delivery.VerdictFail && v.Failed.Machine == "checker" && strings.HasPrefix(v.Failed.Words, UntrunkedWords)
|
||||
}
|
||||
|
||||
// leftOf are the pull requests of a candidate whose repository has no trunk head known, with Trunk empty, in repository and
|
||||
// number order; nil when every repository's is known.
|
||||
func leftOf(c delivery.Candidate, trunk map[string]string) []Pull {
|
||||
var out []Pull
|
||||
for _, p := range c.Pulls {
|
||||
if trunk[p.Repository] == "" {
|
||||
out = append(out, Pull{Repository: p.Repository, Number: p.Number, Head: p.Head.Head, Modules: p.Modules, Fps: p.Fps})
|
||||
}
|
||||
}
|
||||
sort.SliceStable(out, func(i, j int) bool {
|
||||
if out[i].Repository != out[j].Repository {
|
||||
return out[i].Repository < out[j].Repository
|
||||
}
|
||||
return out[i].Number < out[j].Number
|
||||
})
|
||||
return out
|
||||
}
|
||||
|
||||
// GateHeads is the hash the gate matches the members an Approve covers by (mesh-catalog mesh-delivery-gate onestatus.go
|
||||
// headsOf): delivery.HeadsHash over every pull request of the candidate, hq's included, as
|
||||
// `<owner/name lowercase>#<number>=<full head sha lowercase>`. A lone pull request is a group of one. Empty for a candidate
|
||||
|
||||
@@ -79,8 +79,6 @@ type Checker struct {
|
||||
}
|
||||
// Bound is how long one check may run; none when 0 (DefaultBound).
|
||||
Bound time.Duration
|
||||
// Asks keeps the asks each running check put, by verdict key (AsksState), for checker_ask_again; nil keeps none.
|
||||
Asks AsksKV
|
||||
// Link is the base of the address a status points at; the verdict key follows.
|
||||
Link string
|
||||
// Announce says a status was written (EventOwedWritten); nil says nothing.
|
||||
@@ -250,35 +248,6 @@ func (ch *Checker) Candidate(ctx context.Context, key string, value []byte) {
|
||||
ch.consider(ctx, key, c)
|
||||
}
|
||||
|
||||
// VerdictDeleted takes the delete of a verdict key in (the checker's watch of its verdicts, issue 1733): the gate asks the
|
||||
// operator's Approve only on a verdict, so heads a candidate holds are never left without one. A check of them running in
|
||||
// this process writes its verdict anew at its end; otherwise the candidate is considered at once and judged again (or said
|
||||
// again to be unjudgeable). A key no candidate holds is said and left.
|
||||
func (ch *Checker) VerdictDeleted(ctx context.Context, key string) {
|
||||
ch.init()
|
||||
ch.mu.Lock()
|
||||
if ch.inflight[key] {
|
||||
ch.mu.Unlock()
|
||||
ch.logf("%s: deleted while its check runs; its end writes it anew", key)
|
||||
return
|
||||
}
|
||||
var group string
|
||||
var cand delivery.Candidate
|
||||
for k, c := range ch.known {
|
||||
if vk, err := delivery.CheckerVerdictKey(c.Group, c.HeadsHash); err == nil && vk == key {
|
||||
group, cand = k, c
|
||||
break
|
||||
}
|
||||
}
|
||||
ch.mu.Unlock()
|
||||
if group == "" {
|
||||
ch.logf("%s: deleted, and no candidate holds these heads, so nothing is judged", key)
|
||||
return
|
||||
}
|
||||
ch.logf("%s: deleted; its heads are judged again", key)
|
||||
ch.consider(ctx, group, cand)
|
||||
}
|
||||
|
||||
// Forget takes a candidate out (its key was deleted): a check of its heads still running is ended, as no candidate holds
|
||||
// them any more.
|
||||
func (ch *Checker) Forget(key string) {
|
||||
@@ -326,17 +295,9 @@ func (ch *Checker) heads() map[string]string {
|
||||
}
|
||||
|
||||
func (ch *Checker) consider(ctx context.Context, key string, c delivery.Candidate) {
|
||||
heads := ch.heads()
|
||||
if left := leftOf(c, heads); len(left) > 0 {
|
||||
// A pull request the checker cannot claim (its repository's trunk head is not known) leaves its whole group
|
||||
// unjudged, said in a failure verdict (issue 1422, issue 1733): the gate asks the Approve only on a verdict, and a
|
||||
// pass of the other members would let the group merge with that one never checked.
|
||||
ch.untrunked(ctx, c, left)
|
||||
return
|
||||
}
|
||||
cs := CaseOf(c, heads)
|
||||
cs := CaseOf(c, ch.heads())
|
||||
if cs == nil {
|
||||
return // no pull request at all: nothing to judge
|
||||
return // nothing to judge: no trunk known
|
||||
}
|
||||
cs.Queued = ch.now()
|
||||
ch.mu.Lock()
|
||||
@@ -363,11 +324,7 @@ func (ch *Checker) consider(ctx context.Context, key string, c delivery.Candidat
|
||||
delete(ch.again, vkey)
|
||||
delete(ch.live, vkey)
|
||||
ch.mu.Unlock()
|
||||
again := errors.Is(context.Cause(run), errAskedAgain)
|
||||
end(nil)
|
||||
if again && ctx.Err() == nil {
|
||||
ch.consider(ctx, key, ch.candidate(key, c)) // asked again by hand (AskAgainTool): claimed anew
|
||||
}
|
||||
}()
|
||||
for {
|
||||
// Every check is asked at once: the build seats' streams hold the backlog, a new head's check and a check after a
|
||||
@@ -394,76 +351,6 @@ func (ch *Checker) consider(ctx context.Context, key string, c delivery.Candidat
|
||||
}()
|
||||
}
|
||||
|
||||
// untrunked says, loudly, that pull requests of a candidate cannot be judged because the gate's source.main names no trunk
|
||||
// head of their repository (issue 1422): its verdict key holds a failure that says so, whatever it held before (a running
|
||||
// verdict left by a check that can no longer run, or a verdict judged on a trunk that is no longer named: issue 1733), which
|
||||
// checker_verdicts and checker_status show; and every pull request of the candidate gets the status error with
|
||||
// UntrunkedWords, the members whose trunk is known too, as their group is not judged. It is logged once per head: when a
|
||||
// status or the verdict is written, never at a look that finds them already said. Once the trunk head is known the verdict
|
||||
// is stale (it names no trunk) and the heads are judged whole (prepare, claim).
|
||||
func (ch *Checker) untrunked(ctx context.Context, c delivery.Candidate, left []Pull) {
|
||||
key, err := delivery.CheckerVerdictKey(c.Group, c.HeadsHash)
|
||||
if err != nil {
|
||||
ch.logf("candidates of %s: no verdict key for heads %q, so it cannot be said that they cannot be judged: %v", c.Group, c.HeadsHash, err)
|
||||
return
|
||||
}
|
||||
repos := make([]string, 0, len(left))
|
||||
for _, p := range left {
|
||||
repos = append(repos, p.Repository)
|
||||
}
|
||||
words := UntrunkedWordsOf(repos)
|
||||
v := delivery.Verdict{Group: c.Group, Heads: GateHeads(c), Trunk: map[string]string{}, State: delivery.VerdictFail,
|
||||
Failed: delivery.FailedFirst{Machine: "checker", Words: words}}
|
||||
data, err := delivery.EncodeVerdict(v)
|
||||
if err != nil {
|
||||
ch.logf("%s: %s, and the verdict cannot say so: %v", key, words, err)
|
||||
return
|
||||
}
|
||||
said, err := putOnce(ctx, ch.Verdicts, key, data, func(cur []byte) bool {
|
||||
held, err := delivery.DecodeVerdict(cur)
|
||||
return err == nil && IsUntrunked(held) && held.Failed.Words == words // the same words stand; anything else is replaced
|
||||
})
|
||||
if err != nil {
|
||||
ch.logf("%s: %s, and the verdict cannot say so: %v", key, words, err)
|
||||
}
|
||||
for _, cp := range c.Pulls {
|
||||
okey, err := delivery.CheckerOwedKey(cp.Repository, cp.Number, cp.Head.Head)
|
||||
if err != nil {
|
||||
ch.logf("%s: %v", key, err)
|
||||
continue
|
||||
}
|
||||
o := delivery.Owed{Act: delivery.OwedStatus, Context: delivery.StatusContextCheck, State: "error",
|
||||
Target: strings.TrimSuffix(ch.Link, "/") + "/" + key, Text: clipLine(words, MaxLine)}
|
||||
data, err := delivery.EncodeCheckerOwed(o)
|
||||
if err != nil {
|
||||
ch.logf("%s: the status of %s cannot be written: %v", key, okey, err)
|
||||
continue
|
||||
}
|
||||
wrote, err := putOnce(ctx, ch.Owed, okey, data, func(cur []byte) bool {
|
||||
held, err := delivery.DecodeCheckerOwed(cur)
|
||||
o.V = delivery.Version
|
||||
return err == nil && delivery.SameOwed(held, o)
|
||||
})
|
||||
if err != nil {
|
||||
ch.logf("%s: the status of %s cannot be written: %v", key, okey, err)
|
||||
continue
|
||||
}
|
||||
if wrote {
|
||||
said = true
|
||||
if ch.Announce != nil {
|
||||
ch.Announce(ctx, EventOwedWritten, okey)
|
||||
}
|
||||
}
|
||||
}
|
||||
if said {
|
||||
names := make([]string, 0, len(left))
|
||||
for _, p := range left {
|
||||
names = append(names, fmt.Sprintf("%s#%d@%s", p.Repository, p.Number, short12(p.Head)))
|
||||
}
|
||||
ch.logf("%s: %s %s; the verdict and every status of its group say so (error), and it is judged once the trunk head is known", key, strings.Join(names, ", "), words)
|
||||
}
|
||||
}
|
||||
|
||||
// The kinds of a check (ADR 0536 §4): a head never judged and a head asked again after a failure of the nodes are asked on
|
||||
// the build seats' check stream; a carry, a verdict moved to a newer trunk or sibling commit by its package digests (ADR 0548
|
||||
// §5), on the recheck stream, pulled after.
|
||||
@@ -493,8 +380,8 @@ func (ch *Checker) prepare(ctx context.Context, cs *Case) {
|
||||
return
|
||||
case IsConflictFailure(v):
|
||||
return // a head git refused: its next check is whole (ADR 0548 §7)
|
||||
case IsEnded(v), IsUntrunked(v):
|
||||
return // ended unjudged when its heads went (issue 1890), or never judged for want of a trunk head (issue 1422): checked whole
|
||||
case IsEnded(v):
|
||||
return // ended unjudged when its heads went (issue 1890): held again, they are checked whole
|
||||
}
|
||||
cs.Kind = KindTrunk
|
||||
if v.State == delivery.VerdictPass && ch.stale(v, cs) {
|
||||
@@ -747,20 +634,10 @@ func (ch *Checker) try(ctx, run context.Context, key string, cs *Case) {
|
||||
return
|
||||
}
|
||||
began := ch.now()
|
||||
if cs.Watch == nil && ch.Asks != nil {
|
||||
cs.Watch = func(_ string, step *delivery.Step) {
|
||||
if step == nil { // put: kept before its wait starts
|
||||
ch.recordAsks(ctx, cs.Key, cs.Asks())
|
||||
}
|
||||
}
|
||||
defer ch.forgetAsks(ctx, cs.Key)
|
||||
}
|
||||
ch.writeOwed(ctx, cs, delivery.Verdict{State: delivery.VerdictRunning}, Outcome{})
|
||||
out := ch.judge(run, cs)
|
||||
// An ask the build seat gave up on, or whose step the build module's next process ended because it was replaced under
|
||||
// it, judged nothing (issue 734, IsLostAsk): the check is asked again under the same claim, its status pending, and after
|
||||
// LostAskTries it fails as not heard, judged again when the nodes change. A step that ended while its watch was deaf is
|
||||
// read at the next nudge (HeardEnds) and judged.
|
||||
// An ask the build seat gave up on judged nothing (issue 734): the check is asked again under the same claim, its status
|
||||
// pending, and after LostAskTries it fails as not heard, judged again when the nodes change.
|
||||
for try := 1; IsLostAsk(out.Verdict) && run.Err() == nil; try++ {
|
||||
if try >= LostAskTries {
|
||||
out.Verdict.Failed.Words = strings.TrimSpace(out.Verdict.Failed.Words + "\nasked " + fmt.Sprint(LostAskTries) +
|
||||
@@ -770,8 +647,8 @@ func (ch *Checker) try(ctx, run context.Context, key string, cs *Case) {
|
||||
ch.logf("%s: %s; asked again (%d of %d)", cs.Key, out.Verdict.Failed.Words, try+1, LostAskTries)
|
||||
out = ch.judge(run, cs)
|
||||
}
|
||||
if ctx.Err() != nil || errors.Is(context.Cause(run), errAskedAgain) {
|
||||
return // the checker stops, or the verdict was asked again by hand, which wrote it: its next check claims it anew
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
var why superseded
|
||||
if errors.As(context.Cause(run), &why) {
|
||||
@@ -812,13 +689,8 @@ func (ch *Checker) try(ctx, run context.Context, key string, cs *Case) {
|
||||
}
|
||||
}
|
||||
if _, err := ch.Verdicts.Update(ctx, cs.Key, data, cl.rev); err != nil {
|
||||
if _, _, found, gerr := ch.Verdicts.Get(ctx, cs.Key); gerr == nil && !found {
|
||||
_, err = ch.Verdicts.Create(ctx, cs.Key, data) // its key was deleted while it ran (issue 1733): written anew
|
||||
}
|
||||
if err != nil {
|
||||
ch.logf("%s: the verdict could not be written: %v", cs.Key, err)
|
||||
return
|
||||
}
|
||||
ch.logf("%s: the verdict could not be written: %v", cs.Key, err)
|
||||
return
|
||||
}
|
||||
if IsNodesFailure(v) {
|
||||
ch.mu.Lock()
|
||||
|
||||
@@ -239,119 +239,14 @@ func TestAnUnheardForgeFailsWithTheReasonAndIsJudgedAgainWhenTheNodesChange(t *t
|
||||
}
|
||||
}
|
||||
|
||||
// Issue 1422: a pull request of a repository the gate's source.main names no trunk head of (one never merged through the
|
||||
// gate) is never dropped silently. It is not judged, its verdict says it cannot be judged and why, its status is error with
|
||||
// those words, checker_status names it, and the log says it once per head. Once its trunk head is known it is judged whole;
|
||||
// in a group, the members whose trunk is known are judged and the others' status says why they are not.
|
||||
func TestARepositoryWithNoTrunkCannotBeJudgedLoudlyUntilItsTrunkIsKnown(t *testing.T) { // N6/AC6, ADR 0484 clause 11, 1422
|
||||
func TestARepositoryWithNoTrunkGetsNoVerdict(t *testing.T) { // N6/AC6, ADR 0484 clause 11
|
||||
ctx, j := context.Background(), passJudge()
|
||||
r := newRig(t, j)
|
||||
var logged []string
|
||||
r.ch.Logf = func(f string, a ...any) { logged = append(logged, fmt.Sprintf(f, a...)) }
|
||||
const unknown = "novox/unknown"
|
||||
cand := candidateOf(t, "feat/y", pullOf(unknown, 4, head1))
|
||||
key := keyOf(t, "feat/y", cand)
|
||||
r.ch.Candidate(ctx, "feat/y", cand)
|
||||
r.ch.Candidate(ctx, "feat/y", candidateOf(t, "feat/y", pullOf("novox/unknown", 4, head1)))
|
||||
r.ch.Wait()
|
||||
r.ch.Reconsider(ctx) // looked at again: nothing is written or said twice
|
||||
r.ch.NodesChanged(ctx)
|
||||
r.ch.Wait()
|
||||
v := r.verdict(t, key)
|
||||
if j.count() != 0 || !IsUntrunked(v) || !strings.Contains(v.Failed.Words, "the trunk head of "+unknown+" is not known") {
|
||||
t.Fatalf("judged %d times, verdict %+v, want none judged and a verdict that says it cannot be judged", j.count(), v)
|
||||
if j.count() != 0 || len(r.verdicts.vals) != 0 || len(r.owed.vals) != 0 {
|
||||
t.Errorf("judged %d times, %d verdicts, %d statuses, want none", j.count(), len(r.verdicts.vals), len(r.owed.vals))
|
||||
}
|
||||
if s := owedState(t, r, unknown, 4, head1); s != "error" {
|
||||
t.Errorf("%s#4 owes %q, want error", unknown, s)
|
||||
}
|
||||
if n := countWith(logged, "cannot be judged"); n != 1 {
|
||||
t.Errorf("said %d times in the log, want once for the head: %q", n, logged)
|
||||
}
|
||||
reader := func() Reader {
|
||||
ckey, _ := delivery.CandidatesKey("feat/y")
|
||||
return Reader{CandidateState: fakeState{ckey: cand}.read(), VerdictState: fakeState(r.verdicts.vals).read(),
|
||||
Trunks: func() (map[string]string, error) { return r.trunks.Heads(), nil }}
|
||||
}
|
||||
st, err := reader().Status()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if why := st.CannotBeJudged["feat/y"]; !strings.Contains(why, unknown) || len(st.Waiting[KindNew]) != 0 || len(st.Unjudged) != 0 {
|
||||
t.Errorf("cannot_be_judged %v, waiting %v, unjudged %v: want feat/y named as cannot be judged, not as new", st.CannotBeJudged, st.Waiting, st.Unjudged)
|
||||
}
|
||||
if rows, _, err := reader().Verdicts("feat/y"); err != nil || len(rows) != 1 || !strings.Contains(rows[0].Reports[unknown+"#4"], "cannot be judged") {
|
||||
t.Errorf("checker_verdicts %+v (%v), want the report of %s#4 to say it cannot be judged", rows, err, unknown)
|
||||
}
|
||||
|
||||
// Its trunk head becomes known: the heads are judged whole, the same key rewritten, the status success.
|
||||
r.setTrunks(t, map[string]string{repoA: trunkA, repoSDK: trunkS, unknown: trunkA2})
|
||||
r.ch.TrunkMoved(ctx)
|
||||
r.ch.Wait()
|
||||
if v := r.verdict(t, key); j.count() != 1 || v.State != delivery.VerdictPass || v.Trunk[unknown] != trunkA2 {
|
||||
t.Errorf("judged %d times, verdict %+v, want one pass on %s", j.count(), v, trunkA2)
|
||||
}
|
||||
if s := owedState(t, r, unknown, 4, head1); s != "success" {
|
||||
t.Errorf("%s#4 owes %q once its trunk is known, want success", unknown, s)
|
||||
}
|
||||
|
||||
// Issue 1733: the gate asks the Approve only on a verdict, so no candidate is left without one. A group with a member
|
||||
// whose trunk is not known is not judged in part: its verdict fails naming that repository, and every status says so; a
|
||||
// running verdict left for heads that cannot be judged is replaced; a verdict key deleted is judged again on its delete.
|
||||
for _, row := range []struct {
|
||||
name string
|
||||
pulls []delivery.CandidatePull
|
||||
running bool // a running verdict is left under the key first
|
||||
deleted bool // the key is deleted after its verdict
|
||||
judged int
|
||||
want string // the verdict's state, and for a failure the repository it names
|
||||
}{
|
||||
{name: "a member the checker cannot claim", pulls: []delivery.CandidatePull{pullOf(repoA, 9, head2), pullOf("novox/other", 2, head2)}, want: "novox/other"},
|
||||
{name: "a running verdict left for heads that cannot be judged", pulls: []delivery.CandidatePull{pullOf(unknown, 5, head2)}, running: true, want: unknown},
|
||||
{name: "a verdict key deleted", pulls: []delivery.CandidatePull{pullOf(repoA, 5, head2)}, deleted: true, judged: 2, want: delivery.VerdictPass},
|
||||
} {
|
||||
t.Run(row.name, func(t *testing.T) {
|
||||
j := passJudge()
|
||||
r := newRig(t, j)
|
||||
cand := candidateOf(t, "feat/z", row.pulls...)
|
||||
key := keyOf(t, "feat/z", cand)
|
||||
if row.running {
|
||||
group, heads, _ := delivery.ParseCheckerVerdictKey(key)
|
||||
raw, _ := delivery.EncodeVerdict(delivery.Verdict{Group: group, Heads: heads, State: delivery.VerdictRunning})
|
||||
_, _ = r.verdicts.Create(ctx, key, raw)
|
||||
}
|
||||
r.ch.Candidate(ctx, "feat%2Fz", cand)
|
||||
r.ch.Wait()
|
||||
if row.deleted {
|
||||
_ = r.verdicts.Delete(ctx, key)
|
||||
r.ch.VerdictDeleted(ctx, key) // the watch of the verdicts hands the delete over
|
||||
r.ch.Wait()
|
||||
}
|
||||
v := r.verdict(t, key)
|
||||
if row.want == delivery.VerdictPass {
|
||||
if v.State != delivery.VerdictPass || j.count() != row.judged {
|
||||
t.Errorf("verdict %s after %d judgings, want a pass after %d", v.State, j.count(), row.judged)
|
||||
}
|
||||
return
|
||||
}
|
||||
if !IsUntrunked(v) || !strings.Contains(v.Failed.Words, row.want) || j.count() != 0 {
|
||||
t.Errorf("verdict %+v after %d judgings, want a failure naming %s and none judged", v, j.count(), row.want)
|
||||
}
|
||||
for _, p := range row.pulls {
|
||||
if s := owedState(t, r, p.Repository, p.Number, p.Head.Head); s != "error" {
|
||||
t.Errorf("%s#%d owes %q, want error", p.Repository, p.Number, s)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func countWith(lines []string, words string) int {
|
||||
n := 0
|
||||
for _, l := range lines {
|
||||
if strings.Contains(l, words) {
|
||||
n++
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// 1887/A10, ADR 0527 §2: hq is judged as any repository, its checks Go tests, and its pull request gets a verdict and a
|
||||
|
||||
@@ -5,7 +5,6 @@ import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
@@ -273,36 +272,8 @@ func TestACleanMergeIsFetchedAtItsTrunkAndBothLevelsPass(t *testing.T) { // 1679
|
||||
t.Errorf("%s was not kept", k)
|
||||
}
|
||||
}
|
||||
|
||||
// ADR 0567 §12: every check ask carries every node's name for the names check; a check whose names cannot be read, or
|
||||
// that reads none, is not asked and fails saying so.
|
||||
for _, c := range []struct {
|
||||
names []string
|
||||
err error
|
||||
fails string
|
||||
}{{[]string{"anchor", "laptop"}, nil, ""}, {nil, nil, "no node record"}, {nil, errors.New("no bucket"), "no bucket"}} {
|
||||
b := newBench(t)
|
||||
b.ex.Nodes = namesOf{c.names, c.err}
|
||||
out := b.judge(t, caseOf(t, pullOf(repoA, 7, head1)))
|
||||
asks := b.asks.Asks()
|
||||
if c.fails != "" {
|
||||
if out.Verdict.State != delivery.VerdictFail || len(asks) != 0 || !strings.Contains(out.Verdict.Failed.Words, c.fails) {
|
||||
t.Errorf("names %v, %v: verdict %+v, %d asks", c.names, c.err, out.Verdict, len(asks))
|
||||
}
|
||||
} else if len(asks) == 0 || strings.Join(asks[0].Check.NodeNames, ",") != "anchor,laptop" {
|
||||
t.Errorf("the check ask carries the names %v", asks)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// namesOf is the nodes' names as a test gives them.
|
||||
type namesOf struct {
|
||||
names []string
|
||||
err error
|
||||
}
|
||||
|
||||
func (n namesOf) Names(context.Context) ([]string, error) { return n.names, n.err }
|
||||
|
||||
func TestOnlyThePackageWithoutAKeptVerdictRuns(t *testing.T) { // 1288/AC1
|
||||
b := newBench(t)
|
||||
b.digest[delivery.LevelUnit] = Digested{Level: "unit", Packages: map[string]string{"fixture/slow": digA, "fixture/other": digC}}
|
||||
@@ -484,40 +455,6 @@ func TestAHeadThatDoesNotMergeEndsFailedNamingTheFilesAndRunsNoTest(t *testing.T
|
||||
if n := len(b.steps.tool("digest")) + len(b.steps.tool("test")); n != 0 {
|
||||
t.Errorf("%d digest and test steps ran after a conflict", n)
|
||||
}
|
||||
// A group: after the first member's failure every other member is asked its merge alone, so one verdict names each
|
||||
// member that does not merge, with its files, and runs no test of the others.
|
||||
for name, c := range map[string]struct {
|
||||
conflicts map[string]bool // by repository
|
||||
want []string // the members the verdict names in conflict
|
||||
listed int // the verdict's Failures: one each when more than one failed, none for one alone
|
||||
}{
|
||||
"both conflict": {map[string]bool{repoSDK: true, repoA: true}, []string{repoSDK + "#12 ", repoA + "#7 "}, 2},
|
||||
"the second merges": {map[string]bool{repoSDK: true}, []string{repoSDK + "#12 "}, 0},
|
||||
} {
|
||||
b := newBench(t)
|
||||
conflict := captured[Fetched](t, "go-fetch-conflict")
|
||||
b.steps.handlers["build_go_fetch"] = func(sc stepCall) (any, error) {
|
||||
if c.conflicts[sc.args["repository"].(string)] {
|
||||
return conflict, nil
|
||||
}
|
||||
return b.fetch, nil
|
||||
}
|
||||
v := b.judge(t, caseOf(t, pullOf(repoA, 7, head1), pullOf(repoSDK, 12, head2))).Verdict
|
||||
if v.State != delivery.VerdictFail || !IsConflictFailure(v) || len(v.Failures) != c.listed {
|
||||
t.Fatalf("%s: verdict = %+v", name, v)
|
||||
}
|
||||
for n, w := range c.want {
|
||||
if !strings.Contains(v.Failed.Words, w+ConflictWords+"a.go, sub/b.go") || len(v.Failures) > 0 && !strings.HasPrefix(v.Failures[n].Words, w) {
|
||||
t.Errorf("%s: the verdict does not name %s in conflict: %+v", name, w, v)
|
||||
}
|
||||
}
|
||||
if n := len(b.steps.tool("fetch")); n != 2 {
|
||||
t.Errorf("%s: %d fetches, want each member's merge tried", name, n)
|
||||
}
|
||||
if n := len(b.steps.tool("digest")) + len(b.steps.tool("test")); n != 0 {
|
||||
t.Errorf("%s: %d digest and test steps ran after a conflict", name, n)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestALevelWhoseDigestIsNotNamedRunsWholeAndKeepsNothing(t *testing.T) { // 1287/AC4
|
||||
|
||||
@@ -60,11 +60,3 @@ func (m *memKV) Keys(context.Context) ([]string, error) {
|
||||
sort.Strings(keys)
|
||||
return keys, nil
|
||||
}
|
||||
|
||||
func (m *memKV) Delete(_ context.Context, key string) error {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
delete(m.vals, key)
|
||||
delete(m.revs, key)
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -110,14 +110,8 @@ type Examiner struct {
|
||||
// waiting until the assigner wrote a verdict that does not wait, or ctx ends (ADR 0528 clause 5, issue 1297); nil
|
||||
// leaves the composition not run.
|
||||
Compositions CompositionReader
|
||||
// Nodes names every node with a record, read from each `<node>.node` of the node-engine's have (novox/hq ADR 0567
|
||||
// §12): every check ask carries them (CheckJob.NodeNames) for the names check, so no repository lists them. A check
|
||||
// whose names cannot be read, or that reads none, is not asked and fails saying so; nil hands no names.
|
||||
Nodes interface {
|
||||
Names(ctx context.Context) ([]string, error)
|
||||
}
|
||||
Now func() time.Time
|
||||
Logf func(format string, args ...any)
|
||||
Now func() time.Time
|
||||
Logf func(format string, args ...any)
|
||||
}
|
||||
|
||||
// CompositionReader answers the assigner's composition of a check by the check's key, waiting until the assigner wrote
|
||||
@@ -333,104 +327,9 @@ func (st *state) levelsOf() []string {
|
||||
}
|
||||
|
||||
// judgeAll puts one check ask per pull request and build module the repository needs, the mesh-sdk member first, and
|
||||
// judges each answer. A head that does not merge, a pin that is not allowed or a level that fails ends the check (ADR 0484
|
||||
// clause 13); the members not yet judged are then asked their merge alone, so the verdict names every member that does not
|
||||
// merge into its trunk, each with its files, and not only the first (a hidden conflict costs a whole check round).
|
||||
// judges each answer. A head that does not merge, a pin that is not allowed or a level that fails ends the check at once
|
||||
// (ADR 0484 clause 13).
|
||||
func (st *state) judgeAll(ctx context.Context) *delivery.Verdict {
|
||||
pulls, v := st.judgeUntilFailed(ctx)
|
||||
if v == nil {
|
||||
return nil
|
||||
}
|
||||
var more []delivery.FailedFirst
|
||||
for _, p := range pulls {
|
||||
if f, ok := st.mergeOnly(ctx, p); ok {
|
||||
more = append(more, f)
|
||||
}
|
||||
}
|
||||
return withMergeFailures(v, more)
|
||||
}
|
||||
|
||||
// withMergeFailures adds to the verdict v the members asked their merge alone after it failed: what failed first stays
|
||||
// first; a conflict's words name every member in conflict on one line, so the status line names them all, and the
|
||||
// verdict's Failures list each.
|
||||
func withMergeFailures(v *delivery.Verdict, more []delivery.FailedFirst) *delivery.Verdict {
|
||||
if len(more) == 0 {
|
||||
return v
|
||||
}
|
||||
if IsConflictFailure(*v) {
|
||||
words := []string{v.Failed.Words}
|
||||
for _, f := range more {
|
||||
if strings.Contains(f.Words, ConflictWords) {
|
||||
words = append(words, f.Words)
|
||||
}
|
||||
}
|
||||
first := v.Failed
|
||||
v.Failed.Words = strings.Join(words, "; ")
|
||||
v.Failures = []delivery.FailedFirst{first}
|
||||
} else if len(v.Failures) == 0 {
|
||||
v.Failures = []delivery.FailedFirst{v.Failed}
|
||||
}
|
||||
all := append(slices.Clone(v.Failures), more...)
|
||||
if n := len(all); n > delivery.MaxFailures+1 {
|
||||
all = append(all[:delivery.MaxFailures], delivery.FailedFirst{Test: fmt.Sprintf("and %d more", n-delivery.MaxFailures)})
|
||||
}
|
||||
v.Failures = all
|
||||
return v
|
||||
}
|
||||
|
||||
// mergeOnly asks the merge of a member's head onto its trunk alone, of its repository's first build module, and answers the
|
||||
// failure it names when the head does not merge or the ask could not be answered. A build module from before
|
||||
// CheckJob.MergeOnly runs the levels too; only the fetch of its answer is read.
|
||||
func (st *state) mergeOnly(ctx context.Context, p Pull) (delivery.FailedFirst, bool) {
|
||||
bms := st.j.Modules.BuildModules(p.Repository)
|
||||
if len(bms) == 0 || ctx.Err() != nil {
|
||||
return delivery.FailedFirst{}, false
|
||||
}
|
||||
bm := bms[0]
|
||||
args := map[string]any{"repository": p.Repository, "commit": p.Head, "ref": refOf(p.Number), "base": p.Trunk}
|
||||
a := delivery.StepAsk{ID: buildstep.NewID(), Kind: delivery.AskCheck, Language: askLanguage[bm], Asker: delivery.CheckerModule,
|
||||
At: st.j.now().UTC(), Check: &delivery.CheckJob{Fetch: args, Levels: st.levelsOf(), MergeOnly: true}}
|
||||
st.c.Progress.Set(fmt.Sprintf("the merge alone of %s#%d (ask %s), after the check failed", p.Repository, p.Number, a.ID), "")
|
||||
st.c.Asked(a.ID)
|
||||
failed := func(machine, words string) (delivery.FailedFirst, bool) {
|
||||
return delivery.FailedFirst{Machine: machine, Words: words}, true
|
||||
}
|
||||
if err := st.j.Asks.Put(ctx, a); err != nil {
|
||||
return failed(bm, fmt.Sprintf("the merge of %s#%d onto its trunk could not be asked: %v", p.Repository, p.Number, err))
|
||||
}
|
||||
waiting := ctx
|
||||
if watch := st.c.Watch; watch != nil {
|
||||
id := a.ID
|
||||
watch(id, nil)
|
||||
waiting = buildrun.WithStepSeen(ctx, func(s delivery.Step) { watch(id, &s) })
|
||||
}
|
||||
step, err := st.j.Asks.Wait(waiting, a)
|
||||
if err != nil {
|
||||
if ctx.Err() != nil {
|
||||
return delivery.FailedFirst{}, false // the check was ended: what failed first is its verdict
|
||||
}
|
||||
return failed(bm, fmt.Sprintf("the merge of %s#%d onto its trunk: %v", p.Repository, p.Number, err))
|
||||
}
|
||||
st.nodes[step.Node] = true
|
||||
if step.State != delivery.StepPassed {
|
||||
return failed(step.Node, fmt.Sprintf("the merge of %s#%d onto its trunk %s: %s", p.Repository, p.Number, step.State, step.Reason))
|
||||
}
|
||||
var answer delivery.CheckAnswer
|
||||
var f Fetched
|
||||
if err := json.Unmarshal(step.Result, &answer); err != nil || json.Unmarshal(answer.Fetch, &f) != nil {
|
||||
return failed(step.Node, fmt.Sprintf("the merge of %s#%d onto its trunk answered what is not a check: %s", p.Repository, p.Number, step.Result))
|
||||
}
|
||||
if len(f.Conflicts) == 0 {
|
||||
return delivery.FailedFirst{}, false
|
||||
}
|
||||
if st.c.Try == nil {
|
||||
st.j.writePreview(ctx, p, []Fetched{f})
|
||||
}
|
||||
return failed(step.Node, fmt.Sprintf("%s#%d %s%s", p.Repository, p.Number, ConflictWords, strings.Join(f.Conflicts, ", ")))
|
||||
}
|
||||
|
||||
// judgeUntilFailed is judgeAll up to the first failure, answering it and the members after the one that failed.
|
||||
func (st *state) judgeUntilFailed(ctx context.Context) ([]Pull, *delivery.Verdict) {
|
||||
// The mesh-sdk member first: whether its merge changes the TypeScript package's paths decides whether the other members'
|
||||
// Node fetches take the group's package (ADR 0500 §5).
|
||||
pulls := slices.Clone(st.c.Pulls)
|
||||
@@ -444,8 +343,7 @@ func (st *state) judgeUntilFailed(ctx context.Context) ([]Pull, *delivery.Verdic
|
||||
return 0
|
||||
})
|
||||
var groupPackage map[string]any
|
||||
for n, p := range pulls {
|
||||
rest := pulls[n+1:]
|
||||
for _, p := range pulls {
|
||||
var merged []Fetched
|
||||
for _, bm := range st.j.Modules.BuildModules(p.Repository) {
|
||||
t := &target{pull: p, module: bm}
|
||||
@@ -459,7 +357,7 @@ func (st *state) judgeUntilFailed(ctx context.Context) ([]Pull, *delivery.Verdic
|
||||
if bm == buildrun.GoBuildModule {
|
||||
beside, err := st.j.besideFor(ctx, st.c, p)
|
||||
if err != nil {
|
||||
return rest, fail(forgeMachine(err, bm), err.Error()+unheardWords(err))
|
||||
return fail(forgeMachine(err, bm), err.Error()+unheardWords(err))
|
||||
}
|
||||
if len(beside) > 0 {
|
||||
args["beside"] = beside
|
||||
@@ -473,7 +371,7 @@ func (st *state) judgeUntilFailed(ctx context.Context) ([]Pull, *delivery.Verdic
|
||||
t.fetchArgs = args
|
||||
st.targets = append(st.targets, t)
|
||||
if v := st.ask(ctx, t, &merged); v != nil {
|
||||
return rest, v
|
||||
return v
|
||||
}
|
||||
}
|
||||
if st.c.Try == nil { // a try owes the indexer no preview: no pull request holds its head
|
||||
@@ -483,7 +381,7 @@ func (st *state) judgeUntilFailed(ctx context.Context) ([]Pull, *delivery.Verdic
|
||||
groupPackage = map[string]any{"repository": p.Repository, "commit": p.Head, "ref": refOf(p.Number)}
|
||||
}
|
||||
}
|
||||
return nil, nil
|
||||
return nil
|
||||
}
|
||||
|
||||
// DefaultDeclineBound is an Examiner's DeclineBound when it names none: longer than a check runs, so a node with the
|
||||
@@ -505,16 +403,6 @@ func (st *state) askNeeding(ctx context.Context, t *target, merged *[]Fetched, l
|
||||
p := t.pull
|
||||
placed := len(needs) > 0
|
||||
job := &delivery.CheckJob{Fetch: t.fetchArgs, Levels: levels, Kept: true, Packages: packages}
|
||||
if st.j.Nodes != nil {
|
||||
names, err := st.j.Nodes.Names(ctx)
|
||||
if err == nil && len(names) == 0 {
|
||||
err = fmt.Errorf("no node record is on the bus")
|
||||
}
|
||||
if err != nil {
|
||||
return fail(t.module, fmt.Sprintf("the check of %s#%d is not asked: the nodes' names the names check needs cannot be read: %v", p.Repository, p.Number, err))
|
||||
}
|
||||
job.NodeNames = names
|
||||
}
|
||||
if t.runner != "" {
|
||||
job.Runner, job.RunnerWhen = t.runner, buildGoDir
|
||||
}
|
||||
|
||||
@@ -3,7 +3,6 @@ package check
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||
)
|
||||
@@ -20,13 +19,9 @@ import (
|
||||
// judged again when the nodes change.
|
||||
const LostAskTries = 3
|
||||
|
||||
// IsLostAsk says whether a verdict failed because its ask judged nothing: the build seat gave it up (buildrun.LostWords), or
|
||||
// the build module that ran its step was replaced under it, and its next process ended the step it left running
|
||||
// (buildstep.RestartReason; a delivery of the build module, as on 2026-10-28). Neither is the change's failure: the check
|
||||
// is asked again, as for a given-up ask.
|
||||
// IsLostAsk says whether a verdict failed because the build seat gave its ask up (buildrun.LostWords).
|
||||
func IsLostAsk(v delivery.Verdict) bool {
|
||||
return v.State == delivery.VerdictFail && (strings.Contains(v.Failed.Words, buildrun.LostWords) ||
|
||||
strings.Contains(v.Failed.Words, buildstep.RestartReason))
|
||||
return v.State == delivery.VerdictFail && strings.Contains(v.Failed.Words, buildrun.LostWords)
|
||||
}
|
||||
|
||||
// IsNotTaken says whether a verdict failed because no build node took its ask within the bound (buildrun.NotTakenWords).
|
||||
|
||||
@@ -8,7 +8,6 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||
)
|
||||
@@ -142,206 +141,30 @@ func TestAnAskGivenUpOrNeverTakenEndsItsTryAsErrorAndItsHeadIsAskedAnew(t *testi
|
||||
})
|
||||
}
|
||||
|
||||
// troubled is the build seat as a check meets it when its step's end does not reach its wait: the first `restarts` asks'
|
||||
// steps are ended by the build module's next process (buildstep.RestartReason), and the first `deaf` asks' watches hear
|
||||
// nothing though their steps end (the lost check of 2026-10-28). Its steps state is read through AskSteps and Steps.
|
||||
type troubled struct {
|
||||
*buildrun.FakeAsks
|
||||
answer func(delivery.StepAsk) delivery.Step
|
||||
|
||||
mu sync.Mutex
|
||||
restarts, deaf int
|
||||
deafIDs map[string]bool
|
||||
steps map[string]delivery.Step
|
||||
}
|
||||
|
||||
func (tr *troubled) PublishAsk(ctx context.Context, subject, id string, data []byte) error {
|
||||
if err := tr.FakeAsks.PublishAsk(ctx, subject, id, data); err != nil {
|
||||
return err
|
||||
}
|
||||
a, err := delivery.DecodeStepAsk(data)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
step := tr.answer(a)
|
||||
step.What = buildstep.AskWhat(a)
|
||||
tr.mu.Lock()
|
||||
if tr.restarts > 0 {
|
||||
tr.restarts--
|
||||
step.State, step.Reason, step.Result = delivery.StepFailed, buildstep.RestartReason, nil
|
||||
}
|
||||
deaf := tr.deaf > 0
|
||||
if deaf {
|
||||
tr.deaf--
|
||||
tr.deafIDs[id] = true
|
||||
}
|
||||
tr.steps[id] = step
|
||||
tr.mu.Unlock()
|
||||
if !deaf {
|
||||
tr.Write(step)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (tr *troubled) WatchAsk(ctx context.Context, module, askID string) (<-chan delivery.Step, error) {
|
||||
tr.mu.Lock()
|
||||
deaf := tr.deafIDs[askID]
|
||||
tr.mu.Unlock()
|
||||
if deaf {
|
||||
return make(chan delivery.Step), nil // open, and silent for good
|
||||
}
|
||||
return tr.FakeAsks.WatchAsk(ctx, module, askID)
|
||||
}
|
||||
|
||||
func (tr *troubled) AskSteps(_ context.Context, _, askID string) ([]delivery.Step, error) {
|
||||
tr.mu.Lock()
|
||||
defer tr.mu.Unlock()
|
||||
if s, ok := tr.steps[askID]; ok {
|
||||
return []delivery.Step{s}, nil
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// Steps is the steps state as the read tools read it, with running in place of every step the hand has not let end.
|
||||
func (tr *troubled) Steps(running bool) StateRead {
|
||||
tr.mu.Lock()
|
||||
defer tr.mu.Unlock()
|
||||
vals := map[string]json.RawMessage{}
|
||||
for id, s := range tr.steps {
|
||||
if running {
|
||||
s.State, s.Ended, s.Started = delivery.StepRunning, time.Time{}, time.Now()
|
||||
}
|
||||
raw, _ := delivery.EncodeStep(s)
|
||||
vals[s.Node+"."+id] = raw
|
||||
}
|
||||
return StateRead{Keys: func() ([]string, error) {
|
||||
var out []string
|
||||
for k := range vals {
|
||||
out = append(out, k)
|
||||
}
|
||||
return out, nil
|
||||
}, Get: func(k string) (json.RawMessage, error) { return vals[k], nil }}
|
||||
}
|
||||
|
||||
// kvRead is a bucket in memory as the read tools read it, live.
|
||||
func kvRead(kv *memKV) StateRead {
|
||||
return StateRead{Keys: func() ([]string, error) { return kv.Keys(context.Background()) },
|
||||
Get: func(k string) (json.RawMessage, error) {
|
||||
v, _, _, err := kv.Get(context.Background(), k)
|
||||
return v, err
|
||||
}}
|
||||
}
|
||||
|
||||
// asVerdicts is the rig's verdicts as the ask-again tool reads and writes them.
|
||||
type asVerdicts struct{ kv *memKV }
|
||||
|
||||
func (a asVerdicts) Entry(key string) (json.RawMessage, uint64, error) {
|
||||
v, rev, _, err := a.kv.Get(context.Background(), key)
|
||||
return v, rev, err
|
||||
}
|
||||
func (a asVerdicts) Update(key string, value []byte, revision uint64) error {
|
||||
_, err := a.kv.Update(context.Background(), key, value, revision)
|
||||
return err
|
||||
}
|
||||
|
||||
// Issue 734 and the lost check of 2026-10-28: a check whose ask judged nothing is asked again, and one whose step ended
|
||||
// unheard is judged. Given up by the build seat once, or its step ended by the build module's next process after a
|
||||
// delivery replaced it, the check is asked again under its claim and passes; given up every time it fails loudly as not
|
||||
// heard, judged again when the nodes change. A step whose watch went deaf is read at the next step end of its build module
|
||||
// (a nudge) and judged with no new ask; one no event reaches is named lost by checker_status, refused by checker_ask_again
|
||||
// while its step still runs, and asked again by it once its step ended.
|
||||
func TestACheckWhoseAskJudgedNothingIsAskedAgainOrRead(t *testing.T) {
|
||||
// Issue 734: a pull request's check whose ask the build seat gave up on is asked again under its claim, its status
|
||||
// pending, and judged; one given up every time fails as not heard and is judged again when the nodes change.
|
||||
func TestACheckWhoseAskWasGivenUpIsAskedAgain(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
for _, row := range []struct {
|
||||
name string
|
||||
lose, restarts, deaf int
|
||||
how string // "nudge": a step of the build module ends; "hand": checker_ask_again
|
||||
state string
|
||||
asks int
|
||||
lose int
|
||||
state string
|
||||
asks int
|
||||
}{
|
||||
{name: "given up once", lose: 1, state: delivery.VerdictPass, asks: 2},
|
||||
{name: "given up every time", lose: LostAskTries, state: delivery.VerdictFail, asks: LostAskTries},
|
||||
{name: "its build module replaced under it", restarts: 1, state: delivery.VerdictPass, asks: 2},
|
||||
{name: "replaced under every ask", restarts: LostAskTries, state: delivery.VerdictFail, asks: LostAskTries},
|
||||
{name: "its watch deaf, a step of its build module ends", deaf: 1, how: "nudge", state: delivery.VerdictPass, asks: 1},
|
||||
{name: "its watch deaf, asked again by hand", deaf: 1, how: "hand", state: delivery.VerdictPass, asks: 2},
|
||||
{name: "its watch deaf, asked again by hand after its build module replaced it", deaf: 1, restarts: 1, how: "hand", state: delivery.VerdictPass, asks: 2},
|
||||
{1, delivery.VerdictPass, 2},
|
||||
{LostAskTries, delivery.VerdictFail, LostAskTries},
|
||||
} {
|
||||
t.Run(row.name, func(t *testing.T) {
|
||||
b := newBench(t)
|
||||
var asked func() int
|
||||
var heard *buildrun.HeardEnds
|
||||
var tr *troubled
|
||||
if row.lose > 0 {
|
||||
l := losing(b, row.lose)
|
||||
asked = func() int { return len(l.Asks()) }
|
||||
} else {
|
||||
tr = &troubled{FakeAsks: &buildrun.FakeAsks{}, answer: b.answer, restarts: row.restarts, deaf: row.deaf,
|
||||
deafIDs: map[string]bool{}, steps: map[string]delivery.Step{}}
|
||||
heard = &buildrun.HeardEnds{Inner: tr, Read: tr}
|
||||
b.ex.Asks = buildrun.Asker{Publisher: tr, Waiter: heard, Bound: 5 * time.Second}
|
||||
asked = func() int { return len(tr.Asks()) }
|
||||
}
|
||||
r := newRig(t, b.ex)
|
||||
asksKept := newMemKV()
|
||||
r.ch.Asks = asksKept
|
||||
cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
|
||||
key := keyOf(t, "feat/x", cand)
|
||||
r.ch.Candidate(ctx, "feat%2Fx", cand)
|
||||
if row.deaf > 0 {
|
||||
for deadline := time.Now().Add(2 * time.Second); heard.Waiting() == 0 && time.Now().Before(deadline); {
|
||||
time.Sleep(2 * time.Millisecond)
|
||||
}
|
||||
if v := r.verdict(t, key); v.State != delivery.VerdictRunning {
|
||||
t.Fatalf("a check whose step ended unheard = %s, want running until it is read", v.State)
|
||||
}
|
||||
}
|
||||
switch row.how {
|
||||
case "nudge":
|
||||
heard.Nudge(buildrun.NodeBuildModule) // another build module's step: not this one's to read
|
||||
heard.Nudge(buildrun.GoBuildModule)
|
||||
case "hand":
|
||||
cands := newMemKV()
|
||||
ckey, _ := delivery.CandidatesKey("feat/x")
|
||||
_, _ = cands.Create(ctx, ckey, cand)
|
||||
none := StateRead{Keys: func() ([]string, error) { return nil, nil }, Get: func(string) (json.RawMessage, error) { return nil, nil }}
|
||||
rd := Reader{CandidateState: kvRead(cands), VerdictState: kvRead(r.verdicts), AskState: kvRead(asksKept),
|
||||
StepStates: map[string]StateRead{buildrun.GoBuildModule: none}} // no node took its ask yet
|
||||
if _, err := rd.AskAgain(asVerdicts{r.verdicts}, "feat/x", "its watch went deaf"); err == nil || !strings.Contains(err.Error(), "waits in its build seat's stream") {
|
||||
t.Errorf("asked again while its ask waits in the stream (issue 1791's kind): %v", err)
|
||||
}
|
||||
rd.StepStates[buildrun.GoBuildModule] = tr.Steps(true)
|
||||
if _, err := rd.AskAgain(asVerdicts{r.verdicts}, "feat/x", "its watch went deaf"); err == nil || !strings.Contains(err.Error(), "a step of it runs") {
|
||||
t.Errorf("asked again while its step runs: %v", err)
|
||||
}
|
||||
rd.StepStates[buildrun.GoBuildModule] = tr.Steps(false)
|
||||
if st, _ := rd.Status(); !strings.Contains(st.Lost["feat/x"], AskAgainTool) {
|
||||
t.Errorf("status lost = %v, want feat/x named with %s", st.Lost, AskAgainTool)
|
||||
}
|
||||
got, err := rd.AskAgain(asVerdicts{r.verdicts}, "feat/x", "its watch went deaf")
|
||||
if err != nil || got.Key != key || !strings.HasPrefix(got.Said, AskedAgainWords) {
|
||||
t.Fatalf("asked again = %+v, %v", got, err)
|
||||
}
|
||||
raw, _, _, _ := r.verdicts.Get(ctx, key)
|
||||
r.ch.VerdictWritten(ctx, key, raw) // the checker's watch of its verdicts hands it over
|
||||
}
|
||||
r.ch.Wait()
|
||||
v := r.verdict(t, key)
|
||||
if v.State != row.state || asked() != row.asks {
|
||||
t.Errorf("verdict %s (%s) after %d asks, want %s after %d", v.State, v.Failed.Words, asked(), row.state, row.asks)
|
||||
}
|
||||
if _, _, kept, _ := asksKept.Get(ctx, key); kept {
|
||||
t.Errorf("the asks of the ended check are still kept under %s", key)
|
||||
}
|
||||
if row.state == delivery.VerdictFail && (!IsLostAsk(v) || !IsNodesFailure(v)) {
|
||||
t.Errorf("a check that judged nothing every time = %q, want lost and judged again when the nodes change", v.Failed.Words)
|
||||
}
|
||||
if row.how == "hand" {
|
||||
rd := Reader{VerdictState: kvRead(r.verdicts), CandidateState: kvRead(newMemKV())}
|
||||
if _, err := rd.AskAgain(asVerdicts{r.verdicts}, "feat/x", "again"); err == nil {
|
||||
t.Error("a judged check was asked again")
|
||||
}
|
||||
}
|
||||
})
|
||||
b := newBench(t)
|
||||
l := losing(b, row.lose)
|
||||
r := newRig(t, b.ex)
|
||||
cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
|
||||
r.ch.Candidate(ctx, "feat%2Fx", cand)
|
||||
r.ch.Wait()
|
||||
v := r.verdict(t, keyOf(t, "feat/x", cand))
|
||||
if v.State != row.state || len(l.Asks()) != row.asks {
|
||||
t.Errorf("lose %d: verdict %s (%s) after %d asks, want %s after %d", row.lose, v.State, v.Failed.Words, len(l.Asks()), row.state, row.asks)
|
||||
}
|
||||
if row.state == delivery.VerdictFail && (!IsLostAsk(v) || !IsNodesFailure(v)) {
|
||||
t.Errorf("a check given up every time = %q, want lost and judged again when the nodes change", v.Failed.Words)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -47,9 +47,7 @@ type Reader struct {
|
||||
Runs interface{ CommitOf(module string) string }
|
||||
// HearingState is the checker's state hearing: whether it hears DEAD_LETTERS (issue 734).
|
||||
HearingState StateRead
|
||||
// AskState is the checker's state asks: the asks each running check put (AsksState), which checker_ask_again reads.
|
||||
AskState StateRead
|
||||
Now func() time.Time
|
||||
Now func() time.Time
|
||||
}
|
||||
|
||||
// VerdictRead is a verdict as stored with the full report of the status of each pull request it judged (Report), when the
|
||||
@@ -88,11 +86,6 @@ func (r Reader) read(key string, raw json.RawMessage) (VerdictRead, error) {
|
||||
cs := CaseOf(c, v.Trunk)
|
||||
out.Reports = map[string]string{}
|
||||
if cs == nil {
|
||||
if IsUntrunked(v) {
|
||||
for _, p := range leftOf(c, v.Trunk) {
|
||||
out.Reports[pullKey(p)] = v.Failed.Words
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
for _, p := range cs.Pulls {
|
||||
@@ -308,8 +301,6 @@ type RunningStep struct {
|
||||
ID string `json:"id"`
|
||||
Tool string `json:"tool"`
|
||||
Since string `json:"since,omitempty"`
|
||||
// What says what the step is (buildstep.AskWhat): an ask's kind, repository@head and levels.
|
||||
What string `json:"what,omitempty"`
|
||||
}
|
||||
|
||||
// LeftStep is a key a build module left running that no step behind it runs any more, and why.
|
||||
@@ -332,12 +323,6 @@ type Status struct {
|
||||
// HeldByTurn are the candidates whose check an approved merge's turn holds off (ADR 0548 §3), by group: the repository and
|
||||
// the group of the merge that waits. Their check is claimed when that turn goes.
|
||||
HeldByTurn map[string]string `json:"held_by_turn"`
|
||||
// CannotBeJudged are the candidates no pull request of which can be judged, by group, with why: the gate's source.main
|
||||
// names no trunk head of their repositories (issue 1422). They are judged once it does; until then their status is error.
|
||||
CannotBeJudged map[string]string `json:"cannot_be_judged"`
|
||||
// Lost are the candidates whose verdict is running while no step of theirs can still run and their newest step ended, by
|
||||
// group, with that step: nothing will judge them (the lost check of 2026-10-28); checker_ask_again asks one again.
|
||||
Lost map[string]string `json:"lost"`
|
||||
// DeadLetters says whether the checker hears the asks the build seats give up on (issue 734): deaf, a try or check
|
||||
// whose ask is given up waits until its bound.
|
||||
DeadLetters DeadLettersHeard `json:"dead_letters"`
|
||||
@@ -401,11 +386,63 @@ func heldBy(t *Turns, c delivery.Candidate, heads map[string]string) string {
|
||||
|
||||
// Status reads the build modules' steps and the candidates against their verdicts.
|
||||
func (r Reader) Status() (Status, error) {
|
||||
st := Status{RunningByNode: map[string]int{}, Lost: map[string]string{}}
|
||||
read := r.readSteps()
|
||||
st.Running, st.LeftRunning, st.Notes = read.running, read.left, read.notes
|
||||
for _, rs := range st.Running {
|
||||
st.RunningByNode[rs.Node]++
|
||||
st := Status{RunningByNode: map[string]int{}}
|
||||
mods := make([]string, 0, len(r.StepStates))
|
||||
for m := range r.StepStates {
|
||||
mods = append(mods, m)
|
||||
}
|
||||
sort.Strings(mods)
|
||||
st.Running, st.LeftRunning = []RunningStep{}, []LeftStep{}
|
||||
now, bound := time.Now(), r.StepBound
|
||||
if r.Now != nil {
|
||||
now = r.Now()
|
||||
}
|
||||
if bound <= 0 {
|
||||
bound = DefaultStepBound
|
||||
}
|
||||
for _, m := range mods {
|
||||
sr := r.StepStates[m]
|
||||
if sr.Keys == nil || sr.Get == nil {
|
||||
continue
|
||||
}
|
||||
keys, err := sr.Keys()
|
||||
if err != nil {
|
||||
st.Notes = append(st.Notes, "the steps of "+m+" cannot be listed: "+err.Error())
|
||||
continue
|
||||
}
|
||||
// A step that runs started lately: the newest ids are read first, at most MaxReads of them.
|
||||
newestFirst(keys)
|
||||
keys, cut := capKeys(keys, "")
|
||||
if cut != "" {
|
||||
st.Notes = append(st.Notes, "the steps of "+m+": read the newest "+fmt.Sprint(MaxReads)+" of its keys; a step older than those is not shown")
|
||||
}
|
||||
steps := make([]delivery.Step, len(keys))
|
||||
isRunning := make([]bool, len(keys))
|
||||
_ = parallel(len(keys), func(i int) error {
|
||||
raw, err := sr.Get(keys[i])
|
||||
if err != nil || len(raw) == 0 {
|
||||
return nil
|
||||
}
|
||||
step, err := delivery.DecodeStep(raw)
|
||||
steps[i], isRunning[i] = step, err == nil && step.State == delivery.StepRunning
|
||||
return nil
|
||||
})
|
||||
for i := range keys {
|
||||
if !isRunning[i] {
|
||||
continue
|
||||
}
|
||||
step := steps[i]
|
||||
rs := RunningStep{Module: m, Node: step.Node, ID: step.ID, Tool: step.Tool}
|
||||
if !step.Started.IsZero() {
|
||||
rs.Since = step.Started.UTC().Format("2006-01-02T15:04:05Z")
|
||||
}
|
||||
if why := r.leftWhy(m, step, now, bound); why != "" {
|
||||
st.LeftRunning = append(st.LeftRunning, LeftStep{RunningStep: rs, Why: why})
|
||||
continue
|
||||
}
|
||||
st.Running = append(st.Running, rs)
|
||||
st.RunningByNode[step.Node]++
|
||||
}
|
||||
}
|
||||
if r.HaveState.Get == nil {
|
||||
st.Notes = append(st.Notes, "the node-engine's have is not readable here, so a step is not checked against its build module being on its node")
|
||||
@@ -415,7 +452,7 @@ func (r Reader) Status() (Status, error) {
|
||||
}
|
||||
st.Unjudged = []string{}
|
||||
st.Waiting = map[string][]string{KindNew: {}, KindNodes: {}, KindTrunk: {}}
|
||||
st.HeldByTurn, st.CannotBeJudged = map[string]string{}, map[string]string{}
|
||||
st.HeldByTurn = map[string]string{}
|
||||
turns := r.turns(&st)
|
||||
var heads map[string]string
|
||||
if r.Trunks == nil {
|
||||
@@ -466,15 +503,6 @@ func (r Reader) Status() (Status, error) {
|
||||
continue
|
||||
}
|
||||
c, raw := row.c, row.raw
|
||||
if heads != nil && len(leftOf(c, heads)) > 0 {
|
||||
// Nothing of it can be judged now, whatever its verdict says (issue 1422): never counted as waiting.
|
||||
var repos []string
|
||||
for _, p := range leftOf(c, heads) {
|
||||
repos = append(repos, p.Repository)
|
||||
}
|
||||
st.CannotBeJudged[c.Group] = UntrunkedWordsOf(repos)
|
||||
continue
|
||||
}
|
||||
if len(raw) == 0 {
|
||||
st.Unjudged = append(st.Unjudged, c.Group)
|
||||
st.Waiting[KindNew] = append(st.Waiting[KindNew], c.Group)
|
||||
@@ -485,16 +513,9 @@ func (r Reader) Status() (Status, error) {
|
||||
}
|
||||
v, err := delivery.DecodeVerdict(raw)
|
||||
switch {
|
||||
case err != nil:
|
||||
case v.State == delivery.VerdictRunning:
|
||||
if why := read.lostWhy(c); why != "" {
|
||||
st.Lost[c.Group] = why
|
||||
}
|
||||
case err != nil || v.State == delivery.VerdictRunning:
|
||||
case IsNodesFailure(v):
|
||||
st.Waiting[KindNodes] = append(st.Waiting[KindNodes], c.Group)
|
||||
case IsUntrunked(v):
|
||||
// Its trunk head is known now: it waits for its whole check.
|
||||
st.Waiting[KindNew] = append(st.Waiting[KindNew], c.Group)
|
||||
case heads != nil:
|
||||
if cs := CaseOf(c, heads); cs != nil && (!sameTrunk(v.Trunk, cs.Trunk) || besideMoved(v.Beside, besideAt(heads, r.Runs))) {
|
||||
st.Waiting[KindTrunk] = append(st.Waiting[KindTrunk], c.Group)
|
||||
@@ -510,118 +531,6 @@ func (r Reader) Status() (Status, error) {
|
||||
return st, nil
|
||||
}
|
||||
|
||||
// stepsRead is what the build modules' steps states hold: the steps that can still run, those left running, and every
|
||||
// ask step read, by module, for the candidate it belongs to (stepsOf).
|
||||
type stepsRead struct {
|
||||
running []RunningStep
|
||||
left []LeftStep
|
||||
all []moduleStep
|
||||
notes []string
|
||||
}
|
||||
|
||||
type moduleStep struct {
|
||||
module string
|
||||
step delivery.Step
|
||||
}
|
||||
|
||||
// readSteps reads the newest MaxReads keys of each build module's steps state.
|
||||
func (r Reader) readSteps() stepsRead {
|
||||
out := stepsRead{running: []RunningStep{}, left: []LeftStep{}}
|
||||
mods := make([]string, 0, len(r.StepStates))
|
||||
for m := range r.StepStates {
|
||||
mods = append(mods, m)
|
||||
}
|
||||
sort.Strings(mods)
|
||||
now, bound := time.Now(), r.StepBound
|
||||
if r.Now != nil {
|
||||
now = r.Now()
|
||||
}
|
||||
if bound <= 0 {
|
||||
bound = DefaultStepBound
|
||||
}
|
||||
for _, m := range mods {
|
||||
sr := r.StepStates[m]
|
||||
if sr.Keys == nil || sr.Get == nil {
|
||||
continue
|
||||
}
|
||||
keys, err := sr.Keys()
|
||||
if err != nil {
|
||||
out.notes = append(out.notes, "the steps of "+m+" cannot be listed: "+err.Error())
|
||||
continue
|
||||
}
|
||||
// A step that runs started lately: the newest ids are read first, at most MaxReads of them.
|
||||
newestFirst(keys)
|
||||
keys, cut := capKeys(keys, "")
|
||||
if cut != "" {
|
||||
out.notes = append(out.notes, "the steps of "+m+": read the newest "+fmt.Sprint(MaxReads)+" of its keys; a step older than those is not shown")
|
||||
}
|
||||
steps := make([]delivery.Step, len(keys))
|
||||
decoded := make([]bool, len(keys))
|
||||
_ = parallel(len(keys), func(i int) error {
|
||||
raw, err := sr.Get(keys[i])
|
||||
if err != nil || len(raw) == 0 {
|
||||
return nil
|
||||
}
|
||||
step, err := delivery.DecodeStep(raw)
|
||||
steps[i], decoded[i] = step, err == nil
|
||||
return nil
|
||||
})
|
||||
for i := range keys {
|
||||
if !decoded[i] {
|
||||
continue
|
||||
}
|
||||
step := steps[i]
|
||||
out.all = append(out.all, moduleStep{module: m, step: step})
|
||||
if step.State != delivery.StepRunning {
|
||||
continue
|
||||
}
|
||||
rs := RunningStep{Module: m, Node: step.Node, ID: step.ID, Tool: step.Tool, What: step.What}
|
||||
if !step.Started.IsZero() {
|
||||
rs.Since = step.Started.UTC().Format("2006-01-02T15:04:05Z")
|
||||
}
|
||||
if why := r.leftWhy(m, step, now, bound); why != "" {
|
||||
out.left = append(out.left, LeftStep{RunningStep: rs, Why: why})
|
||||
continue
|
||||
}
|
||||
out.running = append(out.running, rs)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// asksOf says whether a step is an ask of one of the candidate's pull requests: its What names `<repository>@<head12>`
|
||||
// (buildstep.AskWhat).
|
||||
func asksOf(c delivery.Candidate, what string) bool {
|
||||
for _, p := range c.Pulls {
|
||||
if p.Repository != "" && strings.Contains(what, p.Repository+"@"+short12(p.Head.Head)) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// lostWhy says why a candidate whose verdict is running has a check that nothing will judge, or "": no step of its pull
|
||||
// requests can still run, and its newest one ended.
|
||||
func (s stepsRead) lostWhy(c delivery.Candidate) string {
|
||||
for _, rs := range s.running {
|
||||
if asksOf(c, rs.What) {
|
||||
return ""
|
||||
}
|
||||
}
|
||||
var last *delivery.Step
|
||||
for i := range s.all {
|
||||
st := s.all[i].step
|
||||
if st.State != delivery.StepRunning && asksOf(c, st.What) && (last == nil || st.Ended.After(last.Ended)) {
|
||||
last = &s.all[i].step
|
||||
}
|
||||
}
|
||||
if last == nil {
|
||||
return "" // its ask waits in its build seat's stream: no node took it yet
|
||||
}
|
||||
return fmt.Sprintf("its verdict is running, but no step of it runs: its last step %s on %s ended %s at %s and no verdict followed; %s asks it again",
|
||||
last.ID, last.Node, last.State, last.Ended.UTC().Format(time.RFC3339), AskAgainTool)
|
||||
}
|
||||
|
||||
// DefaultStepBound is how long one check ask's step may run once a build node took it (its key written running; the fetch,
|
||||
// the digests and the levels), never counting its wait in the stream; past it the check fails naming the node.
|
||||
const DefaultStepBound = 30 * time.Minute
|
||||
|
||||
@@ -36,7 +36,6 @@ func (f fakeState) read() StateRead {
|
||||
func readerOf(t *testing.T) (Reader, string) {
|
||||
t.Helper()
|
||||
r := newRig(t, passJudge())
|
||||
r.setTrunks(t, map[string]string{repoA: trunkA, repoSDK: trunkS, hqRepo: trunkA2})
|
||||
cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1), pullOf(hqRepo, 3, head2))
|
||||
r.ch.Candidate(context.Background(), "feat%2Fx", cand)
|
||||
r.ch.Wait()
|
||||
|
||||
@@ -38,6 +38,11 @@ type stream struct {
|
||||
}
|
||||
|
||||
type manifest struct {
|
||||
Claims []struct {
|
||||
Name string `json:"name"`
|
||||
Scope string `json:"scope"`
|
||||
Serves []string `json:"serves"`
|
||||
} `json:"claims"`
|
||||
Streams []stream `json:"streams"`
|
||||
Tools []string `json:"tools"`
|
||||
Module string `json:"module"`
|
||||
@@ -93,7 +98,7 @@ func TestManifestDeclarations(t *testing.T) { // 1678/AC1
|
||||
t.Errorf("module = %q", m.Module)
|
||||
}
|
||||
// delivery.CheckerKeptHistory, CheckerOwedHistory, PreviewsHistory, CheckerVerdictsHistory; the names follow the SDK's buckets.
|
||||
if want := []state{{Name: "asks", History: 1}, {Name: "hearing", History: 1}, {Name: "kept", History: 1}, {Name: "owed", History: 5}, {Name: "previews", History: 5}, {Name: "tries", History: 1}, {Name: "verdicts", History: 5}}; !reflect.DeepEqual(m.State, want) {
|
||||
if want := []state{{Name: "hearing", History: 1}, {Name: "kept", History: 1}, {Name: "owed", History: 5}, {Name: "previews", History: 5}, {Name: "tries", History: 1}, {Name: "verdicts", History: 5}}; !reflect.DeepEqual(m.State, want) {
|
||||
t.Errorf("state = %+v, want %+v", m.State, want)
|
||||
}
|
||||
if want := []string{"mesh-build-go.steps", "mesh-build-go.toolchains", "mesh-build-node.steps", "mesh-build-node.toolchains", "mesh-controller.assignments",
|
||||
@@ -124,12 +129,16 @@ func TestManifestDeclarations(t *testing.T) { // 1678/AC1
|
||||
t.Errorf("streams = %+v, want %+v", m.Streams, want)
|
||||
}
|
||||
// The read tools (issue 1686) are served by the tools bundle, not by the checker's process.
|
||||
// The try's tools (ADR 0501 §5; the end by hand, issue 734) beside them: mesh-controller's try calls them. A lost check
|
||||
// is asked again by hand (checker_ask_again).
|
||||
if want := []string{"checker_ask_again", "checker_kept", "checker_previews", "checker_pull", "checker_status", "checker_try", "checker_try_end", "checker_try_read", "checker_verdicts"}; !reflect.DeepEqual(sorted(m.Tools), want) {
|
||||
// The try's tools (ADR 0501 §5; the end by hand, issue 734) beside them: mesh-controller's try calls them.
|
||||
if want := []string{"checker_kept", "checker_previews", "checker_pull", "checker_status", "checker_try", "checker_try_end", "checker_try_read", "checker_verdicts"}; !reflect.DeepEqual(sorted(m.Tools), want) {
|
||||
t.Errorf("tools = %v, want %v", m.Tools, want)
|
||||
}
|
||||
for _, k := range []string{"logs", "emits", "claims", "provides", "requires", "defines-seats", "seat", "seats", "bus", "config"} {
|
||||
// The try is the checker's own seat's verbs (ADR 0547 §10, issue 774), served beside the tools by the same code.
|
||||
if len(m.Claims) != 1 || m.Claims[0].Name != "mesh-delivery-checker" || m.Claims[0].Scope != "mesh" ||
|
||||
!reflect.DeepEqual(m.Claims[0].Serves, []string{"try", "try-read", "try-end"}) {
|
||||
t.Errorf("claims = %+v, want the mesh seat mesh-delivery-checker serving try, try-read and try-end", m.Claims)
|
||||
}
|
||||
for _, k := range []string{"logs", "emits", "provides", "requires", "defines-seats", "seat", "seats", "bus", "config"} {
|
||||
if _, ok := top[k]; ok {
|
||||
t.Errorf("the manifest declares %q: the checker has none", k)
|
||||
}
|
||||
|
||||
@@ -2,6 +2,17 @@
|
||||
"module": "mesh-delivery-checker",
|
||||
"version": "1",
|
||||
"slug": "checker",
|
||||
"claims": [
|
||||
{
|
||||
"name": "mesh-delivery-checker",
|
||||
"scope": "mesh",
|
||||
"serves": [
|
||||
"try",
|
||||
"try-read",
|
||||
"try-end"
|
||||
]
|
||||
}
|
||||
],
|
||||
"consumes": [
|
||||
"mesh-build-go.toolchain.reported",
|
||||
"mesh-build-node.toolchain.reported"
|
||||
@@ -10,7 +21,6 @@
|
||||
{ "name": "checks", "events": ["preview.**", "owed.**"] }
|
||||
],
|
||||
"state": [
|
||||
{ "name": "asks", "history": 1 },
|
||||
{ "name": "hearing", "history": 1 },
|
||||
{ "name": "kept", "history": 1 },
|
||||
{ "name": "owed", "history": 5 },
|
||||
@@ -39,7 +49,6 @@
|
||||
"node-build-node"
|
||||
],
|
||||
"tools": [
|
||||
"checker_ask_again",
|
||||
"checker_kept",
|
||||
"checker_previews",
|
||||
"checker_pull",
|
||||
|
||||
@@ -91,6 +91,28 @@ func treesVia(ctx context.Context, call caller) bridge.Trees {
|
||||
}
|
||||
}
|
||||
|
||||
// listedVia asks the forge for the files directly under one directory of a commit through call: gitea_commit_trees with
|
||||
// listed (issue 1126), answer {listed: {directory: {name: blob id}}}.
|
||||
func listedVia(ctx context.Context, call caller) bridge.Listed {
|
||||
return func(owner, repo, sha, dir string) (map[string]string, error) {
|
||||
raw, err := call(ctx, commitTreesTool, map[string]any{"owner": owner, "repo": repo, "sha": sha, "listed": []string{dir}})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var a struct {
|
||||
Listed map[string]map[string]string `json:"listed"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &a); err != nil {
|
||||
return nil, fmt.Errorf("the answer of %s is not {listed}: %w", commitTreesTool, err)
|
||||
}
|
||||
if a.Listed == nil {
|
||||
// A forge module from before listed answers none: placement cannot be read, never read as empty.
|
||||
return nil, fmt.Errorf("the answer of %s lists no directory; the forge module does not list yet", commitTreesTool)
|
||||
}
|
||||
return a.Listed[dir], nil
|
||||
}
|
||||
}
|
||||
|
||||
// getFileTool is the forge's tool for one file at a ref (mesh-catalog modules/gitea tools/index.ts gitea_get_file):
|
||||
// arguments owner, repo, path and ref, answer {file: {path, ref, sha, size, content}}, the content decoded to text
|
||||
// (client.ts getFile). A path the forge does not have fails with `Gitea API <path>: 404 <body>` (client.ts request).
|
||||
|
||||
+1
-7
@@ -72,17 +72,11 @@ accounts {
|
||||
"mesh.mod.gitea.tool.gitea_open_pulls"
|
||||
"mesh.mod.gitea.tool.gitea_open_pulls.*"
|
||||
"$JS.API.CONSUMER.CREATE.KV_mesh-delivery-forge-bridge_performed.>"
|
||||
"$JS.API.CONSUMER.CREATE.KV_mesh-delivery-forge-bridge_inputs.>"
|
||||
"$JS.API.CONSUMER.DELETE.KV_mesh-delivery-forge-bridge_performed.>"
|
||||
"$JS.API.CONSUMER.DELETE.KV_mesh-delivery-forge-bridge_inputs.>"
|
||||
"$JS.API.DIRECT.GET.KV_mesh-delivery-forge-bridge_performed.>"
|
||||
"$JS.API.DIRECT.GET.KV_mesh-delivery-forge-bridge_inputs.>"
|
||||
"$JS.API.STREAM.INFO.KV_mesh-delivery-forge-bridge_performed"
|
||||
"$JS.API.STREAM.INFO.KV_mesh-delivery-forge-bridge_inputs"
|
||||
"$JS.FC.KV_mesh-delivery-forge-bridge_performed.>"
|
||||
"$JS.FC.KV_mesh-delivery-forge-bridge_inputs.>"
|
||||
"$KV.mesh-delivery-forge-bridge_performed.>"
|
||||
"$KV.mesh-delivery-forge-bridge_inputs.>"
|
||||
"$JS.API.CONSUMER.CREATE.KV_mesh-delivery-gate_merges.>"
|
||||
"$JS.API.CONSUMER.DELETE.KV_mesh-delivery-gate_merges.>"
|
||||
"$JS.API.DIRECT.GET.KV_mesh-delivery-gate_merges.>"
|
||||
@@ -225,7 +219,7 @@ func startBed(t *testing.T, openReply string, prepare func(ctx context.Context,
|
||||
}
|
||||
// Its other state bucket, and the gate's buckets the bridge reads, made by the controller from the gate's manifest
|
||||
// (mesh-sdk go/delivery MergesBucket, OwedBucket with their histories).
|
||||
for name, history := range map[string]uint8{performedBucket: 1, inputsBucket: 1, delivery.MergesBucket: delivery.MergesHistory, delivery.OwedBucket: delivery.OwedHistory} {
|
||||
for name, history := range map[string]uint8{performedBucket: 1, delivery.MergesBucket: delivery.MergesHistory, delivery.OwedBucket: delivery.OwedHistory} {
|
||||
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: name, History: history}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -1,81 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"log"
|
||||
"regexp"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
"mesh-delivery-forge-bridge/internal/bridge"
|
||||
)
|
||||
|
||||
// inputsBucket is the module's state bucket inputs (manifest state entry inputs): what the bridge read of a module
|
||||
// directory, its manifest and lock-file hashes, by the directory's git tree hash, so a merge reads the forge only for the
|
||||
// directories it changed (issue 1126; internal/bridge kept.go). A tree hash names its bytes, so a value is never stale;
|
||||
// a merge adds a value only for each directory it changed (a few kilobytes each).
|
||||
const inputsBucket = delivery.ForgeBridgeModule + "_inputs"
|
||||
|
||||
// keptTimeout bounds one read or write of the bucket: the bucket is a saving, never a reason to fail a merge.
|
||||
const keptTimeout = 2 * time.Second
|
||||
|
||||
var treeHash = regexp.MustCompile(`^[0-9a-f]{40}$|^[0-9a-f]{64}$`)
|
||||
|
||||
// kept is the bridge's Kept: in memory always, and in the bucket inputs when it is on the bus, so the first merge after a
|
||||
// start reads only what changed too. A value the bucket cannot give or take is read from the forge, and said.
|
||||
type kept struct {
|
||||
kv jetstream.KeyValue // nil when the bucket is not on the bus
|
||||
logger *log.Logger
|
||||
mu sync.Mutex
|
||||
mem map[string]bridge.Inputs
|
||||
}
|
||||
|
||||
func newKept(kv jetstream.KeyValue, logger *log.Logger) *kept {
|
||||
return &kept{kv: kv, logger: logger, mem: map[string]bridge.Inputs{}}
|
||||
}
|
||||
|
||||
func (k *kept) Get(tree string) (bridge.Inputs, bool) {
|
||||
k.mu.Lock()
|
||||
in, ok := k.mem[tree]
|
||||
k.mu.Unlock()
|
||||
if ok || k.kv == nil || !treeHash.MatchString(tree) {
|
||||
return in, ok
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), keptTimeout)
|
||||
defer cancel()
|
||||
e, err := k.kv.Get(ctx, tree)
|
||||
if err != nil {
|
||||
return bridge.Inputs{}, false // not kept (or not readable): read from the forge
|
||||
}
|
||||
if err := json.Unmarshal(e.Value(), &in); err != nil || in.Manifest == "" {
|
||||
k.logger.Printf("%s.%s cannot be read, so its directory is read from the forge: %v", inputsBucket, tree, err)
|
||||
return bridge.Inputs{}, false
|
||||
}
|
||||
k.mu.Lock()
|
||||
k.mem[tree] = in
|
||||
k.mu.Unlock()
|
||||
return in, true
|
||||
}
|
||||
|
||||
func (k *kept) Put(tree string, in bridge.Inputs) {
|
||||
k.mu.Lock()
|
||||
k.mem[tree] = in
|
||||
k.mu.Unlock()
|
||||
if k.kv == nil || !treeHash.MatchString(tree) {
|
||||
return
|
||||
}
|
||||
data, err := json.Marshal(in)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), keptTimeout)
|
||||
defer cancel()
|
||||
if _, err := k.kv.Put(ctx, tree, data); err != nil {
|
||||
k.logger.Printf("%s.%s is not kept, so the next merge reads its directory from the forge again: %v", inputsBucket, tree, err)
|
||||
}
|
||||
}
|
||||
@@ -67,8 +67,8 @@ type publisher interface {
|
||||
// handle turns one event into its message on FORGE and acknowledges it. An event it cannot read is terminated, never
|
||||
// redelivered, and the log names its subject and why. A message it could not write stays unacknowledged: the server
|
||||
// delivers it again (the server's ack wait is the only clock).
|
||||
func handle(ctx context.Context, m delivered, pub publisher, trees bridge.Trees, files bridge.Files, kept bridge.Kept, logger *log.Logger) {
|
||||
msg, err := bridge.ConvertKept(m.Subject(), m.Data(), trees, files, kept)
|
||||
func handle(ctx context.Context, m delivered, pub publisher, trees bridge.Trees, files bridge.Files, list bridge.Listed, logger *log.Logger) {
|
||||
msg, err := bridge.ConvertWith(m.Subject(), m.Data(), trees, files, list)
|
||||
switch {
|
||||
case errors.Is(err, bridge.ErrRetry):
|
||||
logger.Printf("%v; heard again in %s", err, retryDelay)
|
||||
@@ -96,9 +96,6 @@ func handle(ctx context.Context, m delivered, pub publisher, trees bridge.Trees,
|
||||
logger.Printf("%s: %s could not be written on FORGE, left for redelivery: %v", m.Subject(), msg.Event, err)
|
||||
return
|
||||
}
|
||||
if msg.Said != "" {
|
||||
logger.Printf("%s: %s", m.Subject(), msg.Said)
|
||||
}
|
||||
_ = m.Ack()
|
||||
}
|
||||
|
||||
@@ -163,8 +160,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
return err
|
||||
}
|
||||
go onReconnect(ctx, again, said, logger)
|
||||
// What a merge read of each module directory, by its tree (issue 1126): a merge reads only the directories it changed.
|
||||
inputs := newKept(process.OptionalBucket(ctx, js, inputsBucket, "a merge's module directories are kept in memory only, so the first merge after a start reads every one", logger.Printf), logger)
|
||||
cons, err := cred.EventConsumer(ctx, js, eventsStream)
|
||||
if err != nil {
|
||||
// Essential: the forge's events are what the bridge carries.
|
||||
@@ -175,7 +170,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
// Bounded: the turn is held, and no sync or other event goes on without it.
|
||||
hctx, cancel := context.WithTimeout(ctx, syncDeadline)
|
||||
defer cancel()
|
||||
handle(hctx, m, rec, treesVia(hctx, call), filesVia(hctx, call), inputs, logger)
|
||||
handle(hctx, m, rec, treesVia(hctx, call), filesVia(hctx, call), listedVia(hctx, call), logger)
|
||||
})
|
||||
}, logger.Printf)
|
||||
if err != nil {
|
||||
|
||||
@@ -3,7 +3,7 @@ module mesh-delivery-forge-bridge
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028174713-38dbe1d3a388
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027193820-8ad9372b1952
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028174713-38dbe1d3a388 h1:ycu/sNieTdLpD3bzhZOCXdzcoqXUOxPRmqWV019REpo=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028174713-38dbe1d3a388/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027193820-8ad9372b1952 h1:TmveT3Bija643RGWkilvkzP/h8UWlVvHFZZtQfJK5oU=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027193820-8ad9372b1952/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
|
||||
@@ -54,8 +54,6 @@ var LockFiles = []string{"go.sum", "package-lock.json"}
|
||||
type Message struct {
|
||||
Event string
|
||||
Data []byte
|
||||
// Said is what the bridge did that the log says, beside writing the message; empty for none.
|
||||
Said string
|
||||
}
|
||||
|
||||
// The subjects of the forge module's events the bridge writes pull messages for (mesh-catalog modules/gitea,
|
||||
@@ -86,6 +84,20 @@ type pull struct {
|
||||
PathsTruncated bool `json:"paths_truncated"`
|
||||
}
|
||||
|
||||
// touchesPlacement says whether a pull request may change a placement file: a changed path under
|
||||
// delivery.PlacementDir, or paths the forge did not say or cut (the safe direction: the files are read).
|
||||
func (p pull) touchesPlacement() bool {
|
||||
if p.Paths == nil || p.PathsTruncated {
|
||||
return true
|
||||
}
|
||||
for _, path := range *p.Paths {
|
||||
if strings.HasPrefix(path, delivery.PlacementDir+"/") {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func unreadable(subject, why string, args ...any) error {
|
||||
return fmt.Errorf("%w: %s: %s", ErrUnreadable, subject, fmt.Sprintf(why, args...))
|
||||
}
|
||||
@@ -95,12 +107,19 @@ func unreadable(subject, why string, args ...any) error {
|
||||
// the trunk asks trees once and becomes a merged message; a merge into any other branch is a closed pull message, as the
|
||||
// pull request left its group. A forge that cannot say is ErrRetry; a merge with no module.json is ErrNoModules.
|
||||
func Convert(subject string, data []byte, trees Trees, files Files) (Message, error) {
|
||||
return ConvertKept(subject, data, trees, files, nil)
|
||||
return ConvertWith(subject, data, trees, files, nil)
|
||||
}
|
||||
|
||||
// ConvertKept is Convert; with kept, a merge reads the forge only for the module directories whose tree hash kept
|
||||
// does not hold, and keeps what it read (kept.go).
|
||||
func ConvertKept(subject string, data []byte, trees Trees, files Files, kept Kept) (Message, error) {
|
||||
// Listed asks the forge for the files directly under one directory of a commit, by name to blob id (gitea_commit_trees
|
||||
// with listed); a directory the commit lacks lists none.
|
||||
type Listed func(owner, repo, sha, dir string) (map[string]string, error)
|
||||
|
||||
// ConvertWith is Convert; with list, a merge of the catalogue repository (delivery.CatalogueRepository) also carries every
|
||||
// placement file of its merge commit (novox/hq ADR 0549 §2, issue 1126), read whole or not at all: a forge that cannot
|
||||
// say is ErrRetry, as for the manifests, so the indexer never takes a part of placement for all of it. So does an
|
||||
// updated pull request of the catalogue that touches placement/ (touchesPlacement), with every placement file at its
|
||||
// head, which the indexer writes as placement candidates (ADR 0549 §4: the nodes it places answer it before it merges).
|
||||
func ConvertWith(subject string, data []byte, trees Trees, files Files, list Listed) (Message, error) {
|
||||
var action string
|
||||
switch subject {
|
||||
case UpdatedSubject:
|
||||
@@ -127,16 +146,7 @@ func ConvertKept(subject string, data []byte, trees Trees, files Files, kept Kep
|
||||
return Message{}, unreadable(subject, "no branch")
|
||||
}
|
||||
if subject == MergedSubject && p.Base == Trunk {
|
||||
return merged(subject, p, trees, files, kept)
|
||||
}
|
||||
said := ""
|
||||
if action == delivery.PullUpdated && p.Base != "" && p.Base != Trunk {
|
||||
// The checker judges pull requests into main only, and source.main names main's head only (novox/hq ADR 0484 §11): a
|
||||
// pull request into another base is no candidate, as it is none at the start (OpenPulls) and at its merge. It is
|
||||
// written closed, so a candidate an older head made is taken out, and said (issue 1422).
|
||||
action = delivery.PullClosed
|
||||
said = fmt.Sprintf("%s/%s#%d at %s is into %s, not %s: written closed, so it is no candidate; the checker judges pull requests into %s only (novox/hq ADR 0484 §11)",
|
||||
p.Owner, p.Repo, p.Number, p.HeadSHA[:12], p.Base, Trunk, Trunk)
|
||||
return merged(subject, p, trees, files, list)
|
||||
}
|
||||
f := delivery.Forge{Kind: delivery.ForgePull, Repository: p.Owner + "/" + p.Repo, Branch: p.Head, Number: p.Number,
|
||||
Head: p.HeadSHA, Action: action}
|
||||
@@ -148,6 +158,13 @@ func ConvertKept(subject string, data []byte, trees Trees, files Files, kept Kep
|
||||
return Message{}, fmt.Errorf("%w: %s: the manifests of %s/%s at the head %s: %v", ErrRetry, subject, p.Owner, p.Repo, p.HeadSHA, err)
|
||||
}
|
||||
}
|
||||
if action == delivery.PullUpdated && list != nil && delivery.SameRepository(f.Repository, delivery.CatalogueRepository) && p.touchesPlacement() {
|
||||
var err error
|
||||
if f.Placements, err = placementsOf(p.Owner, p.Repo, p.HeadSHA, list, files); err != nil {
|
||||
return Message{}, fmt.Errorf("%w: %s: the placement files of %s at the head %s: %v", ErrRetry, subject, f.Repository, p.HeadSHA, err)
|
||||
}
|
||||
f.PlacementsRead = true
|
||||
}
|
||||
event, err := delivery.ForgeEvent(f.Repository, delivery.ForgePull)
|
||||
if err != nil {
|
||||
return Message{}, unreadable(subject, "%v", err)
|
||||
@@ -156,7 +173,7 @@ func ConvertKept(subject string, data []byte, trees Trees, files Files, kept Kep
|
||||
if err != nil {
|
||||
return Message{}, unreadable(subject, "%v", err)
|
||||
}
|
||||
return Message{Event: event, Data: body, Said: said}, nil
|
||||
return Message{Event: event, Data: body}, nil
|
||||
}
|
||||
|
||||
// MaxReadsInFlight is how many files of the forge inputsOf asks for at once. A merge into a repository of many
|
||||
@@ -169,7 +186,6 @@ const MaxReadsInFlight = 8
|
||||
type read struct {
|
||||
dir, path string
|
||||
manifest bool
|
||||
asked bool
|
||||
raw []byte
|
||||
err error
|
||||
}
|
||||
@@ -183,14 +199,6 @@ type read struct {
|
||||
// order the calls finish in, and the error is that of the first failing file in the order of dirs and LockFiles. After
|
||||
// a failure no further file is asked for, and inputsOf returns only when every call it started has returned.
|
||||
func inputsOf(owner, repo, ref string, dirs []string, files Files) (manifests map[string]string, locks map[string]map[string]string, err error) {
|
||||
manifests, locks, _, err = inputsRead(owner, repo, ref, dirs, files)
|
||||
return manifests, locks, err
|
||||
}
|
||||
|
||||
// inputsRead is inputsOf, and also answers, on a failure too, the directories whose manifest and lock files were all
|
||||
// read, with their manifests and lock hashes: a merge keeps them by their tree, so its next try reads only the rest
|
||||
// (issue 1126).
|
||||
func inputsRead(owner, repo, ref string, dirs []string, files Files) (manifests map[string]string, locks map[string]map[string]string, whole []string, err error) {
|
||||
at := func(dir, name string) string {
|
||||
if dir == "" {
|
||||
return name
|
||||
@@ -219,7 +227,6 @@ func inputsRead(owner, repo, ref string, dirs []string, files Files) (manifests
|
||||
<-slots
|
||||
break
|
||||
}
|
||||
r.asked = true
|
||||
wg.Add(1)
|
||||
go func(r *read) {
|
||||
defer func() { <-slots; wg.Done() }()
|
||||
@@ -234,16 +241,10 @@ func inputsRead(owner, repo, ref string, dirs []string, files Files) (manifests
|
||||
wg.Wait()
|
||||
manifests = map[string]string{}
|
||||
locks = map[string]map[string]string{}
|
||||
broken := map[string]bool{} // a directory with a file not read (failed, or never asked after a failure)
|
||||
for _, r := range reads {
|
||||
switch {
|
||||
case r.err != nil && (r.manifest || !errors.Is(r.err, ErrNotFound)):
|
||||
broken[r.dir] = true
|
||||
if err == nil {
|
||||
err = fmt.Errorf("%s: %w", r.path, r.err)
|
||||
}
|
||||
case r.err == nil && r.raw == nil && !r.asked:
|
||||
broken[r.dir] = true
|
||||
return nil, nil, fmt.Errorf("%s: %w", r.path, r.err)
|
||||
case r.err != nil: // a lock file the forge does not have
|
||||
case r.manifest:
|
||||
manifests[r.dir] = string(r.raw)
|
||||
@@ -255,29 +256,17 @@ func inputsRead(owner, repo, ref string, dirs []string, files Files) (manifests
|
||||
locks[r.dir][path.Base(r.path)] = hex.EncodeToString(sum[:])
|
||||
}
|
||||
}
|
||||
for _, dir := range dirs {
|
||||
if !broken[dir] {
|
||||
whole = append(whole, dir)
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
for dir := range broken {
|
||||
delete(manifests, dir)
|
||||
delete(locks, dir)
|
||||
}
|
||||
return manifests, locks, whole, err
|
||||
}
|
||||
if len(manifests) == 0 {
|
||||
manifests = nil
|
||||
}
|
||||
if len(locks) == 0 {
|
||||
locks = nil
|
||||
}
|
||||
return manifests, locks, whole, nil
|
||||
return manifests, locks, nil
|
||||
}
|
||||
|
||||
// merged is the merged message of a merge into the trunk: the trees of the merge commit, asked once.
|
||||
func merged(subject string, p pull, trees Trees, files Files, kept Kept) (Message, error) {
|
||||
func merged(subject string, p pull, trees Trees, files Files, list Listed) (Message, error) {
|
||||
if !sha40.MatchString(p.MergeSHA) {
|
||||
return Message{}, unreadable(subject, "the merge commit %q is not 40 hex digits", p.MergeSHA)
|
||||
}
|
||||
@@ -294,20 +283,9 @@ func merged(subject string, p pull, trees Trees, files Files, kept Kept) (Messag
|
||||
names = append(names, dir)
|
||||
}
|
||||
slices.Sort(names)
|
||||
manifests, locks, toRead := keptInputs(names, dirs, kept)
|
||||
read, readLocks, whole, err := inputsRead(p.Owner, p.Repo, p.MergeSHA, toRead, files)
|
||||
manifests, locks, err := inputsOf(p.Owner, p.Repo, p.MergeSHA, names, files)
|
||||
if err != nil {
|
||||
keep(whole, dirs, read, readLocks, kept) // what was read whole is not asked again at the next try
|
||||
return Message{}, fmt.Errorf("%w: %s: the manifests of %s at the merge commit %s (%d of %d module directories read from the forge, the others kept by their tree): %v", ErrRetry, subject, repository, p.MergeSHA, len(toRead), len(names), err)
|
||||
}
|
||||
keep(toRead, dirs, read, readLocks, kept)
|
||||
maps.Copy(manifests, read)
|
||||
maps.Copy(locks, readLocks)
|
||||
if len(manifests) == 0 {
|
||||
manifests = nil
|
||||
}
|
||||
if len(locks) == 0 {
|
||||
locks = nil
|
||||
return Message{}, fmt.Errorf("%w: %s: the manifests of %s at the merge commit %s: %v", ErrRetry, subject, repository, p.MergeSHA, err)
|
||||
}
|
||||
event, err := delivery.ForgeEvent(repository, delivery.ForgeMerged)
|
||||
if err != nil {
|
||||
@@ -315,6 +293,12 @@ func merged(subject string, p pull, trees Trees, files Files, kept Kept) (Messag
|
||||
}
|
||||
f := delivery.Forge{Kind: delivery.ForgeMerged, Repository: repository, Branch: p.Head, Number: p.Number,
|
||||
Head: p.HeadSHA, Merge: p.MergeSHA, Root: root, Trees: dirs, Manifests: manifests, Locks: locks}
|
||||
if list != nil && delivery.SameRepository(repository, delivery.CatalogueRepository) {
|
||||
if f.Placements, err = placementsOf(p.Owner, p.Repo, p.MergeSHA, list, files); err != nil {
|
||||
return Message{}, fmt.Errorf("%w: %s: the placement files of %s at the merge commit %s: %v", ErrRetry, subject, repository, p.MergeSHA, err)
|
||||
}
|
||||
f.PlacementsRead = true
|
||||
}
|
||||
body, err := delivery.EncodeForge(f)
|
||||
if err != nil {
|
||||
return Message{}, unreadable(subject, "%v", err)
|
||||
@@ -412,3 +396,30 @@ func OpenMessage(answer []byte) (Message, error) {
|
||||
}
|
||||
return Message{Event: delivery.OpenEvent(), Data: body}, nil
|
||||
}
|
||||
|
||||
// placementsOf is every placement file of the catalogue at a commit, by node, its bytes exact: each `<node>.json` directly
|
||||
// under delivery.PlacementDir. A file whose name is no node is left out and the rest read; the catalogue's check refuses
|
||||
// such a file before it merges (ADR 0549 §1). Nil when there is none.
|
||||
func placementsOf(owner, repo, sha string, list Listed, files Files) (map[string]string, error) {
|
||||
names, err := list(owner, repo, sha, delivery.PlacementDir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out map[string]string
|
||||
for _, name := range slices.Sorted(maps.Keys(names)) {
|
||||
file := delivery.PlacementDir + "/" + name
|
||||
node, ok := delivery.PlacementFileNode(file)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
raw, err := files(owner, repo, file, sha)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s: %w", file, err)
|
||||
}
|
||||
if out == nil {
|
||||
out = map[string]string{}
|
||||
}
|
||||
out[node] = string(raw)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
@@ -110,21 +110,6 @@ func TestClosedBecomesAClosedPullMessage(t *testing.T) { // 1235/AC4
|
||||
if !delivery.SameForge(f, want) {
|
||||
t.Errorf("message = %+v, want %+v", f, want)
|
||||
}
|
||||
// Issue 1422, ADR 0484 §11: a head pushed to a pull request into another base is written closed, read nothing for, and said;
|
||||
// one into main is no closed message, and an event naming no base is written as before.
|
||||
msg, err = Convert(updatedSubject, updated(map[string]any{"base": "nox-mesh"}), nil, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if f, err = delivery.DecodeForge(msg.Data); err != nil || !delivery.SameForge(f, want) || !strings.Contains(msg.Said, "is into nox-mesh, not main") {
|
||||
t.Errorf("an update into nox-mesh is %+v (%v), said %q; want it closed and said", f, err, msg.Said)
|
||||
}
|
||||
for _, base := range []any{"main", nil} {
|
||||
msg, err = Convert(updatedSubject, updated(map[string]any{"base": base}), nil, anyFiles)
|
||||
if f, _ = delivery.DecodeForge(msg.Data); err != nil || f.Action != delivery.PullUpdated || msg.Said != "" {
|
||||
t.Errorf("an update with base %v is %+v (%v), said %q; want it updated", base, f, err, msg.Said)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestUnreadableEventsWriteNothing(t *testing.T) { // 1235/AC5
|
||||
|
||||
@@ -79,26 +79,47 @@ func TestMergedCarriesManifestsAndLocks(t *testing.T) { // 1448/AC1
|
||||
if want := map[string]map[string]string{"modules/a": {"go.sum": hexSum(sum)}}; !reflect.DeepEqual(f.Locks, want) {
|
||||
t.Errorf("locks = %v, want %v", f.Locks, want)
|
||||
}
|
||||
// ADR 0567: placement is the seat mesh-placement's: a merge of the catalogue carries no placement file and asks the
|
||||
// forge for none.
|
||||
forge.calls = nil
|
||||
msg, err = Convert(mergedSubject, mergedBody(map[string]any{"repo": "mesh-catalog"}), treesOf(&calls, "r0", map[string]string{"modules/a": "t1"}, nil), forge.get)
|
||||
if f.Placements != nil || f.PlacementsRead {
|
||||
t.Errorf("a merge of another repository than the catalogue carries placement: %v", f.Placements)
|
||||
}
|
||||
|
||||
// Issue 1126 (ADR 0549 §2): a merge of the catalogue carries every placement file of its merge commit, by node; a file
|
||||
// whose name is no node is left out; a listing or a file the forge cannot give is a retry, never a part.
|
||||
catalogue := mergedBody(map[string]any{"repo": "mesh-catalog"})
|
||||
forge.have["placement/ace.json"] = []byte(`{"v":1,"node":"ace","modules":[]}`)
|
||||
forge.have["placement/g14.json"] = []byte(`{"v":1,"node":"g14","modules":[{"module":"zsh"}]}`)
|
||||
listing := map[string]string{"ace.json": "b1", "g14.json": "b2", "README.md": "b3", "a.b.json": "b4"}
|
||||
var listedAt []string
|
||||
list := func(owner, repo, sha, dir string) (map[string]string, error) {
|
||||
listedAt = append(listedAt, owner+"/"+repo+"@"+sha+":"+dir)
|
||||
return listing, nil
|
||||
}
|
||||
msg, err = ConvertWith(mergedSubject, catalogue, treesOf(&calls, "r0", map[string]string{"modules/a": "t1"}, nil), forge.get, list)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if f, err = delivery.DecodeForge(msg.Data); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(f.RetiredPlacements) > 0 || f.RetiredRead || strings.Contains(string(msg.Data), `"placements`) {
|
||||
t.Errorf("a merge of the catalogue carries placement: %s", msg.Data)
|
||||
want := map[string]string{"ace": `{"v":1,"node":"ace","modules":[]}`, "g14": `{"v":1,"node":"g14","modules":[{"module":"zsh"}]}`}
|
||||
if !f.PlacementsRead || !reflect.DeepEqual(f.Placements, want) || !reflect.DeepEqual(listedAt, []string{"novox/mesh-catalog@" + merge + ":placement"}) {
|
||||
t.Errorf("placements = %v (read %v), listed at %v", f.Placements, f.PlacementsRead, listedAt)
|
||||
}
|
||||
for _, c := range forge.calls {
|
||||
if strings.HasPrefix(c.path, "placement") {
|
||||
t.Errorf("the forge was asked for %s", c.path)
|
||||
}
|
||||
listing = map[string]string{}
|
||||
if msg, err = ConvertWith(mergedSubject, catalogue, treesOf(&calls, "r0", map[string]string{"modules/a": "t1"}, nil), forge.get, list); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if f, _ = delivery.DecodeForge(msg.Data); !f.PlacementsRead || f.Placements != nil {
|
||||
t.Errorf("a catalogue with no placement file says it read none: %v, %v", f.Placements, f.PlacementsRead)
|
||||
}
|
||||
listing = map[string]string{"gone.json": "b5"}
|
||||
if _, err := ConvertWith(mergedSubject, catalogue, treesOf(&calls, "r0", map[string]string{"modules/a": "t1"}, nil), forge.get, list); !errors.Is(err, ErrRetry) || !strings.Contains(err.Error(), "placement/gone.json") {
|
||||
t.Errorf("a placement file the forge cannot give: %v", err)
|
||||
}
|
||||
failing := func(_, _, _, _ string) (map[string]string, error) { return nil, errors.New("forge down") }
|
||||
if _, err := ConvertWith(mergedSubject, catalogue, treesOf(&calls, "r0", map[string]string{"modules/a": "t1"}, nil), forge.get, failing); !errors.Is(err, ErrRetry) {
|
||||
t.Errorf("a listing the forge cannot give: %v", err)
|
||||
}
|
||||
// Issue 1126: a merge reads the forge only for the module directories whose tree changed.
|
||||
t.Run("a merge reads only the directories whose tree changed", mergeReadsOnlyChangedDirs)
|
||||
}
|
||||
|
||||
func TestPullWithKnownDirsCarriesTheSameAtTheHead(t *testing.T) { // 1448/AC2
|
||||
@@ -125,6 +146,55 @@ func TestPullWithKnownDirsCarriesTheSameAtTheHead(t *testing.T) { // 1448/AC2
|
||||
if f.Manifests != nil || f.Locks != nil || len(forge.calls) != 0 {
|
||||
t.Errorf("manifests %v locks %v calls %v", f.Manifests, f.Locks, forge.calls)
|
||||
}
|
||||
|
||||
// Issue 1126 (ADR 0549 §4): an updated pull request of the catalogue that touches placement/ carries every placement
|
||||
// file at its head, read whole or not at all; one that touches none, or of another repository, carries none and lists
|
||||
// nothing; paths the forge did not say or cut are read as touching; a closed one carries none.
|
||||
forge = &forgeFiles{have: map[string][]byte{"modules/a/module.json": manifest,
|
||||
"placement/ace.json": []byte(`{"v":1,"node":"ace","modules":[]}`), "placement/g14.json": []byte(`{"v":1,"node":"g14","modules":[]}`)}}
|
||||
listing := map[string]string{"ace.json": "b1", "g14.json": "b2", "README.md": "b3"}
|
||||
var listedAt []string
|
||||
list := func(owner, repo, sha, dir string) (map[string]string, error) {
|
||||
listedAt = append(listedAt, owner+"/"+repo+"@"+sha+":"+dir)
|
||||
return listing, nil
|
||||
}
|
||||
all := map[string]string{"ace": `{"v":1,"node":"ace","modules":[]}`, "g14": `{"v":1,"node":"g14","modules":[]}`}
|
||||
catalogue := map[string]any{"repo": "mesh-catalog", "paths": []string{"placement/g14.json"}}
|
||||
for _, row := range []struct {
|
||||
name string
|
||||
subject string
|
||||
body map[string]any
|
||||
want map[string]string
|
||||
read bool
|
||||
}{
|
||||
{"a catalogue pull touching placement", updatedSubject, catalogue, all, true},
|
||||
{"a catalogue pull touching no placement", updatedSubject, map[string]any{"repo": "mesh-catalog"}, nil, false},
|
||||
{"a catalogue pull whose paths are cut", updatedSubject, map[string]any{"repo": "mesh-catalog", "paths_truncated": true}, all, true},
|
||||
{"a catalogue pull whose paths are not said", updatedSubject, map[string]any{"repo": "mesh-catalog", "paths": nil}, all, true},
|
||||
{"another repository's pull touching placement/", updatedSubject, map[string]any{"paths": []string{"placement/g14.json"}}, nil, false},
|
||||
{"a closed catalogue pull", closedSubject, catalogue, nil, false},
|
||||
} {
|
||||
listedAt = nil
|
||||
body := updated(row.body)
|
||||
msg, err := ConvertWith(row.subject, body, nil, forge.get, list)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", row.name, err)
|
||||
}
|
||||
f, err := delivery.DecodeForge(msg.Data)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", row.name, err)
|
||||
}
|
||||
if f.PlacementsRead != row.read || !reflect.DeepEqual(f.Placements, row.want) || (len(listedAt) > 0) != row.read {
|
||||
t.Errorf("%s: placements %v (read %v), listed %v", row.name, f.Placements, f.PlacementsRead, listedAt)
|
||||
}
|
||||
if row.read && listedAt[0] != "novox/mesh-catalog@"+head+":placement" {
|
||||
t.Errorf("%s: listed at %v, want the head", row.name, listedAt)
|
||||
}
|
||||
}
|
||||
listing = map[string]string{"gone.json": "b5"}
|
||||
if _, err := ConvertWith(updatedSubject, updated(catalogue), nil, forge.get, list); !errors.Is(err, ErrRetry) || !strings.Contains(err.Error(), "placement/gone.json") {
|
||||
t.Errorf("a placement file the forge cannot give at the head: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAForgeErrorOrAMissingManifestIsRetryAndALockNotFoundIsLeftOut(t *testing.T) { // 1448/AC3
|
||||
@@ -299,73 +369,3 @@ func TestOneFailingFileIsRetryAndEveryCallHasReturnedBeforeTheReadDoes(t *testin
|
||||
t.Errorf("err = %v, want ErrRetry", err)
|
||||
}
|
||||
}
|
||||
|
||||
// memKept is a Kept in memory.
|
||||
type memKept map[string]Inputs
|
||||
|
||||
func (k memKept) Get(tree string) (Inputs, bool) { in, ok := k[tree]; return in, ok }
|
||||
func (k memKept) Put(tree string, in Inputs) { k[tree] = in }
|
||||
|
||||
// Issue 1126 (mesh-catalog#535; issue 1458 met the kind first): a merge reads the forge only for the module directories
|
||||
// whose tree changed. Its message is the same as one read whole; a directory whose tree is kept costs no read.
|
||||
func mergeReadsOnlyChangedDirs(t *testing.T) {
|
||||
manifest, sum := real(t, "module.json"), real(t, "go.sum")
|
||||
forge := &forgeFiles{have: map[string][]byte{"modules/a/module.json": manifest, "modules/a/go.sum": sum, "modules/b/module.json": []byte(`{"module":"b"}`)}}
|
||||
kept := memKept{}
|
||||
var calls []call
|
||||
first, err := ConvertKept(mergedSubject, mergedBody(nil), treesOf(&calls, "r0", map[string]string{"modules/a": "t1", "modules/b": "t2"}, nil), forge.get, kept)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(forge.calls) != 6 {
|
||||
t.Fatalf("the first merge read %d files, want 6 (a manifest and two lock files for each directory)", len(forge.calls))
|
||||
}
|
||||
// The next merge changes modules/b alone: only its three files are read, and the message equals one read whole.
|
||||
forge.calls = nil
|
||||
forge.have["modules/b/module.json"] = []byte(`{"module":"b","v":2}`)
|
||||
trees := map[string]string{"modules/a": "t1", "modules/b": "t3"}
|
||||
next, err := ConvertKept(mergedSubject, mergedBody(nil), treesOf(&calls, "r1", trees, nil), forge.get, kept)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, c := range forge.calls {
|
||||
if !strings.HasPrefix(c.path, "modules/b/") {
|
||||
t.Errorf("%s was read although its directory's tree is kept", c.path)
|
||||
}
|
||||
}
|
||||
if len(forge.calls) != 3 {
|
||||
t.Errorf("the second merge read %d files, want 3", len(forge.calls))
|
||||
}
|
||||
whole, err := Convert(mergedSubject, mergedBody(nil), treesOf(&calls, "r1", trees, nil), forge.get)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if string(next.Data) != string(whole.Data) {
|
||||
t.Errorf("the message from kept inputs differs from one read whole:\n%s\n%s", next.Data, whole.Data)
|
||||
}
|
||||
if string(first.Data) == string(next.Data) {
|
||||
t.Error("the second merge's message did not change")
|
||||
}
|
||||
// A failed read keeps the directories it read whole and nothing of the one that failed, so the next try reads only the
|
||||
// rest: a merge too large for one try's bound converges over its tries.
|
||||
kept2 := memKept{}
|
||||
failing := &forgeFiles{have: forge.have, errs: map[string]error{"modules/b/module.json": errors.New("context deadline exceeded")}}
|
||||
if _, err := ConvertKept(mergedSubject, mergedBody(nil), treesOf(&calls, "r1", trees, nil), failing.get, kept2); !errors.Is(err, ErrRetry) {
|
||||
t.Fatalf("err = %v, want ErrRetry", err)
|
||||
}
|
||||
if _, ok := kept2["t1"]; !ok || len(kept2) != 1 { // modules/a's files are asked first, before the failure is seen
|
||||
t.Errorf("a failed read kept %v, want modules/a's tree t1 alone", kept2)
|
||||
}
|
||||
failing.calls = nil
|
||||
failing.errs = nil
|
||||
again, err := ConvertKept(mergedSubject, mergedBody(nil), treesOf(&calls, "r1", trees, nil), failing.get, kept2)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if string(again.Data) != string(whole.Data) {
|
||||
t.Errorf("the message of the try after a failure differs from one read whole")
|
||||
}
|
||||
if len(failing.calls) != 3 {
|
||||
t.Errorf("the try after a failure read %d files, want only modules/b's 3", len(failing.calls))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,57 +0,0 @@
|
||||
package bridge
|
||||
|
||||
// A merge of a repository of many modules read three files of the forge for every module, whatever the merge changed:
|
||||
// the catalogue's 140 modules are 420 reads, which no longer fit the 30 s bound of a handled event while the forge is
|
||||
// busy after a merge, so the merged message of mesh-catalog#535 was never written and the indexer never saw it (issue
|
||||
// 1126; issue 1458 met the same kind at 131 modules and raised the reads in flight). A directory's git tree hash names
|
||||
// every byte under it, so the manifest and the lock-file hashes read once at a tree hash are those of every later commit
|
||||
// whose directory has the same tree: a merge reads the forge only for the directories it changed.
|
||||
|
||||
// Inputs is what the bridge read of one module directory at one tree hash: its manifest and its lock files' hex sha256
|
||||
// by name (none when it has none).
|
||||
type Inputs struct {
|
||||
Manifest string `json:"manifest"`
|
||||
Locks map[string]string `json:"locks,omitempty"`
|
||||
}
|
||||
|
||||
// Kept holds Inputs by a directory's git tree hash (the bridge's state bucket inputs). Get answers false for a tree it
|
||||
// does not hold or cannot read; Put is best effort: a value it could not keep is read from the forge next time.
|
||||
type Kept interface {
|
||||
Get(tree string) (Inputs, bool)
|
||||
Put(tree string, in Inputs)
|
||||
}
|
||||
|
||||
// keptInputs splits dirs into the inputs kept for their tree hash and the directories still to read from the forge.
|
||||
func keptInputs(dirs []string, trees map[string]string, kept Kept) (manifests map[string]string, locks map[string]map[string]string, read []string) {
|
||||
manifests, locks = map[string]string{}, map[string]map[string]string{}
|
||||
for _, dir := range dirs {
|
||||
tree := trees[dir]
|
||||
if kept == nil || tree == "" {
|
||||
read = append(read, dir)
|
||||
continue
|
||||
}
|
||||
in, ok := kept.Get(tree)
|
||||
if !ok {
|
||||
read = append(read, dir)
|
||||
continue
|
||||
}
|
||||
manifests[dir] = in.Manifest
|
||||
if len(in.Locks) > 0 {
|
||||
locks[dir] = in.Locks
|
||||
}
|
||||
}
|
||||
return manifests, locks, read
|
||||
}
|
||||
|
||||
// keep puts what was read of each directory under its tree hash.
|
||||
func keep(dirs []string, trees map[string]string, manifests map[string]string, locks map[string]map[string]string, kept Kept) {
|
||||
if kept == nil {
|
||||
return
|
||||
}
|
||||
for _, dir := range dirs {
|
||||
raw, ok := manifests[dir]
|
||||
if tree := trees[dir]; ok && tree != "" {
|
||||
kept.Put(tree, Inputs{Manifest: raw, Locks: locks[dir]})
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -103,12 +103,11 @@ func TestManifestDeclarations(t *testing.T) { // 1235/AC1
|
||||
if m.SecretsOwner != "mesh-delivery-forge-bridge" {
|
||||
t.Errorf("secrets-owner = %q", m.SecretsOwner)
|
||||
}
|
||||
// Three buckets: the head last written on FORGE for each pull request, so a sync writes only what differs (ADR 0443
|
||||
// §5), the revision of each key of the gate's buckets the bridge performed, so a key is performed once per
|
||||
// revision (ADR 0437 §8; issue 1341), and what a merge read of each module directory by its tree, so a merge reads
|
||||
// the forge only for the directories it changed (issue 1126).
|
||||
if !reflect.DeepEqual(m.State, []state{{Name: "heads", History: 1}, {Name: "performed", History: 1}, {Name: "inputs", History: 1}}) { // 1279/AC6, 1341, 1126
|
||||
t.Errorf("state = %+v, want the buckets heads, performed and inputs", m.State)
|
||||
// Two buckets: the head last written on FORGE for each pull request, so a sync writes only what differs (ADR 0443
|
||||
// §5), and the revision of each key of the gate's buckets the bridge performed, so a key is performed once per
|
||||
// revision (ADR 0437 §8; issue 1341).
|
||||
if !reflect.DeepEqual(m.State, []state{{Name: "heads", History: 1}, {Name: "performed", History: 1}}) { // 1279/AC6, 1341
|
||||
t.Errorf("state = %+v, want the buckets heads and performed", m.State)
|
||||
}
|
||||
// No other state, no bus, no seat, no tool, no event (ADR 0437 §4; ADR 0521: no check.status).
|
||||
for _, k := range []string{"logs", "emits", "claims", "provides", "requires", "uses", "defines-seats", "tools", "seat", "seats"} {
|
||||
|
||||
@@ -12,8 +12,7 @@
|
||||
],
|
||||
"state": [
|
||||
{ "name": "heads", "history": 1 },
|
||||
{ "name": "performed", "history": 1 },
|
||||
{ "name": "inputs", "history": 1 }
|
||||
{ "name": "performed", "history": 1 }
|
||||
],
|
||||
"reads": [
|
||||
"mesh-delivery-gate.merges",
|
||||
|
||||
@@ -29,9 +29,7 @@ func Tools(ask asker, now func() time.Time) []stdio.Tool {
|
||||
Description: "Take a registered module in now: the indexer reads its manifest from the repository and commit the controller's " +
|
||||
"build records register it from (at the head the gate's source names, when it names one), whatever it tried before, and " +
|
||||
"writes it to its modules state as a merge would. For a module registered from outside the indexed repositories whose " +
|
||||
"repository the forge could not read (a 404 before the forge's account could read it). A module the indexer holds " +
|
||||
"whose record stands behind the head the gate's source names (a merge whose merged message never reached the " +
|
||||
"indexer) is read again at that head and written when it changed. Answers stored, from which " +
|
||||
"repository the forge could not read (a 404 before the forge's account could read it). Answers stored, from which " +
|
||||
"repository, commit and path, or why not (held already, no registration, the forge's error, no manifest there). " +
|
||||
"The indexer reads such a module again by itself on the next source revision and merged message; this asks now. " +
|
||||
"indexer_module (mesh-delivery-view) reads the result.",
|
||||
@@ -49,6 +47,25 @@ func Tools(ask asker, now func() time.Time) []stdio.Tool {
|
||||
}
|
||||
return a.Result, nil
|
||||
}},
|
||||
{Name: index.PlacementRereadTool,
|
||||
Description: "Read the catalogue's placement files again now (novox/hq ADR 0559 §8): the indexer asks the forge for the trunk's head and " +
|
||||
"every open catalogue pull request's, and writes its states placement (placement.<node>) and placement-candidates (<group>.<node>) " +
|
||||
"as a merge and a pull request's head would, only where a key differs; a candidate key whose file no longer differs, or whose pull " +
|
||||
"request is closed, is deleted. The recovery for a forge message the indexer missed. Answers the heads read, the keys written or " +
|
||||
"deleted in each state, or why nothing was (the forge could not say). indexer_placement (mesh-delivery-view) reads the result.",
|
||||
Input: map[string]any{"type": "object", "properties": map[string]any{
|
||||
"why": map[string]any{"type": "string", "description": "why it is read again now, in plain words"}}},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
key, value, err := index.NewPlacementReread(str(args, "why"), stdio.CallerOf(args), now())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
a, err := ask(key, value)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return a.Reread, nil
|
||||
}},
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -18,7 +18,6 @@ import (
|
||||
"os"
|
||||
"os/signal"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
@@ -146,50 +145,25 @@ type delivered interface {
|
||||
Ack() error
|
||||
DoubleAck(ctx context.Context) error
|
||||
Term() error
|
||||
Metadata() (*jetstream.MsgMetadata, error)
|
||||
}
|
||||
|
||||
// handle takes one message of FORGE into the candidates and acknowledges it. A message it can never read is terminated,
|
||||
// never redelivered, and the log names its subject and why. A write that failed stays unacknowledged: the server
|
||||
// delivers it again (the server's ack wait is the only clock). Each message handled is written to the heard state
|
||||
// (index.Heard; nil, or a state not on the bus yet, writes none), so indexer_heard reads what the indexer heard of FORGE.
|
||||
func handle(ctx context.Context, m delivered, ix *index.Indexer, heard index.Store, logger *log.Logger) {
|
||||
h := index.Heard{Subject: m.Subject()}
|
||||
if md, err := m.Metadata(); err == nil && md != nil {
|
||||
h.Sequence, h.Delivered, h.Published = md.Sequence.Stream, md.NumDelivered, md.Timestamp.UTC()
|
||||
}
|
||||
defer func() {
|
||||
if heard == nil {
|
||||
return
|
||||
}
|
||||
h.Handled = time.Now().UTC()
|
||||
if err := index.WriteHeard(ctx, heard, h); err != nil && !errors.Is(err, process.ErrNotYet) {
|
||||
logger.Printf("%s: sequence %d is not written to the heard state: %v", h.Subject, h.Sequence, err)
|
||||
}
|
||||
}()
|
||||
// delivers it again (the server's ack wait is the only clock).
|
||||
func handle(ctx context.Context, m delivered, ix *index.Indexer, logger *log.Logger) {
|
||||
f, err := delivery.DecodeForge(m.Data())
|
||||
if err != nil {
|
||||
logger.Printf("%s: not a forge message, dropped: %v", m.Subject(), err)
|
||||
h.Outcome, h.Words = index.HeardDropped, err.Error()
|
||||
_ = m.Term()
|
||||
return
|
||||
}
|
||||
h.Kind, h.Repository, h.Number, h.Branch, h.Commit = f.Kind, f.Repository, f.Number, f.Branch, f.Head
|
||||
if f.Merge != "" {
|
||||
h.Commit = f.Merge
|
||||
}
|
||||
applied, err := ix.ApplyMessage(ctx, f)
|
||||
h.Written, h.Late = applied.Written, applied.Late
|
||||
switch {
|
||||
switch err := ix.Apply(ctx, f); {
|
||||
case errors.Is(err, index.ErrUnreadable):
|
||||
logger.Printf("%s: %v; dropped", m.Subject(), err)
|
||||
h.Outcome, h.Words = index.HeardDropped, err.Error()
|
||||
_ = m.Term()
|
||||
case err != nil:
|
||||
logger.Printf("%s: %s#%d left for redelivery: %v", m.Subject(), f.Repository, f.Number, err)
|
||||
h.Outcome, h.Words = index.HeardLeft, err.Error()
|
||||
default:
|
||||
h.Outcome = index.HeardApplied
|
||||
// Waits for the server, so the consumer's info no longer counts the message as unacknowledged when the fill that
|
||||
// follows asks whether the stream is applied (issue 1280); a failed ack is redelivered after the ack wait, which
|
||||
// kicks the fill again.
|
||||
@@ -273,6 +247,12 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
// Its taken state (ADR 0556 clause 1) is bound when it appears, as every state a delivery adds (issue 1790): until then
|
||||
// a merge's taken key is said and not written, and the rest of the indexer runs.
|
||||
taken := process.AwaitBucket(ctx, js, conn, delivery.TakenBucket, "no repository's taken key is written", logger.Printf)
|
||||
// Its placement state (ADR 0549 §2) is bound when it appears, as every state a delivery adds (issue 1790): until then a
|
||||
// catalogue merge's placement is said and not written, and the rest of the indexer runs.
|
||||
placement := process.AwaitBucket(ctx, js, conn, delivery.PlacementBucket, "no catalogue merge's placement is written", logger.Printf)
|
||||
// Its placement candidates (issue 1126, ADR 0549 §4) are bound the same way: until the state appears a catalogue pull
|
||||
// request's placement is said and not written, so no node answers it and its check names every placed node as silent.
|
||||
placementCandidates := process.AwaitBucket(ctx, js, conn, delivery.PlacementCandidatesBucket, "no catalogue pull request's placement candidate is written", logger.Printf)
|
||||
// The controller's build records only name the directory of a module whose key names none (fillPaths): without them
|
||||
// such a module waits for its path, every other is indexed (issue 1945).
|
||||
builds := process.OptionalBucket(ctx, js, delivery.BuildsBucket, "a module whose key names no directory is not given one from the controller's build records until the indexer starts again", logger.Printf)
|
||||
@@ -304,7 +284,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
defer calls.Close()
|
||||
ix := &index.Indexer{Store: kvStore{candidates}, Modules: kvStore{modules}, Manifests: kvStore{manifests}, Taken: awaitedStore{taken}, Builds: known, Logf: logger.Printf,
|
||||
Named: namedVia(calls.Call), Trunk: trunkVia(calls.Call), Contains: containsVia(calls.Call), Read: readVia(calls.Call),
|
||||
ReadOpen: openVia(calls.Call)}
|
||||
Placement: awaitedStore{placement}, PlacementCandidates: awaitedStore{placementCandidates}, ReadPlacement: placementVia(calls.Call), Say: func(event string, body any) error { return calls.Emit(event, body) }}
|
||||
// The gate's source bucket (mesh-delivery-gate_source) comes with the gate (release 3e, novox/hq ADR 0450 §1): its key
|
||||
// main is the mesh's source revision, which stamps the modules (issue 1344). Where it does not exist yet the indexer does
|
||||
// not wait for it (that needs a timer, ADR 0437 §8): it says so, and is started again with the gate.
|
||||
@@ -424,13 +404,17 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
// Essential: the forge's merges are what the indexer indexes.
|
||||
return fmt.Errorf("the consumer %s_%s on %s cannot be read (%v): the indexer hears no merge without it, so it stops; where the bus refused it or did not answer, this module's consumes (made by the controller with its grant) have not reached the bus yet", cred.Node, cred.Module, forgeStream, err)
|
||||
}
|
||||
// A merge applied while the taken state was missing gets its taken key (issue 1280): when the state appears (Ready runs
|
||||
// once, now or then), and after each applied message. The gate's source heads never move a key (issue 1126).
|
||||
// A merge applied while the taken state was missing, and the gate's heads once the forge stream is applied whole, get
|
||||
// their taken key (issue 1280): when the state appears (Ready runs once, now or then), and after each applied message.
|
||||
ix.Drained = func(ctx context.Context) (bool, error) {
|
||||
info, err := cons.Info(ctx)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return info.NumPending == 0 && info.NumAckPending == 0, nil
|
||||
}
|
||||
taken.Ready(func(jetstream.KeyValue) { ix.KickFill(ctx) })
|
||||
// What the indexer heard of FORGE (issue 1126 note 76) is bound when its state appears, as every state a delivery adds
|
||||
// (issue 1790): until then no message is written there, and the rest of the indexer runs.
|
||||
heard := awaitedStore{process.AwaitBucket(ctx, js, conn, index.HeardBucket, "no FORGE message the indexer handles is written for indexer_heard", logger.Printf)}
|
||||
cc, err := process.HearEvents(cons, func(m jetstream.Msg) { handle(ctx, m, ix, heard, logger); ix.KickFill(ctx) }, logger.Printf)
|
||||
cc, err := process.HearEvents(cons, func(m jetstream.Msg) { handle(ctx, m, ix, logger); ix.KickFill(ctx) }, logger.Printf)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -3,13 +3,9 @@ package main
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"log"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
@@ -33,21 +29,6 @@ func (f *fakeMsg) DoubleAck(context.Context) error {
|
||||
}
|
||||
func (f *fakeMsg) Term() error { f.termd++; return nil }
|
||||
|
||||
// Metadata is the stream's word on the message: its sequence 41, delivered a second time (issue 1126 note 76).
|
||||
func (f *fakeMsg) Metadata() (*jetstream.MsgMetadata, error) {
|
||||
return &jetstream.MsgMetadata{Sequence: jetstream.SequencePair{Stream: 41, Consumer: 7}, NumDelivered: 2, Timestamp: time.Date(2026, 10, 28, 13, 7, 42, 0, time.UTC)}, nil
|
||||
}
|
||||
|
||||
// heardOf is the indexer's record of the last FORGE message it handled (index.HeardKey).
|
||||
func heardOf(t *testing.T, st *memStore) index.Heard {
|
||||
t.Helper()
|
||||
var h index.Heard
|
||||
if err := json.Unmarshal(st.vals[index.HeardKey], &h); err != nil {
|
||||
t.Fatalf("heard.%s = %q: %v", index.HeardKey, st.vals[index.HeardKey], err)
|
||||
}
|
||||
return h
|
||||
}
|
||||
|
||||
type memStore struct{ vals map[string][]byte }
|
||||
|
||||
func (m *memStore) Get(_ context.Context, k string) ([]byte, uint64, bool, error) {
|
||||
@@ -76,32 +57,21 @@ func TestAMessageIsIndexedAndAcked(t *testing.T) { // 1238/AC2 (unit part)
|
||||
data, _ := delivery.EncodeForge(delivery.Forge{Kind: delivery.ForgePull, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: head,
|
||||
Action: delivery.PullUpdated, Dirs: []string{"modules/a"}, DirsKnown: true})
|
||||
m := &fakeMsg{subject: "mesh.stream.mesh-delivery-forge-bridge.event.forge.novox.mesh-delivery.pull", data: data}
|
||||
heard := &memStore{vals: map[string][]byte{}}
|
||||
handle(context.Background(), m, ix, heard, log.New(&bytes.Buffer{}, "", 0))
|
||||
handle(context.Background(), m, ix, log.New(&bytes.Buffer{}, "", 0))
|
||||
c, err := delivery.DecodeCandidate(st.vals["feat/x"])
|
||||
if err != nil || len(c.Pulls) != 1 || c.Pulls[0].Modules[0] != "a" || m.acked != 1 || m.doubled != 1 || m.termd != 0 {
|
||||
t.Errorf("candidate %+v err %v acked %d terminated %d", c, err, m.acked, m.termd)
|
||||
}
|
||||
// Issue 1126 note 76: what the indexer heard of FORGE is readable (indexer_heard): subject, sequence, deliveries, time,
|
||||
// repository, commit and what it did.
|
||||
if h := heardOf(t, heard); h.Subject != m.subject || h.Sequence != 41 || h.Delivered != 2 || h.Published.Hour() != 13 || h.Kind != delivery.ForgePull ||
|
||||
h.Repository != "novox/mesh-delivery" || h.Number != 12 || h.Commit != head || h.Outcome != index.HeardApplied || h.Handled.IsZero() {
|
||||
t.Errorf("heard = %+v", h)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAMessageThatIsNoForgeMessageIsTerminatedAndLogged(t *testing.T) { // 1238 (never redelivered)
|
||||
var logs bytes.Buffer
|
||||
st := &memStore{vals: map[string][]byte{}}
|
||||
m := &fakeMsg{subject: "mesh.stream.mesh-delivery-forge-bridge.event.forge.x.y.pull", data: []byte(`{"v":1,"kind":"pull"}`)}
|
||||
heard := &memStore{vals: map[string][]byte{}}
|
||||
handle(context.Background(), m, &index.Indexer{Store: st, Builds: index.NewBuilds()}, heard, log.New(&logs, "", 0))
|
||||
handle(context.Background(), m, &index.Indexer{Store: st, Builds: index.NewBuilds()}, log.New(&logs, "", 0))
|
||||
if m.termd != 1 || m.acked != 0 || len(st.vals) != 0 || !strings.Contains(logs.String(), m.subject) {
|
||||
t.Errorf("terminated %d acked %d keys %d log %q", m.termd, m.acked, len(st.vals), logs.String())
|
||||
}
|
||||
if h := heardOf(t, heard); h.Outcome != index.HeardDropped || h.Sequence != 41 || h.Words == "" {
|
||||
t.Errorf("heard = %+v", h)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheSourceMainKeyIsTakenAtItsRevision(t *testing.T) { // 1344/AC1 (the watch's side: key, revision, value)
|
||||
|
||||
@@ -171,35 +171,108 @@ func readVia(call toolCaller) func(ctx context.Context, repository, sha, dir str
|
||||
}
|
||||
}
|
||||
|
||||
// The forge's tool for the open pull requests, and the trunk branch they are open into.
|
||||
// The forge's tools the placement reread asks (indexer_placement_reread, novox/hq ADR 0559 §8): a repository's branches
|
||||
// with their commits, and every open pull request with its branch and head.
|
||||
const (
|
||||
openPullsTool = "gitea.gitea_open_pulls"
|
||||
trunkBranch = "main"
|
||||
listBranchesTool = "gitea.gitea_list_branches"
|
||||
openPullsTool = "gitea.gitea_open_pulls"
|
||||
placementTrunk = "main"
|
||||
)
|
||||
|
||||
// openVia asks the forge which pull requests into the trunk are open now (gitea_open_pulls), every repository's.
|
||||
func openVia(call toolCaller) func(ctx context.Context) ([]delivery.OpenPull, error) {
|
||||
return func(context.Context) ([]delivery.OpenPull, error) {
|
||||
raw, err := call(openPullsTool, map[string]any{})
|
||||
// placementVia reads the catalogue's placement now through call: the trunk's head (gitea_list_branches), every placement
|
||||
// file at it, and each open pull request of the catalogue into the trunk (gitea_open_pulls) with every placement file at
|
||||
// its head, each read whole or not at all (gitea_commit_trees with listed, gitea_get_file).
|
||||
func placementVia(call toolCaller) func(ctx context.Context) (index.PlacementRead, error) {
|
||||
owner, repo, _ := strings.Cut(delivery.CatalogueRepository, "/")
|
||||
filesAt := func(sha string) (map[string]string, error) {
|
||||
raw, err := call(commitTreesTool, map[string]any{"owner": owner, "repo": repo, "sha": sha, "listed": []string{delivery.PlacementDir}})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var a struct {
|
||||
Listed map[string]map[string]string `json:"listed"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &a); err != nil || a.Listed == nil {
|
||||
return nil, fmt.Errorf("the answer of %s lists no directory: %v", commitTreesTool, err)
|
||||
}
|
||||
var out map[string]string
|
||||
for name := range a.Listed[delivery.PlacementDir] {
|
||||
file := delivery.PlacementDir + "/" + name
|
||||
node, ok := delivery.PlacementFileNode(file)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
raw, err := call(getFileTool, map[string]string{"owner": owner, "repo": repo, "path": file, "ref": sha})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s at %s: %w", file, sha, err)
|
||||
}
|
||||
var f struct {
|
||||
File *struct {
|
||||
Content *string `json:"content"`
|
||||
} `json:"file"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &f); err != nil || f.File == nil || f.File.Content == nil {
|
||||
return nil, fmt.Errorf("the answer of %s for %s is not {file: {content}}: %v", getFileTool, file, err)
|
||||
}
|
||||
if out == nil {
|
||||
out = map[string]string{}
|
||||
}
|
||||
out[node] = *f.File.Content
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
return func(context.Context) (index.PlacementRead, error) {
|
||||
raw, err := call(listBranchesTool, map[string]string{"owner": owner, "repo": repo})
|
||||
if err != nil {
|
||||
return index.PlacementRead{}, err
|
||||
}
|
||||
var b struct {
|
||||
Branches []struct {
|
||||
Name string `json:"name"`
|
||||
Commit string `json:"commit"`
|
||||
} `json:"branches"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &b); err != nil {
|
||||
return index.PlacementRead{}, fmt.Errorf("the answer of %s is not {branches}: %w", listBranchesTool, err)
|
||||
}
|
||||
var out index.PlacementRead
|
||||
for _, br := range b.Branches {
|
||||
if br.Name == placementTrunk {
|
||||
out.Trunk = br.Commit
|
||||
}
|
||||
}
|
||||
if out.Trunk == "" {
|
||||
return index.PlacementRead{}, fmt.Errorf("%s names no branch %s of %s", listBranchesTool, placementTrunk, delivery.CatalogueRepository)
|
||||
}
|
||||
if out.Files, err = filesAt(out.Trunk); err != nil {
|
||||
return index.PlacementRead{}, err
|
||||
}
|
||||
raw, err = call(openPullsTool, map[string]any{})
|
||||
if err != nil {
|
||||
return index.PlacementRead{}, err
|
||||
}
|
||||
var o struct {
|
||||
Pulls *[]struct {
|
||||
Owner string `json:"owner"`
|
||||
Repo string `json:"repo"`
|
||||
Number int `json:"number"`
|
||||
Base string `json:"base"`
|
||||
Owner string `json:"owner"`
|
||||
Repo string `json:"repo"`
|
||||
Number int `json:"number"`
|
||||
Base string `json:"base"`
|
||||
Head string `json:"head"`
|
||||
HeadSHA string `json:"head_sha"`
|
||||
} `json:"pulls"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &o); err != nil || o.Pulls == nil {
|
||||
return nil, fmt.Errorf("the answer of %s is not {pulls}: %v", openPullsTool, err)
|
||||
return index.PlacementRead{}, fmt.Errorf("the answer of %s is not {pulls}: %v", openPullsTool, err)
|
||||
}
|
||||
out := []delivery.OpenPull{}
|
||||
for _, p := range *o.Pulls {
|
||||
if p.Base == trunkBranch && p.Number > 0 {
|
||||
out = append(out, delivery.OpenPull{Repository: p.Owner + "/" + p.Repo, Number: p.Number})
|
||||
if !delivery.SameRepository(p.Owner+"/"+p.Repo, delivery.CatalogueRepository) || p.Base != placementTrunk || p.Number <= 0 || p.Head == "" || p.HeadSHA == "" {
|
||||
continue
|
||||
}
|
||||
files, err := filesAt(p.HeadSHA)
|
||||
if err != nil {
|
||||
return index.PlacementRead{}, fmt.Errorf("%s#%d: %w", delivery.CatalogueRepository, p.Number, err)
|
||||
}
|
||||
out.Pulls = append(out.Pulls, index.PullPlacement{Branch: p.Head, Number: p.Number, Head: p.HeadSHA, Files: files})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
@@ -14,6 +14,7 @@ import (
|
||||
// takerIn is the part of the indexer a takeins value asks through.
|
||||
type takerIn interface {
|
||||
TakeIn(ctx context.Context, module string) index.TakenIn
|
||||
RereadPlacement(ctx context.Context) index.PlacementReread
|
||||
}
|
||||
|
||||
// takenIn answers one value of the takeins state (the tool indexer_take_in, issue 1945): a value not yet answered takes the
|
||||
@@ -31,9 +32,15 @@ func takenIn(ctx context.Context, ix takerIn, kv jetstream.KeyValue, e jetstream
|
||||
if !a.Answered.IsZero() {
|
||||
return
|
||||
}
|
||||
r := ix.TakeIn(ctx, a.Module)
|
||||
a.Result, a.Answered = &r, time.Now().UTC()
|
||||
logf("%s: a take-in asked by %s (%s): %s", a.Module, a.By, a.Why, r.Answer)
|
||||
if a.Placement {
|
||||
r := ix.RereadPlacement(ctx)
|
||||
a.Reread, a.Answered = &r, time.Now().UTC()
|
||||
logf("a placement reread asked by %s (%s): %s", a.By, a.Why, r.Answer)
|
||||
} else {
|
||||
r := ix.TakeIn(ctx, a.Module)
|
||||
a.Result, a.Answered = &r, time.Now().UTC()
|
||||
logf("%s: a take-in asked by %s (%s): %s", a.Module, a.By, a.Why, r.Answer)
|
||||
}
|
||||
data, err := json.Marshal(a)
|
||||
if err != nil {
|
||||
return
|
||||
|
||||
@@ -3,7 +3,7 @@ module mesh-delivery-indexer
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028174713-38dbe1d3a388
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027193820-8ad9372b1952
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028174713-38dbe1d3a388 h1:ycu/sNieTdLpD3bzhZOCXdzcoqXUOxPRmqWV019REpo=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028174713-38dbe1d3a388/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027193820-8ad9372b1952 h1:TmveT3Bija643RGWkilvkzP/h8UWlVvHFZZtQfJK5oU=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027193820-8ad9372b1952/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
|
||||
@@ -65,10 +65,19 @@ type Indexer struct {
|
||||
// Contains says whether a commit is the second or an ancestor of it (gitea.gitea_contains).
|
||||
Contains func(ctx context.Context, repository, commit, in string) (bool, error)
|
||||
Logf func(format string, args ...any)
|
||||
// ReadOpen asks the forge which pull requests into the trunk are open now (gitea_open_pulls). A catalogue pull message
|
||||
// that finds the candidates of another pull request asks it and deletes those of a pull request no longer open, so a
|
||||
// closed message the indexer missed heals at the next one (issue 1126); nil asks none.
|
||||
ReadOpen func(ctx context.Context) ([]delivery.OpenPull, error)
|
||||
// Placement is the placement bucket (delivery.PlacementBucket, ADR 0549 §2); nil writes no placement.
|
||||
Placement Store
|
||||
// PlacementCandidates is the placement candidates bucket (delivery.PlacementCandidatesBucket, issue 1126); nil writes
|
||||
// no candidate (placement_candidates.go).
|
||||
PlacementCandidates Store
|
||||
// ReadPlacement asks the forge for the catalogue's placement now, at the trunk's head and each open pull request's
|
||||
// (indexer_placement_reread, ADR 0559 §8); nil rereads none.
|
||||
ReadPlacement func(ctx context.Context) (PlacementRead, error)
|
||||
// Say says one of the indexer's events (placement-changed); nil says none.
|
||||
Say func(event string, body any) error
|
||||
// Drained says whether the indexer has applied every message of the forge stream (the consumer has none pending and none
|
||||
// unacknowledged); FillTaken writes the source's heads only then. Nil fills none from the source (taken.go).
|
||||
Drained func(ctx context.Context) (bool, error)
|
||||
|
||||
mu sync.Mutex // guards source, toolchains and previews, which the watches and the forge messages all reach
|
||||
source sourceNow
|
||||
@@ -96,54 +105,47 @@ func (ix *Indexer) logf(format string, args ...any) {
|
||||
// by compare-and-set on the revision read, and read again on a conflict. A lone pull request is a group of one. A
|
||||
// merged message also writes the fingerprint of each module of its repository (modules.go).
|
||||
func (ix *Indexer) Apply(ctx context.Context, f delivery.Forge) error {
|
||||
_, err := ix.ApplyMessage(ctx, f)
|
||||
return err
|
||||
}
|
||||
|
||||
// ApplyMessage is Apply, and answers which modules a merged message wrote and kept (for the indexer's heard state). Each
|
||||
// applied merged message is said in one line: its repository, commit and pull request, and the modules written.
|
||||
func (ix *Indexer) ApplyMessage(ctx context.Context, f delivery.Forge) (Applied, error) {
|
||||
var applied Applied
|
||||
if f.Kind == delivery.ForgeOpen {
|
||||
return applied, ix.applyOpen(ctx, f)
|
||||
return ix.applyOpen(ctx, f)
|
||||
}
|
||||
if f.Kind != delivery.ForgePull && f.Kind != delivery.ForgeMerged {
|
||||
return applied, nil
|
||||
return nil
|
||||
}
|
||||
if f.Kind == delivery.ForgeMerged && ix.Modules != nil {
|
||||
w, err := ix.applyMerged(ctx, f)
|
||||
applied = Applied{Written: w.written, Late: w.late}
|
||||
if err != nil {
|
||||
return applied, err
|
||||
if err := ix.applyMerged(ctx, f); err != nil {
|
||||
return err
|
||||
}
|
||||
ix.logf("%s", w.said(f.Number))
|
||||
if w.late == "" { // a late message leaves the taken key, which names a later merge
|
||||
if err := ix.writeTakenKnown(ctx, f, w.asked); err != nil {
|
||||
return applied, err
|
||||
}
|
||||
if err := ix.writeTaken(ctx, f); err != nil {
|
||||
return err
|
||||
}
|
||||
ix.kickRetry(ctx) // a registered module the forge could not read is read again on a merge, in the background (issue 1945)
|
||||
}
|
||||
if err := ix.applyPlacement(ctx, f); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := ix.applyPlacementCandidates(ctx, f); err != nil {
|
||||
return err
|
||||
}
|
||||
var digests map[string]string
|
||||
var held map[string]delivery.Module
|
||||
if f.Kind == delivery.ForgePull {
|
||||
var err error
|
||||
if digests, err = ix.keepManifests(ctx, f); err != nil {
|
||||
return applied, err
|
||||
return err
|
||||
}
|
||||
if held, err = ix.heldModules(ctx); err != nil {
|
||||
return applied, err
|
||||
return err
|
||||
}
|
||||
}
|
||||
if trunkLike[f.Branch] {
|
||||
ix.logf("%s#%d: its branch %q is no delivery group, so no candidate", f.Repository, f.Number, f.Branch)
|
||||
return applied, nil
|
||||
return nil
|
||||
}
|
||||
key, err := delivery.CandidatesKey(f.Branch)
|
||||
if err != nil {
|
||||
return applied, fmt.Errorf("%w: %s#%d: %v", ErrUnreadable, f.Repository, f.Number, err)
|
||||
return fmt.Errorf("%w: %s#%d: %v", ErrUnreadable, f.Repository, f.Number, err)
|
||||
}
|
||||
return applied, write(ctx, ix.Store, key, func(value []byte, found bool) ([]byte, bool, error) {
|
||||
return write(ctx, ix.Store, key, func(value []byte, found bool) ([]byte, bool, error) {
|
||||
var cur *delivery.Candidate
|
||||
if found {
|
||||
c, err := delivery.DecodeCandidate(value)
|
||||
@@ -178,6 +180,14 @@ func (ix *Indexer) applyOpen(ctx context.Context, f delivery.Forge) error {
|
||||
for _, p := range f.Open {
|
||||
named[p] = true
|
||||
}
|
||||
if ix.PlacementCandidates != nil {
|
||||
// A pull request the forge no longer has open keeps no placement candidate (issue 1126).
|
||||
if err := ix.dropPlacementCandidates(ctx, func(_ string, c delivery.PlacementCandidate) bool {
|
||||
return !named[delivery.OpenPull{Repository: c.Pull.Repository, Number: c.Pull.Number}]
|
||||
}, "its pull request is not open"); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
keys, err := ix.Store.Keys(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("listing the candidates: %w", err)
|
||||
|
||||
@@ -1,60 +0,0 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// What the indexer heard of the forge bridge's stream FORGE (issue 1126 note 76): no tool read FORGE's messages, so a merge
|
||||
// written back over a later one could not be traced to a late or repeated message. The indexer writes, for each message
|
||||
// it handles, one value under the key forge of its state heard: the message's subject, stream sequence, how often the
|
||||
// server delivered it, when it was published and handled, what it names and what the indexer did with it. The state keeps
|
||||
// the last 64 values (its history); mesh-delivery-view's indexer_heard reads them, newest first, with their times.
|
||||
const (
|
||||
HeardState = "heard"
|
||||
HeardBucket = delivery.IndexerModule + "_" + HeardState
|
||||
HeardKey = "forge"
|
||||
// What the indexer did with a message.
|
||||
HeardApplied = "applied"
|
||||
HeardLeft = "left for redelivery"
|
||||
HeardDropped = "dropped"
|
||||
)
|
||||
|
||||
// Heard is one FORGE message as the indexer handled it.
|
||||
type Heard struct {
|
||||
V int `json:"v"`
|
||||
Subject string `json:"subject"`
|
||||
Sequence uint64 `json:"sequence"` // the stream's sequence of the message
|
||||
Delivered uint64 `json:"delivered,omitempty"` // how often the server delivered it to the indexer, this time included
|
||||
Published time.Time `json:"published,omitzero"`
|
||||
Handled time.Time `json:"handled"`
|
||||
Kind string `json:"kind,omitempty"`
|
||||
Repository string `json:"repository,omitempty"`
|
||||
Number int `json:"number,omitempty"`
|
||||
Branch string `json:"branch,omitempty"`
|
||||
Commit string `json:"commit,omitempty"` // the merge commit of a merged message, else the pull request's head
|
||||
Outcome string `json:"outcome"`
|
||||
Words string `json:"words,omitempty"` // why it was left or dropped
|
||||
Written []string `json:"written,omitempty"` // the modules a merged message wrote
|
||||
Late string `json:"late,omitempty"` // the taken merge a merged message is older than: nothing of it was written (order.go)
|
||||
}
|
||||
|
||||
// Applied is what Apply did with a merged message's modules: the keys written, and, for a message older than the
|
||||
// repository's taken merge, that merge (nothing was written).
|
||||
type Applied struct {
|
||||
Written []string
|
||||
Late string
|
||||
}
|
||||
|
||||
// WriteHeard writes h as the newest value of heard.forge.
|
||||
func WriteHeard(ctx context.Context, st Store, h Heard) error {
|
||||
h.V = 1
|
||||
data, err := json.Marshal(h)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return write(ctx, st, HeardKey, func([]byte, bool) ([]byte, bool, error) { return data, false, nil })
|
||||
}
|
||||
@@ -43,6 +43,8 @@ func withManifests(f delivery.Forge, manifests map[string]string, locks map[stri
|
||||
}
|
||||
|
||||
func TestAMergedManifestIsKeptByItsDigestAndNamedByTheModule(t *testing.T) { // 1449/AC1
|
||||
// A merge's placement files are written as its manifests are (issue 1126): folded here to hold the count.
|
||||
placementIsWrittenAtAMergeOnlyWhereItChanged(t)
|
||||
avahi := catalogueManifest(t, "avahi")
|
||||
ix, mods, manifests := manifestsIndexer(t, rec(t, "build-1", "avahi", catalogueRepo, "modules/avahi", "main", "", "2026-10-19T09:00:00Z", aInputs))
|
||||
f := withManifests(mergedMsg(catalogueRepo, map[string]string{"modules/avahi": "newtree"}), map[string]string{"modules/avahi": avahi}, nil)
|
||||
|
||||
@@ -24,30 +24,21 @@ import (
|
||||
// clause 7). Two directories naming one module write the one the bucket holds, or the first by name, and say the other.
|
||||
// A value the module already holds but for its commit is not written again: its commit stays the one its fingerprint first
|
||||
// appeared at (writeOverlay), so a merge writes only the modules it changes.
|
||||
//
|
||||
// A merge never moves a module backwards (order.go): a merged message older than the repository's taken merge writes and
|
||||
// deletes nothing, and is said; it is asked once, before anything is written.
|
||||
func (ix *Indexer) applyMerged(ctx context.Context, f delivery.Forge) (*mergeWrite, error) {
|
||||
func (ix *Indexer) applyMerged(ctx context.Context, f delivery.Forge) error {
|
||||
return ix.applyManifests(ctx, f, true)
|
||||
}
|
||||
|
||||
// applyManifests is applyMerged; drop says whether the modules of the repository the message holds no manifest for are
|
||||
// deleted. A message that names only some of a repository's modules (a registered module read on its own, registered.go)
|
||||
// drops none.
|
||||
func (ix *Indexer) applyManifests(ctx context.Context, f delivery.Forge, drop bool) (*mergeWrite, error) {
|
||||
w := newMergeWrite(f)
|
||||
if drop {
|
||||
if err := ix.lateMerge(ctx, w); err != nil || w.late != "" {
|
||||
return w, err
|
||||
}
|
||||
}
|
||||
func (ix *Indexer) applyManifests(ctx context.Context, f delivery.Forge, drop bool) error {
|
||||
digests, err := ix.keepManifests(ctx, f)
|
||||
if err != nil {
|
||||
return w, err
|
||||
return err
|
||||
}
|
||||
held, err := ix.heldModules(ctx)
|
||||
if err != nil {
|
||||
return w, err
|
||||
return err
|
||||
}
|
||||
objects, namedErr := ix.namedObjects(ctx, f)
|
||||
overlay := map[string]delivery.Module{}
|
||||
@@ -102,11 +93,11 @@ func (ix *Indexer) applyManifests(ctx context.Context, f delivery.Forge, drop bo
|
||||
// (issue 1900). A message that carries no manifests (an older writer) or one that cannot be read whole says nothing.
|
||||
if drop && len(f.Manifests) > 0 && every {
|
||||
if err := ix.dropRemoved(ctx, f.Repository, held, func(key string, _ delivery.Module) bool { return named[key] }); err != nil {
|
||||
return w, err
|
||||
return err
|
||||
}
|
||||
}
|
||||
// What this merge moved reaches the modules standing on it (a base's tree, build-agent's fingerprint), in the graph's order.
|
||||
return w, ix.recomputeWith(ctx, overlay, w)
|
||||
return ix.recomputeWith(ctx, overlay)
|
||||
}
|
||||
|
||||
// baseFP answers the new-form fingerprint of a base module as modules.<base> holds it, or why it has none.
|
||||
|
||||
@@ -3,8 +3,6 @@ package index
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"path"
|
||||
"strings"
|
||||
"testing"
|
||||
@@ -99,17 +97,10 @@ func TestAnUnchangedModuleWritesNothing(t *testing.T) { // 1239 (unchanged value
|
||||
commit map[string]string
|
||||
// taken is the merge the repository's taken key names after then (issue 1280, ADR 0556 clause 1).
|
||||
taken string
|
||||
// err says then fails (the message is left for redelivery); said is a line the indexer's log (or that error) must hold.
|
||||
err bool
|
||||
said string
|
||||
// contains, when set, is the forge's order, and asks how often then asks it (one question per merged message).
|
||||
contains func(context.Context, string, string, string) (bool, error)
|
||||
asks int
|
||||
}{
|
||||
{name: "the same merge again", writes: 0, commit: map[string]string{"a": merge, "b": merge, "c": merge}, taken: merge,
|
||||
then: func(ix *Indexer) error { return ix.Apply(context.Background(), mergedMsg(repo, three)) }},
|
||||
{name: "a merge touching one of three modules", writes: 1, commit: map[string]string{"a": next, "b": merge, "c": merge}, taken: next,
|
||||
said: "novox/mesh-delivery at cccccccccccc (#12): merged message applied; modules written: a",
|
||||
then: func(ix *Indexer) error {
|
||||
m := mergedMsg(repo, map[string]string{"modules/a": "treeA2", "modules/b": "treeB", "modules/c": "treeC"})
|
||||
m.Merge = next
|
||||
@@ -148,30 +139,32 @@ func TestAnUnchangedModuleWritesNothing(t *testing.T) { // 1239 (unchanged value
|
||||
{name: "a source head whose merge was not applied is not a taken key", writes: 0, commit: map[string]string{"a": merge, "b": merge, "c": merge}, taken: "",
|
||||
then: func(ix *Indexer) error { // the forge stream still holds the merge: the consumer is not drained
|
||||
delete(ix.Taken.(*memStore).vals, takenKey(repo))
|
||||
ix.Drained = func(context.Context) (bool, error) { return false, nil }
|
||||
if err := ix.ApplySource(context.Background(), 8, sourceOf(repo, next)); err != nil {
|
||||
return err
|
||||
}
|
||||
return ix.FillTaken(context.Background())
|
||||
}},
|
||||
// Issue 1126 (mesh-catalog#535): the forge bridge never wrote the merged message of 9fd9569, so the forge stream was
|
||||
// drained with the merge not applied; the source head moved the key all the same, and the view said the merge was
|
||||
// on the machines. A key moves only by a merge applied (ADR 0556 clause 1), never by the source's word.
|
||||
{name: "a source head of a drained forge stream whose merge was never heard leaves the key", writes: 0, commit: map[string]string{"a": merge, "b": merge, "c": merge}, taken: merge,
|
||||
then: func(ix *Indexer) error {
|
||||
ix.Contains = func(_ context.Context, _, commit, in string) (bool, error) { return commit == merge && in == next, nil }
|
||||
if err := ix.ApplySource(context.Background(), 8, sourceOf(repo, next)); err != nil {
|
||||
return err
|
||||
}
|
||||
return ix.FillTaken(context.Background())
|
||||
}},
|
||||
{name: "a source head of a drained forge stream fills no missing key", writes: 0, commit: map[string]string{"a": merge, "b": merge, "c": merge}, taken: "",
|
||||
{name: "a source head of a drained forge stream fills a missing key", writes: 0, commit: map[string]string{"a": merge, "b": merge, "c": merge}, taken: next,
|
||||
then: func(ix *Indexer) error {
|
||||
delete(ix.Taken.(*memStore).vals, takenKey(repo))
|
||||
ix.Drained = func(context.Context) (bool, error) { return true, nil }
|
||||
if err := ix.ApplySource(context.Background(), 8, sourceOf(repo, next)); err != nil {
|
||||
return err
|
||||
}
|
||||
return ix.FillTaken(context.Background())
|
||||
}},
|
||||
{name: "filling from an older source head leaves a later key", writes: 0, commit: map[string]string{"a": merge, "b": merge, "c": merge}, taken: next,
|
||||
then: func(ix *Indexer) error {
|
||||
ix.Contains = func(_ context.Context, _, commit, in string) (bool, error) { return commit == merge && in == next, nil }
|
||||
m := mergedMsg(repo, three)
|
||||
m.Merge = next
|
||||
if err := ix.Apply(context.Background(), m); err != nil {
|
||||
return err
|
||||
}
|
||||
ix.Drained = func(context.Context) (bool, error) { return true, nil }
|
||||
return ix.FillTaken(context.Background()) // the source still names merge
|
||||
}},
|
||||
{name: "an earlier merge heard again after a later one leaves taken", writes: 0, commit: map[string]string{"a": merge, "b": merge, "c": merge}, taken: merge,
|
||||
then: func(ix *Indexer) error {
|
||||
ix.Contains = func(_ context.Context, _, commit, in string) (bool, error) { return commit == next && in == merge, nil }
|
||||
@@ -179,78 +172,6 @@ func TestAnUnchangedModuleWritesNothing(t *testing.T) { // 1239 (unchanged value
|
||||
m.Merge = next
|
||||
return ix.Apply(context.Background(), m)
|
||||
}},
|
||||
// Issue 1126 note 76: mesh-catalog's merge 9cdac09a was written back over the later 69056948, rolling kernel and
|
||||
// mesh-build-image back on the nodes. A merge never moves a module backwards: the held commit must be in the incoming one.
|
||||
{name: "an earlier merge heard again after a later one moves no module back", writes: 1, commit: map[string]string{"a": next, "b": merge, "c": merge}, taken: next,
|
||||
said: "the merged message is older than taken.novox/mesh-delivery at cccccccccccc",
|
||||
then: func(ix *Indexer) error {
|
||||
ix.Contains = func(_ context.Context, _, commit, in string) (bool, error) {
|
||||
return commit == in || commit == merge && in == next, nil
|
||||
}
|
||||
later := mergedMsg(repo, map[string]string{"modules/a": "treeA2", "modules/b": "treeB", "modules/c": "treeC"})
|
||||
later.Merge = next
|
||||
if err := ix.Apply(context.Background(), later); err != nil {
|
||||
return err
|
||||
}
|
||||
return ix.Apply(context.Background(), mergedMsg(repo, three))
|
||||
}},
|
||||
{name: "an earlier merge that lacks a later merge's module deletes nothing", writes: 1, commit: map[string]string{"a": merge, "d": next}, taken: next,
|
||||
said: "the merged message is older than taken.novox/mesh-delivery at cccccccccccc",
|
||||
then: func(ix *Indexer) error {
|
||||
ix.Contains = func(_ context.Context, _, commit, in string) (bool, error) {
|
||||
return commit == in || commit == merge && in == next, nil
|
||||
}
|
||||
later := mergedMsg(repo, map[string]string{"modules/a": "treeA", "modules/b": "treeB", "modules/c": "treeC", "modules/d": "treeD"})
|
||||
later.Merge = next
|
||||
if err := ix.Apply(context.Background(), later); err != nil {
|
||||
return err
|
||||
}
|
||||
return ix.Apply(context.Background(), mergedMsg(repo, three))
|
||||
}},
|
||||
// Review round 2: a 502 whose words carry "404" inside a sha is no not-found; the late message is left for redelivery.
|
||||
{name: "a late message whose order the forge fails with a 502 writes nothing and is left for redelivery", writes: 1, commit: map[string]string{"a": next, "b": merge, "c": merge}, taken: next, err: true,
|
||||
said: "502 Bad Gateway",
|
||||
then: func(ix *Indexer) error {
|
||||
ix.Contains = func(context.Context, string, string, string) (bool, error) { return false, nil }
|
||||
later := mergedMsg(repo, map[string]string{"modules/a": "treeA2", "modules/b": "treeB", "modules/c": "treeC"})
|
||||
later.Merge = next
|
||||
if err := ix.Apply(context.Background(), later); err != nil {
|
||||
return err
|
||||
}
|
||||
ix.Contains = func(context.Context, string, string, string) (bool, error) {
|
||||
return false, errors.New("Gitea API /repos/x/y/compare/aaa404aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa...bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb: 502 Bad Gateway")
|
||||
}
|
||||
return ix.Apply(context.Background(), mergedMsg(repo, three))
|
||||
}},
|
||||
{name: "a forge that cannot say the order writes nothing and leaves the message for redelivery", writes: 0, commit: map[string]string{"a": merge}, taken: merge, err: true,
|
||||
said: "cannot be read",
|
||||
then: func(ix *Indexer) error {
|
||||
ix.Contains = func(context.Context, string, string, string) (bool, error) {
|
||||
return false, errors.New("the forge did not answer")
|
||||
}
|
||||
later := mergedMsg(repo, map[string]string{"modules/a": "treeA2", "modules/b": "treeB", "modules/c": "treeC"})
|
||||
later.Merge = next
|
||||
return ix.Apply(context.Background(), later)
|
||||
}},
|
||||
// Review of 77d738d: only a merge older than the one taken in is kept back. A diverged or rewritten history, or a
|
||||
// taken merge the forge no longer has, is not older, and one question per message is asked.
|
||||
{name: "a merge of a diverged history is written, asking the forge once", writes: 1, commit: map[string]string{"a": next, "b": merge, "c": merge}, taken: next,
|
||||
contains: func(context.Context, string, string, string) (bool, error) { return false, nil }, asks: 1,
|
||||
then: func(ix *Indexer) error {
|
||||
later := mergedMsg(repo, map[string]string{"modules/a": "treeA2", "modules/b": "treeB", "modules/c": "treeC"})
|
||||
later.Merge = next
|
||||
return ix.Apply(context.Background(), later)
|
||||
}},
|
||||
{name: "a taken merge the forge no longer has stops no later merge", writes: 1, commit: map[string]string{"a": next, "b": merge, "c": merge}, taken: next,
|
||||
said: "is not known to be older",
|
||||
then: func(ix *Indexer) error {
|
||||
ix.Contains = func(context.Context, string, string, string) (bool, error) {
|
||||
return false, errors.New("gitea_contains: the forge answered: 404 Not Found")
|
||||
}
|
||||
later := mergedMsg(repo, map[string]string{"modules/a": "treeA2", "modules/b": "treeB", "modules/c": "treeC"})
|
||||
later.Merge = next
|
||||
return ix.Apply(context.Background(), later)
|
||||
}},
|
||||
{name: "a later source revision of another repository's merge", writes: 0, commit: map[string]string{"a": merge, "b": merge, "c": merge}, taken: merge,
|
||||
then: func(ix *Indexer) error {
|
||||
return ix.ApplySource(context.Background(), 9, delivery.Source{Group: "g", Heads: map[string]string{repo: merge, "novox/mesh-catalog": otherMerge}})
|
||||
@@ -268,27 +189,8 @@ func TestAnUnchangedModuleWritesNothing(t *testing.T) { // 1239 (unchanged value
|
||||
t.Fatal(err)
|
||||
}
|
||||
writes := mods.writes
|
||||
asks := 0
|
||||
if tc.contains != nil {
|
||||
ix.Contains = func(ctx context.Context, repo, commit, in string) (bool, error) {
|
||||
asks++
|
||||
return tc.contains(ctx, repo, commit, in)
|
||||
}
|
||||
}
|
||||
var said strings.Builder
|
||||
ix.Logf = func(format string, args ...any) { fmt.Fprintf(&said, format+"\n", args...) }
|
||||
err := tc.then(ix)
|
||||
if tc.contains != nil && asks != tc.asks {
|
||||
t.Errorf("the forge was asked %d times, want %d", asks, tc.asks)
|
||||
}
|
||||
if (err != nil) != tc.err {
|
||||
t.Fatalf("then = %v, want an error %v", err, tc.err)
|
||||
}
|
||||
if err != nil {
|
||||
said.WriteString(err.Error()) // what handle logs as left for redelivery
|
||||
}
|
||||
if !strings.Contains(said.String(), tc.said) {
|
||||
t.Errorf("the log %q does not say %q", said.String(), tc.said)
|
||||
if err := tc.then(ix); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := mods.writes - writes; got != tc.writes {
|
||||
t.Errorf("%d values written, want %d", got, tc.writes)
|
||||
|
||||
@@ -1,118 +0,0 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
)
|
||||
|
||||
// A merge never moves a module backwards (issue 1126, note 76). The indexer wrote mesh-catalog's merge 9cdac09a back over
|
||||
// the later 69056948 for kernel and mesh-build-image: applyManifests and writeOverlay wrote whatever commit a merged
|
||||
// message carried, so a late or repeated message rolled the modules, and the nodes that want them, back.
|
||||
//
|
||||
// The question is whether the incoming commit is OLDER: the forge says the incoming commit is in the later one and the
|
||||
// two differ (Contains(incoming, in: later)). Only then is it kept back. A diverged or rewritten history answers no, and
|
||||
// is written forward; a commit the forge no longer has (a 404) is not known to be older, is written, and said. A forge
|
||||
// that cannot say for another reason (a timeout) writes nothing and is an error: the server delivers the message again.
|
||||
//
|
||||
// - A merged message asks once, before it writes anything: is its merge older than the repository's taken merge (the
|
||||
// last merge applied whole, taken.go)? Then the whole message is late: no module is written or deleted, the taken key
|
||||
// stays, and the log says so.
|
||||
// - A take-in of a held module asks once for that module: is the source's head older than the module's record
|
||||
// (takeInHeld)?
|
||||
|
||||
// mergeWrite is one merge's write of the modules: its repository and commit, whether it is late (older than the taken
|
||||
// merge), the taken merge it was compared with and the answer, and the module keys written, for the line the indexer says
|
||||
// per applied merged message.
|
||||
type mergeWrite struct {
|
||||
repository, merge string
|
||||
late string // the taken merge this one is older than, or ""
|
||||
asked string // the taken merge the order was asked against, or "" when none was asked
|
||||
written []string
|
||||
}
|
||||
|
||||
func newMergeWrite(f delivery.Forge) *mergeWrite {
|
||||
return &mergeWrite{repository: f.Repository, merge: f.Merge}
|
||||
}
|
||||
|
||||
// notFound says the forge answered that a commit it was asked about does not exist: gitea's status ": 404 ", matched as
|
||||
// readVia matches it (cmd/mesh-delivery-indexer named.go). A bare "404" is not enough: the error's words carry both shas,
|
||||
// and a 502 naming a sha with 404 in it would read as not older and write a late message backwards. The thorough form,
|
||||
// a typed not-found answered by gitea_contains, is noted on issue 1126.
|
||||
func notFound(err error) bool { return err != nil && strings.Contains(err.Error(), ": 404 ") }
|
||||
|
||||
// older says commit is older than later: the forge says commit is in later and the two differ. A commit the forge does not
|
||||
// have is not known to be older (false, said); a forge that cannot say otherwise is an error. Without a forge to ask
|
||||
// (Contains nil) nothing is older.
|
||||
func (ix *Indexer) older(ctx context.Context, repository, commit, later string) (bool, error) {
|
||||
if ix.Contains == nil || commit == "" || later == "" || commit == later {
|
||||
return false, nil
|
||||
}
|
||||
in, err := ix.Contains(ctx, repository, commit, later)
|
||||
if notFound(err) {
|
||||
ix.logf("%s: the forge does not have %.12s or %.12s (a rewritten history?), so %.12s is not known to be older and is written: %v", repository, commit, later, commit, err)
|
||||
return false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("%s: whether %.12s is older than %.12s cannot be read, so nothing of it is written (a merge never moves a module backwards, issue 1126): %w", repository, commit, later, err)
|
||||
}
|
||||
return in, nil
|
||||
}
|
||||
|
||||
// lateMerge asks, before a merged message writes anything, whether its merge is older than the repository's taken merge
|
||||
// (one forge call). A taken state not on the bus yet, or no taken key, asks nothing.
|
||||
func (ix *Indexer) lateMerge(ctx context.Context, w *mergeWrite) error {
|
||||
if ix.Taken == nil || w.merge == "" {
|
||||
return nil
|
||||
}
|
||||
key, err := delivery.RepositoryKey(w.repository)
|
||||
if err != nil {
|
||||
return nil // writeTaken says it
|
||||
}
|
||||
value, _, found, err := ix.Taken.Get(ctx, key)
|
||||
if errors.Is(err, process.ErrNotYet) || !found {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("taken.%s: %w", key, err)
|
||||
}
|
||||
cur, err := delivery.DecodeTaken(value)
|
||||
if err != nil || cur.Merge == "" || cur.Merge == w.merge {
|
||||
return nil
|
||||
}
|
||||
old, err := ix.older(ctx, w.repository, w.merge, cur.Merge)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
w.asked = cur.Merge
|
||||
if old {
|
||||
w.late = cur.Merge
|
||||
ix.logf("%s at %.12s: the merged message is older than taken.%s at %.12s, so no module is written or deleted (a late or repeated message; a merge never moves a module backwards, issue 1126)",
|
||||
w.repository, w.merge, key, cur.Merge)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// wrote records a module key the merge wrote.
|
||||
func (w *mergeWrite) wrote(key string) {
|
||||
if w != nil && !slices.Contains(w.written, key) {
|
||||
w.written = append(w.written, key)
|
||||
}
|
||||
}
|
||||
|
||||
// said is the line of an applied merged message: its repository, commit and pull request, and the modules written.
|
||||
func (w *mergeWrite) said(number int) string {
|
||||
if w.late != "" {
|
||||
return fmt.Sprintf("%s at %.12s (#%d): merged message applied as late: older than the taken %.12s, nothing written", w.repository, w.merge, number, w.late)
|
||||
}
|
||||
written := "none"
|
||||
if len(w.written) > 0 {
|
||||
written = strings.Join(slices.Sorted(slices.Values(w.written)), ", ")
|
||||
}
|
||||
return fmt.Sprintf("%s at %.12s (#%d): merged message applied; modules written: %s", w.repository, w.merge, number, written)
|
||||
}
|
||||
@@ -0,0 +1,123 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"maps"
|
||||
"slices"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
)
|
||||
|
||||
// Placement of modules on nodes (novox/hq ADR 0549 §2, issue 1126): at each merge of the catalogue the forge bridge
|
||||
// carries every placement file of the merge commit, and the indexer writes each node's key `placement.<node>` in its
|
||||
// state `placement` only when it differs (the revision alone is no change), deletes the key of a node whose file is
|
||||
// gone, and says `placement-changed` for each node it wrote or deleted, naming the modules added, removed or changed.
|
||||
// A merge that changes no placement writes nothing and says nothing.
|
||||
//
|
||||
// A file that does not read as a placement, or names another node than its file, is said in the log and its key is left
|
||||
// as it was: the catalogue's own check refuses such a file before it merges (ADR 0549 §1), so it is a fault of that
|
||||
// check, and a node keeps composing from the placement it had rather than from none.
|
||||
|
||||
// applyPlacement takes the placement files of a merged message in; a message that read none (another repository, an
|
||||
// older bridge) changes nothing.
|
||||
func (ix *Indexer) applyPlacement(ctx context.Context, f delivery.Forge) error {
|
||||
if ix.Placement == nil || f.Kind != delivery.ForgeMerged || !f.PlacementsRead {
|
||||
return nil
|
||||
}
|
||||
keys, err := ix.Placement.Keys(ctx)
|
||||
if errors.Is(err, process.ErrNotYet) {
|
||||
// The state is not on the bus yet: said, and the merge's candidates are still taken out (issue 1790's binding
|
||||
// writes the next catalogue merge's placement once it appears).
|
||||
ix.logf("%s at %s: its placement files are not written: %v", f.Repository, f.Merge, err)
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("reading the placement keys: %w", err)
|
||||
}
|
||||
gone := map[string]bool{}
|
||||
for _, key := range keys {
|
||||
if node, err := delivery.ParsePlacementKey(key); err == nil {
|
||||
gone[node] = true
|
||||
}
|
||||
}
|
||||
var refused []string
|
||||
for _, node := range slices.Sorted(maps.Keys(f.Placements)) {
|
||||
delete(gone, node)
|
||||
next, err := delivery.DecodePlacement([]byte(f.Placements[node]))
|
||||
if err == nil && next.Node != node {
|
||||
err = fmt.Errorf("the file of %s names the node %q", node, next.Node)
|
||||
}
|
||||
if err != nil {
|
||||
refused = append(refused, node)
|
||||
ix.logf("%s at %s: the placement file of %s is refused and placement.%s is left as it was (the catalogue's check should have refused it): %v",
|
||||
f.Repository, f.Merge, node, node, err)
|
||||
continue
|
||||
}
|
||||
next.Revision = f.Merge
|
||||
if err := ix.writePlacement(ctx, node, &next); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
for _, node := range slices.Sorted(maps.Keys(gone)) {
|
||||
if err := ix.writePlacement(ctx, node, nil); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if len(refused) > 0 {
|
||||
// Said, and the merge's candidates are still taken out: a refused file holds nobody's delivery.
|
||||
ix.logf("%s at %s: the placement files of %v do not read; their keys are left as they were", f.Repository, f.Merge, refused)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// writePlacement writes one node's key, or deletes it for next nil, only when it differs, and says what changed.
|
||||
func (ix *Indexer) writePlacement(ctx context.Context, node string, next *delivery.Placement) error {
|
||||
key, err := delivery.PlacementKey(node)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var change *delivery.PlacementChange
|
||||
err = write(ctx, ix.Placement, key, func(value []byte, found bool) ([]byte, bool, error) {
|
||||
change = nil
|
||||
var before delivery.Placement
|
||||
if found {
|
||||
cur, err := delivery.DecodePlacement(value)
|
||||
if err != nil {
|
||||
ix.logf("placement.%s holds what is not a placement, written again: %v", node, err)
|
||||
} else {
|
||||
before = cur
|
||||
}
|
||||
}
|
||||
if next == nil {
|
||||
if !found {
|
||||
return nil, false, nil
|
||||
}
|
||||
c := delivery.ChangeOf(before, delivery.Placement{Node: node})
|
||||
change = &c
|
||||
return nil, true, nil
|
||||
}
|
||||
if found && delivery.SamePlacement(before, *next) {
|
||||
return nil, false, nil
|
||||
}
|
||||
data, err := delivery.EncodePlacement(*next)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
c := delivery.ChangeOf(before, *next)
|
||||
change = &c
|
||||
return data, false, nil
|
||||
})
|
||||
if err != nil || change == nil {
|
||||
return err
|
||||
}
|
||||
if ix.Say != nil {
|
||||
if err := ix.Say(delivery.PlacementChangedEvent, *change); err != nil {
|
||||
// The key is written: a reader that watches it hears the change; the event is said loudly as not said.
|
||||
ix.logf("placement.%s is written, and %s could not be said: %v", node, delivery.PlacementChangedEvent, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,302 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"maps"
|
||||
"slices"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
)
|
||||
|
||||
// Placement candidates (issue 1126; novox/hq ADR 0549 §4: a placement change passes the composition check before any
|
||||
// node sees it). An updated pull request of the catalogue that touches placement/ carries every placement file at its
|
||||
// head (the forge bridge, PlacementsRead). For each node whose file at the head differs from the trunk's key
|
||||
// `placement.<node>` (SamePlacement, the revision left out), or whose file the head removes, the indexer writes the key
|
||||
// `<group>.<node>` of its state `placement-candidates`; a node whose file is the trunk's has none, and a key the head no
|
||||
// longer needs is deleted. The key is written only when its value changes, so its write is the node's event to compose
|
||||
// in shadow and answer. When the pull request closes or merges, or an open message no longer names it, its keys are
|
||||
// deleted, and so are they when a head the forge says the changes of (DirsKnown) carries no placement: it touches none.
|
||||
// A pull message that says nothing of its head (a resend of the open pull requests) leaves them as they are.
|
||||
//
|
||||
// A file at the head that does not read as a placement, or names another node, is said in the log and that node gets no
|
||||
// key (one it had is deleted): the catalogue's own check refuses the pull request (ADR 0549 §1), and a node answers no
|
||||
// candidate that is not one.
|
||||
|
||||
// applyPlacementCandidates takes one forge message into the placement candidates.
|
||||
func (ix *Indexer) applyPlacementCandidates(ctx context.Context, f delivery.Forge) error {
|
||||
if ix.PlacementCandidates == nil || !delivery.SameRepository(f.Repository, delivery.CatalogueRepository) || trunkLike[f.Branch] {
|
||||
return nil
|
||||
}
|
||||
if f.Kind == delivery.ForgeMerged || f.Action == delivery.PullClosed {
|
||||
return ix.dropPlacementCandidates(ctx, func(_ string, c delivery.PlacementCandidate) bool {
|
||||
return c.Group == f.Branch && delivery.SameRepository(c.Pull.Repository, f.Repository) && c.Pull.Number == f.Number
|
||||
}, fmt.Sprintf("%s#%d %s", f.Repository, f.Number, map[bool]string{true: "merged", false: "closed"}[f.Kind == delivery.ForgeMerged]))
|
||||
}
|
||||
if f.Kind != delivery.ForgePull {
|
||||
return nil
|
||||
}
|
||||
if !f.PlacementsRead {
|
||||
if f.DirsKnown {
|
||||
// The forge said what this head changes (its directories and paths) and the bridge read no placement: the head
|
||||
// changes no placement file, so a key an earlier head left goes (ADR 0559 §3).
|
||||
return ix.dropPlacementCandidates(ctx, func(_ string, c delivery.PlacementCandidate) bool {
|
||||
return c.Group == f.Branch && delivery.SameRepository(c.Pull.Repository, f.Repository) && c.Pull.Number == f.Number
|
||||
}, fmt.Sprintf("%s#%d at %s changes no placement file", f.Repository, f.Number, f.Head))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
trunk, err := ix.trunkPlacements(ctx)
|
||||
if err != nil {
|
||||
ix.logf("%s#%d at %s: its placement candidates are not written: %v", f.Repository, f.Number, f.Head, err)
|
||||
return nil
|
||||
}
|
||||
nodes := map[string]bool{}
|
||||
for node := range f.Placements {
|
||||
nodes[node] = true
|
||||
}
|
||||
for node := range trunk {
|
||||
nodes[node] = true
|
||||
}
|
||||
pull := delivery.Head{Repository: f.Repository, Number: f.Number, Head: f.Head}
|
||||
for _, node := range slices.Sorted(maps.Keys(nodes)) {
|
||||
key, err := delivery.PlacementCandidateKey(f.Branch, node)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%w: %s#%d: %v", ErrUnreadable, f.Repository, f.Number, err)
|
||||
}
|
||||
var next *delivery.PlacementCandidate // nil: no key for this node
|
||||
raw, atHead := f.Placements[node]
|
||||
before, inTrunk := trunk[node]
|
||||
switch {
|
||||
case atHead:
|
||||
p, err := delivery.DecodePlacement([]byte(raw))
|
||||
if err == nil && p.Node != node {
|
||||
err = fmt.Errorf("the file of %s names the node %q", node, p.Node)
|
||||
}
|
||||
if err != nil {
|
||||
ix.logf("%s#%d at %s: the placement file of %s is refused, so %s has no placement candidate (the catalogue's check refuses the pull request): %v",
|
||||
f.Repository, f.Number, f.Head, node, node, err)
|
||||
break
|
||||
}
|
||||
if inTrunk && delivery.SamePlacement(before, p) {
|
||||
break
|
||||
}
|
||||
next = &delivery.PlacementCandidate{Group: f.Branch, Pull: pull, Placement: &p}
|
||||
case inTrunk:
|
||||
next = &delivery.PlacementCandidate{Group: f.Branch, Pull: pull} // the head removes the node's file
|
||||
}
|
||||
if err := ix.writePlacementCandidate(ctx, key, next); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
// A node the head and the trunk both no longer name keeps no key of this group.
|
||||
return ix.dropPlacementCandidates(ctx, func(node string, c delivery.PlacementCandidate) bool {
|
||||
return c.Group == f.Branch && delivery.SameRepository(c.Pull.Repository, f.Repository) && c.Pull.Number == f.Number && !nodes[node]
|
||||
}, fmt.Sprintf("%s#%d at %s", f.Repository, f.Number, f.Head))
|
||||
}
|
||||
|
||||
// trunkPlacements is every node's placement in the trunk's state placement, by node.
|
||||
func (ix *Indexer) trunkPlacements(ctx context.Context) (map[string]delivery.Placement, error) {
|
||||
out := map[string]delivery.Placement{}
|
||||
if ix.Placement == nil {
|
||||
return nil, errors.New("the indexer reads no placement state")
|
||||
}
|
||||
keys, err := ix.Placement.Keys(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("reading the placement keys: %w", err)
|
||||
}
|
||||
for _, key := range keys {
|
||||
node, err := delivery.ParsePlacementKey(key)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
value, _, found, err := ix.Placement.Get(ctx, key)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("reading %s: %w", key, err)
|
||||
}
|
||||
if !found {
|
||||
continue
|
||||
}
|
||||
p, err := delivery.DecodePlacement(value)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s holds what is not a placement: %w", key, err)
|
||||
}
|
||||
out[node] = p
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// writePlacementCandidate writes one candidate key, or deletes it for next nil, only when it changes.
|
||||
func (ix *Indexer) writePlacementCandidate(ctx context.Context, key string, next *delivery.PlacementCandidate) error {
|
||||
var data []byte
|
||||
if next != nil {
|
||||
var err error
|
||||
if data, err = delivery.EncodePlacementCandidate(*next); err != nil {
|
||||
return fmt.Errorf("%w: %s.%s: %v", ErrUnreadable, delivery.PlacementCandidatesState, key, err)
|
||||
}
|
||||
}
|
||||
err := write(ctx, ix.PlacementCandidates, key, func(value []byte, found bool) ([]byte, bool, error) {
|
||||
switch {
|
||||
case next == nil:
|
||||
return nil, found, nil
|
||||
case found && string(value) == string(data):
|
||||
return nil, false, nil
|
||||
}
|
||||
return data, false, nil
|
||||
})
|
||||
if errors.Is(err, process.ErrNotYet) {
|
||||
ix.logf("%s.%s is not written: %v", delivery.PlacementCandidatesState, key, err)
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// dropPlacementCandidates deletes every candidate key whose node and value gone says, and says each with why.
|
||||
func (ix *Indexer) dropPlacementCandidates(ctx context.Context, gone func(node string, c delivery.PlacementCandidate) bool, why string) error {
|
||||
keys, err := ix.PlacementCandidates.Keys(ctx)
|
||||
if errors.Is(err, process.ErrNotYet) {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("reading the placement candidate keys: %w", err)
|
||||
}
|
||||
for _, key := range keys {
|
||||
_, node, err := delivery.ParsePlacementCandidateKey(key)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
dropped := false
|
||||
err = write(ctx, ix.PlacementCandidates, key, func(value []byte, found bool) ([]byte, bool, error) {
|
||||
if !found {
|
||||
return nil, false, nil
|
||||
}
|
||||
c, err := delivery.DecodePlacementCandidate(value)
|
||||
if err != nil {
|
||||
ix.logf("%s.%s holds what is not a placement candidate, deleted: %v", delivery.PlacementCandidatesState, key, err)
|
||||
return nil, true, nil
|
||||
}
|
||||
dropped = gone(node, c)
|
||||
return nil, dropped, nil
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if dropped {
|
||||
ix.logf("%s.%s is deleted: %s", delivery.PlacementCandidatesState, key, why)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// PlacementRead is the catalogue's placement as the forge has it now (indexer_placement_reread, ADR 0559 §8): the trunk's
|
||||
// head and every placement file at it, by node, and each open pull request into the trunk with its own.
|
||||
type PlacementRead struct {
|
||||
Trunk string
|
||||
Files map[string]string
|
||||
Pulls []PullPlacement
|
||||
}
|
||||
|
||||
// PullPlacement is one open catalogue pull request and every placement file at its head, by node.
|
||||
type PullPlacement struct {
|
||||
Branch string
|
||||
Number int
|
||||
Head string
|
||||
Files map[string]string
|
||||
}
|
||||
|
||||
// PlacementReread is the answer of indexer_placement_reread: the heads read, the keys written or deleted in each state,
|
||||
// and in words what happened.
|
||||
type PlacementReread struct {
|
||||
Trunk string `json:"trunk,omitempty"`
|
||||
Pulls []string `json:"pulls,omitempty"`
|
||||
Placement []string `json:"placement"`
|
||||
Candidates []string `json:"candidates"`
|
||||
Answer string `json:"answer"`
|
||||
}
|
||||
|
||||
// RereadPlacement reads the catalogue's placement now, at the trunk's head and at the head of each open catalogue pull
|
||||
// request, and writes placement and placement-candidates as a merged and a pull message would (ADR 0549 §2, ADR 0559
|
||||
// §3), only where a key differs; a candidate key of a pull request that is no longer open, or whose file no longer
|
||||
// differs at its head, is deleted. It is the recovery for a forge message the indexer missed (ADR 0559 §8).
|
||||
func (ix *Indexer) RereadPlacement(ctx context.Context) PlacementReread {
|
||||
out := PlacementReread{Placement: []string{}, Candidates: []string{}}
|
||||
if ix.ReadPlacement == nil || ix.Placement == nil || ix.PlacementCandidates == nil {
|
||||
out.Answer = "the indexer reads no placement here: it has no forge reader or no placement state"
|
||||
return out
|
||||
}
|
||||
read, err := ix.ReadPlacement(ctx)
|
||||
if err != nil {
|
||||
out.Answer = "the forge could not say the catalogue's placement, nothing is written: " + err.Error()
|
||||
return out
|
||||
}
|
||||
out.Trunk = read.Trunk
|
||||
before := revisions(ctx, ix.Placement)
|
||||
beforeC := revisions(ctx, ix.PlacementCandidates)
|
||||
if err := ix.applyPlacement(ctx, delivery.Forge{Kind: delivery.ForgeMerged, Repository: delivery.CatalogueRepository, Merge: read.Trunk,
|
||||
Placements: read.Files, PlacementsRead: true}); err != nil {
|
||||
out.Answer = "the trunk's placement could not be written: " + err.Error()
|
||||
return out
|
||||
}
|
||||
open := map[int]bool{}
|
||||
var failed []string
|
||||
for _, p := range read.Pulls {
|
||||
open[p.Number] = true
|
||||
out.Pulls = append(out.Pulls, fmt.Sprintf("%s#%d at %s", delivery.CatalogueRepository, p.Number, p.Head))
|
||||
f := delivery.Forge{Kind: delivery.ForgePull, Repository: delivery.CatalogueRepository, Branch: p.Branch, Number: p.Number, Head: p.Head,
|
||||
Action: delivery.PullUpdated, Placements: p.Files, PlacementsRead: true}
|
||||
if err := ix.applyPlacementCandidates(ctx, f); err != nil {
|
||||
failed = append(failed, fmt.Sprintf("#%d: %v", p.Number, err))
|
||||
}
|
||||
}
|
||||
if err := ix.dropPlacementCandidates(ctx, func(_ string, c delivery.PlacementCandidate) bool {
|
||||
return delivery.SameRepository(c.Pull.Repository, delivery.CatalogueRepository) && !open[c.Pull.Number]
|
||||
}, "its pull request is not open (indexer_placement_reread)"); err != nil {
|
||||
failed = append(failed, err.Error())
|
||||
}
|
||||
out.Placement = changed(before, revisions(ctx, ix.Placement))
|
||||
out.Candidates = changed(beforeC, revisions(ctx, ix.PlacementCandidates))
|
||||
switch {
|
||||
case len(failed) > 0:
|
||||
out.Answer = "read at the trunk's head " + read.Trunk + "; some candidates are not written: " + fmt.Sprint(failed)
|
||||
case len(out.Placement)+len(out.Candidates) == 0:
|
||||
out.Answer = "read at the trunk's head " + read.Trunk + " and every open catalogue pull request's: every key was as the forge has it, nothing written"
|
||||
default:
|
||||
out.Answer = fmt.Sprintf("read at the trunk's head %s and every open catalogue pull request's: %d placement keys and %d candidate keys written or deleted",
|
||||
read.Trunk, len(out.Placement), len(out.Candidates))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// revisions is every key of a store with its revision; a state not on the bus yet has none.
|
||||
func revisions(ctx context.Context, st Store) map[string]uint64 {
|
||||
out := map[string]uint64{}
|
||||
keys, err := st.Keys(ctx)
|
||||
if err != nil {
|
||||
return out
|
||||
}
|
||||
for _, key := range keys {
|
||||
if _, rev, found, err := st.Get(ctx, key); err == nil && found {
|
||||
out[key] = rev
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// changed is every key written, created or deleted between two revisions maps, sorted.
|
||||
func changed(before, after map[string]uint64) []string {
|
||||
out := []string{}
|
||||
for key, rev := range after {
|
||||
if before[key] != rev {
|
||||
out = append(out, key)
|
||||
}
|
||||
}
|
||||
for key := range before {
|
||||
if _, ok := after[key]; !ok {
|
||||
out = append(out, key)
|
||||
}
|
||||
}
|
||||
slices.Sort(out)
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,201 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"context"
|
||||
"reflect"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// ADR 0549 §2 (issue 1126): a merge changing one node's file writes that key alone and says one event; a merge with no
|
||||
// placement change writes nothing and says nothing; a node whose file is gone loses its key and says its modules
|
||||
// removed; a file that does not read leaves its key; a message that read no placement changes none.
|
||||
func placementIsWrittenAtAMergeOnlyWhereItChanged(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
ix, _, logs := newIndexer(t)
|
||||
place := newMem()
|
||||
ix.Placement = place
|
||||
var said []delivery.PlacementChange
|
||||
ix.Say = func(event string, body any) error {
|
||||
if event != delivery.PlacementChangedEvent {
|
||||
t.Errorf("said %s", event)
|
||||
}
|
||||
said = append(said, body.(delivery.PlacementChange))
|
||||
return nil
|
||||
}
|
||||
merged := func(merge string, files map[string]string) delivery.Forge {
|
||||
return delivery.Forge{V: delivery.Version, Kind: delivery.ForgeMerged, Repository: delivery.CatalogueRepository, Branch: "main",
|
||||
Number: 7, Head: "h", Merge: merge, Root: "r", Trees: map[string]string{"": "t"}, Placements: files, PlacementsRead: true}
|
||||
}
|
||||
ace := `{"v":1,"node":"ace","modules":[{"module":"plex"}]}`
|
||||
g14 := `{"v":1,"node":"g14","modules":[{"module":"zsh"}]}`
|
||||
written := func(node string) (delivery.Placement, uint64) {
|
||||
t.Helper()
|
||||
v, ok := place.vals["placement."+node]
|
||||
if !ok {
|
||||
return delivery.Placement{}, 0
|
||||
}
|
||||
p, err := delivery.DecodePlacement(v)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return p, place.revs["placement."+node]
|
||||
}
|
||||
|
||||
if err := ix.Apply(ctx, merged("m1", map[string]string{"ace": ace, "g14": g14})); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if p, _ := written("ace"); p.Revision != "m1" || !reflect.DeepEqual(p.Assigned(), []string{"plex"}) || len(said) != 2 {
|
||||
t.Fatalf("first merge: %+v, said %v", p, said)
|
||||
}
|
||||
_, aceRev := written("ace")
|
||||
|
||||
said = nil
|
||||
if err := ix.Apply(ctx, merged("m2", map[string]string{"ace": ace, "g14": g14})); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if p, rev := written("ace"); rev != aceRev || p.Revision != "m1" || len(said) != 0 {
|
||||
t.Errorf("a merge with no placement change wrote %+v (revision %d, was %d) and said %v", p, rev, aceRev, said)
|
||||
}
|
||||
|
||||
said = nil
|
||||
if err := ix.Apply(ctx, merged("m3", map[string]string{"ace": ace, "g14": `{"v":1,"node":"g14","modules":[{"module":"zsh"},{"module":"xterm"}]}`})); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, rev := written("ace"); rev != aceRev || len(said) != 1 || said[0].Node != "g14" || said[0].Revision != "m3" ||
|
||||
!reflect.DeepEqual(said[0].Added, []string{"xterm"}) {
|
||||
t.Errorf("one file changed: ace's revision %d (was %d), said %+v", rev, aceRev, said)
|
||||
}
|
||||
|
||||
said = nil
|
||||
if err := ix.Apply(ctx, merged("m4", map[string]string{"ace": ace, "g14": `{"v":1,"node":"ace","modules":[]}`})); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if p, _ := written("g14"); len(p.Modules) != 2 || len(said) != 0 || !strings.Contains(logs.String(), "the placement file of %s is refused") {
|
||||
t.Errorf("a file naming another node replaced the key: %+v, said %v, logs %s", p, said, logs)
|
||||
}
|
||||
|
||||
said = nil
|
||||
if err := ix.Apply(ctx, merged("m5", map[string]string{"ace": ace})); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, rev := written("g14"); rev != 0 || len(said) != 1 || !reflect.DeepEqual(said[0].Removed, []string{"xterm", "zsh"}) {
|
||||
t.Errorf("a file removed: g14's key at revision %d, said %+v", rev, said)
|
||||
}
|
||||
|
||||
said = nil
|
||||
unread := merged("m6", nil)
|
||||
unread.PlacementsRead = false
|
||||
if err := ix.Apply(ctx, unread); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if p, _ := written("ace"); p.Revision != "m1" || len(said) != 0 {
|
||||
t.Errorf("a message that read no placement changed it: %+v, %v", p, said)
|
||||
}
|
||||
|
||||
// Issue 1126 (ADR 0549 §4), placement candidates: the trunk now places ace alone. A catalogue pull request writes
|
||||
// `<group>.<node>` for each node whose file at its head differs from the trunk's, or that the head removes, and only
|
||||
// when the value changes; a pull message that read no placement, or a pull request changing none, writes nothing; a
|
||||
// file that does not read gets no key; a merge, a close or an open message not naming it deletes its keys.
|
||||
cands := newMem()
|
||||
ix.PlacementCandidates = cands
|
||||
said = nil
|
||||
pullOf := func(branch string, number int, head string, files map[string]string) delivery.Forge {
|
||||
return delivery.Forge{V: delivery.Version, Kind: delivery.ForgePull, Repository: delivery.CatalogueRepository, Branch: branch,
|
||||
Number: number, Head: head, Action: delivery.PullUpdated, Placements: files, PlacementsRead: files != nil}
|
||||
}
|
||||
candidate := func(key string) (delivery.PlacementCandidate, uint64, bool) {
|
||||
t.Helper()
|
||||
v, ok := cands.vals[key]
|
||||
if !ok {
|
||||
return delivery.PlacementCandidate{}, 0, false
|
||||
}
|
||||
c, err := delivery.DecodePlacementCandidate(v)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return c, cands.revs[key], true
|
||||
}
|
||||
keys := func() string { k, _ := cands.Keys(ctx); slices.Sort(k); return strings.Join(k, ",") }
|
||||
g14New := `{"v":1,"node":"g14","modules":[{"module":"zsh"}]}`
|
||||
for _, row := range []struct {
|
||||
name string
|
||||
f delivery.Forge
|
||||
keys string
|
||||
}{
|
||||
{"a pull request adding g14", pullOf("feat/p", 9, h1, map[string]string{"ace": ace, "g14": g14New}), "feat/p.g14"},
|
||||
{"the same head again", pullOf("feat/p", 9, h1, map[string]string{"ace": ace, "g14": g14New}), "feat/p.g14"},
|
||||
{"a pull message that read no placement", pullOf("feat/p", 9, h2, nil), "feat/p.g14"},
|
||||
{"a pull request changing no placement", pullOf("feat/q", 10, h1, map[string]string{"ace": ace}), "feat/p.g14"},
|
||||
{"a new head removing ace", pullOf("feat/p", 9, h2, map[string]string{"g14": g14New}), "feat/p.ace,feat/p.g14"},
|
||||
{"a head whose g14 file does not read", pullOf("feat/p", 9, h3, map[string]string{"g14": `{"v":1,"node":"ace","modules":[]}`}), "feat/p.ace"},
|
||||
{"the head adding g14 again", pullOf("feat/p", 9, h2, map[string]string{"g14": g14New}), "feat/p.ace,feat/p.g14"},
|
||||
{"a head the forge says changes no placement file", func() delivery.Forge { f := pullOf("feat/p", 9, h3, nil); f.DirsKnown = true; return f }(), ""},
|
||||
{"the head adding g14 and removing ace again", pullOf("feat/p", 9, h2, map[string]string{"g14": g14New}), "feat/p.ace,feat/p.g14"},
|
||||
{"an open message not naming it", delivery.Forge{V: delivery.Version, Kind: delivery.ForgeOpen, Open: []delivery.OpenPull{{Repository: delivery.CatalogueRepository, Number: 10}}}, ""},
|
||||
{"the pull request heard again", pullOf("feat/p", 9, h2, map[string]string{"g14": g14New}), "feat/p.ace,feat/p.g14"},
|
||||
{"its merge", func() delivery.Forge {
|
||||
m := merged("m7", map[string]string{"g14": g14New})
|
||||
m.Branch, m.Number = "feat/p", 9
|
||||
return m
|
||||
}(), ""},
|
||||
} {
|
||||
_, revBefore, _ := candidate("feat/p.g14")
|
||||
if err := ix.Apply(ctx, row.f); err != nil {
|
||||
t.Fatalf("%s: %v", row.name, err)
|
||||
}
|
||||
if got := keys(); got != row.keys {
|
||||
t.Errorf("%s: keys %q, want %q", row.name, got, row.keys)
|
||||
}
|
||||
c, rev, ok := candidate("feat/p.g14")
|
||||
switch row.name {
|
||||
case "a pull request adding g14":
|
||||
if !ok || c.Group != "feat/p" || c.Pull != (delivery.Head{Repository: delivery.CatalogueRepository, Number: 9, Head: h1}) || c.Placement == nil ||
|
||||
!reflect.DeepEqual(c.Placement.Assigned(), []string{"zsh"}) {
|
||||
t.Errorf("%s: %+v", row.name, c)
|
||||
}
|
||||
case "the same head again", "a pull message that read no placement", "a pull request changing no placement":
|
||||
if rev != revBefore {
|
||||
t.Errorf("%s: feat/p.g14 written again (revision %d, was %d)", row.name, rev, revBefore)
|
||||
}
|
||||
case "a new head removing ace":
|
||||
if a, _, _ := candidate("feat/p.ace"); a.Placement != nil || a.Pull.Head != h2 || c.Pull.Head != h2 {
|
||||
t.Errorf("%s: ace %+v, g14 at %s", row.name, a, c.Pull.Head)
|
||||
}
|
||||
}
|
||||
}
|
||||
if p, _ := written("g14"); len(p.Modules) != 1 || p.Revision != "m7" {
|
||||
t.Errorf("the merge's trunk placement of g14: %+v", p)
|
||||
}
|
||||
|
||||
// ADR 0559 §8, indexer_placement_reread: a reread with nothing changed writes nothing; one after a missed merge and a
|
||||
// missed head writes those keys; one after a missed revert deletes the candidate; a forge that cannot say writes nothing.
|
||||
g14Other := `{"v":1,"node":"g14","modules":[{"module":"zsh"},{"module":"htop"}]}`
|
||||
for _, row := range []struct {
|
||||
name string
|
||||
read PlacementRead
|
||||
err error
|
||||
placement, cands []string
|
||||
}{
|
||||
{"nothing changed", PlacementRead{Trunk: "m7", Files: map[string]string{"g14": g14New}}, nil, []string{}, []string{}},
|
||||
{"a missed merge and a missed head", PlacementRead{Trunk: "m8", Files: map[string]string{"g14": g14New, "ace": ace},
|
||||
Pulls: []PullPlacement{{Branch: "feat/r", Number: 11, Head: h1, Files: map[string]string{"g14": g14Other, "ace": ace}}}}, nil,
|
||||
[]string{"placement.ace"}, []string{"feat/r.g14"}},
|
||||
{"a missed revert", PlacementRead{Trunk: "m8", Files: map[string]string{"g14": g14New, "ace": ace},
|
||||
Pulls: []PullPlacement{{Branch: "feat/r", Number: 11, Head: h2, Files: map[string]string{"g14": g14New, "ace": ace}}}}, nil,
|
||||
[]string{}, []string{"feat/r.g14"}},
|
||||
{"a forge that cannot say", PlacementRead{}, context.DeadlineExceeded, []string{}, []string{}},
|
||||
} {
|
||||
ix.ReadPlacement = func(context.Context) (PlacementRead, error) { return row.read, row.err }
|
||||
r := ix.RereadPlacement(ctx)
|
||||
if !reflect.DeepEqual(r.Placement, row.placement) || !reflect.DeepEqual(r.Candidates, row.cands) || r.Answer == "" {
|
||||
t.Errorf("%s: %+v", row.name, r)
|
||||
}
|
||||
}
|
||||
if keys() != "" {
|
||||
t.Errorf("after the reread of a revert, candidates %q", keys())
|
||||
}
|
||||
}
|
||||
@@ -45,12 +45,6 @@ type taken struct {
|
||||
}
|
||||
|
||||
func (ix *Indexer) indexRegistered(ctx context.Context, r Registration, force bool) (taken, error) {
|
||||
return ix.indexAt(ctx, r, force, false)
|
||||
}
|
||||
|
||||
// indexAt is indexRegistered; reread also reads a module the modules state holds (a person's take-in of a held module
|
||||
// whose record stands behind the head the gate's source names, issue 1126).
|
||||
func (ix *Indexer) indexAt(ctx context.Context, r Registration, force, reread bool) (taken, error) {
|
||||
if ix.Modules == nil || ix.Read == nil {
|
||||
return taken{why: "the indexer reads no forge or writes no modules"}, nil
|
||||
}
|
||||
@@ -61,7 +55,7 @@ func (ix *Indexer) indexAt(ctx context.Context, r Registration, force, reread bo
|
||||
}
|
||||
if _, _, found, err := ix.Modules.Get(ctx, key); err != nil {
|
||||
return taken{}, fmt.Errorf("modules.%s: %w", key, err)
|
||||
} else if found && !reread {
|
||||
} else if found {
|
||||
ix.markRead(r.Module)
|
||||
return taken{held: true, why: "the modules state holds it already: a merge to its repository writes it from now on"}, nil
|
||||
}
|
||||
@@ -129,7 +123,7 @@ func (ix *Indexer) indexAt(ctx context.Context, r Registration, force, reread bo
|
||||
if l := read.Locks[dir]; len(l) > 0 {
|
||||
f.Locks = map[string]map[string]string{dir: l}
|
||||
}
|
||||
if _, err := ix.applyManifests(ctx, f, false); err != nil {
|
||||
if err := ix.applyManifests(ctx, f, false); err != nil {
|
||||
return t, err
|
||||
}
|
||||
if _, _, found, err := ix.Modules.Get(ctx, key); err != nil {
|
||||
@@ -210,9 +204,6 @@ type TakenIn struct {
|
||||
// controller's records, or whose every registration could not be read or holds no manifest of it, is answered with why.
|
||||
func (ix *Indexer) TakeIn(ctx context.Context, module string) TakenIn {
|
||||
out := TakenIn{Module: module}
|
||||
if again, ok := ix.takeInHeld(ctx, module); ok {
|
||||
return again
|
||||
}
|
||||
if ix.Builds == nil {
|
||||
out.Answer = "the indexer reads no controller's build records, so it knows no registration"
|
||||
return out
|
||||
@@ -282,69 +273,3 @@ func (ix *Indexer) IndexRegisteredAll(ctx context.Context) error {
|
||||
}
|
||||
return first
|
||||
}
|
||||
|
||||
// takeInHeld reads again, at the head the gate's source names for its repository, a module the modules state holds whose
|
||||
// record stands at another commit, and writes it as a merge would (only when it changed). It is the recovery for a merge
|
||||
// whose merged message never reached the indexer (issue 1126: mesh-catalog#535's message was given up by the forge bridge
|
||||
// after its reads timed out, and mesh-vault's record stayed at the merge before). It answers false where it does not
|
||||
// apply: no such module held, or no head named, or the record at that head already.
|
||||
func (ix *Indexer) takeInHeld(ctx context.Context, module string) (TakenIn, bool) {
|
||||
out := TakenIn{Module: module}
|
||||
key, err := delivery.ModulesKey(module)
|
||||
if err != nil || ix.Modules == nil {
|
||||
return out, false
|
||||
}
|
||||
value, _, found, err := ix.Modules.Get(ctx, key)
|
||||
if err != nil || !found {
|
||||
return out, false
|
||||
}
|
||||
cur, err := delivery.DecodeModule(value)
|
||||
if err != nil || cur.Repository == "" || cur.Path == "" {
|
||||
return out, false
|
||||
}
|
||||
head := ""
|
||||
ix.mu.Lock()
|
||||
for repo, h := range ix.source.heads {
|
||||
if delivery.SameRepository(repo, cur.Repository) && h != "" {
|
||||
head = h
|
||||
}
|
||||
}
|
||||
ix.mu.Unlock()
|
||||
if head == "" || head == cur.Commit {
|
||||
return out, false
|
||||
}
|
||||
// Never backwards (order.go): a source head older than the record (a merge applied before the source moved) is never
|
||||
// read and written back over it. A diverged or rewritten history, or a record whose commit the forge no longer has, is
|
||||
// not older and is read; where the forge cannot say otherwise, nothing is read. One forge call.
|
||||
old, why := true, "the forge cannot be asked"
|
||||
if ix.Contains != nil {
|
||||
var err error
|
||||
if old, err = ix.older(ctx, cur.Repository, head, cur.Commit); err != nil {
|
||||
old, why = true, err.Error()
|
||||
} else if old {
|
||||
why = fmt.Sprintf("%.12s is older than its record at %.12s", head, cur.Commit)
|
||||
}
|
||||
}
|
||||
if old {
|
||||
out.Held, out.Repository, out.Commit, out.Path = true, cur.Repository, cur.Commit, cur.Path
|
||||
out.Answer = fmt.Sprintf("held; the source names another head, %.12s, and not one newer than its record (%s), so it is not read again", head, why)
|
||||
return out, true
|
||||
}
|
||||
t, err := ix.indexAt(ctx, Registration{Module: module, Repository: cur.Repository, Path: cur.Path, Commit: head}, true, true)
|
||||
out.Held, out.Repository, out.Commit, out.Path = true, t.repository, t.commit, pathOf(t.dir)
|
||||
switch {
|
||||
case err != nil:
|
||||
out.Answer = "held, and not read again at the source's head: " + err.Error()
|
||||
case !t.stored:
|
||||
out.Answer = "held, and not read again at the source's head: " + t.why
|
||||
default:
|
||||
after, _, _, _ := ix.Modules.Get(ctx, key)
|
||||
out.Stored = true
|
||||
if string(after) == string(value) {
|
||||
out.Answer = fmt.Sprintf("held, and read again from %s at %s: unchanged there, so its record stands", t.repository, t.commit)
|
||||
} else {
|
||||
out.Answer = fmt.Sprintf("held, read again from %s at %s and written to the modules state: it changed since %.12s", t.repository, t.commit, cur.Commit)
|
||||
}
|
||||
}
|
||||
return out, true
|
||||
}
|
||||
|
||||
@@ -111,64 +111,6 @@ func TestARegisteredModuleIsIndexedFromItsRepository(t *testing.T) { // the gran
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// Issue 1126: mesh-catalog#535 changed mesh-vault, but its merged message never reached the indexer, so the record
|
||||
// stood at the merge before. A take-in of the held module reads it again at the source's head, only when the record is
|
||||
// behind that head: a head older than the record (a merge applied before the source moved), or one whose order the forge
|
||||
// cannot say, is never written back.
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
contains func(context.Context, string, string, string) (bool, error)
|
||||
moved bool
|
||||
}{
|
||||
{name: "a held module behind the source's head is read again", moved: true,
|
||||
contains: func(_ context.Context, _, commit, in string) (bool, error) { return commit == merge && in == h2, nil }},
|
||||
{name: "a source head older than the record is not read", contains: func(_ context.Context, _, commit, in string) (bool, error) { return commit == h2 && in == merge, nil }},
|
||||
// Review of 77d738d: a diverged or rewritten history, or a record whose commit the forge no longer has, is not older.
|
||||
{name: "a source head of a diverged history is read again", moved: true,
|
||||
contains: func(context.Context, string, string, string) (bool, error) { return false, nil }},
|
||||
{name: "a record whose commit the forge no longer has is read again", moved: true, contains: func(context.Context, string, string, string) (bool, error) {
|
||||
return false, errors.New("gitea_contains: 404 Not Found")
|
||||
}},
|
||||
{name: "a forge that cannot say the order reads nothing", contains: func(context.Context, string, string, string) (bool, error) {
|
||||
return false, errors.New("timeout")
|
||||
}},
|
||||
{name: "no forge to ask reads nothing"},
|
||||
} {
|
||||
t.Run("take-in: "+tc.name, func(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
ix, mods := modulesIndexer(t)
|
||||
ix.Manifests = newMem()
|
||||
if err := ix.Apply(ctx, mergedMsg("novox/mesh-catalog", map[string]string{"modules/a": "treeA"})); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ix.source = sourceNow{rev: 2, heads: map[string]string{"novox/mesh-catalog": h2}}
|
||||
ix.Contains = tc.contains
|
||||
var asked []string
|
||||
ix.Read = func(_ context.Context, _, sha, dir string) (RepoRead, error) {
|
||||
asked = append(asked, sha+":"+dir)
|
||||
return RepoRead{Root: "r1", Trees: map[string]string{"modules/a": "treeA2"}, Manifests: map[string]string{"modules/a": `{"module":"a","version":"2"}`}}, nil
|
||||
}
|
||||
got := ix.TakeIn(ctx, "a")
|
||||
m, ok := moduleOf(t, mods, "a")
|
||||
if !tc.moved {
|
||||
if got.Stored || !got.Held || len(asked) != 0 || !strings.Contains(got.Answer, "another head") || !ok || m.Commit != merge {
|
||||
t.Errorf("take-in answered %+v, asked %v, modules.a at %.8s", got, asked, m.Commit)
|
||||
}
|
||||
return
|
||||
}
|
||||
if !got.Stored || !got.Held || got.Commit != h2 || got.Path != "modules/a" || !strings.Contains(got.Answer, "written") {
|
||||
t.Errorf("take-in answered %+v", got)
|
||||
}
|
||||
if !ok || m.Commit != h2 || m.Inputs2 == nil || m.Inputs2.Tree != "treeA2" {
|
||||
t.Errorf("modules.a = %+v (%v), want it at %.8s with tree treeA2 (asked %v)", m, ok, h2, asked)
|
||||
}
|
||||
n := len(asked) // at the head now: a take-in answers held and reads nothing
|
||||
if again := ix.TakeIn(ctx, "a"); again.Stored || !again.Held || len(asked) != n {
|
||||
t.Errorf("a take-in of a module at the source's head answered %+v and asked %v", again, asked)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func mustModule(t *testing.T, m delivery.Module) []byte {
|
||||
|
||||
@@ -17,6 +17,10 @@ import (
|
||||
// of the indexer's states stay in mesh-delivery-view.
|
||||
const (
|
||||
TakeInTool = "indexer_take_in"
|
||||
// PlacementRereadTool reads the catalogue's placement again now (ADR 0559 §8); it asks through the same state, under
|
||||
// PlacementRereadKey, a key no module has (a module's name holds no dot).
|
||||
PlacementRereadTool = "indexer_placement_reread"
|
||||
PlacementRereadKey = "placement.reread"
|
||||
// TakeInsState is the indexer's state of take-in asks: the bucket mesh-delivery-indexer_takeins, keyed as the modules state.
|
||||
TakeInsState = "takeins"
|
||||
TakeInsBucket = delivery.IndexerModule + "_" + TakeInsState
|
||||
@@ -32,10 +36,19 @@ type TakeInAsk struct {
|
||||
At time.Time `json:"at"`
|
||||
Result *TakenIn `json:"result,omitempty"`
|
||||
Answered time.Time `json:"answered,omitzero"`
|
||||
// Placement asks indexer_placement_reread instead of a module's take-in; Reread is its answer.
|
||||
Placement bool `json:"placement,omitempty"`
|
||||
Reread *PlacementReread `json:"reread,omitempty"`
|
||||
}
|
||||
|
||||
// Done says the ask is answered.
|
||||
func (a TakeInAsk) Done() bool { return !a.Answered.IsZero() && a.Result != nil }
|
||||
func (a TakeInAsk) Done() bool { return !a.Answered.IsZero() && (a.Result != nil || a.Reread != nil) }
|
||||
|
||||
// NewPlacementReread is the value indexer_placement_reread writes: its key and the value.
|
||||
func NewPlacementReread(why, by string, at time.Time) (string, []byte, error) {
|
||||
data, err := json.Marshal(TakeInAsk{V: 1, Placement: true, Why: strings.TrimSpace(why), By: by, At: at.UTC()})
|
||||
return PlacementRereadKey, data, err
|
||||
}
|
||||
|
||||
// DecodeTakeIn reads a takeins value.
|
||||
func DecodeTakeIn(data []byte) (TakeInAsk, error) {
|
||||
@@ -46,7 +59,7 @@ func DecodeTakeIn(data []byte) (TakeInAsk, error) {
|
||||
if a.V != 1 {
|
||||
return a, fmt.Errorf("a takeins value of version %d, not 1", a.V)
|
||||
}
|
||||
if a.Module == "" {
|
||||
if a.Module == "" && !a.Placement {
|
||||
return a, errors.New("a takeins value naming no module")
|
||||
}
|
||||
return a, nil
|
||||
|
||||
@@ -20,12 +20,6 @@ import (
|
||||
|
||||
// writeTaken writes the repository's taken key for the merge f carries, unless it names that merge or a later one.
|
||||
func (ix *Indexer) writeTaken(ctx context.Context, f delivery.Forge) error {
|
||||
return ix.writeTakenKnown(ctx, f, "")
|
||||
}
|
||||
|
||||
// writeTakenKnown is writeTaken where the merge is known not to be older than notOlder (lateMerge asked it, issue 1126): a
|
||||
// key naming that merge is written over without asking the forge again.
|
||||
func (ix *Indexer) writeTakenKnown(ctx context.Context, f delivery.Forge, notOlder string) error {
|
||||
if ix.Taken == nil || f.Merge == "" {
|
||||
return nil
|
||||
}
|
||||
@@ -40,7 +34,7 @@ func (ix *Indexer) writeTakenKnown(ctx context.Context, f delivery.Forge, notOld
|
||||
if cur.Merge == f.Merge {
|
||||
return nil, false, nil
|
||||
}
|
||||
if ix.Contains != nil && cur.Merge != notOlder {
|
||||
if ix.Contains != nil {
|
||||
later, err := ix.Contains(ctx, f.Repository, f.Merge, cur.Merge)
|
||||
if err != nil {
|
||||
ix.logf("%s: whether %s is in %s cannot be read, so taken.%s stays at %s: %v", f.Repository, f.Merge, cur.Merge, key, cur.Merge, err)
|
||||
@@ -74,13 +68,13 @@ func (ix *Indexer) writeTakenKnown(ctx context.Context, f delivery.Forge, notOld
|
||||
return err
|
||||
}
|
||||
|
||||
// FillTaken writes the taken keys of the merges applied while the state was not on the bus (kept by writeTaken), through
|
||||
// writeTaken as a live merge does (issue 1280), whenever the state is there. It never writes a merge the indexer has not
|
||||
// applied whole (a key names a merge only after it was applied: ADR 0556 clause 1, and the view reads it as the upper bound
|
||||
// of what is reached). The gate's source heads are not such merges: a drained forge stream does not say a merge was applied,
|
||||
// since the forge bridge may not have written its merged message yet (issue 1126: mesh-catalog#535's merged message was
|
||||
// still being retried by the bridge when the indexer started, and the source's head moved the key, so the view said the
|
||||
// merge was on the machines while the indexer held mesh-vault at the merge before).
|
||||
// FillTaken writes the taken keys that were left unwritten, never a merge the indexer has not applied whole (a key names
|
||||
// a merge only after it was applied: ADR 0556 clause 1, and the view reads it as the upper bound of what is reached). Two
|
||||
// sources, both through writeTaken as a live merge does (issue 1280):
|
||||
// - the merges applied while the state was not on the bus (kept by writeTaken), whenever the state is there;
|
||||
// - the gate's source heads, only when Drained says the indexer has applied every message of the forge stream that was
|
||||
// there before the heads were read. Otherwise it says so, and the next kick of the fill (after a message is applied)
|
||||
// tries again.
|
||||
//
|
||||
// It is idempotent: a key naming the merge or a later one is left. A state not on the bus yet writes nothing and says
|
||||
// nothing (the fill runs again when it appears). The first error is returned after every repository was tried.
|
||||
@@ -93,6 +87,7 @@ func (ix *Indexer) FillTaken(ctx context.Context) error {
|
||||
}
|
||||
ix.mu.Lock()
|
||||
pending := maps.Clone(ix.untaken)
|
||||
heads := maps.Clone(ix.source.heads)
|
||||
ix.mu.Unlock()
|
||||
var first error
|
||||
note := func(repository string, err error) {
|
||||
@@ -111,6 +106,25 @@ func (ix *Indexer) FillTaken(ctx context.Context) error {
|
||||
}
|
||||
note(repository, err)
|
||||
}
|
||||
if ix.Drained == nil {
|
||||
return first
|
||||
}
|
||||
if len(heads) == 0 {
|
||||
ix.logf("the source names no repository head, so no taken key is filled from it")
|
||||
return first
|
||||
}
|
||||
done, err := ix.Drained(ctx)
|
||||
if err != nil {
|
||||
ix.logf("whether the forge stream is applied cannot be read, so no taken key is filled from the source: %v", err)
|
||||
return first
|
||||
}
|
||||
if !done {
|
||||
ix.logf("the forge stream is not applied whole yet, so no taken key is filled from the source; the next applied message tries again")
|
||||
return first
|
||||
}
|
||||
for _, repository := range slices.Sorted(maps.Keys(heads)) {
|
||||
note(repository, ix.writeTaken(ctx, delivery.Forge{Repository: repository, Merge: heads[repository]}))
|
||||
}
|
||||
return first
|
||||
}
|
||||
|
||||
|
||||
@@ -123,29 +123,28 @@ type known struct {
|
||||
// the order the bucket lists them in; modules that stand on each other have a cycle as their reason. A module is written
|
||||
// only when its fingerprint or its reason changes, and only if its manifest is still the one computed from. A module with
|
||||
// no kept inputs (a message that carried no manifest) is left as it is.
|
||||
func (ix *Indexer) Recompute(ctx context.Context) error { return ix.recomputeWith(ctx, nil, nil) }
|
||||
func (ix *Indexer) Recompute(ctx context.Context) error { return ix.recomputeWith(ctx, nil) }
|
||||
|
||||
// recomputeWith is Recompute for a merge as well: the records the merge gives (overlay, by module key) stand in place
|
||||
// of what the bucket holds, are computed with the others, and are written once, with their final fingerprint or reason
|
||||
// (issue 1444: a merge writes each module once). A record of the overlay that has no kept inputs is written as it is. w is
|
||||
// the merge's write (order.go), nil for none; it is told each module written.
|
||||
func (ix *Indexer) recomputeWith(ctx context.Context, overlay map[string]delivery.Module, w *mergeWrite) error {
|
||||
// (issue 1444: a merge writes each module once). A record of the overlay that has no kept inputs is written as it is.
|
||||
func (ix *Indexer) recomputeWith(ctx context.Context, overlay map[string]delivery.Module) error {
|
||||
if err := ix.fillPaths(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := ix.recomputeModules(ctx, overlay, w); err != nil {
|
||||
if err := ix.recomputeModules(ctx, overlay); err != nil {
|
||||
return err
|
||||
}
|
||||
// What the modules moved to reaches the candidates' fingerprints too (a base, the builder, a toolchain).
|
||||
return ix.RecomputeCandidates(ctx)
|
||||
}
|
||||
|
||||
func (ix *Indexer) recomputeModules(ctx context.Context, overlay map[string]delivery.Module, w *mergeWrite) error {
|
||||
func (ix *Indexer) recomputeModules(ctx context.Context, overlay map[string]delivery.Module) error {
|
||||
if ix.Modules == nil {
|
||||
return nil
|
||||
}
|
||||
if ix.Manifests == nil {
|
||||
return ix.writeOverlay(ctx, overlay, nil, w)
|
||||
return ix.writeOverlay(ctx, overlay, nil)
|
||||
}
|
||||
ix.recompute.Lock()
|
||||
defer ix.recompute.Unlock()
|
||||
@@ -295,7 +294,6 @@ func (ix *Indexer) recomputeModules(ctx context.Context, overlay map[string]deli
|
||||
continue
|
||||
}
|
||||
digest, inputs, paths := k.cur.Manifest, k.cur.Inputs2, k.cur.ArtifactPaths
|
||||
wrote := false
|
||||
err := write(ctx, ix.Modules, key, func(value []byte, found bool) ([]byte, bool, error) {
|
||||
if !found {
|
||||
return nil, false, nil
|
||||
@@ -310,17 +308,13 @@ func (ix *Indexer) recomputeModules(ctx context.Context, overlay map[string]deli
|
||||
cur.FP2, cur.None2, cur.Toolchains = r.fp, r.none, r.tcs
|
||||
cur.Inputs2, cur.ArtifactPaths = inputs, paths
|
||||
data, err := delivery.EncodeModule(cur)
|
||||
wrote = err == nil
|
||||
return data, false, err
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("modules.%s: %w", key, err)
|
||||
}
|
||||
if wrote {
|
||||
w.wrote(key)
|
||||
}
|
||||
}
|
||||
return ix.writeOverlay(ctx, overlay, finals, w)
|
||||
return ix.writeOverlay(ctx, overlay, finals)
|
||||
}
|
||||
|
||||
// writeOverlay writes the records of a merge, each once and only if it differs from what the bucket holds; finals has
|
||||
@@ -331,32 +325,24 @@ func (ix *Indexer) recomputeModules(ctx context.Context, overlay map[string]deli
|
||||
// appeared at: when the merge gives a module the value it holds but for the commit (and the source revision stamped on it),
|
||||
// the held value stands, its commit unchanged, so a catalogue merge touching one module writes one key, not one per module of
|
||||
// the repository, and the controller's take-in hears only that one. The repository's trunk head is the gate's source.main.
|
||||
//
|
||||
// A merged message older than the repository's taken merge never reaches here (order.go): it is asked before any write.
|
||||
func (ix *Indexer) writeOverlay(ctx context.Context, overlay, finals map[string]delivery.Module, w *mergeWrite) error {
|
||||
func (ix *Indexer) writeOverlay(ctx context.Context, overlay, finals map[string]delivery.Module) error {
|
||||
for _, key := range slices.Sorted(maps.Keys(overlay)) {
|
||||
next := overlay[key]
|
||||
if f, ok := finals[key]; ok {
|
||||
next = f
|
||||
}
|
||||
wrote := false
|
||||
err := write(ctx, ix.Modules, key, func(value []byte, found bool) ([]byte, bool, error) {
|
||||
wrote = false
|
||||
if found {
|
||||
if cur, err := delivery.DecodeModule(value); err == nil && sameButWhere(cur, next) {
|
||||
return nil, false, nil
|
||||
}
|
||||
}
|
||||
data, err := delivery.EncodeModule(next)
|
||||
wrote = err == nil
|
||||
return data, false, err
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("modules.%s: %w", key, err)
|
||||
}
|
||||
if wrote {
|
||||
w.wrote(key)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -81,12 +81,11 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build
|
||||
if m.Module != "mesh-delivery-indexer" {
|
||||
t.Errorf("module = %q", m.Module)
|
||||
}
|
||||
// 1126: placement is the seat mesh-placement's (ADR 0567): the indexer holds no placement state and says no event.
|
||||
if len(m.Emits) != 0 {
|
||||
t.Errorf("emits = %v, want none", m.Emits)
|
||||
// 1126 (ADR 0549 §2): the indexer's placement state, and the event it says when a node's placement changed.
|
||||
if want := []string{"placement-changed"}; !reflect.DeepEqual(m.Emits, want) {
|
||||
t.Errorf("emits = %v, want %v", m.Emits, want)
|
||||
}
|
||||
// 1126 note 76: heard keeps the last 64 FORGE messages the indexer handled, for indexer_heard.
|
||||
if want := []state{{Name: "candidates", History: 10}, {Name: "heard", History: 64}, {Name: "manifests", History: 1}, {Name: "modules", History: 20}, {Name: "taken", History: 1}, {Name: "takeins", History: 1}}; !reflect.DeepEqual(m.State, want) {
|
||||
if want := []state{{Name: "candidates", History: 10}, {Name: "manifests", History: 1}, {Name: "modules", History: 20}, {Name: "placement", History: 10}, {Name: "placement-candidates", History: 5}, {Name: "taken", History: 1}, {Name: "takeins", History: 1}}; !reflect.DeepEqual(m.State, want) {
|
||||
t.Errorf("state = %+v, want %+v", m.State, want)
|
||||
}
|
||||
if want := []string{"mesh-build-go.toolchains", "mesh-build-node.toolchains", "mesh-controller.assignments", "mesh-controller.builds", "mesh-delivery-checker.previews", "mesh-delivery-gate.source"}; !reflect.DeepEqual(sorted(m.Reads), want) {
|
||||
@@ -106,14 +105,15 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build
|
||||
// Three invokes: the file of a registered module's manifest and lock files (registered.go), the forge's trees (the named paths' objects, since the indexer is the one reader of a manifest's paths:
|
||||
// ADR 0499 clauses 2 and 3, ADR 0495 clause 1; and the module directories of the trunk, issue 1900) and whether a
|
||||
// commit is in another (issue 1900).
|
||||
if want := []string{"gitea.gitea_commit_trees", "gitea.gitea_contains", "gitea.gitea_get_file", "gitea.gitea_open_pulls"}; !reflect.DeepEqual(m.Invokes, want) {
|
||||
// ADR 0559 §8: the placement reread also asks the catalogue's branches and open pull requests.
|
||||
if want := []string{"gitea.gitea_commit_trees", "gitea.gitea_contains", "gitea.gitea_get_file", "gitea.gitea_list_branches", "gitea.gitea_open_pulls"}; !reflect.DeepEqual(m.Invokes, want) {
|
||||
t.Errorf("invokes = %v, want %v", m.Invokes, want)
|
||||
}
|
||||
// 1945: indexer_take_in, served by its tools bundle through the takeins state; the process serves none.
|
||||
if want := []string{"indexer_take_in"}; !reflect.DeepEqual(m.Tools, want) {
|
||||
// 1945: indexer_take_in, and indexer_placement_reread (ADR 0559 §8), served by its tools bundle through the takeins state; the process serves none.
|
||||
if want := []string{"indexer_take_in", "indexer_placement_reread"}; !reflect.DeepEqual(m.Tools, want) {
|
||||
t.Errorf("tools = %v, want %v", m.Tools, want)
|
||||
}
|
||||
// No streams, no bus, no seat (ADR 0437 §4); it says no event.
|
||||
// No streams, no bus, no seat (ADR 0437 §4); its one event is placement-changed (ADR 0549 §2), checked above.
|
||||
for _, k := range []string{"streams", "logs", "claims", "provides", "requires", "uses", "defines-seats", "seat", "seats", "bus"} {
|
||||
if _, ok := top[k]; ok {
|
||||
t.Errorf("the manifest declares %q: the indexer has none", k)
|
||||
|
||||
@@ -3,7 +3,11 @@
|
||||
"version": "1",
|
||||
"slug": "indexer",
|
||||
"tools": [
|
||||
"indexer_take_in"
|
||||
"indexer_take_in",
|
||||
"indexer_placement_reread"
|
||||
],
|
||||
"emits": [
|
||||
"placement-changed"
|
||||
],
|
||||
"consumes": [
|
||||
"mesh-build-go.toolchain.reported",
|
||||
@@ -13,9 +17,10 @@
|
||||
],
|
||||
"state": [
|
||||
{ "name": "candidates", "history": 10 },
|
||||
{ "name": "heard", "history": 64 },
|
||||
{ "name": "manifests", "history": 1 },
|
||||
{ "name": "modules", "history": 20 },
|
||||
{ "name": "placement", "history": 10 },
|
||||
{ "name": "placement-candidates", "history": 5 },
|
||||
{ "name": "taken", "history": 1 },
|
||||
{ "name": "takeins", "history": 1 }
|
||||
],
|
||||
@@ -31,6 +36,7 @@
|
||||
"gitea.gitea_commit_trees",
|
||||
"gitea.gitea_contains",
|
||||
"gitea.gitea_get_file",
|
||||
"gitea.gitea_list_branches",
|
||||
"gitea.gitea_open_pulls"
|
||||
],
|
||||
"own-secrets": {
|
||||
|
||||
@@ -3,7 +3,7 @@ module mesh-delivery-teller
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028174713-38dbe1d3a388
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027193820-8ad9372b1952
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028174713-38dbe1d3a388 h1:ycu/sNieTdLpD3bzhZOCXdzcoqXUOxPRmqWV019REpo=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028174713-38dbe1d3a388/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027193820-8ad9372b1952 h1:TmveT3Bija643RGWkilvkzP/h8UWlVvHFZZtQfJK5oU=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027193820-8ad9372b1952/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
|
||||
@@ -31,29 +31,6 @@ func Tools(r view.Reader, u view.UsageReader) []stdio.Tool {
|
||||
repository, _ := args["repository"].(string)
|
||||
return r.Reach(repository)
|
||||
}},
|
||||
{Name: view.ModuleHistoryTool,
|
||||
Description: "The history of one module's value in the indexer's modules state (modules.<module>), with times: each value it " +
|
||||
"still keeps (up to 20), newest first, with its revision, when it was written, put or del, the repository and commit " +
|
||||
"it was computed at, its source revision (src), its new-form fingerprint (fp2) or why it has none (none2), its manifest " +
|
||||
"digest and path. Shows when a merge wrote a module, and when an older merge wrote it back. Read-only.",
|
||||
Input: map[string]any{"type": "object", "required": []string{"module"}, "properties": map[string]any{
|
||||
"module": map[string]any{"type": "string", "description": "the module's name"}}},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
name, _ := args["module"].(string)
|
||||
return r.ModuleHistory(name)
|
||||
}},
|
||||
{Name: view.HeardTool,
|
||||
Description: "The forge stream's (FORGE, the forge bridge's) recent messages as the indexer heard them: for each merged, pull, " +
|
||||
"closed or open message it handled, newest first (its last 64), the subject, the stream sequence, how often the server " +
|
||||
"delivered it (a redelivery shows above 1), when it was published and handled, its kind, repository, pull request, " +
|
||||
"branch and commit (the merge commit, or the head), and what the indexer did: applied (with the modules written, or, " +
|
||||
"for a late merged message, the later merge already taken in, when nothing was written), left for redelivery or dropped, with why. Read-only.",
|
||||
Input: map[string]any{"type": "object", "properties": map[string]any{
|
||||
"limit": map[string]any{"type": "integer", "description": "at most this many messages, newest first (default every one kept)"}}},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
limit, _ := args["limit"].(float64)
|
||||
return r.Heard(int(limit))
|
||||
}},
|
||||
{Name: view.BuildUsageTool,
|
||||
Description: "How the build nodes are used, and whether a node is eligible to build, per node (every node holding a build module or named by a capacity): each build seat it holds (node-build-go, -node, -image) with its capacity " +
|
||||
"(mesh-config's resolved value, or the modules' default) and pause, and whether it is eligible to build and why not; the asks " +
|
||||
@@ -85,6 +62,17 @@ func Tools(r view.Reader, u view.UsageReader) []stdio.Tool {
|
||||
mods, err := r.Modules(only)
|
||||
return map[string]any{"modules": mods}, err
|
||||
}},
|
||||
{Name: view.PlacementReadTool,
|
||||
Description: "Where modules run, as the indexer read it from the catalogue's placement files at the last merge (placement.<node>, novox/hq ADR 0549): " +
|
||||
"each node's modules in order with their host ports, bindings and settings placement, its pins and the seats its modules hold, and the catalogue revision. " +
|
||||
"With node, that node's; with module, every node placing it; with neither, every node's. Keys that do not read are listed as unreadable. " +
|
||||
"Beside it, candidates: each open catalogue pull request's placement of a node that differs from the trunk's (placement-candidates <group>.<node>, issue 1126), " +
|
||||
"narrowed the same way, with the node's answer from its <node>.node have (answered: same, differs, refused, stale for an earlier head, none when it has not answered). Read-only.",
|
||||
Input: map[string]any{"type": "object", "properties": nodeModule},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
node, module := nodeModuleOf(args)
|
||||
return r.Placement(node, module)
|
||||
}},
|
||||
{Name: view.CandidateReadTool,
|
||||
Description: "What the indexer stored for one candidate (the open pull requests of a branch) under candidates.<group>, as stored: " +
|
||||
"the pull requests and heads, the base, and per pull request the touched modules with their new-form fingerprint (fp2) or why " +
|
||||
@@ -198,14 +186,7 @@ func backlogOf(path string) func() ([]view.BacklogRow, error) {
|
||||
}
|
||||
|
||||
func read(state stdio.KeptState) view.StateRead {
|
||||
return view.StateRead{Keys: state.Keys, History: func(key string) ([]view.Revision, error) {
|
||||
revs, err := state.History(key)
|
||||
out := make([]view.Revision, 0, len(revs))
|
||||
for _, rev := range revs {
|
||||
out = append(out, view.Revision{Revision: rev.Revision, At: rev.At, Op: rev.Op, Value: rev.Value})
|
||||
}
|
||||
return out, err
|
||||
}, GetEntry: func(key string) (json.RawMessage, uint64, error) {
|
||||
return view.StateRead{Keys: state.Keys, GetEntry: func(key string) (json.RawMessage, uint64, error) {
|
||||
e, err := state.Get(key)
|
||||
if err != nil || e == nil {
|
||||
return nil, 0, err
|
||||
@@ -223,8 +204,9 @@ func read(state stdio.KeptState) view.StateRead {
|
||||
func main() {
|
||||
r := view.Reader{State: read(stdio.State("mesh-delivery-indexer.modules")), Builds: read(stdio.State("mesh-delivery-builder.builds")), CandidatesState: read(stdio.State("mesh-delivery-indexer.candidates")),
|
||||
WantsState: read(stdio.State("mesh-delivery-assigner.wants")), DifferencesState: read(stdio.State("mesh-delivery-assigner.differences")), HavesState: read(stdio.State("node-engine.have")),
|
||||
ReachState: read(stdio.State("reach")),
|
||||
HeardState: read(stdio.State("mesh-delivery-indexer.heard"))}
|
||||
ReachState: read(stdio.State("reach")),
|
||||
PlacementState: read(stdio.State("mesh-delivery-indexer.placement")),
|
||||
PlacementCandidatesState: read(stdio.State("mesh-delivery-indexer.placement-candidates"))}
|
||||
u := view.UsageReader{Haves: r.HavesState, Resolved: read(stdio.State("mesh-config.resolved")), Tries: read(stdio.State("mesh-delivery-checker.tries")),
|
||||
Steps: map[string]view.StateRead{}, Pauses: map[string]view.StateRead{}, Backlog: backlogOf(socket())}
|
||||
for _, lang := range delivery.AskLanguages {
|
||||
|
||||
@@ -84,36 +84,6 @@ func TestModuleToolAnswersTheStoredValue(t *testing.T) {
|
||||
if _, err := run(t, r, view.ReachReadTool, map[string]any{"repository": "novox/gone"}); err == nil || !strings.Contains(err.Error(), "no reach of novox/gone") {
|
||||
t.Errorf("a repository with no reach: %v", err)
|
||||
}
|
||||
// Issue 1126 note 76: indexer_module_history answers modules.<module>'s kept values newest first, each with its
|
||||
// revision, time, commit, source revision and fp2; indexer_heard the FORGE messages the indexer handled, newest first.
|
||||
at := time.Date(2026, 10, 28, 13, 7, 46, 0, time.UTC)
|
||||
r.State.History = func(key string) ([]view.Revision, error) {
|
||||
if key != "a" {
|
||||
return nil, nil
|
||||
}
|
||||
return []view.Revision{{Revision: 538, At: at.Add(-time.Minute), Op: "put", Value: json.RawMessage(withNone)},
|
||||
{Revision: 539, At: at, Op: "put", Value: json.RawMessage(withFP2)}, {Revision: 540, At: at.Add(time.Minute), Op: "del"}}, nil
|
||||
}
|
||||
out, err := run(t, r, view.ModuleHistoryTool, map[string]any{"module": "a"})
|
||||
h, _ := out.(view.ModuleHistory)
|
||||
if err != nil || len(h.Values) != 3 || h.Values[0].Revision != 540 || h.Values[0].Op != "del" || h.Values[1].Revision != 539 || !h.Values[1].At.Equal(at) ||
|
||||
h.Values[1].Commit != "c1" || h.Values[1].FP2 != "b2" || h.Values[2].Commit != "c2" || h.Values[2].None2 == "" {
|
||||
t.Errorf("indexer_module_history a = %+v, %v", out, err)
|
||||
}
|
||||
if _, err := run(t, r, view.ModuleHistoryTool, map[string]any{"module": "gone"}); err == nil || !strings.Contains(err.Error(), "no history of gone") {
|
||||
t.Errorf("a module with no history: %v", err)
|
||||
}
|
||||
const heard1, heard2 = `{"v":1,"subject":"s","sequence":41,"repository":"novox/mesh-catalog","commit":"9cdac09a"}`, `{"v":1,"subject":"s","sequence":42,"repository":"novox/mesh-catalog","commit":"69056948"}`
|
||||
r.HeardState = view.StateRead{History: func(key string) ([]view.Revision, error) {
|
||||
return []view.Revision{{Revision: 1, At: at, Op: "put", Value: json.RawMessage(heard1)}, {Revision: 2, At: at, Op: "put", Value: json.RawMessage(heard2)}}, nil
|
||||
}}
|
||||
out, err = run(t, r, view.HeardTool, map[string]any{})
|
||||
if rows, _ := out.(view.HeardAnswer); err != nil || len(rows.Heard) != 2 || rows.Heard[0].Revision != 2 || string(rows.Heard[0].Heard) != heard2 || !rows.Heard[1].At.Equal(at) {
|
||||
t.Errorf("indexer_heard = %+v, %v", out, err)
|
||||
}
|
||||
if out, err = run(t, r, view.HeardTool, map[string]any{"limit": float64(1)}); err != nil || len(out.(view.HeardAnswer).Heard) != 1 {
|
||||
t.Errorf("indexer_heard limit 1 = %+v, %v", out, err)
|
||||
}
|
||||
}
|
||||
|
||||
// 1562/AC1: indexer_modules answers every key as stored, and with none2 only those that say why they have no FP2.
|
||||
@@ -135,6 +105,77 @@ func TestModulesToolAnswersEveryKeyAsStored(t *testing.T) {
|
||||
if len(only) != 1 || string(only["b"]) != withNone {
|
||||
t.Errorf("none2 modules = %v", only)
|
||||
}
|
||||
|
||||
// 1126 (ADR 0549 §2): indexer_placement answers a node's key as stored, every node placing a module, or every node's,
|
||||
// and lists a key that does not read.
|
||||
const ace, g14 = `{"v":1,"node":"ace","revision":"m1","modules":[{"module":"plex"}]}`, `{"v":1,"node":"g14","revision":"m1","modules":[{"module":"zsh"},{"module":"plex"}]}`
|
||||
p := view.Reader{PlacementState: fake{"placement.ace": ace, "placement.g14": g14, "placement.bad": `{"v":1}`, "other": "{}"}.reader().State}
|
||||
got := func(args map[string]any) map[string]json.RawMessage {
|
||||
t.Helper()
|
||||
out, err := run(t, p, view.PlacementReadTool, args)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return out.(map[string]any)["placement"].(map[string]json.RawMessage)
|
||||
}
|
||||
if one := got(map[string]any{"node": "ace"}); len(one) != 1 || string(one["ace"]) != ace {
|
||||
t.Errorf("node ace = %v", one)
|
||||
}
|
||||
if zsh := got(map[string]any{"module": "zsh"}); len(zsh) != 1 || string(zsh["g14"]) != g14 {
|
||||
t.Errorf("module zsh = %v", zsh)
|
||||
}
|
||||
out, err = run(t, p, view.PlacementReadTool, map[string]any{})
|
||||
if all := out.(map[string]any); err != nil || len(all["placement"].(map[string]json.RawMessage)) != 2 || len(all["unreadable"].([]string)) != 2 {
|
||||
t.Errorf("every node = %v, %v", all, err)
|
||||
}
|
||||
if _, err := run(t, p, view.PlacementReadTool, map[string]any{"node": "novox"}); err == nil || !strings.Contains(err.Error(), "no placement of novox") {
|
||||
t.Errorf("a node with no key: %v", err)
|
||||
}
|
||||
if c := out.(map[string]any)["candidates"].([]view.CandidateRow); len(c) != 0 {
|
||||
t.Errorf("with no candidate open: %v", c)
|
||||
}
|
||||
|
||||
// ADR 0559 §7 (issue 1126): beside the trunk's, each open candidate key and the node's answer from its <node>.node:
|
||||
// answered at the key's head and revision, at an earlier head (stale), or not at all (none); a node the trunk does not
|
||||
// place yet is shown by its candidate alone; ace, placed on the trunk with its file unchanged (no key of its own), is
|
||||
// shown with its answer, counted by the head alone.
|
||||
const h1, h2 = "1111111111111111111111111111111111111111", "2222222222222222222222222222222222222222"
|
||||
cand := func(node, head string, placed bool) string {
|
||||
pl := `,"placement":{"v":1,"node":"` + node + `","modules":[{"module":"plex"}]}`
|
||||
if !placed {
|
||||
pl = ""
|
||||
}
|
||||
return `{"v":1,"group":"feat/p","pull":{"repository":"novox/mesh-catalog","number":9,"head":"` + head + `"}` + pl + `}`
|
||||
}
|
||||
cands := fake{"feat/p.g14": cand("g14", h2, false), "feat/p.novox": cand("novox", h2, true)}
|
||||
revs := map[string]uint64{"feat/p.g14": 5, "feat/p.novox": 6}
|
||||
p.PlacementCandidatesState = view.StateRead{Keys: cands.reader().State.Keys, GetEntry: func(key string) (json.RawMessage, uint64, error) {
|
||||
raw, _ := cands.reader().State.Get(key)
|
||||
return raw, revs[key], nil
|
||||
}}
|
||||
p.HavesState = fake{
|
||||
"ace.node": `{"v":1,"composed":"x","candidates":{"feat/p":{"head":"` + h2 + `","verdict":"same","same":["plex"]}}}`,
|
||||
"g14.node": `{"v":1,"composed":"x","candidates":{"feat/p":{"head":"` + h1 + `","revision":2,"verdict":"same"}}}`,
|
||||
}.reader().State
|
||||
out, err = run(t, p, view.PlacementReadTool, map[string]any{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
answered := map[string]string{}
|
||||
for _, c := range out.(map[string]any)["candidates"].([]view.CandidateRow) {
|
||||
answered[c.Node] = c.Answered
|
||||
}
|
||||
if want := map[string]string{"ace": "same", "g14": view.AnswerStale, "novox": view.AnswerNone}; fmt.Sprint(answered) != fmt.Sprint(want) {
|
||||
t.Errorf("answers %v, want %v", answered, want)
|
||||
}
|
||||
out, err = run(t, p, view.PlacementReadTool, map[string]any{"node": "novox"})
|
||||
if c := out.(map[string]any)["candidates"].([]view.CandidateRow); err != nil || len(c) != 1 || c[0].Pull.Head != h2 || len(out.(map[string]any)["placement"].(map[string]json.RawMessage)) != 0 {
|
||||
t.Errorf("a node placed by its candidate alone: %v, %v", out, err)
|
||||
}
|
||||
out, _ = run(t, p, view.PlacementReadTool, map[string]any{"module": "plex"})
|
||||
if c := out.(map[string]any)["candidates"].([]view.CandidateRow); len(c) != 2 {
|
||||
t.Errorf("the candidates placing plex: %v (g14's removes its file)", c)
|
||||
}
|
||||
}
|
||||
|
||||
const (
|
||||
|
||||
@@ -75,14 +75,12 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
}
|
||||
defer conn.Close()
|
||||
model.SetWriter(kvWriter{ctx: ctx, js: js})
|
||||
model.SetMotionEvent(func(e view.MotionEvent) error {
|
||||
err := emitEvent(ctx, js, cred, e.Type, e.Body, e.ID)
|
||||
if err != nil {
|
||||
logger.Printf("the %s of %s on %s could not be said (said again at its next change): %v", e.Type, e.Body.Module, e.Body.Node, err)
|
||||
model.SetMotionEvent(func(e view.MotionEvent) {
|
||||
if err := emitEvent(ctx, js, cred, e.Type, e.Body, e.ID); err != nil {
|
||||
logger.Printf("the %s of %s on %s could not be said: %v", e.Type, e.Body.Module, e.Body.Node, err)
|
||||
}
|
||||
return err
|
||||
})
|
||||
model.SetDeliveredEvent(func(e view.DeliveredEvent) error {
|
||||
model.SetDeliveredEvent(func(e view.DeliveredEvent) {
|
||||
id := fmt.Sprintf("delivered.%d", e.Revision)
|
||||
if e.Repository != "" { // a repository's reached head (ADR 0556 clause 3)
|
||||
key, _ := delivery.RepositoryKey(e.Repository)
|
||||
@@ -92,18 +90,14 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
id += "." + hex.EncodeToString(sum[:6])
|
||||
}
|
||||
}
|
||||
err := emitEvent(ctx, js, cred, DeliveredEventType, e, id)
|
||||
if err != nil {
|
||||
logger.Printf("the delivery of source revision %d could not be said (said again at the next change the view takes in): %v", e.Revision, err)
|
||||
if err := emitEvent(ctx, js, cred, DeliveredEventType, e, id); err != nil {
|
||||
logger.Printf("the delivery of source revision %d could not be said: %v", e.Revision, err)
|
||||
}
|
||||
return err
|
||||
})
|
||||
model.SetReachWriter(func(w view.ReachWrite) error {
|
||||
err := putReach(ctx, js, w)
|
||||
if err != nil {
|
||||
logger.Printf("the reach of %s could not be written (written again at the next change the view takes in): %v", w.Key, err)
|
||||
model.SetReachWriter(func(w view.ReachWrite) {
|
||||
if err := putReach(ctx, js, w); err != nil {
|
||||
logger.Printf("the reach of %s could not be written (written again at its next change): %v", w.Key, err)
|
||||
}
|
||||
return err
|
||||
})
|
||||
model.SetBacklog(view.BacklogOver(js, 5*time.Second))
|
||||
for _, bucket := range []string{delivery.HaveBucket, view.AssignmentsBucket, view.BuildsBucket, delivery.PresenceBucket, delivery.StuckBucket, delivery.CandidatesBucket, delivery.ModulesBucket, delivery.AsksBucket, delivery.AnswersBucket, delivery.BuilderBuildsBucket, delivery.AssignerWantsBucket, delivery.SourceBucket, delivery.ControllerComposedBucket, delivery.AssignerAsksBucket, delivery.ViewAnswersBucket, delivery.TakenBucket} {
|
||||
|
||||
@@ -3,7 +3,7 @@ module mesh-delivery-view
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028174713-38dbe1d3a388
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027193820-8ad9372b1952
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028174713-38dbe1d3a388 h1:ycu/sNieTdLpD3bzhZOCXdzcoqXUOxPRmqWV019REpo=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028174713-38dbe1d3a388/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027193820-8ad9372b1952 h1:TmveT3Bija643RGWkilvkzP/h8UWlVvHFZZtQfJK5oU=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027193820-8ad9372b1952/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
|
||||
@@ -1,141 +0,0 @@
|
||||
package view
|
||||
|
||||
// Two read tools for tracing a module's value back to the message that wrote it (issue 1126 note 76: mesh-catalog's merge
|
||||
// 9cdac09a was written back over the later 69056948, and no tool read a modules key's history with times, nor what the
|
||||
// indexer heard of the forge stream FORGE).
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
const (
|
||||
// ModuleHistoryTool reads modules.<module>'s kept values with their times.
|
||||
ModuleHistoryTool = "indexer_module_history"
|
||||
// HeardTool reads the FORGE messages the indexer handled (its state heard).
|
||||
HeardTool = "indexer_heard"
|
||||
// heardKey is the indexer's one key of its heard state (mesh-delivery-indexer internal/index heard.go HeardKey).
|
||||
heardKey = "forge"
|
||||
)
|
||||
|
||||
// Revision is one value a state still keeps for a key (mesh-sdk stdio StateRevision): its revision, when it was written,
|
||||
// put or del, and the value (none for a delete).
|
||||
type Revision struct {
|
||||
Revision uint64 `json:"revision"`
|
||||
At time.Time `json:"at"`
|
||||
Op string `json:"op"`
|
||||
Value json.RawMessage `json:"value,omitempty"`
|
||||
}
|
||||
|
||||
// ModuleRevision is one kept value of modules.<module>, read: where it was computed and what it computed.
|
||||
type ModuleRevision struct {
|
||||
Revision uint64 `json:"revision"`
|
||||
At time.Time `json:"at"`
|
||||
Op string `json:"op"`
|
||||
Repository string `json:"repository,omitempty"`
|
||||
Commit string `json:"commit,omitempty"`
|
||||
Src uint64 `json:"src,omitempty"`
|
||||
FP2 string `json:"fp2,omitempty"`
|
||||
None2 string `json:"none2,omitempty"`
|
||||
Manifest string `json:"manifest,omitempty"`
|
||||
Path string `json:"path,omitempty"`
|
||||
Unreadable string `json:"unreadable,omitempty"`
|
||||
}
|
||||
|
||||
// ModuleHistory is a module's kept values, newest first.
|
||||
type ModuleHistory struct {
|
||||
Module string `json:"module"`
|
||||
Values []ModuleRevision `json:"values"`
|
||||
Note string `json:"note"`
|
||||
}
|
||||
|
||||
// ModuleHistory reads every value the modules state keeps for module, newest first, each with its revision, time, commit,
|
||||
// source revision and fingerprint.
|
||||
func (r Reader) ModuleHistory(module string) (ModuleHistory, error) {
|
||||
module = strings.TrimSpace(module)
|
||||
out := ModuleHistory{Module: module, Note: "newest first; the modules state keeps 20 values a key (its history), so an older one is gone"}
|
||||
key, err := delivery.ModulesKey(module)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
if r.State.History == nil {
|
||||
return out, fmt.Errorf("the modules state's history cannot be read here")
|
||||
}
|
||||
revs, err := r.State.History(key)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
if len(revs) == 0 {
|
||||
return out, fmt.Errorf("no history of %s in the modules state", module)
|
||||
}
|
||||
for _, rev := range newestFirst(revs) {
|
||||
row := ModuleRevision{Revision: rev.Revision, At: rev.At, Op: rev.Op}
|
||||
if len(rev.Value) > 0 && string(rev.Value) != "null" {
|
||||
// Read field by field, whatever the value's version, so an older or newer writer's value still shows.
|
||||
var m delivery.Module
|
||||
if err := json.Unmarshal(rev.Value, &m); err != nil {
|
||||
row.Unreadable = err.Error()
|
||||
} else {
|
||||
row.Repository, row.Commit, row.Src, row.FP2, row.None2, row.Manifest, row.Path = m.Repository, m.Commit, m.Src, m.FP2, m.None2, m.Manifest, m.Path
|
||||
}
|
||||
}
|
||||
out.Values = append(out.Values, row)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// HeardRow is one FORGE message as the indexer handled it, with the revision and time of its value.
|
||||
type HeardRow struct {
|
||||
Revision uint64 `json:"revision"`
|
||||
At time.Time `json:"at"`
|
||||
Heard json.RawMessage `json:"heard"`
|
||||
}
|
||||
|
||||
// HeardAnswer is the FORGE messages the indexer handled, newest first.
|
||||
type HeardAnswer struct {
|
||||
Heard []HeardRow `json:"heard"`
|
||||
Note string `json:"note"`
|
||||
}
|
||||
|
||||
// Heard reads the indexer's heard state: the last FORGE messages it handled (at most limit, 0 for every one kept), newest
|
||||
// first.
|
||||
func (r Reader) Heard(limit int) (HeardAnswer, error) {
|
||||
out := HeardAnswer{Heard: []HeardRow{}, Note: "newest first; the indexer keeps its last 64 FORGE messages. A message the stream holds and the indexer never handled is not here; " +
|
||||
"delivered above 1 is a message the server delivered again"}
|
||||
if r.HeardState.History == nil {
|
||||
return out, fmt.Errorf("the indexer's heard state cannot be read here")
|
||||
}
|
||||
revs, err := r.HeardState.History(heardKey)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
for _, rev := range newestFirst(revs) {
|
||||
if limit > 0 && len(out.Heard) == limit {
|
||||
break
|
||||
}
|
||||
if len(rev.Value) == 0 || string(rev.Value) == "null" {
|
||||
continue
|
||||
}
|
||||
out.Heard = append(out.Heard, HeardRow{Revision: rev.Revision, At: rev.At, Heard: rev.Value})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func newestFirst(revs []Revision) []Revision {
|
||||
revs = slices.Clone(revs)
|
||||
slices.SortFunc(revs, func(a, b Revision) int {
|
||||
switch {
|
||||
case a.Revision > b.Revision:
|
||||
return -1
|
||||
case a.Revision < b.Revision:
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
})
|
||||
return revs
|
||||
}
|
||||
@@ -31,9 +31,8 @@ type ReachWrite struct {
|
||||
Value []byte
|
||||
}
|
||||
|
||||
// SetReachWriter sets where the view's reach values go; nothing is written until it is set. A value it answers an error for is
|
||||
// written again at the next change the view takes in (sendReach).
|
||||
func (m *Model) SetReachWriter(put func(ReachWrite) error) {
|
||||
// SetReachWriter sets where the view's reach values go; nothing is written until it is set.
|
||||
func (m *Model) SetReachWriter(put func(ReachWrite)) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
m.reachPut = put
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user