diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/deliveries.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/deliveries.go index 24780ce18..42f700bd5 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/deliveries.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/deliveries.go @@ -69,6 +69,8 @@ type DeliveryStore interface { type ForgeReads interface { OpenPulls() (gitseat.OpenPullsAnswer, error) Pull(repository string, number int64) (gitseat.PullAnswer, error) + // Branch is a repository branch's head (the seat git's verb branch). + Branch(repository, branch string) (gitseat.BranchAnswer, error) } // kvKept is the store over the runtime. diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/deliveries_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/deliveries_test.go index 7710e6fe1..7fd363538 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/deliveries_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/deliveries_test.go @@ -41,6 +41,7 @@ func (k *memKept) Delete(state, key string) error { delete(k.m[state], key); ret type fakeForge struct { open []gitseat.OpenPull pulls map[string]gitseat.Pull + heads map[string]string // "@" -> head fail bool } @@ -50,6 +51,16 @@ func (f *fakeForge) OpenPulls() (gitseat.OpenPullsAnswer, error) { } return gitseat.OpenPullsAnswer{Pulls: f.open}, nil } +func (f *fakeForge) Branch(repository, branch string) (gitseat.BranchAnswer, error) { + if f.fail { + return gitseat.BranchAnswer{}, errors.New("the forge is away") + } + head, ok := f.heads[repository+"@"+branch] + if !ok { + return gitseat.BranchAnswer{Error: "no such branch"}, nil + } + return gitseat.BranchAnswer{Branch: branch, Repository: repository, Head: head}, nil +} func (f *fakeForge) Pull(repository string, number int64) (gitseat.PullAnswer, error) { p, ok := f.pulls[fmt.Sprintf("%s#%d", repository, number)] if !ok { diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/gate.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/gate.go index 94de1c9b7..05222b46c 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/gate.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/gate.go @@ -23,11 +23,14 @@ type Holder struct { // Merges and Old are the old sections the switch reads (switch.go); Source is the gate's source (merges.go). Merges MergeStore // Kept is the gate's states deliveries and groups, and Forge the seat git's reads (deliveries.go, §716). - Kept DeliveryStore - Forge ForgeReads - dmu sync.Mutex - Old OldState - Source SourceStore + Kept DeliveryStore + Forge ForgeReads + // Catalog reads the seat mesh-catalog's registrations (sourcestart.go); startMu keeps one start of the sources at a time. + Catalog Registered + startMu sync.Mutex + dmu sync.Mutex + Old OldState + Source SourceStore // Owed says what the gate owes the forge as the seat's event owed (owed.go). Owed OwedSayer // Records keeps the delivery records on the gate's ledger (record.go, ADR 0656 §7); nil keeps none. diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/main.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/main.go index 60984b018..b9bc97998 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/main.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/main.go @@ -32,7 +32,7 @@ func (l *listening) get() string { l.mu.Lock(); defer l.mu.Unlock(); return l.n // The events this owner follows: the forge bridge's word on an act owed, and the step events the delivery records are built // from (consumer.go). -var followed = append(append([]string{owedEndedEvent}, stepPatterns()...), pullUpdatedEvent, pullMergedEvent, pullClosedEvent) +var followed = append(append([]string{owedEndedEvent}, stepPatterns()...), pullUpdatedEvent, pullMergedEvent, pullClosedEvent, registrationChangedEvent) func main() { settingsFile := os.Getenv("MESH_DELIVERY_GATE_SETTINGS") @@ -68,6 +68,7 @@ func main() { h.Old = kvOld{} h.Kept = kvKept{} h.Forge = seatGit{ask: stdio.Ask} + h.Catalog = seatCatalog{ask: stdio.Ask} h.Consumer = &StepConsumer{Records: h.Records, Families: stepFamilies, Now: h.Now, Logf: logf, Taken: h.TakenIn} // The gate's one held writer of its own findings (design 47 G A-39, A-37, A-40): made at once, started once the runtime serves. fw, err := NewFindingWriter() @@ -116,12 +117,27 @@ func run(h *Holder, l *listening) { } l.set("listening") logf("[mesh-delivery] listening: %s", strings.Join(followed, ", ")) + // A watched branch no source key holds starts from the forge, once at each start and at each registration change (A-401). + go startSourcesAtStart(h) // A missed pull event is put right from the forge's own record, once at each start (§716). if err := h.ReadForge(); err != nil { logf("[mesh-delivery] the open pull requests were not read from the forge, they are read again at the next start: %v", err) } } +// startSourcesAtStart starts the sources of the watched branches no key holds, retrying until the registrations can be read. +func startSourcesAtStart(h *Holder) { + for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) { + started, err := h.StartSources() + if err == nil { + logf("[mesh-delivery] the sources started from the forge at the start: %v", started) + return + } + logf("[mesh-delivery] the sources cannot be started yet (%v); asking again in %s", err, wait) + time.Sleep(wait) + } +} + // switchToRecords carries the groups in flight into their delivery records (switch.go), retrying until it can: no event is // taken before, so no outcome reaches a record the switch has not opened yet. func switchToRecords(h *Holder, l *listening) { @@ -172,6 +188,10 @@ func take(h *Holder, e stdio.Envelope) error { if e.Key == pullMergedEvent { return errors.Join(h.DeliveryMerged(e.Body), h.pullEnded(e.Body, "merged")) } + if e.Key == registrationChangedEvent { + _, err := h.StartSources() + return err + } if e.Key == pullClosedEvent { return errors.Join(h.DeliveryClosed(e.Body), h.pullEnded(e.Body, "closed")) } diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/manifest_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/manifest_test.go index a679e1fa7..7e060757d 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/manifest_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/manifest_test.go @@ -181,7 +181,7 @@ func TestItClaimsTheDeliverySeatAndFollowsWhatItHandles(t *testing.T) { } // It calls no controller verb and no forge verb: the forge's reads and writes are the bridge's (ADR 0437 §8; design 47 §592). // It uses operator-channel only to say its conditions (ADR 0437 §9, issue 1275): the teller asks no more (§431). - want := []string{"seat:git.open-pulls", "seat:git.pull", "seat:mesh-hand-acts.hand-act", WantsEntryVerb, EngineCompositionVerb} + want := []string{"seat:git.branch", "seat:git.open-pulls", "seat:git.pull", "seat:mesh-catalog.registrations", "seat:mesh-hand-acts.hand-act", WantsEntryVerb, EngineCompositionVerb} if !reflect.DeepEqual(m.Uses, []string{"operator-channel"}) { t.Fatalf("uses %v: operator-channel, for its conditions alone", m.Uses) } diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/sourcestart.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/sourcestart.go new file mode 100644 index 000000000..b264da7a7 --- /dev/null +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/sourcestart.go @@ -0,0 +1,200 @@ +package main + +// A watched branch starts from the forge, as a clone does (design 47 G A-401, §430, §15): at its start and on each +// registration-changed of the seat mesh-catalog, the gate writes source. for every watched branch the registrations name, +// each registered repository at the head of that branch, asked of the seat git's verb branch, the group left empty, and says +// source-moved. A repository whose head cannot be read is left out and said. A head a key holds is never rewritten here: from +// then on only a take-in moves it (A-34). + +import ( + "encoding/json" + "errors" + "fmt" + "sort" + "strconv" + "strings" + + "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient" + gitseat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/git" + meshcatalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" + seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery" + "git.novox.be/novox/mesh-sdk/go/keys" +) + +// registrationChangedEvent is the seat mesh-catalog's event, as the gate's manifest consumes it. +var registrationChangedEvent = declared(meshcatalog.EventRegistrationChanged) + +// Registered is what the gate reads of the seat mesh-catalog: its registrations. +type Registered interface { + Registrations() (meshcatalog.RegistrationsAnswer, error) +} + +// seatCatalog reads the seat mesh-catalog through the tool runner. +type seatCatalog struct{ ask asking } + +func (s seatCatalog) Registrations() (meshcatalog.RegistrationsAnswer, error) { + var a meshcatalog.RegistrationsAnswer + raw, err := s.ask("seat:"+meshcatalog.Seat+"."+meshcatalog.VerbRegistrations, meshcatalog.RegistrationsInput{}) + if err != nil { + return a, err + } + answer, said, ok := answerOf(raw) + if !ok { + return a, fmt.Errorf("registrations refused: %s", lastLine(said)) + } + if err := json.Unmarshal(answer, &a); err != nil { + return a, fmt.Errorf("the answer of registrations is not readable: %w", err) + } + return a, nil +} + +// Branch is the seat git's verb branch: a repository's branch head. +func (s seatGit) Branch(repository, branch string) (gitseat.BranchAnswer, error) { + var a gitseat.BranchAnswer + return a, s.call(gitseat.VerbBranch, gitseat.BranchInput{Repository: repository, Branch: branch}, &a) +} + +// watchedRepositories are the repositories each watched branch is registered for, each once, sorted. +func watchedRepositories(regs []meshcatalog.Registration) map[string][]string { + seen := map[string]map[string]bool{} + for _, r := range regs { + if strings.TrimSpace(r.Origin) == "" { + continue + } + for _, b := range r.Branches { + if strings.TrimSpace(b) == "" { + continue + } + if seen[b] == nil { + seen[b] = map[string]bool{} + } + seen[b][r.Origin] = true + } + } + out := map[string][]string{} + for b, rs := range seen { + for r := range rs { + out[b] = append(out[b], r) + } + sort.Strings(out[b]) + } + return out +} + +// StartSources writes the source key of every watched branch the registrations name that no key holds yet, and answers the +// branches it wrote. A registrations answer that cannot be read is an error (the event is offered again); a repository whose +// head cannot be read is said and left out. +func (h *Holder) StartSources() ([]string, error) { + started, _, err := h.startSources("") + return started, err +} + +// startSources starts the branches no key holds, or only branch when it is named; it answers the branches written and the +// repositories left out as "@". +func (h *Holder) startSources(only string) (started, leftOut []string, err error) { + if h.Source == nil || h.Catalog == nil || h.Forge == nil { + return nil, nil, errors.New("this gate keeps no source, reads no registrations or reads no forge") + } + h.startMu.Lock() + defer h.startMu.Unlock() + regs, err := h.Catalog.Registrations() + if err != nil { + return nil, nil, fmt.Errorf("the registrations cannot be read: %w", err) + } + watched := watchedRepositories(regs.Registrations) + if only != "" { + if _, ok := watched[only]; !ok { + return nil, nil, fmt.Errorf("no registration names the branch %s", only) + } + } + names := make([]string, 0, len(watched)) + for b := range watched { + if only == "" || b == only { + names = append(names, b) + } + } + sort.Strings(names) + var errs []error + for _, branch := range names { + held, _, err := h.Source.Read(branch) + if err != nil { + errs = append(errs, err) + continue + } + if held != nil { + continue + } + heads := map[string]string{} + for _, repository := range watched[branch] { + a, err := h.Forge.Branch(repository, branch) + if err == nil && a.Error != "" { + err = errors.New(a.Error) + } + if err == nil && a.Head == "" { + err = errors.New("the forge named no head") + } + if err != nil { + leftOut = append(leftOut, repository+"@"+branch) + h.logf("[mesh-delivery] source.%s starts without %s: its head cannot be read (%v); it is added at the next registration change or by gate_source_start", branch, repository, err) + continue + } + heads[repository] = a.Head + } + if len(heads) == 0 { + continue + } + wrote := false + rev, err := h.Source.Set(branch, func(prev *Source) (Source, bool) { + wrote = prev == nil // a key written meanwhile is never rewritten + return Source{Heads: heads}, wrote + }) + if err != nil { + errs = append(errs, fmt.Errorf("source.%s cannot be started: %w", branch, err)) + continue + } + if !wrote { + continue + } + started = append(started, branch) + if h.SaySeat == nil { + continue + } + at := h.Now().UTC() + if err := h.SaySeat(seat.SubjectSourceMoved(keys.EscapeName(branch)), + seat.SourceMovedEvent{Branch: branch, Revision: int64(rev), At: &at}, + "source-moved."+keys.EscapeName(branch)+"."+strconv.FormatUint(rev, 10)); err != nil { + errs = append(errs, err) + } + } + return started, leftOut, errors.Join(errs...) +} + +func (h *Holder) logf(format string, a ...any) { + if h.Logf != nil { + h.Logf(format, a...) + } +} + +// SourceStart starts one watched branch by hand, after the hand-act log took it: the same as the start does, for a branch no +// key holds yet. +func (h *Holder) SourceStart(branch, why, by string, acts handActs) (map[string]any, error) { + if strings.TrimSpace(branch) == "" || strings.TrimSpace(why) == "" { + return nil, errors.New("say the branch and why its source is started") + } + if h.Source == nil || h.Catalog == nil || h.Forge == nil { + return nil, errors.New("this gate keeps no source, reads no registrations or reads no forge") + } + if acts == nil { + return nil, errors.New("the hand-act log cannot be reached here, so nothing was started") + } + if _, err := acts.Record(meshhandactsclient.Request{What: "started the source of the branch " + branch, + Why: why, Cause: "gate_source_start", Verb: "source-start"}); err != nil { + return nil, fmt.Errorf("the hand act was not recorded through mesh-hand-acts, so nothing was started: %w", err) + } + started, leftOut, err := h.startSources(branch) + out := map[string]any{"branch": branch, "started": started, "left-out": leftOut, "by": by} + if err == nil && len(started) == 0 && len(leftOut) == 0 { + out["note"] = "the gate holds a source key of the branch already: only a take-in moves it" + } + return out, err +} diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/sourcetools.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/sourcetools.go index 68c982da0..6d3f7586c 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/sourcetools.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/sourcetools.go @@ -79,5 +79,15 @@ func sourceTools(h *Holder, acts handActs) []stdio.Tool { Run: func(a map[string]any) (any, error) { return h.SourceAgain(strArg(a, "branch"), strArg(a, "why"), stdio.CallerOf(a), acts) }}, + {Name: "gate_source_start", + Description: "Start the source of one watched branch the registrations name and no source key holds yet, from the forge: each " + + "registered repository at the branch's head, the group empty, then source-moved; recorded in the hand-act log of the seat " + + "mesh-hand-acts first. A key the gate holds is never rewritten.", + Input: map[string]any{"type": "object", "properties": map[string]any{ + "branch": str("the watched branch"), "why": str("why it is started by hand, in plain words")}, + "required": []string{"branch", "why"}}, + Run: func(a map[string]any) (any, error) { + return h.SourceStart(strArg(a, "branch"), strArg(a, "why"), stdio.CallerOf(a), acts) + }}, } } diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/sourcetools_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/sourcetools_test.go index 2094ff7b5..758645bfc 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/sourcetools_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/sourcetools_test.go @@ -2,7 +2,10 @@ package main import ( "errors" + "fmt" + meshcatalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery" + "reflect" "strings" "testing" "time" @@ -103,3 +106,109 @@ func TestTheSourceAnswerNamesTheBranchItWasAskedAndStatusSaysEveryBranch(t *test t.Fatalf("status says every branch's source: %+v", h.SourceAll()) } } + +type fakeCatalog struct { + regs []meshcatalog.Registration + err error +} + +func (f fakeCatalog) Registrations() (meshcatalog.RegistrationsAnswer, error) { + return meshcatalog.RegistrationsAnswer{Registrations: f.regs}, f.err +} + +// Design 47 G A-401: a watched branch starts from the forge as a clone does: each registered repository at the branch's head, +// the group empty, source-moved said; a repository whose head cannot be read is left out; a key held is never rewritten. +func TestAWatchedBranchStartsFromTheForge(t *testing.T) { + regs := []meshcatalog.Registration{ + {Module: "car-hunter", Origin: "o/car-hunter", Branches: []string{"main"}}, + {Module: "mesh-a", Origin: "o/mesh", Branches: []string{"main"}}, + {Module: "mesh-b", Origin: "o/mesh", Branches: []string{"main"}}, // the same repository twice: once + {Module: "f1", Origin: "o/f1-circuits", Branches: []string{"master"}}, + {Module: "l2p", Origin: "o/link2pay", Branches: []string{"nox-mesh"}}, + {Module: "ph", Origin: "o/photos", Branches: []string{"nox-mesh", "main"}}, + } + forge := &fakeForge{heads: map[string]string{ + "o/car-hunter@main": "c1", "o/mesh@main": "c2", "o/f1-circuits@master": "c3", "o/link2pay@nox-mesh": "c4", + "o/photos@main": "c6", // o/photos@nox-mesh cannot be read + }} + for _, c := range []struct { + name string + held map[string]Source + catalog fakeCatalog + only string + started []string + heads map[string]map[string]string + events int + wantErr bool + }{ + {name: "every branch no key holds", catalog: fakeCatalog{regs: regs}, + started: []string{"main", "master", "nox-mesh"}, events: 3, + heads: map[string]map[string]string{"main": {"o/car-hunter": "c1", "o/mesh": "c2", "o/photos": "c6"}, + "master": {"o/f1-circuits": "c3"}, "nox-mesh": {"o/link2pay": "c4"}}}, + {name: "a held key is never rewritten", catalog: fakeCatalog{regs: regs}, + held: map[string]Source{"main": {Group: "g", Heads: map[string]string{"o/mesh": "old"}}}, + started: []string{"master", "nox-mesh"}, events: 2, + heads: map[string]map[string]string{"main": {"o/mesh": "old"}, "master": {"o/f1-circuits": "c3"}, "nox-mesh": {"o/link2pay": "c4"}}}, + {name: "by hand, one branch", catalog: fakeCatalog{regs: regs}, only: "master", + started: []string{"master"}, events: 1, heads: map[string]map[string]string{"master": {"o/f1-circuits": "c3"}}}, + {name: "registrations unreadable", catalog: fakeCatalog{err: errors.New("away")}, wantErr: true}, + {name: "no registration names the branch", catalog: fakeCatalog{regs: regs}, only: "nope", wantErr: true}, + } { + t.Run(c.name, func(t *testing.T) { + src := &branchSource{memSource{vals: map[string]Source{}}} + for b, s := range c.held { + src.vals[b] = s + } + events := 0 + var logged []string + h := &Holder{Now: time.Now, Source: src, Forge: forge, Catalog: c.catalog, + Logf: func(f string, a ...any) { logged = append(logged, fmt.Sprintf(f, a...)) }, + SaySeat: func(subject string, body any, id string) error { + events++ + if ev := body.(seat.SourceMovedEvent); ev.Group != "" { + t.Errorf("the group is empty, got %q", ev.Group) + } + return nil + }} + started, _, err := h.startSources(c.only) + if (err != nil) != c.wantErr { + t.Fatalf("err %v", err) + } + if c.wantErr { + return + } + if !reflect.DeepEqual(started, c.started) || events != c.events { + t.Fatalf("started %v (%d events), want %v (%d)", started, events, c.started, c.events) + } + for b, want := range c.heads { + if got := src.vals[b].Heads; !reflect.DeepEqual(got, want) { + t.Errorf("source.%s heads %v, want %v", b, got, want) + } + } + if c.only == "" && c.held == nil && !strings.Contains(strings.Join(logged, "\n"), "o/photos") { + t.Errorf("the repository left out was not said: %v", logged) + } + }) + } +} + +// Design 47 G A-401: gate_source_start is recorded in the hand-act log first, and starts nothing when the log refuses it. +func TestSourceStartByHandIsRecordedFirst(t *testing.T) { + src := &branchSource{memSource{vals: map[string]Source{}}} + h := &Holder{Now: time.Now, Source: src, Forge: &fakeForge{heads: map[string]string{"o/f1@master": "c3"}}, + Catalog: fakeCatalog{regs: []meshcatalog.Registration{{Module: "f1", Origin: "o/f1", Branches: []string{"master"}}}}} + acts := &handActsFake{refuse: errors.New("log down")} + if _, err := h.SourceStart("master", "newly watched", "me", acts); err == nil || len(src.vals) != 0 { + t.Fatalf("started without the log: %v %v", err, src.vals) + } + acts.refuse = nil + if _, err := h.SourceStart("master", "", "me", acts); err == nil { + t.Fatal("started with no reason") + } + if _, err := h.SourceStart("master", "newly watched", "me", acts); err != nil { + t.Fatal(err) + } + if len(acts.took) != 1 || acts.took[0].Cause != "gate_source_start" || src.vals["master"].Heads["o/f1"] != "c3" { + t.Fatalf("log %+v source %+v", acts.took, src.vals) + } +} diff --git a/modules/mesh-delivery-gate/module.json b/modules/mesh-delivery-gate/module.json index aad02e5cf..2d170922d 100644 --- a/modules/mesh-delivery-gate/module.json +++ b/modules/mesh-delivery-gate/module.json @@ -22,6 +22,7 @@ "git.pull-updated", "git.pull-merged", "git.pull-closed", + "mesh-catalog.registration-changed", "mesh-conditions.findings-asked" ], "emits": [ @@ -351,8 +352,10 @@ "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg": "0.1" }, "invokes": [ + "seat:git.branch", "seat:git.open-pulls", "seat:git.pull", + "seat:mesh-catalog.registrations", "seat:mesh-hand-acts.hand-act", "seat:mesh-wants.entry", "seat:node-engine.composition" @@ -379,7 +382,8 @@ "gate_record_repair", "gate_records", "gate_source", - "gate_source_again" + "gate_source_again", + "gate_source_start" ], "config": { "record-retention": {