package view import ( "fmt" "maps" "slices" "strings" "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/trunk" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-view/viewstate" ) // How far each repository reached (novox/hq ADR 0556 clauses 2 and 3, issue 1280). The view keeps, per repository the gate's // source names, how far each of its modules reached on the nodes that want it, and derives the repository's reached head: the // newest head of the repository in the source's history, not past the merge the indexer took in, at which every wanted module's // floor is at or after its last change at or before that head. It writes that as its state reach and says each new reached // head as its event delivered, per repository. Nothing here records a merge as delivered (ADR 0429 §1): a merge at or before // the reached head is on the machines, derived by whoever reads it. // ReachHistory names the watch of a bucket with its history, as Initial is told it: the modules' history gives each // fingerprint's first commit. Each head's source revision is from the branch's sources heard since the start (design 47 G // A-173, A-225), laid over the reach the view's own state kept. func ReachHistory(bucket string) string { return bucket + "#history" } // reachBuckets are the watches the derivation waits for before it writes or says anything, so a start does not write a reach // derived from half its inputs; the view's own reach first, which a restart lays the derivation over (A-225). func (m *Model) reachBuckets() []string { return []string{ModulesBucket, AssignerWantsBucket, HaveBucket, TakenBucket, ReachHistory(ModulesBucket), m.own.Reach} } // ReachWrite is a reach value to write under its key in the view's state reach. type ReachWrite struct { Key string Value []byte } // SetReachWriter sets where the view's reach values go; nothing is written until it is set. A value it answers an error for is // written again at the next change the view takes in (sendReach). func (m *Model) SetReachWriter(put func(ReachWrite) error) { m.mu.Lock() defer m.mu.Unlock() m.reachPut = put } // applyHistory takes one revision of a watch with history: of the catalog half's modules (each module's values in order). func (m *Model) applyHistory(e Entry) { switch e.Bucket { case ModulesBucket: if e.Deleted { delete(m.modHist, e.Key) return } mod, err := meshcatalogclient.DecodeModule(e.Value) if err != nil || mod.Fp2 == "" || mod.Commit == "" { return } h := m.modHist[e.Key] if n := len(h); n > 0 && h[n-1].Fp2 == mod.Fp2 && h[n-1].Commit == mod.Commit { if mod.Src != 0 { h[n-1].Src = mod.Src } return } h = append(h, mod) if len(h) > 2*viewstate.MostChanges { h = h[len(h)-viewstate.MostChanges:] } m.modHist[e.Key] = h } } // headAt is one head of a repository and the first source revision that named it. type headAt struct { commit string rev uint64 } // headsOf are the heads of a repository in the history of one branch's source (escaped), oldest first, each with the first // revision naming it. func (m *Model) headsOf(branch, repository string) []headAt { var out []headAt hist := m.srcHist[branch] for _, rev := range slices.Sorted(maps.Keys(hist)) { for r, head := range hist[rev] { if !trunk.SameRepository(r, repository) || head == "" { continue } if n := len(out); n == 0 || !sameCommit(out[n-1].commit, head) { out = append(out, headAt{commit: head, rev: rev}) } } } return out } // revOf is the first source revision naming a commit as the repository's head, or 0 when the history holds none. func revOf(heads []headAt, commit string) uint64 { for _, h := range heads { if sameCommit(h.commit, commit) { return h.rev } } return 0 } // sameCommit is whether two spellings of a commit are one (either may be abbreviated to at least seven characters). func sameCommit(a, b string) bool { a, b = strings.ToLower(a), strings.ToLower(b) if len(a) > len(b) { a, b = b, a } return len(a) >= 7 && strings.HasPrefix(b, a) } // noBound is a module's bound when its floor is its newest change: it holds no head back. const noBound = ^uint64(0) // wantKeys are the wants' keys by module, each sorted: read once for every repository a derivation looks at. func (m *Model) wantKeys() map[string][]string { out := map[string][]string{} for _, key := range slices.Sorted(maps.Keys(m.wants)) { _, module := splitKey(key) out[module] = append(out[module], key) } return out } // deriveReach is the reach of one repository from the source of its branch (escaped) as the model holds it; wants are the // wants' keys by module (wantKeys). func (m *Model) deriveReach(branch, repository string, wants map[string][]string) viewstate.Reach { r := viewstate.Reach{Repository: trunk.RepositoryName(repository), Modules: map[string]viewstate.ModuleReach{}} heads := m.headsOf(branch, repository) key, _ := trunk.RepositoryKey(repository) if t, ok := m.taken[key]; ok { r.Indexed = t.Merge } bound := noBound var holds []string for _, name := range slices.Sorted(maps.Keys(m.modules)) { me := m.modules[name] if me.err != "" || me.value.Fp2 == "" || !trunk.SameRepository(me.value.Repository, repository) { continue } mr, b, words := m.moduleReach(name, me.value, heads, wants[name]) r.Modules[name] = mr if b < bound { bound = b } if b != noBound { holds = append(holds, words) } } if len(r.Modules) == 0 { r.Modules = nil } indexed := uint64(0) if r.Indexed != "" { indexed = revOf(heads, r.Indexed) } switch { case r.Indexed == "": holds = append(holds, "the indexer has taken in no merge of "+r.Repository) case indexed == 0: holds = append(holds, fmt.Sprintf("the merge the indexer took in, %.12s, is not a head of %s in the source's history", r.Indexed, r.Repository)) } for _, h := range heads { if h.rev <= indexed && h.rev < bound { r.Reached, r.ReachedSrc = h.commit, h.rev } } // The history is the revisions heard since the start: the reach the view's own state kept at the start stands until one // derived from them is newer (A-225). if k, ok := m.kept[key]; ok && k.Reached != "" && (r.Reached == "" || r.ReachedSrc < k.ReachedSrc) { r.Reached, r.ReachedSrc = k.Reached, k.ReachedSrc } if n := len(heads); n > 0 && !sameCommit(heads[n-1].commit, r.Reached) { if indexed != 0 && indexed < heads[n-1].rev { holds = append(holds, fmt.Sprintf("the indexer has not yet taken in %.12s", heads[n-1].commit)) } slices.Sort(holds) r.Holds = slices.Compact(holds) } return r } // moduleReach is how far one module reached, the source revision it holds the reached head before (noBound when none), and // what holds it, in words. A node counts for the fingerprint its running have runs, healthy or not (ADR 0556 clause 2). func (m *Model) moduleReach(name string, mod meshcatalog.ModulesByModuleValue, heads []headAt, wantKeys []string) (viewstate.ModuleReach, uint64, string) { mr := viewstate.ModuleReach{FP: mod.Fp2} hist := m.modHist[name] if n := len(hist); n == 0 || hist[n-1].Fp2 != mod.Fp2 || hist[n-1].Commit != mod.Commit { hist = append(slices.Clone(hist), mod) // the value watched now is the newest, whatever the history watch has given } var values []meshcatalog.ModulesByModuleValue // the catalogue's value of each change, in step with mr.Changes for i := len(hist) - 1; i >= 0 && len(mr.Changes) < viewstate.MostChanges; i-- { c := viewstate.Change{Commit: hist[i].Commit, FP: hist[i].Fp2, Src: uint64(hist[i].Src)} if c.Src == 0 { c.Src = revOf(heads, c.Commit) } if n := len(mr.Changes); n > 0 && mr.Changes[n-1].FP == c.FP { mr.Changes[n-1] = c // the same fingerprint again, back to back: its first appearance is the older commit values[n-1] = hist[i] continue } mr.Changes = append(mr.Changes, c) values = append(values, hist[i]) } var waiting []string away := m.awayNodes() floor, floorNode, unset := -1, "", "" for _, key := range wantKeys { node, _ := splitKey(key) w := m.wants[key] if w.err != "" || w.value.Removed || w.value.Fp == "" { continue } if away[node] { // A node away holds no head back (ADR 0603 §7); it catches up by its wants and counts once it runs them. waiting = append(waiting, fmt.Sprintf("%s on %s is away", name, node)) continue } mr.Nodes++ h, ok := m.haves[key] switch { case !ok || h.err != "" || h.have.Fp == "": waiting = append(waiting, fmt.Sprintf("%s on %s runs nothing yet", name, node)) unset = waiting[len(waiting)-1] continue case !runs(h.have.Phase): waiting = append(waiting, fmt.Sprintf("%s on %s is %s at %s, not running", name, node, h.have.Phase, short(h.have.Fp))) unset = waiting[len(waiting)-1] continue case h.have.Fp != w.value.Fp: waiting = append(waiting, fmt.Sprintf("%s on %s runs %s, not yet the %s it wants", name, node, short(h.have.Fp), short(w.value.Fp))) case h.have.Health == nodeengineclient.HealthFailed: waiting = append(waiting, fmt.Sprintf("%s on %s runs %s and its health check failed", name, node, short(h.have.Fp))) case h.have.Health != nodeengineclient.HealthOK: waiting = append(waiting, fmt.Sprintf("%s on %s runs %s, not yet healthy", name, node, short(h.have.Fp))) } i := slices.IndexFunc(mr.Changes, func(c viewstate.Change) bool { return c.FP == h.have.Fp }) if i < 0 { unset = fmt.Sprintf("%s on %s runs %s, which is not among its %d known changes", name, node, short(h.have.Fp), len(mr.Changes)) waiting = append(waiting, unset) continue } if i > floor { floor, floorNode = i, node } } mr.Waiting = strings.Join(waiting, "; ") switch { case mr.Nodes == 0: return mr, noBound, "" case unset != "": // A node that wants the module runs nothing countable of it (no have, not running, or a build older than the kept // history): the module holds no head of the repository back, which would make every apply anywhere, or one module // stuck failed, freeze the whole repository (review of issue 1280, ADR 0556 clause 2). Its words name it, and a merge // among its changes is not yet on the machines (notYet). return mr, noBound, "" } mr.Floor = mr.Changes[floor].Commit if floor == 0 { return mr, noBound, "" } next := mr.Changes[floor-1] words := m.laggingWords(name, floorNode, mr.Changes[floor], next, values[floor], values[floor-1], floor == 1) if next.Src == 0 { // The next change's order is not known: the head is held at the floor's own revision, or before every head when that is // not known either. if own := mr.Changes[floor].Src; own != 0 { return mr, own + 1, words } return mr, 0, words } return mr, next.Src, words } // laggingWords says what a node's running build lacks of the module's next change: both builds by fingerprint with their // commits, which input moved when the commit is the same (the catalogue holds the module's inputs, not the version a mesh // package resolved to), and a next build that failed, with the first words of its error, instead of "not yet". func (m *Model) laggingWords(name, node string, run, next viewstate.Change, runV, nextV meshcatalog.ModulesByModuleValue, newest bool) string { words := fmt.Sprintf("%s on %s runs build %s (%.8s)", name, node, shortFP(run.FP), run.Commit) nextIs := fmt.Sprintf("%s (%.8s)", shortFP(next.FP), next.Commit) if run.Commit == next.Commit { if moved := inputsMoved(runV, nextV); moved != "" { nextIs = fmt.Sprintf("%s (%.8s, %s)", shortFP(next.FP), next.Commit, moved) } } if key, err := builds.BuilderBuildKey(name, next.FP); err == nil { if k, ok := m.keys[key]; ok && k.err == "" && k.value.State == builds.BuildFailed { which := "the build it waits for" if newest { which = "its newest build" } return fmt.Sprintf("%s, %s %s failed: %s", words, which, nextIs, firstWords(k.value.Error)) } } return fmt.Sprintf("%s, not yet %s", words, nextIs) } // inputsMoved names what of a module's inputs differs between two values with the same commit, or "". func inputsMoved(a, b meshcatalog.ModulesByModuleValue) string { if a.Inputs2 == nil || b.Inputs2 == nil { return "" } x, y := a.Inputs2, b.Inputs2 var moved []string if x.Tree != y.Tree { moved = append(moved, "tree") } for _, name := range slices.Sorted(maps.Keys(mergedKeys(x.Locks, y.Locks))) { if x.Locks[name] != y.Locks[name] { moved = append(moved, "lock "+name) } } if !slices.Equal(x.Bases, y.Bases) { moved = append(moved, "a base module") } if len(moved) == 0 && x.Builder != y.Builder { moved = append(moved, "builder") } if len(moved) == 0 && len(y.Packages) > 0 { // Nothing of the module's own moved: a mesh package it resolves at its latest published version did. moved = append(moved, "a mesh package of "+strings.Join(y.Packages, ", ")) } return strings.Join(moved, ", ") } func mergedKeys(a, b map[string]string) map[string]bool { out := map[string]bool{} for k := range a { out[k] = true } for k := range b { out[k] = true } return out } // firstWords is the start of an error, up to eighty characters cut at a word. func firstWords(s string) string { s = strings.Join(strings.Fields(s), " ") if len(s) <= 80 { return s } s = s[:80] if i := strings.LastIndexByte(s, ' '); i > 0 { s = s[:i] } return s + "..." } // shortFP is a fingerprint's first eight characters, after its form. func shortFP(fp string) string { fp = strings.TrimPrefix(fp, meshcatalogclient.SourcePrefix2) if len(fp) > 8 { fp = fp[:8] } return fp } // runs is whether a have in this phase runs the fingerprint it names: running, being checked after its apply, or held after a // failed backup, which keeps running what it has (ADR 0539 clause 16). A have backing up or applying, failed or removed runs // nothing the derivation can count. func runs(p string) bool { return p == nodeengineclient.PhaseRunning || p == nodeengineclient.PhaseChecking || p == nodeengineclient.PhaseHeld } // short is a fingerprint's first twelve characters, after its form. func short(fp string) string { fp = strings.TrimPrefix(fp, meshcatalogclient.SourcePrefix2) if len(fp) > 12 { fp = fp[:12] } return fp } // reachReady is whether every watch the derivation reads handed over what stood at its start. func (m *Model) reachReady() bool { for _, b := range m.reachBuckets() { if !m.initial[b] { return false } } return true } // branchOf is the escaped branch a repository's reach is derived from: the first branch of its modules' registration (by module // name, the first that has one), before the first registrations the first branch whose source names the repository (design 47 // G A-173). "" when none. func (m *Model) branchOf(repository string) string { for _, name := range slices.Sorted(maps.Keys(m.modules)) { if me := m.modules[name]; me.err == "" && trunk.SameRepository(me.value.Repository, repository) && m.modBranch[name] != "" { return m.modBranch[name] } } for _, b := range slices.Sorted(maps.Keys(m.sources)) { for r := range m.sources[b].heads { if trunk.SameRepository(r, repository) { return b } } } return "" } // reachNow derives the reach of every repository a watched branch's source names, from the source of the repository's own // branch (branchOf), and answers the values that changed and the reached heads not yet said. It is called with the lock held. func (m *Model) reachNow() ([]ReachWrite, []DeliveredEvent) { if (m.reachPut == nil && m.deliveredEvent == nil) || !m.reachReady() { return nil, nil } var writes []ReachWrite var events []DeliveredEvent wants := m.wantKeys() type repoAt struct{ branch, repository string } var repos []repoAt for _, b := range slices.Sorted(maps.Keys(m.sources)) { if m.sources[b].err != "" { continue } for _, repository := range slices.Sorted(maps.Keys(m.sources[b].heads)) { if m.branchOf(repository) == b { repos = append(repos, repoAt{b, repository}) } } } for _, ra := range repos { repository := ra.repository key, err := trunk.RepositoryKey(repository) if err != nil { continue } r := m.deriveReach(ra.branch, repository, wants) if prev, ok := m.reach[key]; !ok || !viewstate.SameReach(prev, r) { if data, err := viewstate.EncodeReach(r); err == nil { m.reach[key] = r writes = append(writes, ReachWrite{Key: key, Value: data}) } } notYet := notYetOf(r) said := strings.Join(notYet, ",") if r.Reached != "" && (r.ReachedSrc > m.reachSaid[key] || said != m.reachNotYet[key]) { m.reachSaid[key], m.reachNotYet[key] = r.ReachedSrc, said events = append(events, DeliveredEvent{Branch: m.sources[ra.branch].branch, Revision: r.ReachedSrc, Heads: map[string]string{r.Repository: r.Reached}, Repository: r.Repository, NotYet: notYet}) } } return writes, events } // notYetOf are the merges at or before a repository's reached head that are not yet on the machines: the changes of each module // a node wants with no floor (a node runs nothing countable of it), sorted. A consumer takes every other merge at or before // the reached head as on the machines. func notYetOf(r viewstate.Reach) []string { var out []string for _, mr := range r.Modules { if mr.Nodes == 0 || mr.Floor != "" { continue } for _, c := range mr.Changes { out = append(out, c.Commit) } } slices.Sort(out) return slices.Compact(out) } // Reach is the reach of every repository as the view derived it last, by key. func (m *Model) Reach() map[string]viewstate.Reach { m.mu.Lock() defer m.mu.Unlock() return maps.Clone(m.reach) }