package index import ( "context" "fmt" "maps" "reflect" "slices" "strings" "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/delivery" ) // The new-form fingerprint of a module is written, and written again with no merge, when what it is of changes // (novox/hq ADR 0465 clause 2; issue 1444): a module's tree and lock hashes come with a merge and are kept in // modules..Inputs2; its builder's, its bases' and its toolchain's come from here. The indexer holds no graph, so each // recompute reads the manifest of every module from manifests. and takes the modules in the order their bases // and the builder come first. const ( // BuilderModule is the module that builds every module (novox/hq ADR 0453 clause 5): each module's Builder input is // its new-form fingerprint. It does not exist before release R4c-4, and a module then has none for want of it. BuilderModule = "mesh-delivery-builder" // BuilderBuiltBy is the module that builds BuilderModule until release R4g: its fingerprint of today's form is the // builder's own Builder input, so a change of build-agent moves the builder's value (ADR 0453 clause 5). BuilderBuiltBy = "build-agent" ) // ToolchainModules are the build module of each language, whose state toolchains (delivery.ToolchainsBucket) the // indexer reads: the language's agreed toolchain is the one every node's entry in it agrees on. var ToolchainModules = map[string]string{"go": "mesh-build-go", "typescript": "mesh-build-node"} // ToolchainStreams are the stream each build module carries its toolchain.reported event on, by module, as the bus names it // (a stream's local name in capitals, catalogue.StreamBusName): `toolchains-go` and `toolchains-node`, declared by the // build modules (streams, with the event toolchain.reported) and heard by this module (consumes). var ToolchainStreams = map[string]string{"mesh-build-go": "TOOLCHAINS_GO", "mesh-build-node": "TOOLCHAINS_NODE"} // PutToolchain takes one change of a build module's toolchains bucket in, without writing any module: a put of a node's // toolchain of a language (key ., delivery.ToolchainKey), or its delete. It answers whether what the // language's toolchain is of (version and recipe) changed. A key, or a value, that is not a toolchain of the module's // language is said in the log and left out. func (ix *Indexer) PutToolchain(module, key string, value []byte, deleted bool) bool { language := "" for l, m := range ToolchainModules { if m == module { language = l } } if language == "" { ix.logf("toolchains of %q are not read: it is no build module the indexer knows", module) return false } node, keyLanguage, err := delivery.ParseToolchainKey(key) if err != nil || keyLanguage != language { ix.logf("%s_toolchains.%s is not a %s toolchain's key, left out: %v", module, key, language, err) return false } ix.mu.Lock() defer ix.mu.Unlock() if deleted { if _, had := ix.toolchains[language][node]; !had { return false } delete(ix.toolchains[language], node) return true } tc, err := delivery.DecodeToolchain(value) if err != nil || tc.Language != language { ix.logf("%s_toolchains.%s is not a %s toolchain, left out: %v", module, key, language, err) return false } if ix.toolchains == nil { ix.toolchains = map[string]map[string]delivery.Toolchain{} } if ix.toolchains[language] == nil { ix.toolchains[language] = map[string]delivery.Toolchain{} } cur, had := ix.toolchains[language][node] ix.toolchains[language][node] = tc return !had || cur.Version != tc.Version || cur.Recipe != tc.Recipe } // ApplyToolchain takes one change of a build module's toolchains in (PutToolchain) and, when the language's toolchain // changed, writes the modules again that it moves (Recompute). The same version and recipe seen again writes nothing. func (ix *Indexer) ApplyToolchain(ctx context.Context, module, key string, value []byte, deleted bool) error { if !ix.PutToolchain(module, key, value, deleted) { return nil } return ix.Recompute(ctx) } // agreed is the toolchain of each language every reporting node agrees on, or why there is none. func (ix *Indexer) agreed() map[string]toolchainOrNone { ix.mu.Lock() defer ix.mu.Unlock() out := map[string]toolchainOrNone{} for language := range ToolchainModules { var entries []delivery.NodeToolchain for _, node := range slices.Sorted(maps.Keys(ix.toolchains[language])) { entries = append(entries, delivery.NodeToolchain{Node: node, Toolchain: ix.toolchains[language][node]}) } tc, none := delivery.AgreedToolchain(entries) out[language] = toolchainOrNone{tc, none} } return out } type toolchainOrNone struct { tc delivery.Toolchain2 none string } // known is a module the indexer holds the inputs of: its record as read, the manifest's shape and the key it is written at. type known struct { key string cur delivery.Module languages map[string]string bases []string contexts bool } // Recompute writes again every module whose new-form fingerprint, or reason for none, is no longer what it holds: from // the inputs kept at its merge (Inputs2), its manifest (manifests.), the fingerprints of its bases and of the // builder, and the toolchains the build modules report. A module is computed after its bases and the builder, whatever // the order the bucket lists them in; modules that stand on each other have a cycle as their reason. A module is written // only when its fingerprint or its reason changes, and only if its manifest is still the one computed from. A module with // no kept inputs (a message that carried no manifest) is left as it is. func (ix *Indexer) Recompute(ctx context.Context) error { return ix.recomputeWith(ctx, nil) } // recomputeWith is Recompute for a merge as well: the records the merge gives (overlay, by module key) stand in place // of what the bucket holds, are computed with the others, and are written once, with their final fingerprint or reason // (issue 1444: a merge writes each module once). A record of the overlay that has no kept inputs is written as it is. func (ix *Indexer) recomputeWith(ctx context.Context, overlay map[string]delivery.Module) error { if ix.Modules == nil { return nil } if ix.Manifests == nil { return ix.writeOverlay(ctx, overlay, nil) } ix.recompute.Lock() defer ix.recompute.Unlock() keys, err := ix.Modules.Keys(ctx) if err != nil { return fmt.Errorf("listing the modules: %w", err) } for key := range overlay { if !slices.Contains(keys, key) { keys = append(keys, key) } } slices.Sort(keys) held := map[string]delivery.Module{} all := map[string]*known{} for _, key := range keys { cur, ok := overlay[key] if !ok { value, _, found, err := ix.Modules.Get(ctx, key) if err != nil { return fmt.Errorf("modules.%s: %w", key, err) } if !found { continue } if cur, err = delivery.DecodeModule(value); err != nil { ix.logf("modules.%s holds what is not a module, left: %v", key, err) continue } } held[key] = cur if cur.Manifest == "" || cur.Inputs2 == nil { continue } m, err := ix.manifestOf(ctx, cur.Manifest) if err != nil { ix.logf("modules.%s: its manifest %s cannot be read, left: %v", key, cur.Manifest, err) continue } all[key] = &known{key: key, cur: cur, languages: languagesOf(m), bases: basesOf(m), contexts: hasContext(m)} } tools := ix.agreed() type result struct { fp, none string tcs map[string]delivery.ToolchainUsed // by language, when there is a fingerprint } done := map[string]result{} visiting := map[string]bool{} var of func(module string, chain []string) result of = func(module string, chain []string) result { if r, ok := done[module]; ok { return r } k, ours := all[module] if !ours { // a module the indexer holds no inputs of: its record, as written cur, has := held[module] switch { case !has: return result{fp: "", none: "the indexer holds no record of it"} case cur.FP2 != "": return result{fp: cur.FP2, none: ""} case cur.None2 != "": return result{fp: "", none: cur.None2} } return result{fp: "", none: "its record carries no fingerprint of the new form"} } if visiting[module] { return result{fp: "", none: "a cycle of bases: " + strings.Join(append(chain, module), " -> ")} } visiting[module] = true defer delete(visiting, module) chain = append(chain, module) kept := *k.cur.Inputs2 kept.Bases, kept.Builder = nil, "" var missing []string for _, b := range k.bases { if r := of(b, chain); r.fp != "" { kept.Bases = append(kept.Bases, r.fp) } else { missing = append(missing, fmt.Sprintf("the base %s has no fingerprint (%s)", b, r.none)) } } slices.Sort(kept.Bases) slices.Sort(missing) toolchains := map[string]delivery.Toolchain2{} var noToolchain string for _, artifact := range slices.Sorted(maps.Keys(k.languages)) { t := tools[k.languages[artifact]] if t.none != "" { if noToolchain == "" { noToolchain = t.none } continue } toolchains[artifact] = t.tc } var noBuilder string if module == BuilderModule { // Built by build-agent until release R4g: its value of today's form. if a := held[BuilderBuiltBy]; a.FP != "" { kept.Builder = a.FP } else { noBuilder = fmt.Sprintf("%s, which builds it, has no fingerprint (%s)", BuilderBuiltBy, a.None) } } else if r := of(BuilderModule, chain); r.fp != "" { kept.Builder = r.fp } else { noBuilder = None2Builder if _, has := held[BuilderModule]; has { noBuilder += ": " + r.none } } var r result switch { case k.contexts: r.none = None2Context case len(missing) > 0: r.none = strings.Join(missing, "; ") case noToolchain != "": r.none = noToolchain case noBuilder != "": r.none = noBuilder default: r.fp, r.none = delivery.SourceFingerprint2(kept.With(module, toolchains)) if r.fp != "" { r.tcs = toolchainsUsed(toolchains) } } done[module] = r k.cur.Inputs2 = &kept return r } finals := map[string]delivery.Module{} for _, key := range slices.Sorted(maps.Keys(all)) { k := all[key] r := of(key, nil) if _, merged := overlay[key]; merged { final := overlay[key] final.FP2, final.None2, final.Inputs2, final.Toolchains = r.fp, r.none, k.cur.Inputs2, r.tcs // What the bases and the builder gave is kept as it was while the value it gave does not change, so a merge // that moves nothing for the module writes nothing (as Recompute does). if value, _, found, err := ix.Modules.Get(ctx, key); err == nil && found { if old, err := delivery.DecodeModule(value); err == nil && old.Inputs2 != nil && old.FP2 == r.fp && old.None2 == r.none { in := *final.Inputs2 in.Bases, in.Builder = old.Inputs2.Bases, old.Inputs2.Builder final.Inputs2 = &in } } finals[key] = final continue } if k.cur.FP2 == r.fp && k.cur.None2 == r.none && reflect.DeepEqual(k.cur.Toolchains, r.tcs) { continue } digest, inputs := k.cur.Manifest, k.cur.Inputs2 err := write(ctx, ix.Modules, key, func(value []byte, found bool) ([]byte, bool, error) { if !found { return nil, false, nil } cur, err := delivery.DecodeModule(value) if err != nil || cur.Manifest != digest || cur.Inputs2 == nil { return nil, false, nil // a merge changed it meanwhile; its own recompute follows } if cur.FP2 == r.fp && cur.None2 == r.none && reflect.DeepEqual(cur.Toolchains, r.tcs) { return nil, false, nil } cur.FP2, cur.None2, cur.Toolchains = r.fp, r.none, r.tcs cur.Inputs2 = inputs data, err := delivery.EncodeModule(cur) return data, false, err }) if err != nil { return fmt.Errorf("modules.%s: %w", key, err) } } return ix.writeOverlay(ctx, overlay, finals) } // writeOverlay writes the records of a merge, each once and only if it differs from what the bucket holds; finals has // the ones that were computed, the others are written as the merge gave them. func (ix *Indexer) writeOverlay(ctx context.Context, overlay, finals map[string]delivery.Module) error { for _, key := range slices.Sorted(maps.Keys(overlay)) { next := overlay[key] if f, ok := finals[key]; ok { next = f } err := write(ctx, ix.Modules, key, func(value []byte, found bool) ([]byte, bool, error) { if found { if cur, err := delivery.DecodeModule(value); err == nil && reflect.DeepEqual(cur, withV(next)) { return nil, false, nil } } data, err := delivery.EncodeModule(next) return data, false, err }) if err != nil { return fmt.Errorf("modules.%s: %w", key, err) } } return nil } // manifestOf is the parsed manifest manifests. holds. func (ix *Indexer) manifestOf(ctx context.Context, digest string) (catalogue.Manifest, error) { key, err := delivery.ManifestsKey(digest) if err != nil { return catalogue.Manifest{}, err } value, _, found, err := ix.Manifests.Get(ctx, key) if err != nil { return catalogue.Manifest{}, err } if !found { return catalogue.Manifest{}, fmt.Errorf("manifests.%s is not there", key) } kept, err := delivery.DecodeManifest(value) if err != nil { return catalogue.Manifest{}, err } return catalogue.ParseManifest([]byte(kept.JSON)) } // hasContext is whether an artifact of the manifest reads another repository's tree. func hasContext(m catalogue.Manifest) bool { if m.Build == nil { return false } for _, a := range m.Build.Artifacts { if a.Context != nil { return true } } return false } // toolchainsUsed are the toolchains a fingerprint was computed with, by language (an artifact's toolchain is its language's). func toolchainsUsed(by map[string]delivery.Toolchain2) map[string]delivery.ToolchainUsed { if len(by) == 0 { return nil } out := map[string]delivery.ToolchainUsed{} for _, t := range by { out[t.Language] = delivery.ToolchainUsed{Version: t.Version, Recipe: t.Recipe} } return out }