package main import ( "archive/tar" "bytes" "compress/gzip" "context" "encoding/base64" "encoding/json" "errors" "fmt" "io" "path" "strings" "time" "git.novox.be/novox/mesh-sdk/go/buildstep" "git.novox.be/novox/mesh-sdk/go/catalogue" sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery" "github.com/nats-io/nats.go/jetstream" "github.com/novox/mesh-controller/internal/artifacts" "github.com/novox/mesh-controller/internal/builderkeys" "github.com/novox/mesh-controller/internal/buildreads" "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/link" ) // The take-in from the builder's done key (novox/hq ADR 0501, ADR 0498 §8, ADR 0453 §5; issue 1851). // // From R4e the controller asks no build but the builder's own module's: a module of a merge is taken from the builder's // key for the fingerprint the indexer's trunk value of the module names (ADR 0501 §2). Only a done key whose every // artifact names a reference is taken (ADR 0498 §8); a key not done yet, one waiting for a reference, a templates // archive the store does not answer, or a list of predefined keys the forge does not answer is waited for and said, // never failed and never delivered from. A failed key fails, in the builder's own words. The take-in then records the // key as a build of today's record layout, so a controller of before reads it on a rollback, and judges it as every // build is judged (registerJudged). // errKeyWaits says a take-in from a key waits: for the builder, a reference, the store or the forge. Never a failure. var errKeyWaits = errors.New("waits") // errKeyFailed says the module's key failed: the builder's own words follow. var errKeyFailed = errors.New("the builder's key failed") // keyReads is what a take-in from a key reads of the builder and the indexer (builderkeys.Reader, or a test's). type keyReads interface { Build(ctx context.Context, module, fp string) (builderkeys.Key, error) Manifest(ctx context.Context, digest string) (sdkdelivery.Manifest, error) } // keyBuildID is the id a key's build is recorded under: the key and its revision, so the same revision recorded twice // is one record, and a rebuild of the same fingerprint (a new revision, ADR 0429 §18) another. func keyBuildID(k builderkeys.Key) string { part, err := sdkdelivery.FingerprintKeyPart(k.Fingerprint) if err != nil { part = strings.TrimPrefix(k.Fingerprint, sdkdelivery.SourcePrefix2) } return fmt.Sprintf("key-%s-%d", part, k.Revision) } // readArchive is the archive a key's reference names, read from the artifact store and checked against its digest. A // variable so a test needs no store. var readArchive = func(ctx context.Context, inv *inventory.Inventory, reference string) ([]byte, error) { shelf, err := inv.Catalogue(ctx) if err != nil { return nil, err } address, err := artifactStoreAddress(ctx, inv, shelf, "") if err != nil { return nil, err } if address == "" { return nil, errors.New("the artifact store is on no network the controller reaches") } return artifacts.Store{Address: address}.GetBlob(ctx, reference) } // predefinedHead is the list of predefined keys at the head of the catalogue repository's trunk, and that head. type predefinedHead struct { List catalogue.PredefinedList Repository string Commit string // Generation orders two heads of the trunk: the head's commit time in seconds. A list kept from a build's // generation (its commit count) is always older than a head read here. Generation int } // predefinedAtTrunk is the list of predefined keys at the head of the catalogue repository's trunk (ADR 0501 §8), read // from the forge. A variable so a test needs no forge. var predefinedAtTrunk = func(ctx context.Context, inv *inventory.Inventory) (predefinedHead, error) { entries, err := inv.Catalogued(ctx) if err != nil { return predefinedHead{}, err } repository := "" for _, e := range entries { if e.Source.Seat == gitSeat && path.Base(strings.Trim(e.Source.Repository, "/")) == catalogue.CatalogueRepository { repository = strings.Trim(e.Source.Repository, "/") break } } if repository == "" { return predefinedHead{}, fmt.Errorf("no module of the catalogue names the repository %s on the forge", catalogue.CatalogueRepository) } var heads []struct { SHA string `json:"sha"` Commit struct { Committer struct { Date time.Time `json:"date"` } `json:"committer"` } `json:"commit"` } if err := askGitea(ctx, "gitea_api", map[string]any{"path": "/repos/" + repository + "/commits?limit=1&stat=false&verification=false&files=false"}, &heads); err != nil { return predefinedHead{}, err } if len(heads) == 0 || heads[0].SHA == "" { return predefinedHead{}, fmt.Errorf("the forge named no head of %s's trunk", repository) } head := predefinedHead{Repository: repository, Commit: heads[0].SHA, Generation: int(heads[0].Commit.Committer.Date.Unix())} var file struct { Content string `json:"content"` Encoding string `json:"encoding"` } if err := askGitea(ctx, "gitea_api", map[string]any{"path": "/repos/" + repository + "/contents/" + catalogue.PredefinedFile + "?ref=" + head.Commit}, &file); err != nil { return predefinedHead{}, err } if file.Encoding != "base64" { return predefinedHead{}, fmt.Errorf("the forge answered %s at %.8s in the encoding %q", catalogue.PredefinedFile, head.Commit, file.Encoding) } raw, err := base64.StdEncoding.DecodeString(file.Content) if err != nil { return predefinedHead{}, err } if head.List, err = catalogue.ParsePredefined(raw); err != nil { return predefinedHead{}, fmt.Errorf("%s at %.8s is refused: %w", catalogue.PredefinedFile, head.Commit, err) } return head, nil } // askGitea asks the forge's module one tool on the bus and reads its result into into. func askGitea(ctx context.Context, tool string, args map[string]any, into any) error { js, err := aBus() if err != nil { return err } defer js.Close() raw, _ := json.Marshal(args) answer, err := link.Ask(ctx, link.OverNATS{Conn: js.Conn()}, "gitea", tool, raw, 30*time.Second) if err != nil { return err } if answer.Error != "" { return fmt.Errorf("gitea.%s: %s", tool, answer.Error) } var wrapped struct { Result json.RawMessage `json:"result"` } if err := json.Unmarshal(answer.Result, &wrapped); err != nil { return err } return json.Unmarshal(wrapped.Result, into) } // keyTake is one take-in from a key: the module, the indexer's trunk value naming the key's fingerprint, and the // registration's warrant. type keyTake struct { Module string Trunk sdkdelivery.Module Ask registrationAsk } // takeInKey takes a module in from the builder's key for the fingerprint its trunk value names: records the key as a // build and registers the module, or says why not. errKeyWaits when it waits, errKeyFailed when the key failed. func takeInKey(ctx context.Context, inv *inventory.Inventory, keys keyReads, t keyTake) ( catalogue.Manifest, inventory.Build, error) { head, err := predefinedAtTrunk(ctx, inv) if err != nil { return catalogue.Manifest{}, inventory.Build{}, fmt.Errorf("%w: the list of predefined keys at the catalogue's "+ "trunk head cannot be read, so %s is not judged: %v", errKeyWaits, t.Module, err) } keepPredefinedHead(ctx, inv, head) previous, err := inv.Builds(ctx, t.Module, 5) if err != nil { return catalogue.Manifest{}, inventory.Build{}, err } kept, manifest, err := buildOfKey(ctx, keys, t, keyContext{list: head.List, previous: previous, archive: func(ctx context.Context, reference string) ([]byte, error) { return readArchive(ctx, inv, reference) }}) if err != nil { return catalogue.Manifest{}, kept, err } if err := inv.RecordBuild(ctx, kept); err != nil { return catalogue.Manifest{}, kept, err } recorded := inventory.Source{Repository: t.Trunk.Repository, Seat: gitSeat, Path: keyPath(t.Trunk), BuiltFrom: t.Trunk.Commit, Head: t.Trunk.Commit, Against: kept.Against} trunk := "main" if was, err := inv.SourceOf(ctx, manifest.Module); err == nil { recorded.Ref = followedBranch(was.Ref) if recorded.Ref != "" { trunk = recorded.Ref } } registered, err := registerJudged(ctx, inv, takenBuild{kept: kept, manifest: manifest, recorded: recorded, ask: t.Ask, path: recorded.Path, trunk: trunk, by: "the builder", what: t.Trunk.Repository, commit: t.Trunk.Commit}) return registered, kept, err } // keyContext is what a key's build is made with besides the key: the list of predefined keys its manifest is judged // against, the module's recent builds (the bases its last one stood on are carried) and how an archive is read. type keyContext struct { list catalogue.PredefinedList previous []inventory.Build archive func(ctx context.Context, reference string) ([]byte, error) } // keyPath is a module's directory as the catalogue records it: empty for a module at its repository's root. func keyPath(m sdkdelivery.Module) string { if m.Path == sdkdelivery.RootPath { return "" } return strings.Trim(m.Path, "/") } // buildOfKey reads the key of the trunk value's fingerprint and makes its build record and the manifest it resolves to // (ADR 0498 §8, ADR 0501 §1, §8): the module's manifest by the key's digest, judged against the list of predefined keys // at the catalogue's trunk head, resolved with the key's artifacts. func buildOfKey(ctx context.Context, keys keyReads, t keyTake, kc keyContext) ( inventory.Build, catalogue.Manifest, error) { waits := func(format string, a ...any) (inventory.Build, catalogue.Manifest, error) { return inventory.Build{}, catalogue.Manifest{}, fmt.Errorf("%w: "+format, append([]any{errKeyWaits}, a...)...) } if t.Trunk.FP2 == "" { why := t.Trunk.None2 if why == "" { why = "the indexer wrote no fingerprint of the new form" } return waits("%s at %.8s has no fingerprint to take a build by: %s", t.Module, t.Trunk.Commit, why) } k, err := keys.Build(ctx, t.Module, t.Trunk.FP2) if err != nil { return waits("%s's key cannot be read: %v", t.Module, err) } switch { case k.Failed(): return inventory.Build{}, catalogue.Manifest{}, fmt.Errorf("%w: %s", errKeyFailed, k.Says()) case !k.Done(): return waits("%s", k.Says()) case len(k.Unreferenced()) > 0: return waits("%s", k.Says()) } src, err := keys.Manifest(ctx, k.Build.Manifest) if err != nil { return waits("%s's manifest: %v", t.Module, err) } manifest, err := catalogue.ParseManifestWith([]byte(src.JSON), kc.list) if err != nil { // **A word the running controller does not read waits for the controller that reads it** (ADR 0501 §7): the // key stays done, and the controller that reads the word registers it with no second build. if word := refusedWord(err.Error()); word != "" { return waits("%s's manifest uses the word %q, which this controller does not read: it is registered "+ "when a controller that reads it runs", t.Module, word) } return inventory.Build{}, catalogue.Manifest{}, fmt.Errorf("the builder built %s and its manifest is "+ "refused: %w", t.Module, err) } if manifest.Module != t.Module { return inventory.Build{}, catalogue.Manifest{}, fmt.Errorf("the key of %s names the manifest of %s", t.Module, manifest.Module) } built, err := builtOfKey(ctx, kc.archive, manifest, k) if err != nil { return waits("%v", err) } resolved, err := manifest.Resolve(built) if err != nil { return inventory.Build{}, catalogue.Manifest{}, fmt.Errorf("the builder built %s and it does not resolve: %w", t.Module, err) } raw, err := json.Marshal(resolved) if err != nil { return inventory.Build{}, catalogue.Manifest{}, err } kept := inventory.Build{ID: keyBuildID(k), Repository: t.Trunk.Repository, Module: t.Module, Commit: t.Trunk.Commit, On: k.Build.By, Path: keyPath(t.Trunk), Manifest: raw, SourceFingerprint: k.Fingerprint, At: time.Now().UTC()} // **Read from the manifest, fresh at every key** (the same rule build-agent says, internal/buildreads). Sources are // left unsaid but for a module whose every artifact names its paths: the planner then reads the module's whole // directory, over-planning and never under-planning, and a fingerprint that did not change moves nothing (R4e A9). // **Its build source where its every artifact names its paths** (novox/hq ADR 0499 §1, §5; issue 1758, B1): the // planner reads a merge's changed files against those paths and the manifest alone, so a change elsewhere in the // module's directory touches nothing it builds from. if own := pathsBuildSource(manifest, keyPath(t.Trunk)); own != nil { kept.Sources = []inventory.BuildSource{{Paths: own}} } for _, c := range buildreads.Of(manifest) { kept.Read = append(kept.Read, inventory.ReadRepository{Repository: c.Repository, Ref: c.Ref}) } // **The bases its last build stood on, carried** (R4e, lead decision): what the collector keeps a mirrored base by. // A key names none; the module's newest successful build's are carried so the keep set does not change. for _, b := range kc.previous { if b.Worked() && b.Module == t.Module && b.ID != kept.ID { kept.Against = append([]string(nil), b.Against...) break } } for _, a := range k.Build.Artifacts { kept.Made = append(kept.Made, inventory.Artifact{Name: a.Name, Kind: kindOf(manifest, a), Reference: catalogue.Recorded(a.Reference)}) } return kept, resolved, nil } // kindOf is an artifact's kind: the key's, else the manifest's. func kindOf(m catalogue.Manifest, a sdkdelivery.BuiltArtifact) string { if a.Kind != "" || m.Build == nil { return a.Kind } for _, d := range m.Build.Artifacts { if d.Name == a.Name { return d.Kind } } return "" } // builtOfKey is each of a done key's artifacts as Resolve reads it: its reference and digest, a TypeScript bundle's // launchers by the build modules' one rule (mesh-sdk buildstep.Launchers), and, for the archive a module packs its // templates into, each template's text read back from the archive the store holds under the key's reference. func builtOfKey(ctx context.Context, archive func(context.Context, string) ([]byte, error), m catalogue.Manifest, k builderkeys.Key) ( []catalogue.Built, error) { declared := map[string]catalogue.Artifact{} if m.Build != nil { for _, a := range m.Build.Artifacts { declared[a.Name] = a } } templates := "" if m.Templated() { templates = catalogue.TemplatesArtifact(m) } var out []catalogue.Built for _, a := range k.Build.Artifacts { d := declared[a.Name] b := catalogue.Built{Name: a.Name, Kind: kindOf(m, a), Reference: catalogue.Recorded(a.Reference), Digest: a.Digest} if launchers := buildstep.Launchers(d.Language, d.Entrypoints); len(launchers) > 0 { b.Launchers = launchers } if a.Name == templates && templates != "" { body, err := archive(ctx, b.Reference) if err != nil { return nil, fmt.Errorf("the templates archive %q of %s cannot be read from the artifact store: %v", a.Name, m.Module, err) } if b.Texts, err = unpackTexts(body); err != nil { return nil, fmt.Errorf("the templates archive %q of %s cannot be read back: %v", a.Name, m.Module, err) } } out = append(out, b) } return out, nil } // unpackTexts is every file of a packed templates archive (gzip over tar), by its path inside it. func unpackTexts(body []byte) (map[string]string, error) { zipped, err := gzip.NewReader(bytes.NewReader(body)) if err != nil { return nil, err } defer zipped.Close() reader := tar.NewReader(zipped) texts := map[string]string{} for { header, err := reader.Next() if errors.Is(err, io.EOF) { return texts, nil } if err != nil { return nil, err } if header.Typeflag != tar.TypeReg { continue } text, err := io.ReadAll(io.LimitReader(reader, 16<<20)) if err != nil { return nil, err } texts[strings.TrimPrefix(header.Name, "./")] = string(text) } } // keepPredefinedHead keeps the list read at the catalogue's trunk head when it is newer than the one kept, as a build of // the catalogue's did (ADR 0348 §7), and judges every manifest read again against it. Said, never fatal: the list read // is the one this take-in judges against either way. func keepPredefinedHead(ctx context.Context, inv *inventory.Inventory, head predefinedHead) { kept, err := inv.KeepPredefined(ctx, inventory.PredefinedRecord{Keys: head.List, Repository: head.Repository, Commit: head.Commit, Generation: head.Generation, Build: "the trunk head " + short(head.Commit)}) if err != nil { fmt.Printf("the list of predefined keys at %s's trunk head %s is not kept: %v\n", head.Repository, short(head.Commit), err) return } if kept { fmt.Printf("the list of predefined keys at %s's trunk head %s is published: %s\n", head.Repository, short(head.Commit), strings.Join(head.List.Keys(), ", ")) } if err := loadPredefined(ctx, inv); err != nil { fmt.Printf("the list of predefined keys kept cannot be read: %v\n", err) } } // openBuilderKeys is the builder's and the indexer's buckets, read on a connection for the moment. func openBuilderKeys() (builderkeys.Reader, func(), error) { js, err := aBus() if err != nil { return builderkeys.Reader{}, nil, err } api, err := jetstream.New(js.Conn()) if err != nil { js.Close() return builderkeys.Reader{}, nil, err } return builderkeys.Reader{JS: api}, js.Close, nil } // pathsBuildSource is the build source of a module whose every artifact names the paths it is built from (ADR 0499 // §1): its manifest and each path, relative to its repository's root, a path read as a file or everything below a // directory of that name (`/**` holds both). Nil when an artifact names none: the module reads its directory. func pathsBuildSource(m catalogue.Manifest, dir string) []string { if m.Build == nil || len(m.Build.Artifacts) == 0 { return nil } in := func(p string) string { if dir == "" { return p } return dir + "/" + p } seen := map[string]bool{} out := []string{in("module.json")} for _, a := range m.Build.Artifacts { if len(a.Paths) == 0 { return nil } for _, p := range a.Paths { entry := in(strings.Trim(path.Clean(p), "/")) + "/**" if !seen[entry] { seen[entry] = true out = append(out, entry) } } } return out }