Keep builds on the bus under their module, so no lookup reads the bucket whole (issue 1121, ADR 0427)
mesh/merge-gate head c79591531575 merged into trunk aa02e7bf4e78 — pass: builds build-agent, mesh-controller → ace, g14, novox, shanks; no bus step; every …
mesh/repo-check head c79591531575 merged into trunk aa02e7bf4e78 — pass: its merge-check.sh passed
mesh/delivery delivered
mesh/merge-gate head c79591531575 merged into trunk aa02e7bf4e78 — pass: builds build-agent, mesh-controller → ace, g14, novox, shanks; no bus step; every …
mesh/repo-check head c79591531575 merged into trunk aa02e7bf4e78 — pass: its merge-check.sh passed
mesh/delivery delivered
Milestone 5 moved 8,599 build records onto the bus under bare build ids, each carrying its module's whole manifest. Every build lookup then read the whole bucket, and a delivery plan's send did that per module per machine while it held the plans and the machine's declaration: the send of build-agent never finished, and every other act and the status waited behind it. Builds now live under <module>.build.<id> (requests under req.<id>, id.<id> names a build's module), with pointer keys newest, worked, held, commit.<sha> and source.<reuse key> that move only to a newer build and are read as hints, repaired when wrong. Manifests are kept once in the same bucket under manifest.sha256-<hex>, so no grant the running bus lacks is needed. The old keys are laid out once at start, idempotently, and left unread. The calls a plan look makes on the silent clocks while it holds the plans are bounded, since the vendored kv.Keys ignores its context.
This commit is contained in:
@@ -0,0 +1,97 @@
|
||||
package chain
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"slices"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// The builds bucket's keys (novox/hq issue 1121). A build is under its module, so a read of one module is a read of the keys
|
||||
// `<module>.>` and no other, and a lookup that has what it looks up by is one Get of a pointer key, whose value is a build's id.
|
||||
//
|
||||
// <module>.build.<id> a build's record, its outcome (a records-only read is the filter `<module>.build.>`)
|
||||
// req.<id> a request, and a build that knew no module
|
||||
// id.<id> the module of the build with that id (the pointer a lookup by id alone reads)
|
||||
// <module>.newest the newest build of the module, of any state
|
||||
// <module>.worked the newest successful build of the module
|
||||
// <module>.held the build standing for the module's newest successful build (issue 280)
|
||||
// <module>.commit.<sha> the newest successful build from that commit: a CommitPointer
|
||||
// <module>.source.<key> the newest successful build with that reuse key (ADR 0350 §1), spelled by SpellReuse
|
||||
// manifest.sha256-<hex> a manifest, once, by the digest of its bytes
|
||||
//
|
||||
// **A pointer is a hint** (issue 1121): the record is written first, then each pointer by compare-and-set; a reader Gets the
|
||||
// record a pointer names and checks it, and when the pointer is missing or wrong reads the module's records, repairs the
|
||||
// pointers and says so.
|
||||
//
|
||||
// No module's name contains a dot. A key is never removed (ADR 0408 §4); the bare `<id>` keys of before the move stay, and are
|
||||
// never read once the kind `builds-layout` is moved.
|
||||
const (
|
||||
RequestPrefix = "req"
|
||||
IDPrefix = "id"
|
||||
)
|
||||
|
||||
// BuildKey is a build's record under its module; a build that knew none is the request's key.
|
||||
func BuildKey(module, id string) string {
|
||||
if module == "" {
|
||||
return RequestKey(id)
|
||||
}
|
||||
return module + ".build." + id
|
||||
}
|
||||
|
||||
func RequestKey(id string) string { return RequestPrefix + "." + id }
|
||||
func IDKey(id string) string { return IDPrefix + "." + id }
|
||||
func NewestKey(module string) string { return module + ".newest" }
|
||||
func WorkedKey(module string) string { return module + ".worked" }
|
||||
func HeldKey(module string) string { return module + ".held" }
|
||||
func CommitKey(module, sha string) string { return module + ".commit." + sha }
|
||||
func SourceKey(module, fp string) string { return module + ".source." + fp }
|
||||
func ModuleFilter(module string) string { return module + ".build.>" }
|
||||
|
||||
// ManifestPrefix is where a build's manifest is kept, in the builds bucket itself: a new bucket needs a grant the bus gets
|
||||
// only at a later push, and the push needs the builds. No module is named so.
|
||||
const ManifestPrefix = "manifest"
|
||||
|
||||
// ReservedModules are the names that are not a module's: a key under one is not a build's record.
|
||||
var ReservedModules = []string{ManifestPrefix, RequestPrefix, IDPrefix}
|
||||
|
||||
// ManifestKey is where a manifest is kept: `manifest.sha256-<hex of its bytes>`.
|
||||
func ManifestKey(raw []byte) string {
|
||||
sum := sha256.Sum256(raw)
|
||||
return ManifestPrefix + ".sha256-" + hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
// CommitPointer is the value of `<module>.commit.<sha>`: the build, and what the surveys of every module's commits read
|
||||
// without a Get of each build: its source fingerprint and what it would put on a machine.
|
||||
type CommitPointer struct {
|
||||
Build string `json:"build"`
|
||||
Source string `json:"source,omitempty"`
|
||||
Fingerprint string `json:"fingerprint,omitempty"`
|
||||
}
|
||||
|
||||
// IsRecordKey is whether a key of the builds bucket is a build's record under a module: `<module>.build.<id>`.
|
||||
func IsRecordKey(key string) bool {
|
||||
parts := strings.Split(key, ".")
|
||||
return len(parts) == 3 && parts[1] == "build" && !slices.Contains(ReservedModules, parts[0])
|
||||
}
|
||||
|
||||
// SpellReuse is a reuse key (`v1:<hex>`) as a part of a key the bus takes: every character it does not take becomes `-`.
|
||||
func SpellReuse(key string) string {
|
||||
return strings.Map(func(r rune) rune {
|
||||
if r >= 'a' && r <= 'z' || r >= 'A' && r <= 'Z' || r >= '0' && r <= '9' || r == '-' || r == '_' {
|
||||
return r
|
||||
}
|
||||
return '-'
|
||||
}, key)
|
||||
}
|
||||
|
||||
// ModuleOf is the module a record key is under.
|
||||
func ModuleOf(key string) string { m, _, _ := strings.Cut(key, "."); return m }
|
||||
|
||||
// DecodeKept is a watched entry as a typed record, for a read that took the entries of several kinds and decodes those
|
||||
// that hold a record.
|
||||
func DecodeKept[T any](e Entry) (Kept[T], error) { return decode[T](e) }
|
||||
|
||||
// AsAnswered is the state of a request record whose outcome has been heard: the record keeps the request and says which
|
||||
// build answered it, so a read of the requests needs no read of the builds.
|
||||
const BuildAnswered = "answered"
|
||||
Reference in New Issue
Block a user