Compare commits
10
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
fc21f2d1bd | ||
|
|
bf7540cd9e | ||
|
|
173c49fc87 | ||
|
|
3f1d9cecba | ||
|
|
a072d60cae | ||
|
|
efdfe1f755 | ||
|
|
9073190646 | ||
|
|
3a63813b3d | ||
|
|
03164e853e | ||
|
|
2819b05838 |
@@ -485,7 +485,7 @@ type StepAskCompose struct {
|
||||
Fetch Fetch `json:"fetch"`
|
||||
Package string `json:"package"`
|
||||
Reader string `json:"reader,omitempty"`
|
||||
// the compose-candidate request
|
||||
// the compose-candidate request, in one of two forms: the request as a JSON object, or, for one near the bus's message limit, a JSON string "gzip+base64:" followed by the base64 of the request object gzip-compressed; a holder that does not know the packed form refuses it, naming that it holds no request object
|
||||
Request any `json:"request"`
|
||||
}
|
||||
|
||||
|
||||
@@ -485,7 +485,7 @@ type StepAskCompose struct {
|
||||
Fetch Fetch `json:"fetch"`
|
||||
Package string `json:"package"`
|
||||
Reader string `json:"reader,omitempty"`
|
||||
// the compose-candidate request
|
||||
// the compose-candidate request, in one of two forms: the request as a JSON object, or, for one near the bus's message limit, a JSON string "gzip+base64:" followed by the base64 of the request object gzip-compressed; a holder that does not know the packed form refuses it, naming that it holds no request object
|
||||
Request any `json:"request"`
|
||||
}
|
||||
|
||||
|
||||
@@ -485,7 +485,7 @@ type StepAskCompose struct {
|
||||
Fetch Fetch `json:"fetch"`
|
||||
Package string `json:"package"`
|
||||
Reader string `json:"reader,omitempty"`
|
||||
// the compose-candidate request
|
||||
// the compose-candidate request, in one of two forms: the request as a JSON object, or, for one near the bus's message limit, a JSON string "gzip+base64:" followed by the base64 of the request object gzip-compressed; a holder that does not know the packed form refuses it, naming that it holds no request object
|
||||
Request any `json:"request"`
|
||||
}
|
||||
|
||||
|
||||
@@ -161,7 +161,7 @@
|
||||
"type": "string"
|
||||
},
|
||||
"request": {
|
||||
"description": "the compose-candidate request"
|
||||
"description": "the compose-candidate request, in one of two forms: the request as a JSON object, or, for one near the bus's message limit, a JSON string \"gzip+base64:\" followed by the base64 of the request object gzip-compressed; a holder that does not know the packed form refuses it, naming that it holds no request object"
|
||||
},
|
||||
"reader": {
|
||||
"type": "string"
|
||||
|
||||
@@ -293,7 +293,7 @@ export interface StepAskCompose {
|
||||
"fetch": Fetch;
|
||||
"package": string;
|
||||
"reader"?: string;
|
||||
/** the compose-candidate request */
|
||||
/** the compose-candidate request, in one of two forms: the request as a JSON object, or, for one near the bus's message limit, a JSON string "gzip+base64:" followed by the base64 of the request object gzip-compressed; a holder that does not know the packed form refuses it, naming that it holds no request object */
|
||||
"request": unknown;
|
||||
}
|
||||
|
||||
|
||||
@@ -293,7 +293,7 @@ export interface StepAskCompose {
|
||||
"fetch": Fetch;
|
||||
"package": string;
|
||||
"reader"?: string;
|
||||
/** the compose-candidate request */
|
||||
/** the compose-candidate request, in one of two forms: the request as a JSON object, or, for one near the bus's message limit, a JSON string "gzip+base64:" followed by the base64 of the request object gzip-compressed; a holder that does not know the packed form refuses it, naming that it holds no request object */
|
||||
"request": unknown;
|
||||
}
|
||||
|
||||
|
||||
@@ -293,7 +293,7 @@ export interface StepAskCompose {
|
||||
"fetch": Fetch;
|
||||
"package": string;
|
||||
"reader"?: string;
|
||||
/** the compose-candidate request */
|
||||
/** the compose-candidate request, in one of two forms: the request as a JSON object, or, for one near the bus's message limit, a JSON string "gzip+base64:" followed by the base64 of the request object gzip-compressed; a holder that does not know the packed form refuses it, naming that it holds no request object */
|
||||
"request": unknown;
|
||||
}
|
||||
|
||||
|
||||
@@ -1233,7 +1233,18 @@ func TestAProposalIsAnsweredApprovedAndWritten(t *testing.T) {
|
||||
t.Fatal("anchor in the form before was not converted")
|
||||
}
|
||||
}
|
||||
verbs := newSeatTools(bedStore{b}, func(string, any) (json.RawMessage, error) { return json.RawMessage(`{"ok":true}`), nil })
|
||||
// The verbs ask the indexer's first views of modules and manifests (ADR 0615 §1), as the tool runner answers them from the
|
||||
// indexer's buckets; every other ask the controller's, answered ok.
|
||||
verbs := newSeatTools(bedStore{b}, func(tool string, _ any) (json.RawMessage, error) {
|
||||
if state, ok := strings.CutPrefix(tool, buckets.IndexerModule+"."); ok && indexerFirstViews[state] != "" {
|
||||
a, err := views.FirstView(buckets.IndexerModule, state, bedSource{b.ctx, b.buckets[indexerFirstViews[state]]}, nil, "", time.Now())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return json.Marshal(map[string]any{"result": a})
|
||||
}
|
||||
return json.RawMessage(`{"ok":true}`), nil
|
||||
})
|
||||
propose := func() mma.Proposal {
|
||||
t.Helper()
|
||||
got, err := verbs.doPropose(map[string]any{"why": "move web", "changes": `[{"op":"take-off","node":"anchor","module":"web"},` +
|
||||
|
||||
@@ -378,7 +378,7 @@ func RequestTools(r check.StateRead) []stdio.Tool {
|
||||
Description: "The composition request a check sent to a node: the composer's whole input for that node (the node-engine's composition input, the case's " +
|
||||
"manifests and verdicts, the seat list), kept exactly as sent so a session reproduces the check's composition (the composer run by hand over the same " +
|
||||
"request gives the check's answer). Kept in the checker's state compositions, per check key (group.heads, the 12-digit heads hash) and node, " +
|
||||
"about 430 KB each (kept compressed, so a value fits the state's 256 KiB), for 3 days (the state's ttl-seconds). With a check and a node, that request whole; with a check, " +
|
||||
"0.7 to 0.8 MB each (kept compressed, in pieces that each fit the state's 256 KiB value), for 3 days (the state's ttl-seconds). With a check and a node, that request whole; with a check, " +
|
||||
"the nodes it kept one for; with neither, every kept request, each with its size, digest and when it was kept. Read-only.",
|
||||
Input: map[string]any{"type": "object", "properties": map[string]any{
|
||||
"check": map[string]any{"type": "string", "description": "the check's key, <group>.<heads hash>, as the verdict and composition name it"},
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"math/rand"
|
||||
"os"
|
||||
"regexp"
|
||||
"slices"
|
||||
@@ -48,7 +49,36 @@ func TestTheToolsServedAreTheFiveReadTools(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
// Issue 1966: a request whose compressed bytes pass the state's value limit is kept in pieces and read back whole; the
|
||||
// pieces are no requests in the list.
|
||||
rnd := make([]byte, 600<<10)
|
||||
_, _ = rand.New(rand.NewSource(1)).Read(rnd)
|
||||
big := []byte(`{"input":"` + base64.StdEncoding.EncodeToString(rnd) + `"}`)
|
||||
if err := keeper.Keep(context.Background(), "big.0123456789ab", "anchor", big); err != nil {
|
||||
t.Fatalf("a request of %d bytes: %v", len(big), err)
|
||||
}
|
||||
rt := RequestTools(check.StateRead{Keys: store.Keys, Get: store.Get})
|
||||
if one, err := rt[0].Run(map[string]any{"check": "big.0123456789ab", "node": "anchor"}); err != nil || !strings.Contains(func() string { b, _ := json.Marshal(one); return string(b) }(), base64.StdEncoding.EncodeToString(rnd)[:64]) {
|
||||
t.Errorf("the big request read back: %v", err)
|
||||
}
|
||||
// Listing reads no piece, and a request of another version is refused by name.
|
||||
store.vals["v1.0123456789ab/anchor"] = json.RawMessage(`{"v":1,"check":"v1.0123456789ab","node":"anchor","gzip":"x"}`)
|
||||
noPieces := check.StateRead{Keys: store.Keys, Get: func(k string) (json.RawMessage, error) {
|
||||
if strings.HasPrefix(k, "pieces/") {
|
||||
return nil, errors.New("a piece was read")
|
||||
}
|
||||
return store.Get(k)
|
||||
}}
|
||||
if _, err := RequestTools(noPieces)[0].Run(map[string]any{}); err != nil {
|
||||
t.Errorf("listing: %v", err)
|
||||
}
|
||||
if _, err := rt[0].Run(map[string]any{"check": "v1.0123456789ab", "node": "anchor"}); err == nil || !strings.Contains(err.Error(), "version 1") {
|
||||
t.Errorf("a version 1 request: %v", err)
|
||||
}
|
||||
delete(store.vals, "v1.0123456789ab/anchor")
|
||||
if all, _ := rt[0].Run(map[string]any{}); strings.Count(func() string { b, _ := json.Marshal(all); return string(b) }(), `"node"`) != 4 {
|
||||
t.Errorf("the list holds a piece as a request: %v", all)
|
||||
}
|
||||
if len(rt) != 1 || rt[0].Name != check.RequestsReadTool || !strings.Contains(rt[0].Description, "composition request a check sent to a node") {
|
||||
t.Fatalf("request tools = %+v", rt)
|
||||
}
|
||||
@@ -312,6 +342,9 @@ func TestTheTryToolsAskAndReadATry(t *testing.T) { // ADR 0501 §5, the agreed i
|
||||
type memPut struct{ m *memStore }
|
||||
|
||||
func (p memPut) Put(_ context.Context, key string, value []byte) (uint64, error) {
|
||||
if len(value) > 262144 { // the state's value limit, headers included (issue 1966)
|
||||
return 0, errors.New("the value is over the 262144 bytes the bucket takes")
|
||||
}
|
||||
p.m.vals[key] = value
|
||||
return 1, nil
|
||||
}
|
||||
|
||||
@@ -228,6 +228,10 @@ func (e EngineCompositions) inSandbox(ctx context.Context, c *Case, head, node s
|
||||
}
|
||||
job := buildstep.ComposeJob{Fetch: buildstep.Fetch{Repository: nodeEngineRepo, Commit: head, Packages: packagesFor(e.Published, c, c.Tagged, nodeEngineRepo)}, Dir: meshcatalogclient.RootPath,
|
||||
Package: composer.Package, Request: raw, Reader: "the node-engine head of " + c.Group}
|
||||
// The ask travels on the bus, whose message limit a whole request passes (1.2 MB for a large node): it goes packed.
|
||||
if err := job.PackRequest(raw); err != nil {
|
||||
return a, err
|
||||
}
|
||||
verdict, err := e.Sandbox.Compose(ctx, job)
|
||||
if err != nil {
|
||||
return a, fmt.Errorf("the composer at the node-engine's head %s did not answer for %s: %w", head, node, err)
|
||||
|
||||
@@ -59,7 +59,11 @@ type fakeSandbox struct {
|
||||
|
||||
func (f *fakeSandbox) Compose(_ context.Context, job buildstep.ComposeJob) (composer.Verdict, error) {
|
||||
var req composer.Request
|
||||
if err := json.Unmarshal(job.Request, &req); err != nil {
|
||||
body, err := job.RequestBytes()
|
||||
if err != nil {
|
||||
return composer.Verdict{}, err
|
||||
}
|
||||
if err := json.Unmarshal(body, &req); err != nil {
|
||||
return composer.Verdict{}, err
|
||||
}
|
||||
f.mu.Lock()
|
||||
@@ -206,7 +210,11 @@ func TestTheCompositionIsMadeFromTheEnginesAnswers(t *testing.T) {
|
||||
}
|
||||
job := sb.jobs[0]
|
||||
var req composer.Request
|
||||
_ = json.Unmarshal(job.Request, &req)
|
||||
if len(job.Request) == 0 || job.Request[0] != '"' {
|
||||
t.Errorf("the request went whole, not packed (issue 1966): %.60s", job.Request)
|
||||
}
|
||||
body, _ := job.RequestBytes()
|
||||
_ = json.Unmarshal(body, &req)
|
||||
if job.Fetch.Repository != nodeEngineRepo || job.Fetch.Commit != head2 || job.Package != composer.Package ||
|
||||
string(req.Input) != `{"nodes":[{"name":"anchor"}]}` || req.Cases[0].Manifests["web"].Digest == "" ||
|
||||
len(req.Seats) != 1 || string(req.Seats[0]) != string(host.Seats[0]) {
|
||||
@@ -220,8 +228,9 @@ func TestTheCompositionIsMadeFromTheEnginesAnswers(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
keyOfHost, _ := nodeengineclient.CompositionKey(host.Group, host.Heads)
|
||||
lastBody, _ := sb.jobs[len(sb.jobs)-1].RequestBytes()
|
||||
got, err := ReadRequests(store.read(), keyOfHost, "anchor")
|
||||
if b, _ := json.Marshal(got); err != nil || !strings.Contains(string(b), `"request":`+string(sb.jobs[len(sb.jobs)-1].Request)) {
|
||||
if b, _ := json.Marshal(got); err != nil || !strings.Contains(string(b), `"request":`+string(lastBody)) {
|
||||
t.Errorf("kept %s, %v: want the request of the last job under %s/anchor", b, err, keyOfHost)
|
||||
}
|
||||
store.err = errors.New("the state is full")
|
||||
@@ -311,7 +320,8 @@ func TestTheCompositionIsMadeFromTheEnginesAnswers(t *testing.T) {
|
||||
t.Fatalf("jobs %+v: want one job at the node-engine's trunk", sb.jobs)
|
||||
}
|
||||
var req composer.Request
|
||||
_ = json.Unmarshal(sb.jobs[0].Request, &req)
|
||||
b0, _ := sb.jobs[0].RequestBytes()
|
||||
_ = json.Unmarshal(b0, &req)
|
||||
if got := string(req.Cases[0].Verdicts["mesh-build-go"]); got != string(brings.Judged["mesh-build-go"]) || len(req.Cases[0].Verdicts) != 2 {
|
||||
t.Errorf("the request's verdicts %s: want the check's judge's for mesh-build-go and the indexer's for mail", req.Cases[0].Verdicts)
|
||||
}
|
||||
@@ -321,7 +331,8 @@ func TestTheCompositionIsMadeFromTheEnginesAnswers(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var none composer.Request
|
||||
_ = json.Unmarshal(sb.jobs[len(sb.jobs)-1].Request, &none)
|
||||
bn, _ := sb.jobs[len(sb.jobs)-1].RequestBytes()
|
||||
_ = json.Unmarshal(bn, &none)
|
||||
if _, kept := none.Cases[0].Verdicts["mesh-build-go"]; kept {
|
||||
t.Errorf("the running indexer's verdict on the change's manifest was kept: %s", none.Cases[0].Verdicts)
|
||||
}
|
||||
|
||||
@@ -20,20 +20,28 @@ import (
|
||||
// session reproduces a check's composition exactly. The checker is a foundation part (novox/hq ADR 0619 §1-3), so its
|
||||
// kept data is a section of its manifest's old form, here `state`, and not an object store asked in `requires`.
|
||||
//
|
||||
// **What bounds it.** A request is the node-engine's whole composition input of one node, about 430 KB as JSON. A
|
||||
// state's value holds at most 256 KiB (kvlimit.ModuleStateValueBytes), so it is kept gzip-compressed (JSON shrinks about
|
||||
// tenfold) in a document of its own; a request whose compressed bytes still do not fit is refused by the put, which
|
||||
// fails the node's composition naming the size (never dropped). The manifest's ttl-seconds is its lifetime (3 days):
|
||||
// the bus drops a value past it, so nothing waits on a timer here. A check kept again overwrites the key.
|
||||
// **What bounds it.** A request is the node-engine's whole composition input of one node, 0.7 to 0.8 MB as JSON and
|
||||
// growing with the mesh. A state's value holds at most 256 KiB (kvlimit.ModuleStateValueBytes, headers included), and
|
||||
// the compressed request already passed it (issue 1966: 300127 bytes), so it is kept gzip-compressed in pieces: the key
|
||||
// of the check and node holds a small document naming the request's digest, size and number of pieces, and each piece is
|
||||
// a value of its own under RequestChunkKey, of at most RequestChunkBytes. The kept data is the checker's, a foundation
|
||||
// part's, so it stays in the old section `state` (ADR 0619 §2) and is read through the tool runner's state read, which
|
||||
// serves no object store of the old sections. The manifest's ttl-seconds is the lifetime (3 days) of the pointer and of
|
||||
// every piece: the bus drops a value past it, so nothing waits on a timer here. A check kept again overwrites the key;
|
||||
// a request whose pieces cannot all be put is refused naming the size (never dropped, never half kept: the pointer is
|
||||
// put after its pieces).
|
||||
const (
|
||||
// RequestsStateName is the checker's state of kept requests and RequestsBucket its bucket.
|
||||
RequestsStateName = "compositions"
|
||||
RequestsBucket = CheckerModule + "_" + RequestsStateName
|
||||
// RequestsReadTool is the tool that reads a kept request.
|
||||
RequestsReadTool = "checker_composition_requests"
|
||||
// RequestChunkBytes is the most compressed bytes one piece holds: base64 of it, in its document, stays well under
|
||||
// the state's 256 KiB value.
|
||||
RequestChunkBytes = 128 << 10
|
||||
)
|
||||
|
||||
// KeptRequest is a request as the state holds it: whose it is, and the request's JSON, compressed.
|
||||
// KeptRequest is the pointer to a request as the state holds it: whose it is, and what its pieces make.
|
||||
type KeptRequest struct {
|
||||
V int `json:"v"`
|
||||
Check string `json:"check"`
|
||||
@@ -42,8 +50,32 @@ type KeptRequest struct {
|
||||
// Size and Digest are the request's, uncompressed: its bytes and their sha256.
|
||||
Size int `json:"size"`
|
||||
Digest string `json:"digest"`
|
||||
// Gzip is the request's bytes, gzip-compressed and base64-encoded.
|
||||
Gzip string `json:"gzip"`
|
||||
// Pieces is how many pieces (RequestChunkKey of the digest, 0 to Pieces-1) make the request's gzip bytes.
|
||||
Pieces int `json:"pieces"`
|
||||
}
|
||||
|
||||
// KeptPiece is one piece of a request's gzip bytes, base64-encoded.
|
||||
type KeptPiece struct {
|
||||
V int `json:"v"`
|
||||
Digest string `json:"digest"`
|
||||
Index int `json:"index"`
|
||||
Gzip string `json:"gzip"`
|
||||
}
|
||||
|
||||
// isPieceKey is whether a key is a piece's: under pieces/ and a sha256 digest, which no check's key (group.heads/node) is.
|
||||
func isPieceKey(k string) bool {
|
||||
rest, ok := strings.CutPrefix(k, "pieces/")
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
digest, _, _ := strings.Cut(rest, "/")
|
||||
return len(digest) == 64
|
||||
}
|
||||
|
||||
// RequestChunkKey is the key of a request's piece: under the request's digest, so a request kept again for the same
|
||||
// bytes writes the same pieces.
|
||||
func RequestChunkKey(digest string, index int) string {
|
||||
return fmt.Sprintf("pieces/%s/%d", digest, index)
|
||||
}
|
||||
|
||||
// RequestKey is the key a check's request to a node is kept under: the check's key (group.heads) and the node.
|
||||
@@ -75,14 +107,28 @@ func (k RequestKeeper) Keep(ctx context.Context, key, node string, body []byte)
|
||||
now = k.Now
|
||||
}
|
||||
sum := sha256.Sum256(body)
|
||||
doc, err := json.Marshal(KeptRequest{V: 1, Check: key, Node: node, KeptAt: now().UTC(), Size: len(body),
|
||||
Digest: hex.EncodeToString(sum[:]), Gzip: base64.StdEncoding.EncodeToString(packed.Bytes())})
|
||||
digest := hex.EncodeToString(sum[:])
|
||||
pieces := (packed.Len() + RequestChunkBytes - 1) / RequestChunkBytes
|
||||
fail := func(err error) error {
|
||||
return fmt.Errorf("the composition request of check %s to %s (%d bytes, %d compressed in %d pieces) could not be kept in the state %s: %w",
|
||||
key, node, len(body), packed.Len(), pieces, RequestsStateName, err)
|
||||
}
|
||||
for i := 0; i < pieces; i++ {
|
||||
end := min((i+1)*RequestChunkBytes, packed.Len())
|
||||
doc, err := json.Marshal(KeptPiece{V: 2, Digest: digest, Index: i, Gzip: base64.StdEncoding.EncodeToString(packed.Bytes()[i*RequestChunkBytes : end])})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := k.Store.Put(ctx, RequestChunkKey(digest, i), doc); err != nil {
|
||||
return fail(err)
|
||||
}
|
||||
}
|
||||
doc, err := json.Marshal(KeptRequest{V: 2, Check: key, Node: node, KeptAt: now().UTC(), Size: len(body), Digest: digest, Pieces: pieces})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := k.Store.Put(ctx, RequestKey(key, node), doc); err != nil {
|
||||
return fmt.Errorf("the composition request of check %s to %s (%d bytes, %d compressed) could not be kept in the state %s: %w",
|
||||
key, node, len(body), packed.Len(), RequestsStateName, err)
|
||||
return fail(err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -104,7 +150,7 @@ func ReadRequests(s StateRead, key, node string) (any, error) {
|
||||
if len(raw) == 0 {
|
||||
return nil, fmt.Errorf("no composition request of check %s to %s is kept (none was sent, or it is past its lifetime)", key, node)
|
||||
}
|
||||
kept, body, err := unpackRequest(raw)
|
||||
kept, body, err := unpackRequest(s, raw)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the kept request %s: %w", RequestKey(key, node), err)
|
||||
}
|
||||
@@ -120,6 +166,9 @@ func ReadRequests(s StateRead, key, node string) (any, error) {
|
||||
if key != "" && !strings.HasPrefix(k, key+"/") {
|
||||
continue
|
||||
}
|
||||
if isPieceKey(k) {
|
||||
continue // a piece of a request, not a request: never read here
|
||||
}
|
||||
raw, err := s.Get(k)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -134,15 +183,30 @@ func ReadRequests(s StateRead, key, node string) (any, error) {
|
||||
return map[string]any{"kept": listed}, nil
|
||||
}
|
||||
|
||||
// unpackRequest reads a kept document and its request's bytes, which must match the digest they were kept with.
|
||||
func unpackRequest(raw []byte) (KeptRequest, []byte, error) {
|
||||
// unpackRequest reads a kept pointer's pieces and the request's bytes, which must match the digest they were kept with.
|
||||
func unpackRequest(s StateRead, raw []byte) (KeptRequest, []byte, error) {
|
||||
var kept KeptRequest
|
||||
if err := json.Unmarshal(raw, &kept); err != nil {
|
||||
return kept, nil, fmt.Errorf("is no kept request: %w", err)
|
||||
}
|
||||
packed, err := base64.StdEncoding.DecodeString(kept.Gzip)
|
||||
if err != nil {
|
||||
return kept, nil, fmt.Errorf("holds no base64: %w", err)
|
||||
if kept.V != 2 {
|
||||
return kept, nil, fmt.Errorf("is a kept request of version %d, which this reader does not read (it reads version 2; an older one is past its lifetime within 3 days)", kept.V)
|
||||
}
|
||||
var packed []byte
|
||||
for i := 0; i < kept.Pieces; i++ {
|
||||
raw, err := s.Get(RequestChunkKey(kept.Digest, i))
|
||||
if err != nil {
|
||||
return kept, nil, err
|
||||
}
|
||||
var piece KeptPiece
|
||||
if err := json.Unmarshal(raw, &piece); err != nil || len(raw) == 0 {
|
||||
return kept, nil, fmt.Errorf("lacks its piece %d of %d (past its lifetime, or never kept)", i, kept.Pieces)
|
||||
}
|
||||
part, err := base64.StdEncoding.DecodeString(piece.Gzip)
|
||||
if err != nil {
|
||||
return kept, nil, fmt.Errorf("holds no base64 in its piece %d: %w", i, err)
|
||||
}
|
||||
packed = append(packed, part...)
|
||||
}
|
||||
zr, err := gzip.NewReader(bytes.NewReader(packed))
|
||||
if err != nil {
|
||||
|
||||
@@ -278,6 +278,10 @@ func main() {
|
||||
tools = append(tools, catalogTools()...)
|
||||
tools = append(tools, viewTools(os.Getenv, keptSource, time.Now)...)
|
||||
tools = append(tools, index.PendingTools(keptRead{stdio.State(index.PendingState)}, keptRead{stdio.State(index.OwnBuildStateName)}, asks, time.Now)...)
|
||||
tools = append(tools, index.FingerprintsTools(asks, time.Now, func(what, why string) error {
|
||||
_, err := acts.Record(meshhandactsclient.Request{What: what, Why: why, Cause: index.FingerprintsAgainTool, Verb: "fingerprints-again"})
|
||||
return err
|
||||
}, 10*time.Second)...)
|
||||
tools = append(tools, index.TryCandidateTools(keptRead{stdio.State("candidates")}, keptRead{stdio.State(index.TryRecoveryState)}, asks, time.Now)...)
|
||||
tools = append(tools, configReadersTool(stdio.NewWatchedStates()), busStateTool(keptSource), requirementsTool(keptSource),
|
||||
packageUsersTool(keptSource), seatDefinitionTool(keptSource), outOfStepTool(keptSource),
|
||||
|
||||
@@ -52,7 +52,14 @@ func main() {
|
||||
|
||||
// awaitedStore is a state bound when it appears (process.AwaitBucket) as the indexer's store: until then every call is
|
||||
// process.ErrNotYet.
|
||||
type awaitedStore struct{ b *process.AwaitedBucket }
|
||||
type awaitedStore struct {
|
||||
b *process.AwaitedBucket
|
||||
// js and name, when set, tell a bucket that does not exist from one that cannot be reached: the awaited bucket is nil for
|
||||
// both (the SDK drops the bus's error), and only the first is "not yet". The packages state is read this way: an unreached
|
||||
// state is no package published, and the version line written over every import form (issue 1966).
|
||||
js process.BucketOpener
|
||||
name string
|
||||
}
|
||||
|
||||
// expiringKeys is index.Bounds on a state whose keys take a time to live (key-ttl in the manifest): a key armed expires
|
||||
// at its end, which its watcher hears as a purge; one standing keeps its end.
|
||||
@@ -88,6 +95,11 @@ func (e expiringKeys) Disarm(ctx context.Context, key string) error {
|
||||
func (s awaitedStore) store() (kvStore, error) {
|
||||
kv := s.b.KV()
|
||||
if kv == nil {
|
||||
if s.js != nil {
|
||||
if _, err := s.js.KeyValue(context.Background(), s.name); err != nil && !errors.Is(err, jetstream.ErrBucketNotFound) {
|
||||
return kvStore{}, fmt.Errorf("the bucket %s cannot be reached: %w", s.name, err)
|
||||
}
|
||||
}
|
||||
return kvStore{}, process.ErrNotYet
|
||||
}
|
||||
return kvStore{kv}, nil
|
||||
@@ -411,17 +423,17 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
seatBus := &seatBus{defs: awaitedSource{ctx, seatDefs}, marks: awaitedSource{ctx, marks}}
|
||||
seatReader, saySeats := followSeatList(seatBus, saySeatDefinition(js, cred), logger.Printf)
|
||||
readWhenBound(seatReader, seatDefs, marks)
|
||||
ix := &index.Indexer{Findings: awaitedStore{findings}, OldSections: awaitedStore{oldSections}, Requirements: awaitedStore{requirements}, Store: kvStore{candidates}, Modules: kvStore{modules}, Manifests: kvStore{manifests}, Taken: awaitedStore{taken}, Builds: known,
|
||||
Verdicts: awaitedStore{verdicts}, SayRefused: sayRefused(conn, cred, logger.Printf), SayAccepted: sayAccepted(conn, cred),
|
||||
Predefined: awaitedStore{predefined}, ConfigCandidates: awaitedStore{configCandidates}, Logf: logger.Printf,
|
||||
ix := &index.Indexer{Findings: awaitedStore{b: findings}, OldSections: awaitedStore{b: oldSections}, Requirements: awaitedStore{b: requirements}, Store: kvStore{candidates}, Modules: kvStore{modules}, Manifests: kvStore{manifests}, Taken: awaitedStore{b: taken}, Builds: known,
|
||||
Verdicts: awaitedStore{b: verdicts}, SayRefused: sayRefused(conn, cred, logger.Printf), SayAccepted: sayAccepted(conn, cred),
|
||||
Predefined: awaitedStore{b: predefined}, ConfigCandidates: awaitedStore{b: configCandidates}, Logf: logger.Printf,
|
||||
Named: namedVia(calls.Call), Trunk: trunkVia(calls.Call), Contains: containsVia(calls.Call), Read: readVia(calls.Call),
|
||||
ReadOpen: openVia(calls.Call), Packages: awaitedStore{packages}, Files: filesVia(calls.Call), GoSum: goSumVia(calls.Call), GoMod: goModVia(calls.Call),
|
||||
OwnBuild: ownBuildVia(calls.Call, cred.Node), OpenTries: openTriesVia(calls.Call), OwnBuildState: awaitedStore{ownBuildState}, TryRecovery: awaitedStore{tryRecovery}, Graph: awaitedStore{graph},
|
||||
SeatDefinitions: awaitedStore{seatDefs}, ReadFile: readFileVia(calls.Call), SaySeatDefinition: saySeats,
|
||||
ReadOpen: openVia(calls.Call), Packages: awaitedStore{b: packages, js: js, name: index.PackagesBucket}, Files: filesVia(calls.Call), GoSum: goSumVia(calls.Call), GoMod: goModVia(calls.Call),
|
||||
OwnBuild: ownBuildVia(calls.Call, cred.Node), OpenTries: openTriesVia(calls.Call), OwnBuildState: awaitedStore{b: ownBuildState}, TryRecovery: awaitedStore{b: tryRecovery}, Graph: awaitedStore{b: graph},
|
||||
SeatDefinitions: awaitedStore{b: seatDefs}, ReadFile: readFileVia(calls.Call), SaySeatDefinition: saySeats,
|
||||
Message: messageVia(calls.Call),
|
||||
// The take-in step (ADR 0633): its state pending and its stream take-in, both this delivery's, bound when they appear;
|
||||
// until then every merge is taken in at once, as before.
|
||||
Pending: awaitedStore{process.AwaitBucket(ctx, js, conn, index.PendingBucket, "no merge is held for the indexer its revision names: each is taken in at once", logger.Printf)},
|
||||
Pending: awaitedStore{b: process.AwaitBucket(ctx, js, conn, index.PendingBucket, "no merge is held for the indexer its revision names: each is taken in at once", logger.Printf)},
|
||||
SayTakeIn: sayTakeIn(js, cred)}
|
||||
// Which build this program is, fixed here once, before it hears any merge and never asked again (index.FixOwnBuild): the
|
||||
// node-engine names the successor while it replaces this process, and an old process that learnt it later would take
|
||||
@@ -483,7 +495,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
// The holds, new with ADR 0632 (issue 1790's kind), bound when they appear: until then a merge that would hold a
|
||||
// manifest is left for redelivery, said, rather than write what a counted engine cannot read.
|
||||
holds := process.AwaitBucket(ctx, js, conn, index.HoldsBucket, "a merge whose manifest every node reads, needing what not every counted node-engine states, is left for redelivery", logger.Printf)
|
||||
ix.Holds = awaitedStore{holds}
|
||||
ix.Holds = awaitedStore{b: holds}
|
||||
holds.Ready(func(jetstream.KeyValue) { ix.KickHolds(ctx) })
|
||||
// Each standing hold's bound, a key that expires at its end (ADR 0632 §2, the indexer's own state): its expiry weighs
|
||||
// the holds again, which raises manifest-held; until the state is bound a hold past its bound is raised at its next
|
||||
@@ -696,7 +708,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
taken.Ready(func(jetstream.KeyValue) { ix.KickFill(ctx) })
|
||||
// What the indexer heard of FORGE (issue 1126 note 76) is bound when its state appears, as every state a delivery adds
|
||||
// (issue 1790): until then no message is written there, and the rest of the indexer runs.
|
||||
heard := awaitedStore{process.AwaitBucket(ctx, js, conn, index.HeardBucket, "no FORGE message the indexer handles is written for indexer_heard", logger.Printf)}
|
||||
heard := awaitedStore{b: process.AwaitBucket(ctx, js, conn, index.HeardBucket, "no FORGE message the indexer handles is written for indexer_heard", logger.Printf)}
|
||||
// Each change of the states its readers read is said on its views family (ADR 0615 §5), from the mark each state was
|
||||
// said up to; a first view (the tools bundle) answers at that mark.
|
||||
startViews(ctx, js, cred, marks,
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"log"
|
||||
"strings"
|
||||
"testing"
|
||||
@@ -151,8 +152,28 @@ func TestAMessageIsIndexedAndAcked(t *testing.T) { // 1238/AC2 (unit part)
|
||||
t.Errorf("forge/group's merged message read as a group's end: %q", logs.String())
|
||||
}
|
||||
})
|
||||
// Issue 1966: a bucket the bus has not made is "not yet"; one it cannot reach (jetstream not enabled, no responders) is the
|
||||
// bus's own error, which stops a pass instead of reading as nothing published.
|
||||
t.Run("a bucket not found is not yet, one the bus cannot reach is its error", func(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
err error
|
||||
notYet bool
|
||||
}{{"not found", jetstream.ErrBucketNotFound, true}, {"not reachable", jetstream.ErrJetStreamNotEnabled, false}} {
|
||||
b := process.AwaitBucket(context.Background(), opener{c.err}, nil, "packages", "none", t.Logf)
|
||||
_, err := awaitedStore{b: b, js: opener{c.err}, name: "packages"}.Keys(context.Background())
|
||||
if errors.Is(err, process.ErrNotYet) != c.notYet || err == nil || (!c.notYet && !errors.Is(err, c.err)) {
|
||||
t.Errorf("%s: Keys answered %v, want not-yet %v", c.name, err, c.notYet)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// opener is a bus that answers every bucket with err.
|
||||
type opener struct{ err error }
|
||||
|
||||
func (o opener) KeyValue(context.Context, string) (jetstream.KeyValue, error) { return nil, o.err }
|
||||
|
||||
func TestTheSourceMainKeyIsTakenAtItsRevision(t *testing.T) { // 1344/AC1 (the watch's side: key, revision, value)
|
||||
var logs bytes.Buffer
|
||||
logger := log.New(&logs, "", 0)
|
||||
|
||||
@@ -24,6 +24,7 @@ type takerIn interface {
|
||||
TakeInMerge(ctx context.Context, repository, merge string, seq uint64, by, why string) index.TakenIn
|
||||
DropTryCandidate(ctx context.Context, id, by, why string) index.TakenIn
|
||||
RecoverTriesByHand(ctx context.Context, by, why string) index.TakenIn
|
||||
FingerprintsAgain(ctx context.Context, module string) index.TakenIn
|
||||
}
|
||||
|
||||
// takenIn answers one value of the takeins state (the tool indexer_take_in, issue 1945): a value not yet answered takes the
|
||||
@@ -42,7 +43,9 @@ func takenIn(ctx context.Context, ix takerIn, kv jetstream.KeyValue, e jetstream
|
||||
return
|
||||
}
|
||||
asked, r := "take-in", index.TakenIn{}
|
||||
if a.RecoverTries {
|
||||
if a.Fingerprints {
|
||||
asked, r = "computing of fingerprints again", ix.FingerprintsAgain(ctx, a.Module)
|
||||
} else if a.RecoverTries {
|
||||
asked, r = "recovery of the tries", ix.RecoverTriesByHand(ctx, a.By, a.Why)
|
||||
a.Module = "tries"
|
||||
} else if a.TryCandidate != "" && a.Drop {
|
||||
|
||||
@@ -150,13 +150,15 @@ type Indexer struct {
|
||||
retrying atomic.Bool // a RetryUnread runs in the background (kickRetry)
|
||||
retryAgain atomic.Bool // an event asked for one more RetryUnread after the running one
|
||||
retryWG sync.WaitGroup // the background RetryUnread, for tests to wait on
|
||||
recompute sync.Mutex // one Recompute at a time
|
||||
untaken map[string]string // repository to the latest merge applied while the taken state was missing (taken.go)
|
||||
filling atomic.Bool // a FillTaken runs in the background (KickFill)
|
||||
fillAgain atomic.Bool // a kick arrived while it ran
|
||||
atHeads map[string]map[string]meshcatalog.PackagesByModuleValue // a member head's published Go modules, by repository@head (candidatePublished)
|
||||
saidAt map[string]string // the head of a repository whose failed read was said, by repository: said once per head
|
||||
failedAt map[string]string // the head of a repository whose tree lacks a directory asked, by repository: not asked again until the head changes
|
||||
fresh map[string]bool // modules whose import form is computed again by hand (FingerprintsAgain), "*" for every one; under freshMu
|
||||
freshMu sync.Mutex
|
||||
recompute sync.Mutex // one Recompute at a time
|
||||
untaken map[string]string // repository to the latest merge applied while the taken state was missing (taken.go)
|
||||
filling atomic.Bool // a FillTaken runs in the background (KickFill)
|
||||
fillAgain atomic.Bool // a kick arrived while it ran
|
||||
atHeads map[string]map[string]meshcatalog.PackagesByModuleValue // a member head's published Go modules, by repository@head (candidatePublished)
|
||||
saidAt map[string]string // the head of a repository whose failed read was said, by repository: said once per head
|
||||
failedAt map[string]string // the head of a repository whose tree lacks a directory asked, by repository: not asked again until the head changes
|
||||
// OwnBuild answers the fingerprint of the build this program runs, from the node-engine's seat; nil knows none.
|
||||
OwnBuild func(ctx context.Context) (string, error)
|
||||
underSeats bool // a dry run inside judge.WithSeats: its judge passes do not hold the seat list (verdicts.go)
|
||||
|
||||
@@ -0,0 +1,167 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"maps"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
)
|
||||
|
||||
// FingerprintsAgainTool computes the fingerprints again now (issue 1966; the instruction "tools are how a module is spoken
|
||||
// to"): the indexer reads the packages state, takes no held fingerprint for the build's, and writes the import form for
|
||||
// every module whose imports are all published, as a merge's recompute would. For the module whose fingerprint a pass
|
||||
// wrote while the packages state was unread (the version line over the import form) and that nothing else moves.
|
||||
const FingerprintsAgainTool = "indexer_fingerprints_again"
|
||||
|
||||
// FingerprintMove is one module's fingerprint before and after a recompute by hand.
|
||||
type FingerprintMove struct {
|
||||
Module string `json:"module"`
|
||||
Old string `json:"old"`
|
||||
New string `json:"new"`
|
||||
}
|
||||
|
||||
// FingerprintsAgain computes the fingerprints of module again ("all" or empty: every held module) from the published
|
||||
// packages as they are read now, and answers each fingerprint that moved. A read of the packages state that failed
|
||||
// writes nothing, and says so.
|
||||
func (ix *Indexer) FingerprintsAgain(ctx context.Context, module string) TakenIn {
|
||||
module = strings.TrimSpace(module)
|
||||
if module == "" {
|
||||
module = "all"
|
||||
}
|
||||
out := TakenIn{Module: module}
|
||||
if ix.Modules == nil || ix.Manifests == nil {
|
||||
out.Answer = "the indexer writes no fingerprints"
|
||||
return out
|
||||
}
|
||||
before, err := ix.fingerprintsHeld(ctx)
|
||||
if err != nil {
|
||||
out.Answer = "the modules state cannot be read: " + err.Error()
|
||||
return out
|
||||
}
|
||||
if module != "all" {
|
||||
if _, ok := before[module]; !ok {
|
||||
out.Answer = "the indexer holds no fingerprint of " + module
|
||||
return out
|
||||
}
|
||||
}
|
||||
key := module
|
||||
if module == "all" {
|
||||
key = "*"
|
||||
}
|
||||
ix.freshMu.Lock()
|
||||
ix.fresh = map[string]bool{key: true}
|
||||
ix.freshMu.Unlock()
|
||||
// The path of a merge's recompute: the modules, then the candidates their fingerprints reach (recomputeWith), then the
|
||||
// pending merges a changed fingerprint settles (PendingAgain, as after a toolchain report).
|
||||
err = ix.recomputeWith(ctx, nil, nil)
|
||||
ix.freshMu.Lock()
|
||||
ix.fresh = nil
|
||||
ix.freshMu.Unlock()
|
||||
if err != nil {
|
||||
out.Answer = "nothing was written: " + err.Error()
|
||||
return out
|
||||
}
|
||||
if err := ix.PendingAgain(ctx); err != nil {
|
||||
ix.logf("the pending merges are not evaluated again after the fingerprints were computed again: %v", err)
|
||||
}
|
||||
after, err := ix.fingerprintsHeld(ctx)
|
||||
if err != nil {
|
||||
out.Answer = "computed, but the modules state cannot be read back: " + err.Error()
|
||||
return out
|
||||
}
|
||||
for _, m := range slices.Sorted(maps.Keys(after)) {
|
||||
if module != "all" && m != module {
|
||||
continue
|
||||
}
|
||||
if before[m] != after[m] {
|
||||
out.Fingerprints = append(out.Fingerprints, FingerprintMove{Module: m, Old: before[m], New: after[m]})
|
||||
}
|
||||
}
|
||||
out.Stored = true
|
||||
out.Answer = fmt.Sprintf("%d fingerprint(s) moved", len(out.Fingerprints))
|
||||
return out
|
||||
}
|
||||
|
||||
// isFresh says the module's import form is computed again by hand in this pass.
|
||||
func (ix *Indexer) isFresh(module string) bool {
|
||||
ix.freshMu.Lock()
|
||||
defer ix.freshMu.Unlock()
|
||||
return ix.fresh[module] || ix.fresh["*"]
|
||||
}
|
||||
|
||||
// fingerprintsHeld is each held module's fingerprint of the new form.
|
||||
func (ix *Indexer) fingerprintsHeld(ctx context.Context) (map[string]string, error) {
|
||||
keys, err := ix.Modules.Keys(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := map[string]string{}
|
||||
for _, k := range keys {
|
||||
value, _, found, err := ix.Modules.Get(ctx, k)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !found {
|
||||
continue
|
||||
}
|
||||
if m, err := meshcatalogclient.DecodeModule(value); err == nil {
|
||||
out[k] = m.Fp2
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// FingerprintsTools is the tool indexer_fingerprints_again over ask (the takeins state). It acts first and says the hand
|
||||
// act after: record is told what was done and answers an error when it could not be recorded, which the answer says and
|
||||
// which never refuses or delays the recovery beyond bound.
|
||||
func FingerprintsTools(ask func(key string, value []byte) (TakeInAsk, error), now func() time.Time, record func(what, why string) error, bound time.Duration) []stdio.Tool {
|
||||
return []stdio.Tool{{Name: FingerprintsAgainTool,
|
||||
Description: "Compute the fingerprints of one module (or all) again now from the published packages as the indexer reads them: " +
|
||||
"the import form is written for every module whose imports are all published, whatever fingerprint the module holds. For the " +
|
||||
"fingerprints a pass wrote while the packages state was unread (the version line over the import form: builds then fail on an " +
|
||||
"older package). Answers each module whose fingerprint moved, old and new; when the packages state cannot be read it writes " +
|
||||
"nothing and says why. Runs as a merge's recompute does (the modules, the candidates they reach, the pending merges they settle). Acts first, then records the hand act through mesh-hand-acts; when that cannot be recorded the answer says so.",
|
||||
Input: map[string]any{"type": "object", "required": []string{"why"}, "properties": map[string]any{
|
||||
"module": map[string]any{"type": "string", "description": "the module's name, or all (the default)"},
|
||||
"why": map[string]any{"type": "string", "description": "why it is computed again now, in plain words"}}},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
module, _ := args["module"].(string)
|
||||
why, _ := args["why"].(string)
|
||||
module, why = strings.TrimSpace(module), strings.TrimSpace(why)
|
||||
if why == "" {
|
||||
return nil, errors.New("say why: a hand act is said in the journal")
|
||||
}
|
||||
if module == "" {
|
||||
module = "all"
|
||||
}
|
||||
data, err := json.Marshal(TakeInAsk{V: 1, Module: module, Fingerprints: true, Why: why, By: stdio.CallerOf(args), At: now().UTC()})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
a, err := ask("fingerprints."+module+"."+now().UTC().Format("20060102T150405.000000000"), data)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
res := map[string]any{"result": a.Result}
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- record("computed the fingerprints of "+module+" again by hand", why) }()
|
||||
wait, cancel := context.WithTimeout(context.Background(), bound) // a context's bound, no timer (ADR 0437 §8)
|
||||
defer cancel()
|
||||
select {
|
||||
case err := <-done:
|
||||
if err != nil {
|
||||
res["hand-act"] = "not recorded: " + err.Error()
|
||||
}
|
||||
case <-wait.Done(): // a bound on the record, never on the recovery
|
||||
res["hand-act"] = fmt.Sprintf("not recorded within %s: mesh-hand-acts did not answer", bound)
|
||||
}
|
||||
return res, nil
|
||||
}}}
|
||||
}
|
||||
@@ -57,6 +57,9 @@ func formOf(module, repository string, kept *meshcatalog.ModulesByModuleValueInp
|
||||
logf("%s: no published package holds %s, so its fingerprint takes the version line", module, strings.Join(r.Missing, ", "))
|
||||
}
|
||||
}
|
||||
// The fingerprint written here is the version line's, not the import form's: no import form is held for the inputs it was
|
||||
// computed from, so a later pass whose import form computes the same does not take this one for the build's (issue 1966).
|
||||
kept.ImportForm = ""
|
||||
others := fingerprint.OtherPackages(kept.Packages, repository)
|
||||
versions, unpublished := fingerprint.HeldVersionsOf(others, held)
|
||||
for p, v := range pins { // an accepted series' newest published version (acceptedSeries, DLV-I6)
|
||||
|
||||
@@ -2,13 +2,16 @@ package index
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
|
||||
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdeliverychecker"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
)
|
||||
|
||||
const meshSDK = "git.novox.be/novox/mesh-sdk/go"
|
||||
@@ -51,6 +54,7 @@ func (m *missingDir) Error() string { return "no directory " + m.dir }
|
||||
func TestImportsReportedMoveTheFingerprintOnlyWithAnImportedPackage(t *testing.T) {
|
||||
t.Run("a build of an older fingerprint keeps no imports", buildOfAnOlderFingerprintKeepsNoImports)
|
||||
t.Run("a change of a sibling directory alone moves its importer", aSiblingDirectoryAloneMovesItsImporter)
|
||||
t.Run("an unread packages state writes no fingerprint, and they are computed again by hand (issue 1966)", anUnreadPackagesStateWritesNoFingerprintAndFingerprintsAreComputedAgain)
|
||||
t.Run("an accepted series' newest published version pins it, by that version's contents", anAcceptedSeriesPinsItsNewest)
|
||||
ctx := context.Background()
|
||||
ix, mods := newFormIndexer(t, mod{"avahi", catalogueManifest(t, "avahi")})
|
||||
@@ -270,3 +274,114 @@ func anAcceptedSeriesPinsItsNewest(t *testing.T) {
|
||||
t.Errorf("a trunk merge moved avahi off its accepted series' newest: %q %v", m.Fp2, m.Packages)
|
||||
}
|
||||
}
|
||||
|
||||
// Issue 1966 (the indexer started while the bus's storage was down): a read of the packages state that failed, or that
|
||||
// answered no published package where held modules were computed from published ones, writes no fingerprint (the version
|
||||
// line over every import form is what an empty state computes); a version line written leaves no import form behind
|
||||
// that a later pass takes for the build's; and indexer_fingerprints_again computes the import form again by hand.
|
||||
func anUnreadPackagesStateWritesNoFingerprintAndFingerprintsAreComputedAgain(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
// avahi at the import form, pinned to mesh-sdk at c3, as the test above leaves it.
|
||||
setup := func(t *testing.T) (*Indexer, *memStore, meshcatalog.ModulesByModuleValue) {
|
||||
ix, mods := newFormIndexer(t, mod{"avahi", catalogueManifest(t, "avahi")})
|
||||
reportToolchain(t, ix, "mesh-build-go", "novox", "go", "go1.27.1")
|
||||
ix.Packages, ix.Files = newMem(), forge{c1: sdkFiles("a1", "b1"), c3: sdkFiles("a2", "b2")}.files
|
||||
setPackages(t, mods, "avahi", []string{meshSDK})
|
||||
raw, err := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Repository: "novox/mesh-sdk", Path: ".", Commit: c1, None2: "a TypeScript package"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
mods.put("mesh-sdk", raw)
|
||||
held, _ := ix.heldModules(ctx)
|
||||
if err := ix.publish(ctx, "novox/mesh-sdk", c1, held, false); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := ix.Recompute(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
first, _ := moduleOf(t, mods, "avahi")
|
||||
if err := ix.ApplyBuildEnd(ctx, buildEnd(t, "avahi", first.Fp2, meshSDK, meshSDK+"/announce")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
held, _ = ix.heldModules(ctx)
|
||||
if err := ix.publish(ctx, "novox/mesh-sdk", c3, held, false); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := ix.Recompute(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m, _ := moduleOf(t, mods, "avahi")
|
||||
if m.Fp2 == first.Fp2 || m.Fp2 != m.Inputs2.ImportForm {
|
||||
t.Fatalf("setup: fp2 %q (was %q), import form %q; want the import form", m.Fp2, first.Fp2, m.Inputs2.ImportForm)
|
||||
}
|
||||
return ix, mods, m
|
||||
}
|
||||
t.Run("a pass whose read of the packages state failed or answered nothing writes no fingerprint", func(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
bad func(ix *Indexer)
|
||||
why string
|
||||
}{
|
||||
{"the listing fails", func(ix *Indexer) { ix.Packages.(*memStore).keysErr = errors.New("jetstream not enabled") }, "cannot be listed"},
|
||||
{"the state is not on the bus yet", func(ix *Indexer) { ix.Packages.(*memStore).keysErr = process.ErrNotYet }, "answers no published package"}, // a mesh with no import form computes as before: TestIndexerWritesModulesAtAMerge
|
||||
{"the state answers no package", func(ix *Indexer) { ix.Packages = newMem() }, "answers no published package"},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
ix, mods, _ := setup(t)
|
||||
was, _ := moduleOf(t, mods, "avahi")
|
||||
c.bad(ix)
|
||||
err := ix.Recompute(ctx)
|
||||
if err == nil || !strings.Contains(err.Error(), c.why) {
|
||||
t.Fatalf("recompute with an unread packages state: err %v, want one saying %q", err, c.why)
|
||||
}
|
||||
if now, _ := moduleOf(t, mods, "avahi"); now.Fp2 != was.Fp2 {
|
||||
t.Fatalf("a pass over an unread packages state moved avahi's fingerprint to %q (was %q)", now.Fp2, was.Fp2)
|
||||
}
|
||||
})
|
||||
}
|
||||
})
|
||||
t.Run("a version line written keeps no import form, and a fingerprint computed again by hand is the import form", func(t *testing.T) {
|
||||
ix, mods, good := setup(t)
|
||||
// What the pass that read nothing wrote on novox: the version line's fingerprint, the import form still kept.
|
||||
bad := good
|
||||
bad.Fp2 = "src2:" + strings.Repeat("d", 64)
|
||||
raw, err := meshcatalogclient.EncodeModule(bad)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
mods.put("avahi", raw)
|
||||
if err := ix.Recompute(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if now, _ := moduleOf(t, mods, "avahi"); now.Fp2 != bad.Fp2 {
|
||||
t.Fatalf("the held version line is not kept by the build's shortcut (fp2 %q); the case the tool recovers has changed", now.Fp2)
|
||||
}
|
||||
r := ix.FingerprintsAgain(ctx, "avahi")
|
||||
if !r.Stored || len(r.Fingerprints) != 1 || r.Fingerprints[0].Old != bad.Fp2 || r.Fingerprints[0].New != good.Fp2 {
|
||||
t.Fatalf("fingerprints again: %+v, want avahi from %q to %q", r, bad.Fp2, good.Fp2)
|
||||
}
|
||||
if now, _ := moduleOf(t, mods, "avahi"); now.Fp2 != good.Fp2 || ix.fresh != nil {
|
||||
t.Fatalf("avahi holds %q after it, want %q (fresh %v)", now.Fp2, good.Fp2, ix.fresh)
|
||||
}
|
||||
// It runs a merge's recompute, candidates included: a candidate store that cannot be listed fails it, said.
|
||||
ix.Store, ix.previews = &memStore{keysErr: errors.New("no bus")}, map[string]meshdeliverychecker.PreviewsByRepositoryHeadTrunkValue{"k": {}}
|
||||
if r := ix.FingerprintsAgain(ctx, "avahi"); r.Stored || !strings.Contains(r.Answer, "listing the candidates") {
|
||||
t.Fatalf("fingerprints again over candidates that cannot be listed: %+v", r)
|
||||
}
|
||||
if r := ix.FingerprintsAgain(ctx, "nothing"); r.Stored {
|
||||
t.Fatalf("fingerprints again of a module not held answered stored: %+v", r)
|
||||
}
|
||||
})
|
||||
t.Run("formOf clears the import form when it writes the version line", func(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
imports []string
|
||||
}{{"an import no package holds", []string{meshSDK + "/announce"}}, {"no import reported", nil}} {
|
||||
kept := meshcatalog.ModulesByModuleValueInputs2{Tree: "t", Imports: c.imports, ImportForm: "src2:" + strings.Repeat("a", 64)}
|
||||
f := formOf("avahi", "novox/mesh-catalog", &kept, nil, meshcatalog.ModulesByModuleValue{}, nil, nil, nil, nil)
|
||||
if kept.ImportForm != "" {
|
||||
t.Errorf("%s: kept import form %q after the version line %q", c.name, kept.ImportForm, f.fp)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
"maps"
|
||||
"path"
|
||||
"reflect"
|
||||
@@ -31,21 +32,40 @@ import (
|
||||
|
||||
// publishedAll is every value of the packages state, by Go module path; empty with no state or one that does not answer.
|
||||
func (ix *Indexer) publishedAll(ctx context.Context) map[string]meshcatalog.PackagesByModuleValue {
|
||||
out := map[string]meshcatalog.PackagesByModuleValue{}
|
||||
if ix.Packages == nil {
|
||||
return out
|
||||
}
|
||||
keys, err := ix.Packages.Keys(ctx)
|
||||
out, err := ix.publishedRead(ctx)
|
||||
if err != nil {
|
||||
ix.logf("the packages state cannot be listed, so every fingerprint takes the version line: %v", err)
|
||||
return out
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// publishedRead is publishedAll with a read that failed said as an error, never as an empty map: a pass that takes an
|
||||
// unread state for "no package is published" writes the version line over every module's fingerprint (issue 1966, the
|
||||
// indexer started while the bus's storage was down). No state wired is no error: nothing is published then.
|
||||
func (ix *Indexer) publishedRead(ctx context.Context) (map[string]meshcatalog.PackagesByModuleValue, error) {
|
||||
out := map[string]meshcatalog.PackagesByModuleValue{}
|
||||
if ix.Packages == nil {
|
||||
return out, nil
|
||||
}
|
||||
keys, err := ix.Packages.Keys(ctx)
|
||||
if errors.Is(err, process.ErrNotYet) {
|
||||
return out, nil // the state is not on the bus yet: nothing is published, as before
|
||||
}
|
||||
if err != nil {
|
||||
return out, fmt.Errorf("the packages state cannot be listed: %w", err)
|
||||
}
|
||||
for _, key := range keys {
|
||||
if IsSeriesKey(key) {
|
||||
continue // a package's series (series.go), not a Go module's digests
|
||||
}
|
||||
value, _, found, err := ix.Packages.Get(ctx, key)
|
||||
if err != nil || !found {
|
||||
if errors.Is(err, process.ErrNotYet) {
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return out, fmt.Errorf("packages.%s cannot be read: %w", key, err)
|
||||
}
|
||||
if !found {
|
||||
continue
|
||||
}
|
||||
p, err := DecodePublishedPackages(value)
|
||||
@@ -55,7 +75,32 @@ func (ix *Indexer) publishedAll(ctx context.Context) map[string]meshcatalog.Pack
|
||||
}
|
||||
out[p.Module] = p
|
||||
}
|
||||
return out
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// publishedForPass is the packages state a recompute computes fingerprints from, or why it must write none: a read that
|
||||
// failed, or that answered no published module where the held modules show packages were published before (a module
|
||||
// whose fingerprint was computed from the import form, or that holds resolved package versions). The first start of a
|
||||
// mesh, whose held modules show none, computes from an empty state as before.
|
||||
func (ix *Indexer) publishedForPass(ctx context.Context, held map[string]meshcatalog.ModulesByModuleValue) (map[string]meshcatalog.PackagesByModuleValue, error) {
|
||||
published, err := ix.publishedRead(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(published) > 0 {
|
||||
return published, nil
|
||||
}
|
||||
var had []string
|
||||
for key, m := range held {
|
||||
if m.Inputs2 != nil && m.Inputs2.ImportForm != "" { // not Packages: a version line holds those too, with nothing published
|
||||
had = append(had, key)
|
||||
}
|
||||
}
|
||||
if len(had) > 0 {
|
||||
slices.Sort(had)
|
||||
return nil, fmt.Errorf("the packages state answers no published package, though %d held modules (%s ...) were computed from published ones", len(had), had[0])
|
||||
}
|
||||
return published, nil
|
||||
}
|
||||
|
||||
// reached are, by repository, the mesh Go module paths the held modules' go.sum names and the imports their builds
|
||||
|
||||
@@ -135,6 +135,40 @@ func sha(c byte) string { return strings.Repeat(string(c), 40) }
|
||||
|
||||
func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633)
|
||||
ctx := context.Background()
|
||||
t.Run("indexer_fingerprints_again acts before it records the hand act and says a failed record", func(t *testing.T) { // issue 1966
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
record func(what, why string) error
|
||||
said string
|
||||
}{
|
||||
{"recorded", func(string, string) error { return nil }, ""},
|
||||
{"record refused", func(string, string) error { return errors.New("no ledger") }, "not recorded: no ledger"},
|
||||
{"record never answers", func(string, string) error { select {} }, "not recorded within"},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
var asked []string
|
||||
ask := func(key string, _ []byte) (TakeInAsk, error) {
|
||||
asked = append(asked, key)
|
||||
return TakeInAsk{Result: &TakenIn{Module: "all", Stored: true}}, nil
|
||||
}
|
||||
tool := FingerprintsTools(ask, time.Now, c.record, 10*time.Millisecond)[0]
|
||||
if _, err := tool.Run(map[string]any{"module": "avahi"}); err == nil {
|
||||
t.Fatal("a hand act with no why was done")
|
||||
}
|
||||
if len(asked) != 0 {
|
||||
t.Fatalf("asked the indexer without a why: %v", asked)
|
||||
}
|
||||
got, err := tool.Run(map[string]any{"module": "avahi", "why": "the packages read came back empty"})
|
||||
if err != nil || len(asked) != 1 || !strings.HasPrefix(asked[0], "fingerprints.avahi.") {
|
||||
t.Fatalf("run: %v, asked %v", err, asked)
|
||||
}
|
||||
said, _ := got.(map[string]any)["hand-act"].(string)
|
||||
if !strings.Contains(said, c.said) || (c.said == "") != (said == "") {
|
||||
t.Fatalf("hand-act %q, want one saying %q", said, c.said)
|
||||
}
|
||||
})
|
||||
}
|
||||
})
|
||||
t.Run("T6 a merge while the own build is unknown writes nothing", func(t *testing.T) {
|
||||
b := newTakeInBed(t)
|
||||
b.ix.ownFP, b.ix.ownFixed = "", false // this process is started again and asks the engine, which does not answer
|
||||
|
||||
@@ -220,6 +220,8 @@ type TakenIn struct {
|
||||
// and deleted.
|
||||
Handed []string `json:"handed,omitempty"`
|
||||
Dropped []string `json:"dropped,omitempty"`
|
||||
// Fingerprints are the modules whose fingerprint a recompute by hand moved (FingerprintsAgain), each old and new.
|
||||
Fingerprints []FingerprintMove `json:"fingerprints,omitempty"`
|
||||
}
|
||||
|
||||
// TakeIn reads a registered module's manifest now, whatever was tried before, and writes it as a merge would (the tool
|
||||
|
||||
@@ -56,6 +56,9 @@ type TakeInAsk struct {
|
||||
Sequence uint64 `json:"sequence,omitempty"`
|
||||
// TryCandidate asks the candidate of this try's group to be dropped (indexer_drop_try_candidate), trycandidates.go.
|
||||
TryCandidate string `json:"try-candidate,omitempty"`
|
||||
// Fingerprints asks the fingerprints of Module ("all": every module) to be computed again now, from the published
|
||||
// packages as they are read (indexer_fingerprints_again, fingerprints.go).
|
||||
Fingerprints bool `json:"fingerprints,omitempty"`
|
||||
// RecoverTries asks the open tries' candidates to be recovered now (indexer_recover_tries).
|
||||
RecoverTries bool `json:"recover-tries,omitempty"`
|
||||
Drop bool `json:"drop,omitempty"`
|
||||
@@ -77,7 +80,7 @@ func DecodeTakeIn(data []byte) (TakeInAsk, error) {
|
||||
if a.V != 1 {
|
||||
return a, fmt.Errorf("a takeins value of version %d, not 1", a.V)
|
||||
}
|
||||
if a.Module == "" && a.Repository == "" && a.Pending == "" && a.Merge == "" && a.TryCandidate == "" && !a.RecoverTries {
|
||||
if a.Module == "" && a.Repository == "" && a.Pending == "" && a.Merge == "" && a.TryCandidate == "" && !a.RecoverTries && !a.Fingerprints {
|
||||
return a, errors.New("a takeins value naming no module, repository, pending wait, merge or try")
|
||||
}
|
||||
return a, nil
|
||||
|
||||
@@ -223,7 +223,13 @@ func (ix *Indexer) recomputeLocked(ctx context.Context, overlay map[string]meshc
|
||||
all[key] = &known{key: key, cur: cur, manifest: m, languages: languagesOf(m), bases: basesOf(m), contexts: hasContext(m), held: before}
|
||||
}
|
||||
tools := ix.agreed()
|
||||
published := ix.publishedAll(ctx)
|
||||
published, err := ix.publishedForPass(ctx, held)
|
||||
if err != nil {
|
||||
// No module's fingerprint is written from a state that was not read: the version line over every import form is
|
||||
// what an empty packages state would compute (issue 1966). The next pass, or indexer_fingerprints_again, computes them.
|
||||
ix.logf("no fingerprint is written by this pass: %v", err)
|
||||
return fmt.Errorf("no fingerprint is written: %w", err)
|
||||
}
|
||||
series := ix.SeriesAll(ctx) // each package's series, the newest a user accepting it takes (DLV-I6)
|
||||
type result struct {
|
||||
fp, none string
|
||||
@@ -258,6 +264,9 @@ func (ix *Indexer) recomputeLocked(ctx context.Context, overlay map[string]meshc
|
||||
chain = append(chain, module)
|
||||
kept := *k.cur.Inputs2
|
||||
kept.Bases, kept.Builder = nil, ""
|
||||
if ix.isFresh(module) { // indexer_fingerprints_again: the import form is written again, the held one stands for nothing
|
||||
kept.ImportForm = ""
|
||||
}
|
||||
var missing []string
|
||||
for _, b := range k.bases {
|
||||
if r := of(b, chain); r.fp != "" {
|
||||
|
||||
@@ -166,15 +166,6 @@ func BusStateProblems(m catalogue.Manifest) []string {
|
||||
"%s: the name is also kept in the old section %s; one local name is one form, so a module that moves an "+
|
||||
"entry removes it from the old section (novox/hq ADR 0595 §11)", at, section))
|
||||
}
|
||||
// A key-value entry kept one bucket key per machine is not issued in the new form yet (issue 1966, the lead's
|
||||
// ruling): its binding names no per-machine key, so neither the grants part nor the tool runner could hold each
|
||||
// machine to its own key (ADR 0260). Refused with the way that holds, never issued as a whole bucket.
|
||||
if e.Storage == catalogue.StorageKeyValue && string(e.Options[catalogue.OptionPerMachine]) == "true" {
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s: per-machine is not issued in `requires` yet, since its binding names no machine's key and each "+
|
||||
"machine would reach every key; keep it in the old section state with per-machine until the binding "+
|
||||
"carries it (novox/hq ADR 0260, ADR 0595 §1)", at))
|
||||
}
|
||||
problems = append(problems, busEntryProblems(at, e, terminal)...)
|
||||
}
|
||||
return problems
|
||||
|
||||
@@ -28,14 +28,6 @@ func TestBusStateIsJudgedOnce(t *testing.T) {
|
||||
{"within bounds", manifest(`{"name":"cache","storage":"key-value","template":"state","options":{"history":8,"key-ttl":true}},` +
|
||||
`{"name":"audit","storage":"stream","template":"log","options":{"max-mib":64,"max-age-days":365}},` +
|
||||
`{"name":"files","storage":"object-store","template":"objects","options":{"max-mib":2048}},` + ledger), nil},
|
||||
// Issue 1966: a process's bus state names a key-value entry of `requires` by its local name, as an old state's
|
||||
// (ADR 0595 §11); one per machine is refused with the way that holds, its binding naming no machine's key (Q3).
|
||||
{"a process naming a requires key-value entry", `{"module":"a","version":"1","requires":[{"name":"cache","storage":"key-value","template":"state"}],` +
|
||||
`"resources":[{"id":"u","type":"user","name":"a"},{"id":"p","type":"process","user":"a","run":["./a"],"bus":{"state":["cache"]}}]}`, nil},
|
||||
{"a process naming a state it keeps in neither form", `{"module":"a","version":"1","requires":[{"name":"cache","storage":"key-value","template":"state"}],` +
|
||||
`"resources":[{"id":"u","type":"user","name":"a"},{"id":"p","type":"process","user":"a","run":["./a"],"bus":{"state":["other"]}}]}`, []string{"bus reads and writes state \"other\""}},
|
||||
{"per-machine in requires", manifest(`{"name":"x","storage":"key-value","template":"state","options":{"per-machine":true}}`),
|
||||
[]string{"\"x\"", "per-machine is not issued in `requires` yet", "old section state"}},
|
||||
// 1966/AC7: each refused case, naming the module, the entry and the field
|
||||
{"unknown storage type", manifest(`{"name":"x","storage":"tape"}`), []string{"a's bus state \"x\"", "storage type \"tape\""}},
|
||||
{"unknown template", manifest(`{"name":"x","storage":"stream","template":"pile"}`), []string{"\"x\"", "template \"pile\""}},
|
||||
|
||||
@@ -3913,11 +3913,9 @@ func ProcessBusProblems(m catalogue.Manifest) []string {
|
||||
for _, s := range m.Streams {
|
||||
streams[s.Name] = true
|
||||
}
|
||||
// What the module keeps, in either form (catalogue.KeptStates, issue 1966): a process names a key-value entry of
|
||||
// `requires` by its local name as it names an old state (ADR 0595 §11).
|
||||
state := map[string]bool{}
|
||||
for _, s := range catalogue.KeptStates(m) {
|
||||
state[s] = true
|
||||
for _, s := range m.State {
|
||||
state[s.Name] = true
|
||||
}
|
||||
emits := map[string]bool{}
|
||||
for _, e := range m.EmitsAll() {
|
||||
@@ -4308,8 +4306,8 @@ func ownStateProblems(m catalogue.Manifest, i int, r catalogue.Receivable, c cat
|
||||
return nil
|
||||
}
|
||||
keeps := map[string]bool{}
|
||||
for _, s := range catalogue.KeptStates(m) { // either form (issue 1966)
|
||||
keeps[s] = true
|
||||
for _, s := range m.State {
|
||||
keeps[s.Name] = true
|
||||
}
|
||||
var problems []string
|
||||
for _, sr := range r.Shape.Reads(c.Data) {
|
||||
@@ -5120,7 +5118,10 @@ func trafficProblems(m catalogue.Manifest, s catalogue.SeatDeclaration) []string
|
||||
m.Module, s.Name))
|
||||
}
|
||||
for _, r := range s.Records {
|
||||
kept := slices.Contains(catalogue.KeptStates(m), r) // either form (issue 1966)
|
||||
kept := false
|
||||
for _, st := range m.State {
|
||||
kept = kept || st.Name == r
|
||||
}
|
||||
if !kept {
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s says %s's users read its records %q, which it keeps no state of", m.Module, s.Name, r))
|
||||
|
||||
@@ -1403,12 +1403,9 @@ func caseAModuleOffersItsOwnStateOnly(t *testing.T) {
|
||||
t.Errorf("%s: accepted, or refused without %q: %v", raw, want, err)
|
||||
}
|
||||
}
|
||||
// Issue 1966: its own state in either form, the old section's or a key-value entry of `requires` (ADR 0595 §11).
|
||||
for _, ok := range []string{`{"module":"power","state":["draw"],"contributions":[` + fmt.Sprintf(block, "power.draw") + `]}`,
|
||||
`{"module":"power","requires":[{"name":"draw","storage":"key-value","template":"state"}],"contributions":[` + fmt.Sprintf(block, "power.draw") + `]}`} {
|
||||
if _, err := ParseManifest([]byte(ok)); err != nil {
|
||||
t.Errorf("its own state was refused: %v", err)
|
||||
}
|
||||
ok := `{"module":"power","state":["draw"],"contributions":[` + fmt.Sprintf(block, "power.draw") + `]}`
|
||||
if _, err := ParseManifest([]byte(ok)); err != nil {
|
||||
t.Errorf("its own state was refused: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4738,11 +4735,6 @@ func caseTheNewRulesAreHeldToWhatTheSeatSays(t *testing.T) {
|
||||
t.Errorf("not refused: %q in %s", want, got)
|
||||
}
|
||||
}
|
||||
// Issue 1966: records kept as a key-value entry of `requires` are kept state, as the old section's are.
|
||||
m.BusState = []catalogue.BusStateEntry{{Name: "nothing", Storage: catalogue.StorageKeyValue, Template: catalogue.TemplateState}}
|
||||
if got := strings.Join(declaredSeatProblems(m), "; "); strings.Contains(got, "keeps no state of") {
|
||||
t.Errorf("records kept in requires refused: %s", got)
|
||||
}
|
||||
}
|
||||
|
||||
// **A build machine has no store, so it may not judge a seat.** The set is data the control plane
|
||||
|
||||
@@ -175,8 +175,8 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build
|
||||
want = append(want, "indexer_first_views", "indexer_first_view_again", index.ConfigReadersTool, index.BusStateTool, index.RequirementsTool, index.PackageUsersTool, index.SayAcceptedTool, // DLV-F3r, DLV-I10, DLV-I11, DLV-I7, DLV-I8, DLV-I9c
|
||||
index.SeatDefinitionTool, index.SeatDefinitionsAgainTool, index.OutOfStepTool, // DLV-I19
|
||||
index.HoldsTool, index.EngineReadsTool, // ADR 0632 §2: the mesh-wide holds and the engines' reads they weigh
|
||||
index.PendingTool, index.TakeInPendingTool, index.DropPendingTool, index.TryCandidatesTool, index.DropTryCandidateTool, index.RecoverTriesTool, index.TakeInMergeTool) // 1585/T12, T13 (ADR 0633 §5)
|
||||
if !reflect.DeepEqual(m.Tools, want) || len(want) != 38 {
|
||||
index.PendingTool, index.TakeInPendingTool, index.DropPendingTool, index.TryCandidatesTool, index.DropTryCandidateTool, index.RecoverTriesTool, index.TakeInMergeTool, index.FingerprintsAgainTool) // issue 1966; 1585/T12, T13 (ADR 0633 §5)
|
||||
if !reflect.DeepEqual(m.Tools, want) || len(want) != 39 {
|
||||
t.Errorf("tools = %v, want %v", m.Tools, want)
|
||||
}
|
||||
// Every read tool of the indexer a reader of this repository invokes is one it serves (the checker's seventh first
|
||||
|
||||
@@ -54,7 +54,8 @@
|
||||
"indexer_try_candidates",
|
||||
"indexer_drop_try_candidate",
|
||||
"indexer_recover_tries",
|
||||
"indexer_take_in_merge"
|
||||
"indexer_take_in_merge",
|
||||
"indexer_fingerprints_again"
|
||||
],
|
||||
"consumes": [
|
||||
"go-package-registry.published",
|
||||
|
||||
@@ -216,14 +216,6 @@ func (s ownStore) Put(state, key string, value []byte) (uint64, error) {
|
||||
|
||||
func (s ownStore) Keys(state string) ([]string, error) { return s.c.State(state).Keys() }
|
||||
|
||||
// Update writes at a revision: a create where it is 0, refused as stdio.ErrChanged when another write came first.
|
||||
func (s ownStore) Update(state, key string, value []byte, revision uint64) (uint64, error) {
|
||||
if revision == 0 {
|
||||
return s.c.State(state).Create(key, json.RawMessage(value))
|
||||
}
|
||||
return s.c.State(state).Update(key, json.RawMessage(value), revision)
|
||||
}
|
||||
|
||||
func (s ownStore) Delete(state, key string) error { return s.c.State(state).Delete(key) }
|
||||
|
||||
// emitting is the process connection as the part's emitter: its events, with no options.
|
||||
|
||||
@@ -16,7 +16,6 @@ import (
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-grants/grantsstate"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/membership"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
"git.novox.be/novox/mesh-sdk/go/seal"
|
||||
"github.com/nats-io/nats-server/v2/server"
|
||||
@@ -38,10 +37,7 @@ func theBusNodeFollowsTheSealedListUnderTheComposedGrants(t *testing.T) {
|
||||
"novox": {{Module: buscompose.GrantsModule, State: []buscompose.Bucket{{Module: buscompose.GrantsModule, Name: "user-list"}}}},
|
||||
"ace": {{Module: "plex", Reads: []string{buscompose.ComposedRead}},
|
||||
{Module: "mesh-delivery-view", Reads: []string{buscompose.ComposedRead}, ReadsObjects: []string{"mesh-knowledge_sections"}},
|
||||
{Module: "mesh-knowledge", Objects: []buscompose.ObjectBucket{{Module: "mesh-knowledge", Name: "sections", MaxMiB: 1, Restricted: true}}},
|
||||
// Issue 1966: a module whose state is asked in `requires`, bound as the bus module said it created it.
|
||||
{Module: "mod-c", Bindings: []membership.BusBinding{{Local: "kept", Storage: catalogue.StorageKeyValue,
|
||||
Template: catalogue.TemplateState, Bucket: "mod-c_kept"}}}},
|
||||
{Module: "mesh-knowledge", Objects: []buscompose.ObjectBucket{{Module: "mesh-knowledge", Name: "sections", MaxMiB: 1, Restricted: true}}}},
|
||||
},
|
||||
}
|
||||
// The bus module and the router as the part composes them from their own manifests (testdata: the catalogue's
|
||||
@@ -54,10 +50,6 @@ func theBusNodeFollowsTheSealedListUnderTheComposedGrants(t *testing.T) {
|
||||
}
|
||||
records.Assigned["novox"] = append(records.Assigned["novox"], fromManifests.Assigned["novox"]...)
|
||||
records.BusState = fromManifests.BusState
|
||||
if records.BusState == nil {
|
||||
records.BusState = map[string][]catalogue.BusStateEntry{}
|
||||
}
|
||||
records.BusState["mod-c"] = []catalogue.BusStateEntry{{Name: "kept", Storage: catalogue.StorageKeyValue, Template: catalogue.TemplateState}}
|
||||
// The node-engine's state as the part's records carry it: every node's compositions (ADR 0593 §6).
|
||||
records.Engine = &buscompose.Declared{Module: buscompose.EngineModule, State: buscompose.EngineComposedBuckets(records.Nodes)}
|
||||
users, err := buscompose.Users(records)
|
||||
@@ -396,34 +388,6 @@ func theBusNodeFollowsTheSealedListUnderTheComposedGrants(t *testing.T) {
|
||||
if _, err := plex.GetLastMsg(sections.Stream(), "$O."+sections.Bucket()+".M.x", nats.DirectGet()); err == nil {
|
||||
t.Error("a module the grant does not name read the bucket")
|
||||
}
|
||||
// Issue 1966 (ADR 0595 §1, design 25 § 12.7, the binding's usable half): the bus module's keeper creates a key-value
|
||||
// entry asked in `requires` under its own account, and the module's own account, granted it from its binding, binds the
|
||||
// bucket by its bus name and writes and reads a key; another module reads nothing of it.
|
||||
keeperJS, err := jetstream.New(dial("novox.nats.keeper"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := keeperJS.CreateStream(ctx, jetstream.StreamConfig{Name: "KV_mod-c_kept", Subjects: []string{"$KV.mod-c_kept.>"},
|
||||
MaxMsgsPerSubject: 1, AllowDirect: true, AllowRollup: true, DenyDelete: true, Discard: jetstream.DiscardNew}); err != nil {
|
||||
t.Fatalf("the keeper creates mod-c's bound bucket: %v", err)
|
||||
}
|
||||
modC, err := jetstream.New(dial("ace.mod-c"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
bound, err := modC.KeyValue(ctx, "mod-c_kept")
|
||||
if err != nil {
|
||||
t.Fatalf("mod-c's own account binds its bound bucket: %v", err)
|
||||
}
|
||||
if _, err := bound.Put(ctx, "k", []byte("v")); err != nil {
|
||||
t.Fatalf("mod-c's own account writes its bound bucket: %v", err)
|
||||
}
|
||||
if e, err := bound.Get(ctx, "k"); err != nil || string(e.Value()) != "v" {
|
||||
t.Errorf("mod-c reads back %v, %v", e, err)
|
||||
}
|
||||
if _, err := plex.GetLastMsg("KV_mod-c_kept", "$KV.mod-c_kept.k", nats.DirectGet()); err == nil {
|
||||
t.Error("another module read mod-c's bound bucket")
|
||||
}
|
||||
}
|
||||
|
||||
// manifestIn is a module's manifest as testdata keeps it.
|
||||
|
||||
@@ -1,75 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/go/seats/meshbroker"
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"mesh-grants/internal/buscompose"
|
||||
"mesh-grants/internal/grants"
|
||||
)
|
||||
|
||||
// readAgainAtMost bounds one re-read of the seat mesh-broker's events (grants_bindings_again): a stream holding more of
|
||||
// them than this is read up to here, and the answer says the rest was not read.
|
||||
const readAgainAtMost = 20000
|
||||
|
||||
// brokerHistory reads the seat mesh-broker's events again from the shared events stream, by sequence and subject
|
||||
// (STREAM.MSG.GET with next_by_subj, the grant buscompose.SeatEventsReadAgain): a get per event, no consumer, oldest first.
|
||||
type brokerHistory struct {
|
||||
ctx context.Context
|
||||
nc *nats.Conn
|
||||
}
|
||||
|
||||
// msgGetAnswer is the bus's answer to a STREAM.MSG.GET.
|
||||
type msgGetAnswer struct {
|
||||
Message *struct {
|
||||
Subject string `json:"subject"`
|
||||
Seq uint64 `json:"seq"`
|
||||
Data []byte `json:"data"`
|
||||
Time time.Time `json:"time"`
|
||||
} `json:"message"`
|
||||
Error *struct {
|
||||
Code int `json:"code"`
|
||||
ErrCode int `json:"err_code"`
|
||||
Description string `json:"description"`
|
||||
} `json:"error"`
|
||||
}
|
||||
|
||||
// noMessage is the bus's error code for a get past the last message of the subject.
|
||||
const noMessage = 10037
|
||||
|
||||
func (h brokerHistory) Said() ([]grants.SaidEvent, error) {
|
||||
subject := "mesh.seat." + meshbroker.Seat + ".event.*"
|
||||
var out []grants.SaidEvent
|
||||
for seq := uint64(1); ; {
|
||||
if len(out) >= readAgainAtMost {
|
||||
return out, fmt.Errorf("the stream holds more than %d of the seat's events: read up to sequence %d, the rest not", readAgainAtMost, seq)
|
||||
}
|
||||
body, _ := json.Marshal(map[string]any{"seq": seq, "next_by_subj": subject})
|
||||
ctx, cancel := context.WithTimeout(h.ctx, requestWithin)
|
||||
reply, err := h.nc.RequestWithContext(ctx, buscompose.SeatEventsReadAgain, body)
|
||||
cancel()
|
||||
if err != nil {
|
||||
return out, fmt.Errorf("%s at sequence %d: %w", buscompose.SeatEventsReadAgain, seq, err)
|
||||
}
|
||||
var a msgGetAnswer
|
||||
if err := json.Unmarshal(reply.Data, &a); err != nil {
|
||||
return out, fmt.Errorf("%s answered what does not read: %w", buscompose.SeatEventsReadAgain, err)
|
||||
}
|
||||
if a.Error != nil {
|
||||
if a.Error.ErrCode == noMessage {
|
||||
return out, nil
|
||||
}
|
||||
return out, fmt.Errorf("%s at sequence %d: %s", buscompose.SeatEventsReadAgain, seq, a.Error.Description)
|
||||
}
|
||||
if a.Message == nil {
|
||||
return out, nil
|
||||
}
|
||||
out = append(out, grants.SaidEvent{Subject: a.Message.Subject, Data: a.Message.Data, At: a.Message.Time})
|
||||
seq = a.Message.Seq + 1
|
||||
}
|
||||
}
|
||||
@@ -316,24 +316,6 @@ func TestTheGrantsPartWhole(t *testing.T) {
|
||||
})
|
||||
sayBroker("dropped", membership.BusBinding{Local: "kept"})
|
||||
until("mod-c's binding gone after dropped", func() bool { return len(bindingOf()) == 0 })
|
||||
// Issue 1966, grants_bindings_again: asked as the tool asks, the process reads the seat's events again from EVENTS by
|
||||
// sequence (brokerHistory) and keeps mod-c's created and dropped in their order, so the dropped entry stays dropped.
|
||||
ask, _ := json.Marshal(grants.Ask{By: "jochen@shanks", At: time.Now(), Bindings: "mod-c"})
|
||||
if _, err := kvs[grants.Module+"."+grants.StateUserList].Put(ctx, grants.KeyAsked, ask); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
until("mod-c's bindings read again from EVENTS", func() bool {
|
||||
e, err := kvs[grants.Module+"."+grants.StateUserList].Get(ctx, grants.KeyBindingsAgain)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
var again grants.BindingsAgain
|
||||
return json.Unmarshal(e.Value(), &again) == nil && again.Module == "mod-c" && again.Read == 2 && again.Error == "" &&
|
||||
slices.Equal(again.Dropped, []string{"kept"}) && len(again.Kept) == 0
|
||||
})
|
||||
if b := bindingOf(); len(b) != 0 {
|
||||
t.Errorf("a dropped binding came back from the re-read: %+v", b)
|
||||
}
|
||||
for _, bucket := range []string{"mesh-controller_lease", grants.Module + "_lease", grants.Module + "_takeover"} {
|
||||
if _, err := js.KeyValue(ctx, bucket); !errors.Is(err, jetstream.ErrBucketNotFound) {
|
||||
t.Errorf("the bucket %s: %v, want none", bucket, err)
|
||||
|
||||
@@ -44,10 +44,6 @@ import (
|
||||
"mesh-grants/internal/grants"
|
||||
)
|
||||
|
||||
// EventRetryDelay is how long a seat event the part could not apply waits before the bus says it again: long enough for a
|
||||
// refused write of its own state to clear, short against the pass that follows it.
|
||||
const EventRetryDelay = 5 * time.Second
|
||||
|
||||
func main() {
|
||||
if os.Getenv("MESH_BROKER_FILE") == "" {
|
||||
if err := serveTools(); err != nil {
|
||||
@@ -135,7 +131,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
Objects: sealedObjects{ctx: ctx, js: js}, Purging: ownPurging{ctx: ctx, js: js, module: account.Module}, Logf: logger.Printf,
|
||||
Bus: grants.BusAddress{URL: account.URL, Fingerprint: account.Fingerprint}}
|
||||
f.Watch = fv
|
||||
f.Part.History = brokerHistory{ctx: ctx, nc: nc}
|
||||
// The seat list: read over the connection and the hub, given to catalogue at every change, and a pass composes again then.
|
||||
gate := &seatGate{}
|
||||
gate.follow(seatlist.ProcessBus(conn.Call, hub), f.Wake, logger.Printf)
|
||||
@@ -159,7 +154,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
// there yet is opened at the next change, and one opened is never opened twice.
|
||||
opened := map[string]bool{}
|
||||
registered := false
|
||||
f.Hear = func(handle func(string, []byte) error) error {
|
||||
f.Hear = func(handle func(string, []byte)) error {
|
||||
if !registered {
|
||||
registered = true
|
||||
// An owner's views event goes to the follower of its state, which acknowledges it (firstviews.go).
|
||||
@@ -187,14 +182,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
}
|
||||
// The other seats' events the part follows its passes by (Consumes).
|
||||
for _, subject := range grants.SeatEventSubjects() {
|
||||
// An event not applied is left unacknowledged, said again after a while (issue 1966), never acknowledged unread.
|
||||
if err := hub.Handle(subject, func(m jetstream.Msg) {
|
||||
if handle(m.Subject(), m.Data()) != nil {
|
||||
_ = m.NakWithDelay(EventRetryDelay)
|
||||
return
|
||||
}
|
||||
_ = m.Ack()
|
||||
}); err != nil {
|
||||
if err := hub.Handle(subject, func(m jetstream.Msg) { handle(m.Subject(), m.Data()); _ = m.Ack() }); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
@@ -132,11 +132,6 @@ func (runtimeStore) Put(state, key string, value []byte) (uint64, error) {
|
||||
|
||||
func (runtimeStore) Keys(state string) ([]string, error) { return stdio.State(state).Keys() }
|
||||
|
||||
// Update is refused: the runtime writes no key at a revision, and a tool asks the part's process for what it writes so.
|
||||
func (runtimeStore) Update(state, key string, _ []byte, _ uint64) (uint64, error) {
|
||||
return 0, fmt.Errorf("%s %s is written at its revision by the part's process alone; a tool asks it", state, key)
|
||||
}
|
||||
|
||||
func (runtimeStore) Delete(state, key string) error { return stdio.State(state).Delete(key) }
|
||||
|
||||
func (runtimeStore) PutTTL(state, key string, value []byte, ttl time.Duration) (uint64, error) {
|
||||
@@ -251,31 +246,13 @@ func toolsOf(p *grants.Part) []stdio.Tool {
|
||||
Run: func(a map[string]any) (any, error) {
|
||||
return p.AskFirstViewAgain(stdio.CallerOf(a), grants.Answer(a, "state"), grants.Answer(a, "why"), recordHandAct)
|
||||
}},
|
||||
{Name: grants.ToolBindings, Description: "The bindings the grants part keeps of a module's kept data on the bus (novox/hq " +
|
||||
"ADR 0595 §1, §7): each entry the bus module said created or recreated, by local name with its names on the bus, beside " +
|
||||
"every membership of the module the part wrote, and the kept bindings a membership lacks. One module, or every one. Read-only.",
|
||||
Input: schema(map[string]string{"module": "one module, e.g. distribution; none for every one"}),
|
||||
Run: func(a map[string]any) (any, error) { return p.BindingsOf(grants.Answer(a, "module")) }},
|
||||
{Name: grants.ToolBindingsAgain, Description: "Read one module's bindings again from the seat mesh-broker's created, " +
|
||||
"recreated and dropped that the shared events stream still holds, kept as if heard now (issue 1966), by a person's act " +
|
||||
"with why, recorded as a hand act: the step when grants_bindings lacks an entry the bus module made. Asks the part's " +
|
||||
"process; grants_bindings shows what it kept, what was already kept, and when the stream no longer holds the created.",
|
||||
Input: schema(map[string]string{"module": "the module, e.g. distribution", "why": "why, kept in the hand act"}, "module", "why"),
|
||||
Run: func(a map[string]any) (any, error) {
|
||||
return p.AskBindingsAgain(stdio.CallerOf(a), grants.Answer(a, "module"), grants.Answer(a, "why"), recordHandActFor("bindings-again"))
|
||||
}},
|
||||
}
|
||||
}
|
||||
|
||||
// recordHandAct records a person's act of grants_first_view_again through the seat mesh-hand-acts.
|
||||
func recordHandAct(what, why string) error { return recordHandActFor("first-view-again")(what, why) }
|
||||
|
||||
// recordHandActFor records a person's act through the seat mesh-hand-acts, with its cause.
|
||||
func recordHandActFor(cause string) func(what, why string) error {
|
||||
return func(what, why string) error {
|
||||
_, err := meshhandactsclient.Recorder{Ask: stdio.Ask}.Record(meshhandactsclient.Request{What: what, Why: why, Cause: cause})
|
||||
return err
|
||||
}
|
||||
// recordHandAct records a person's act through the seat mesh-hand-acts.
|
||||
func recordHandAct(what, why string) error {
|
||||
_, err := meshhandactsclient.Recorder{Ask: stdio.Ask}.Record(meshhandactsclient.Request{What: what, Why: why, Cause: "first-view-again"})
|
||||
return err
|
||||
}
|
||||
|
||||
// serveTools serves the tools to the runtime until it stops.
|
||||
|
||||
@@ -8,10 +8,6 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"git.novox.be/novox/mesh-contracts/go/seats/meshbroker"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"go/ast"
|
||||
"go/parser"
|
||||
"go/token"
|
||||
"io/fs"
|
||||
"maps"
|
||||
"os"
|
||||
@@ -55,19 +51,17 @@ import (
|
||||
// tools, the part's own states and its events. It records every write, call and event, so a row can prove nothing was
|
||||
// published or asserted.
|
||||
type fakeBus struct {
|
||||
// beforeUpdate, when set, runs once before the next Update: a second writer heard at once.
|
||||
beforeUpdate func()
|
||||
mu sync.Mutex
|
||||
states map[string]map[string][]byte
|
||||
refused map[string]bool
|
||||
loaded []grants.User
|
||||
streams []map[string]any
|
||||
own map[string]map[string][]byte
|
||||
revs map[string]uint64
|
||||
refuseTo map[string]bool
|
||||
calls []string
|
||||
events []string
|
||||
writes []string
|
||||
mu sync.Mutex
|
||||
states map[string]map[string][]byte
|
||||
refused map[string]bool
|
||||
loaded []grants.User
|
||||
streams []map[string]any
|
||||
own map[string]map[string][]byte
|
||||
revs map[string]uint64
|
||||
refuseTo map[string]bool
|
||||
calls []string
|
||||
events []string
|
||||
writes []string
|
||||
// Each membership's subject: the last membership said there, every notice, and whether it is refused.
|
||||
stream map[string][]byte
|
||||
publishes []string
|
||||
@@ -338,23 +332,6 @@ func (b *fakeBus) ownGet(state, key string) ([]byte, uint64, bool, error) {
|
||||
|
||||
type ownStore struct{ b *fakeBus }
|
||||
|
||||
// Update writes at a revision as the bus does: refused, wrapping stdio.ErrChanged, when the key is at another; and the
|
||||
// fake's beforeUpdate, when set, writes first (a second writer heard at once).
|
||||
func (s ownStore) Update(state, key string, value []byte, revision uint64) (uint64, error) {
|
||||
if f := s.b.beforeUpdate; f != nil {
|
||||
s.b.beforeUpdate = nil
|
||||
f()
|
||||
}
|
||||
s.b.mu.Lock()
|
||||
rev := s.b.revs[state+"/"+key]
|
||||
_, has := s.b.own[state][key]
|
||||
s.b.mu.Unlock()
|
||||
if (revision == 0 && has) || (revision != 0 && rev != revision) {
|
||||
return 0, fmt.Errorf("%s: %w", key, stdio.ErrChanged)
|
||||
}
|
||||
return s.Put(state, key, value)
|
||||
}
|
||||
|
||||
func (s ownStore) Get(state, key string) ([]byte, uint64, bool, error) { return s.b.ownGet(state, key) }
|
||||
func (s ownStore) Put(state, key string, value []byte) (uint64, error) {
|
||||
b := s.b
|
||||
@@ -640,14 +617,6 @@ func TestGrants(t *testing.T) {
|
||||
}
|
||||
src = []byte(strings.ReplaceAll(string(src), "CreateObjectStore", ""))
|
||||
}
|
||||
// The one key created: a kept binding's first write in the part's own state, at revision 0 (bus.go's
|
||||
// ownStore.Update, issue 1966), a key and never a bucket or a stream.
|
||||
if path == filepath.Join("cmd", "mesh-grants", "bus.go") {
|
||||
if strings.Count(string(src), "s.c.State(state).Create(key, ") != 1 {
|
||||
t.Errorf("bus.go creates other than one key of its own state")
|
||||
}
|
||||
src = []byte(strings.Replace(string(src), "s.c.State(state).Create(key, ", "", 1))
|
||||
}
|
||||
for _, f := range forbidden {
|
||||
if strings.Contains(string(src), f) {
|
||||
t.Errorf("%s holds %s: the part holds no timer and creates or asserts nothing", path, f)
|
||||
@@ -1033,9 +1002,9 @@ func TestGrants(t *testing.T) {
|
||||
}
|
||||
// A root-not-free condition raised or cleared wakes a pass; another kind does not (never a timer).
|
||||
wb := fixture()
|
||||
var handle func(string, []byte) error
|
||||
var handle func(string, []byte)
|
||||
wf := &grants.Follower{Part: partOn(wb), Watch: &fakeWatcher{opened: make(chan string, 16)}, Logf: func(string, ...any) {},
|
||||
Hear: func(h func(string, []byte) error) error { handle = h; return nil }}
|
||||
Hear: func(h func(string, []byte)) error { handle = h; return nil }}
|
||||
wctx, wcancel := context.WithCancel(context.Background())
|
||||
wdone := make(chan struct{})
|
||||
go func() { wf.Run(wctx); close(wdone) }()
|
||||
@@ -2873,68 +2842,6 @@ func TestGrants(t *testing.T) {
|
||||
}
|
||||
})
|
||||
|
||||
// Issue 1966 (ADR 0595 §1, §11): the state a manifest keeps and a membership issues is read through mesh-sdk's two rules,
|
||||
// catalogue.KeptStates and Membership.StateIssued or StateNames, which answer the old `state` section and a key-value
|
||||
// entry asked in `requires` alike. A reader of a manifest's or a membership's `State` beside them misses the new form,
|
||||
// the kind of this issue's faults; only the rules that judge or compose the old section itself read it. Read by the
|
||||
// syntax tree across the repository, since the field's name alone is shared by other types.
|
||||
t.Run("the state a manifest keeps or a membership issues is read through mesh-sdk's two rules", func(t *testing.T) {
|
||||
judgesTheOldSection := map[string]bool{
|
||||
"modules/mesh-delivery-indexer/judge/busstate.go BusStateProblems": true,
|
||||
"modules/mesh-delivery-indexer/judge/oldsections.go SectionsUsed": true,
|
||||
"modules/mesh-delivery-indexer/judge/rules.go readOnlyStateProblems": true,
|
||||
"modules/mesh-delivery-indexer/judge/rules.go parseManifest": true,
|
||||
"modules/mesh-delivery-indexer/judge/rules.go StateProblems": true,
|
||||
"modules/mesh-delivery-indexer/judge/rules.go ObjectsProblems": true,
|
||||
"modules/mesh-grants/internal/grants/records.go bucketsOf": true,
|
||||
}
|
||||
planted := "package p\n\nimport \"git.novox.be/novox/mesh-sdk/go/catalogue\"\n\n" +
|
||||
"func planted(m catalogue.Manifest) (n int) {\n\tfor range m.State {\n\t\tn++\n\t}\n\treturn n\n}\n"
|
||||
if got, err := oldStateReaders("p.go", []byte(planted)); err != nil || !slices.Equal(got, []string{"p.go planted"}) {
|
||||
t.Fatalf("a planted range m.State is read as %v (%v), want it found", got, err)
|
||||
}
|
||||
files, seen := 0, map[string]bool{}
|
||||
err := filepath.WalkDir(filepath.Join("..", ".."), func(path string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if d.IsDir() && path != filepath.Join("..", "..") && (d.Name() == "testdata" || d.Name() == "node_modules" || (len(d.Name()) > 1 && d.Name()[0] == '.')) {
|
||||
return filepath.SkipDir
|
||||
}
|
||||
if d.IsDir() || !strings.HasSuffix(path, ".go") || strings.HasSuffix(path, "_test.go") {
|
||||
return nil
|
||||
}
|
||||
src, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
files++
|
||||
rel, _ := filepath.Rel(filepath.Join("..", ".."), path)
|
||||
readers, err := oldStateReaders(filepath.ToSlash(rel), src)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, r := range readers {
|
||||
seen[r] = true
|
||||
if !judgesTheOldSection[r] {
|
||||
t.Errorf("%s reads a manifest's or a membership's State beside catalogue.KeptStates and Membership.StateIssued", r)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if files < 100 {
|
||||
t.Fatalf("read %d source files, want the repository's", files)
|
||||
}
|
||||
for r := range judgesTheOldSection {
|
||||
if !seen[r] {
|
||||
t.Errorf("%s no longer reads the old section: take it off the list", r)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
// 774/S29/AC1 (ADR 0549 §5, ADR 0571 §10): the memberships composed from the assignments alone, with no read of the
|
||||
// controller, are the ones the union of assignments and assignments gave on the same input: the fixture's (two nodes,
|
||||
// the bus and mod-x on a, mod-y on b, as placed on the live mesh's own shape of nats beside its modules), whose
|
||||
@@ -3117,53 +3024,6 @@ func TestGrants(t *testing.T) {
|
||||
if !hear("mesh.seat.mesh-broker.event.created", first) || hear("mesh.seat.mesh-broker.event.created", first) {
|
||||
t.Fatal("created: not kept once")
|
||||
}
|
||||
// Issue 1966: grants_bindings reads what the part keeps: a binding kept before the pass that writes it into the
|
||||
// memberships is named lacking there, and no longer once the pass has written it; a module with none says so.
|
||||
lacking := func() string {
|
||||
got, err := p.BindingsOf("mod-x")
|
||||
if err != nil {
|
||||
t.Fatalf("grants_bindings: %v", err)
|
||||
}
|
||||
raw, _ := json.Marshal(got)
|
||||
if !strings.Contains(string(raw), `"bucket":"mod-x_records"`) {
|
||||
t.Errorf("grants_bindings names no kept binding: %s", raw)
|
||||
}
|
||||
return string(raw)
|
||||
}
|
||||
if got := lacking(); !strings.Contains(got, `"lacking":{"a.mod-x":["records"]}`) {
|
||||
t.Errorf("a binding kept and not written is not named lacking: %s", got)
|
||||
}
|
||||
p.Compose()
|
||||
if got := lacking(); strings.Contains(got, `"lacking"`) || !strings.Contains(got, `"a.mod-x":["records"]`) {
|
||||
t.Errorf("a binding written is named lacking: %s", got)
|
||||
}
|
||||
if got, err := p.BindingsOf("mod-y"); err != nil || !strings.Contains(fmt.Sprint(got), "keeps no binding of mod-y") {
|
||||
t.Errorf("grants_bindings of a module with none = %v, %v", got, err)
|
||||
}
|
||||
// Issue 1966 (F3): two created of one module heard at once are both kept: the second writer comes between the
|
||||
// first's read and its write, and the first, refused at its revision, reads again and keeps both.
|
||||
holds := membership.BusBinding{Local: "holds", Storage: "key-value", Template: "state", Bucket: "mod-x_holds"}
|
||||
b.beforeUpdate = func() {
|
||||
body, _ := json.Marshal(grants.BrokerEvent{V: 1, Module: "mod-x", Local: "holds", Storage: "key-value", Template: "state", Binding: holds})
|
||||
if _, err := p.HearBroker("mesh.seat.mesh-broker.event.created", body, p.Brokers()); err != nil {
|
||||
t.Errorf("the second writer: %v", err)
|
||||
}
|
||||
}
|
||||
lost := first
|
||||
lost.Local, lost.Bucket = "lost", "mod-x_lost"
|
||||
body, _ := json.Marshal(grants.BrokerEvent{V: 1, Module: "mod-x", Local: "lost", Storage: "key-value", Template: "state", Binding: lost})
|
||||
if _, err := p.HearBroker("mesh.seat.mesh-broker.event.created", body, p.Brokers()); err != nil {
|
||||
t.Fatalf("created beside another writer: %v", err)
|
||||
}
|
||||
p.Compose()
|
||||
if got := bindingsOn("a.mod-x"); len(got) != 3 {
|
||||
t.Errorf("a binding was lost to a write it did not see: %+v", got)
|
||||
}
|
||||
for _, local := range []string{"holds", "lost"} {
|
||||
if _, err := p.HearBroker("mesh.seat.mesh-broker.event.dropped", []byte(`{"module":"mod-x","local-name":"`+local+`"}`), p.Brokers()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
b.assign("b", "mod-x")
|
||||
p.Compose()
|
||||
for _, key := range []string{"a.mod-x", "b.mod-x"} {
|
||||
@@ -3189,64 +3049,6 @@ func TestGrants(t *testing.T) {
|
||||
if got := bindingsOn("a.mod-x"); len(got) != 0 {
|
||||
t.Errorf("after dropped: %+v", got)
|
||||
}
|
||||
// Issue 1966, grants_bindings_again (the lead's ruling, option b): the process reads the seat's events the shared
|
||||
// stream still holds again, oldest first, and keeps one module's created, recreated and dropped as if heard now: a
|
||||
// binding never kept comes back, one dropped since stays dropped, another module's is left; the hand act is recorded
|
||||
// before the ask, and an ask without why or a module is refused.
|
||||
event := func(name, module, local, bucket string) grants.SaidEvent {
|
||||
body, _ := json.Marshal(grants.BrokerEvent{V: 1, Module: module, LocalName: local, StorageType: "key-value", Template: "state",
|
||||
Binding: membership.BusBinding{Local: local, Storage: "key-value", Template: "state", Bucket: bucket}})
|
||||
return grants.SaidEvent{Subject: "mesh.seat.mesh-broker.event." + name, Data: body, At: time.Date(2026, 11, 3, 0, 0, 0, 0, time.UTC)}
|
||||
}
|
||||
p.Create = b
|
||||
p.History = saidHistory{event("created", "mod-x", "windows", "mod-x_windows"), event("refused", "mod-x", "mirrors", ""),
|
||||
event("created", "mod-y", "other", "mod-y_other"), event("created", "mod-x", "gone", "mod-x_gone"),
|
||||
event("dropped", "mod-x", "gone", "")}
|
||||
var acts []string
|
||||
record := func(what, why string) error { acts = append(acts, what+": "+why); return nil }
|
||||
for _, bad := range [][2]string{{"mod-x", ""}, {"", "x"}, {"a.mod-x", "x"}} {
|
||||
if _, err := p.AskBindingsAgain("jochen@shanks", bad[0], bad[1], record); err == nil || len(acts) != 0 {
|
||||
t.Errorf("grants_bindings_again %q %q was asked: %v %v", bad[0], bad[1], err, acts)
|
||||
}
|
||||
}
|
||||
readAgain := func(module string) string {
|
||||
if _, err := p.AskBindingsAgain("jochen@shanks", module, "distribution's lost bindings", record); err != nil {
|
||||
t.Fatalf("grants_bindings_again %s: %v", module, err)
|
||||
}
|
||||
p.Compose()
|
||||
got, err := p.BindingsOf(module)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
raw, _ := json.Marshal(got)
|
||||
return string(raw)
|
||||
}
|
||||
if got := readAgain("mod-x"); !strings.Contains(got, `"kept":["windows"],"already":[],"dropped":["gone"]`) || !strings.Contains(got, `"read":3`) {
|
||||
t.Errorf("the first re-read: %s", got)
|
||||
}
|
||||
if got := bindingsOn("a.mod-x"); len(got) != 1 || got[0].Bucket != "mod-x_windows" {
|
||||
t.Errorf("after the re-read, mod-x's membership carries %+v", got)
|
||||
}
|
||||
if raw, _, found, _ := b.ownGet(grants.StateMemberships, grants.BoundKey("mod-y")); found {
|
||||
t.Errorf("another module's binding was kept by mod-x's re-read: %s", raw)
|
||||
}
|
||||
if got := readAgain("mod-x"); !strings.Contains(got, `"kept":[],"already":["windows"]`) {
|
||||
t.Errorf("the second re-read: %s", got)
|
||||
}
|
||||
if got := readAgain("mod-z"); !strings.Contains(got, "holds no created or recreated of mod-z") || !strings.Contains(got, "seat mesh-broker's list") {
|
||||
t.Errorf("a module whose created the stream no longer holds: %s", got)
|
||||
}
|
||||
// A re-read that fails partway applies nothing it read: the created whose dropped lay past the failure is not kept.
|
||||
p.History = failingHistory{saidHistory{event("created", "mod-x", "late", "mod-x_late")}}
|
||||
if got := readAgain("mod-x"); !strings.Contains(got, "could not be read, so nothing was kept") || !strings.Contains(got, `"read":0`) {
|
||||
t.Errorf("a failed re-read: %s", got)
|
||||
}
|
||||
if got := bindingsOn("a.mod-x"); len(got) != 1 || got[0].Bucket != "mod-x_windows" {
|
||||
t.Errorf("a failed re-read kept what it read before the failure: %+v", got)
|
||||
}
|
||||
if len(acts) != 4 || !strings.HasPrefix(acts[0], "grants_bindings_again mod-x: ") {
|
||||
t.Errorf("hand acts %v", acts)
|
||||
}
|
||||
})
|
||||
|
||||
// Issue 1966 (F2), the conformance row: an entry asked in `requires` is granted where its old section is. For a
|
||||
@@ -3734,10 +3536,10 @@ func TestGrants(t *testing.T) {
|
||||
})
|
||||
t.Run("the vault's account-issued composes the list again", func(t *testing.T) {
|
||||
b := fixture()
|
||||
var handle func(string, []byte) error
|
||||
var handle func(string, []byte)
|
||||
p := partOn(b)
|
||||
f := &grants.Follower{Part: p, Watch: &fakeWatcher{opened: make(chan string, 16)}, Logf: func(string, ...any) {},
|
||||
Hear: func(h func(string, []byte) error) error { handle = h; return nil }}
|
||||
Hear: func(h func(string, []byte)) error { handle = h; return nil }}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan struct{})
|
||||
go func() { f.Run(ctx); close(done) }()
|
||||
@@ -3759,21 +3561,6 @@ func TestGrants(t *testing.T) {
|
||||
b.mu.Unlock()
|
||||
handle(buscompose.AccountIssuedSubject, []byte(`{"user":"a.mod-x","kind":"module","node":"a","module":"mod-x","generation":1}`))
|
||||
waitFor(t, held)
|
||||
// Issue 1966 (F3): a bus module's created the part cannot keep is answered, so the hearer leaves it unacknowledged
|
||||
// and the bus says it again; one kept is answered nil.
|
||||
created := []byte(`{"module":"mod-x","local-name":"records","storage-type":"key-value","binding":{"bucket":"mod-x_records"}}`)
|
||||
b.mu.Lock()
|
||||
b.refuseTo[grants.StateMemberships] = true
|
||||
b.mu.Unlock()
|
||||
if err := handle("mesh.seat.mesh-broker.event.created", created); err == nil {
|
||||
t.Error("a created not kept was answered nil, so it would be acknowledged and lost")
|
||||
}
|
||||
b.mu.Lock()
|
||||
b.refuseTo[grants.StateMemberships] = false
|
||||
b.mu.Unlock()
|
||||
if err := handle("mesh.seat.mesh-broker.event.created", created); err != nil {
|
||||
t.Errorf("a created kept was answered %v", err)
|
||||
}
|
||||
if !slices.Contains(grants.Consumes, grants.EventAccountIssued) || !grants.IsAccountIssued(grants.EventAccountIssued) ||
|
||||
grants.IsAccountIssued("mesh.mod.mesh-vault.event.secret-changed") {
|
||||
t.Errorf("the part hears %v", grants.Consumes)
|
||||
@@ -4622,76 +4409,3 @@ func channelTakes(t *testing.T, say func(oc.Condition) error) func(oc.Condition)
|
||||
return say(c)
|
||||
}
|
||||
}
|
||||
|
||||
// oldStateReaders names, as "<path> <function>", every function of the source that reads the field State of a
|
||||
// catalogue.Manifest or a membership.Membership: a receiver, parameter or declared variable of either type, or a field
|
||||
// named Manifest or Membership (issue 1966's source guard).
|
||||
func oldStateReaders(path string, src []byte) ([]string, error) {
|
||||
f, err := parser.ParseFile(token.NewFileSet(), path, src, 0)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
typed := func(e ast.Expr) bool {
|
||||
if s, ok := e.(*ast.StarExpr); ok {
|
||||
e = s.X
|
||||
}
|
||||
sel, ok := e.(*ast.SelectorExpr)
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
x, ok := sel.X.(*ast.Ident)
|
||||
return ok && ((x.Name == "catalogue" && sel.Sel.Name == "Manifest") || (x.Name == "membership" && sel.Sel.Name == "Membership"))
|
||||
}
|
||||
var out []string
|
||||
for _, decl := range f.Decls {
|
||||
fn, ok := decl.(*ast.FuncDecl)
|
||||
if !ok || fn.Body == nil {
|
||||
continue
|
||||
}
|
||||
names := map[string]bool{}
|
||||
var fields []*ast.Field
|
||||
if fn.Recv != nil {
|
||||
fields = append(fields, fn.Recv.List...)
|
||||
}
|
||||
for _, fl := range append(fields, fn.Type.Params.List...) {
|
||||
if typed(fl.Type) {
|
||||
for _, n := range fl.Names {
|
||||
names[n.Name] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
reads := false
|
||||
ast.Inspect(fn.Body, func(n ast.Node) bool {
|
||||
if vs, ok := n.(*ast.ValueSpec); ok && vs.Type != nil && typed(vs.Type) {
|
||||
for _, n := range vs.Names {
|
||||
names[n.Name] = true
|
||||
}
|
||||
}
|
||||
if s, ok := n.(*ast.SelectorExpr); ok && s.Sel.Name == "State" {
|
||||
switch x := s.X.(type) {
|
||||
case *ast.Ident:
|
||||
reads = reads || names[x.Name]
|
||||
case *ast.SelectorExpr:
|
||||
reads = reads || x.Sel.Name == "Manifest" || x.Sel.Name == "Membership"
|
||||
}
|
||||
}
|
||||
return true
|
||||
})
|
||||
if reads {
|
||||
out = append(out, path+" "+fn.Name.Name)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// saidHistory is the seat mesh-broker's events a shared events stream holds, oldest first (grants.BrokerHistory).
|
||||
type saidHistory []grants.SaidEvent
|
||||
|
||||
func (h saidHistory) Said() ([]grants.SaidEvent, error) { return h, nil }
|
||||
|
||||
// failingHistory is a re-read that fails partway: the events read so far, and the error.
|
||||
type failingHistory struct{ read saidHistory }
|
||||
|
||||
func (h failingHistory) Said() ([]grants.SaidEvent, error) {
|
||||
return h.read, errors.New("$JS.API.STREAM.MSG.GET.EVENTS at sequence 2: context deadline exceeded")
|
||||
}
|
||||
|
||||
@@ -600,18 +600,13 @@ func TestTheCopiedComposerComposesAsTheControllers(t *testing.T) {
|
||||
t.Errorf("a one-filter consumer's grants = %v: its update is the no-filter create form", one)
|
||||
}
|
||||
for _, u := range users {
|
||||
if u.Module != GrantsModule || u.Kind != KindModule {
|
||||
continue
|
||||
}
|
||||
perms, err := PermissionsFor(u)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Issue 1966: the grants part alone reads the shared events stream by sequence, to read the seat mesh-broker's
|
||||
// events again (grants_bindings_again).
|
||||
if may := MayPublish(perms, "$JS.API.STREAM.MSG.GET.EVENTS"); may != (u.Module == GrantsModule && u.Kind == KindModule) {
|
||||
t.Errorf("%s %s may read EVENTS by sequence: %v", u.Kind, u.Module, may)
|
||||
}
|
||||
if u.Module != GrantsModule || u.Kind != KindModule {
|
||||
continue
|
||||
}
|
||||
if !MayPublish(perms, "$JS.API.CONSUMER.CREATE.EVENTS.a_m") || MayPublish(perms, "$JS.API.CONSUMER.CREATE.EVENTS.a_quiet") ||
|
||||
MayPublish(perms, "$JS.API.CONSUMER.DELETE.EVENTS.a_m") || MayPublish(perms, "$JS.API.CONSUMER.CREATE.EVENTS.a_m.mesh.mod.x.event.happened") {
|
||||
t.Errorf("the grants part creates consumers as %v", u.CreatesConsumers)
|
||||
|
||||
@@ -859,10 +859,6 @@ func PermissionsFor(p Principal) (Permissions, error) {
|
||||
pub = append(pub, CreateConsumerGrants(p.CreatesConsumers)...)
|
||||
// And its own object bucket, where it writes the user list sealed to the bus node (ADR 0547 clause 7).
|
||||
pub = append(pub, GrantsUserListWrites()...)
|
||||
// And the read of one message of the shared events stream by its sequence and subject, where the part reads the
|
||||
// seat mesh-broker's created, recreated and dropped again for grants_bindings_again (issue 1966): a get, never a
|
||||
// consumer, so no other account's consumer is touched.
|
||||
pub = append(pub, SeatEventsReadAgain)
|
||||
}
|
||||
// The view alone reads the build seats' workers for the backlog of asks (issue 1887 A8a).
|
||||
if p.Module == ViewModule {
|
||||
@@ -1550,10 +1546,6 @@ func CreateConsumerGrants(consumers []ModuleConsumer) []string {
|
||||
return out
|
||||
}
|
||||
|
||||
// SeatEventsReadAgain is the grants part's read of the shared events stream by sequence (STREAM.MSG.GET with
|
||||
// next_by_subj): a binding the bus module said that the part did not keep is read again from it (issue 1966).
|
||||
const SeatEventsReadAgain = "$JS.API.STREAM.MSG.GET." + EventsStream
|
||||
|
||||
// GrantsModule is the one module whose principal publishes the modules' memberships (novox/hq ADR 0547 §5): the grants
|
||||
// part, which composes them from the manifests and assignments. It defines no stream and is granted no other subject here.
|
||||
const GrantsModule = "mesh-grants"
|
||||
|
||||
File diff suppressed because one or more lines are too long
@@ -2,15 +2,10 @@ package grants
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/go/seats/meshbroker"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/membership"
|
||||
"mesh-grants/internal/buscompose"
|
||||
@@ -98,18 +93,7 @@ func (p *Part) HearBroker(event string, body []byte, brokers map[string]bool) (b
|
||||
if e.Module == "" || local == "" || strings.Contains(e.Module, ".") {
|
||||
return false, fmt.Errorf("the bus module's %s names no module or no entry (%q, %q)", name, e.Module, local)
|
||||
}
|
||||
// Written at the revision read, and read again once when another write came first (two `created` of one module heard
|
||||
// at once): a binding is never lost to a write it did not see (issue 1966).
|
||||
changed, err := p.keepBinding(e, name, local, storage)
|
||||
if errors.Is(err, stdio.ErrChanged) {
|
||||
changed, err = p.keepBinding(e, name, local, storage)
|
||||
}
|
||||
return changed, err
|
||||
}
|
||||
|
||||
// keepBinding applies one event to the kept bindings of its module, written at the revision it read them at.
|
||||
func (p *Part) keepBinding(e BrokerEvent, name, local, storage string) (bool, error) {
|
||||
b, rev, err := p.boundAt(e.Module)
|
||||
b, err := p.bound(e.Module)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
@@ -135,7 +119,7 @@ func (p *Part) keepBinding(e BrokerEvent, name, local, storage string) (bool, er
|
||||
b.Bindings[local] = bind
|
||||
}
|
||||
raw, _ := json.Marshal(b)
|
||||
if _, err := p.Store.Update(StateMemberships, BoundKey(e.Module), raw, rev); err != nil {
|
||||
if _, err := p.Store.Put(StateMemberships, BoundKey(e.Module), raw); err != nil {
|
||||
return false, fmt.Errorf("its own %s %s cannot be written: %w", StateMemberships, BoundKey(e.Module), err)
|
||||
}
|
||||
return true, nil
|
||||
@@ -143,26 +127,20 @@ func (p *Part) keepBinding(e BrokerEvent, name, local, storage string) (bool, er
|
||||
|
||||
// bound reads the kept bindings of module; none kept is an empty value.
|
||||
func (p *Part) bound(module string) (Bound, error) {
|
||||
b, _, err := p.boundAt(module)
|
||||
return b, err
|
||||
}
|
||||
|
||||
// boundAt is bound with the revision it was read at, 0 when none is kept.
|
||||
func (p *Part) boundAt(module string) (Bound, uint64, error) {
|
||||
b := Bound{V: 1, Module: module, Bindings: map[string]membership.BusBinding{}}
|
||||
raw, rev, found, err := p.Store.Get(StateMemberships, BoundKey(module))
|
||||
raw, _, found, err := p.Store.Get(StateMemberships, BoundKey(module))
|
||||
if err != nil {
|
||||
return b, 0, fmt.Errorf("its own %s %s cannot be read: %w", StateMemberships, BoundKey(module), err)
|
||||
return b, fmt.Errorf("its own %s %s cannot be read: %w", StateMemberships, BoundKey(module), err)
|
||||
}
|
||||
if found {
|
||||
if err := json.Unmarshal(raw, &b); err != nil {
|
||||
return b, 0, fmt.Errorf("its own %s %s does not read: %w", StateMemberships, BoundKey(module), err)
|
||||
return b, fmt.Errorf("its own %s %s does not read: %w", StateMemberships, BoundKey(module), err)
|
||||
}
|
||||
if b.Bindings == nil {
|
||||
b.Bindings = map[string]membership.BusBinding{}
|
||||
}
|
||||
}
|
||||
return b, rev, nil
|
||||
return b, nil
|
||||
}
|
||||
|
||||
// bindingsOf is module's kept bindings in local-name order, for its memberships; nil when none.
|
||||
@@ -183,241 +161,6 @@ func (p *Part) bindingsOf(module string) ([]membership.BusBinding, error) {
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// BindingsOf answers the tool grants_bindings: every binding the part keeps of one module, or of every module when module
|
||||
// is empty (its own state memberships, key BoundKey), each beside the memberships `<node>.<module>` it wrote, and by
|
||||
// membership the kept local names its bindings lack, so a binding kept but not yet written into a membership is named.
|
||||
func (p *Part) BindingsOf(module string) (any, error) {
|
||||
keys, err := p.Store.Keys(StateMemberships)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("its own %s cannot be listed: %w", StateMemberships, err)
|
||||
}
|
||||
type kept struct {
|
||||
Module string `json:"module"`
|
||||
Bindings []membership.BusBinding `json:"bindings"`
|
||||
Memberships map[string][]string `json:"memberships"`
|
||||
Lacking map[string][]string `json:"lacking,omitempty"`
|
||||
}
|
||||
out := []kept{}
|
||||
for _, key := range keys {
|
||||
mod, ok := strings.CutPrefix(key, BoundPrefix)
|
||||
if !ok || (module != "" && mod != module) {
|
||||
continue
|
||||
}
|
||||
binds, err := p.bindingsOf(mod)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
k := kept{Module: mod, Bindings: binds, Memberships: map[string][]string{}}
|
||||
if k.Bindings == nil {
|
||||
k.Bindings = []membership.BusBinding{}
|
||||
}
|
||||
for _, mk := range keys {
|
||||
if _, of, _ := strings.Cut(mk, "."); strings.HasPrefix(mk, BoundPrefix) || of != mod {
|
||||
continue
|
||||
}
|
||||
raw, _, found, err := p.Store.Get(StateMemberships, mk)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("its own membership %s cannot be read: %w", mk, err)
|
||||
}
|
||||
if !found {
|
||||
continue
|
||||
}
|
||||
bare, err := membership.BareGrantsMembership(raw)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("its own membership %s does not read: %w", mk, err)
|
||||
}
|
||||
var m membership.Membership
|
||||
if err := json.Unmarshal(bare, &m); err != nil {
|
||||
return nil, fmt.Errorf("its own membership %s does not read: %w", mk, err)
|
||||
}
|
||||
written := []string{}
|
||||
for _, b := range m.Bindings {
|
||||
written = append(written, b.Local)
|
||||
}
|
||||
k.Memberships[mk] = written
|
||||
for _, b := range binds {
|
||||
if !slices.Contains(written, b.Local) {
|
||||
if k.Lacking == nil {
|
||||
k.Lacking = map[string][]string{}
|
||||
}
|
||||
k.Lacking[mk] = append(k.Lacking[mk], b.Local)
|
||||
}
|
||||
}
|
||||
}
|
||||
out = append(out, k)
|
||||
}
|
||||
again, err := p.lastBindingsAgain(module)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
answer := map[string]any{"modules": out,
|
||||
"says": "each binding the part kept from the bus module's created and recreated, beside the memberships it wrote; lacking names a kept binding a membership does not carry yet"}
|
||||
if module != "" && len(out) == 0 {
|
||||
answer["says"] = "the part keeps no binding of " + module + ": the bus module said no created for it that the part heard"
|
||||
}
|
||||
if again != nil {
|
||||
answer["read-again"] = again
|
||||
}
|
||||
return answer, nil
|
||||
}
|
||||
|
||||
// **A binding the part did not keep is read again from the shared events stream** (issue 1966; the lead's ruling, option
|
||||
// b): grants_bindings_again asks the part's process, by a person's act recorded as a hand act, to read the seat
|
||||
// mesh-broker's events the stream still holds, oldest first, and to keep each of one module's created, recreated and
|
||||
// dropped through HearBroker, as if heard now, so a binding dropped since is not brought back. The stream keeps its events
|
||||
// for a bounded age: a created older than that is not there, and the answer says so, naming the path then (the seat's
|
||||
// list answering each entry's binding, a record extending ADR 0595 §7).
|
||||
|
||||
// KeyBindingsAgain is, in StateUserList, the last re-read of one module's bindings: BindingsAgain.
|
||||
const KeyBindingsAgain = "bindings-again"
|
||||
|
||||
// BrokerHistory reads the seat mesh-broker's events again from the shared events stream.
|
||||
type BrokerHistory interface {
|
||||
// Said is every event of the seat the stream still holds, oldest first.
|
||||
Said() ([]SaidEvent, error)
|
||||
}
|
||||
|
||||
// SaidEvent is one event of the seat mesh-broker as the shared events stream holds it.
|
||||
type SaidEvent struct {
|
||||
Subject string `json:"subject"`
|
||||
Data []byte `json:"data"`
|
||||
At time.Time `json:"at"`
|
||||
}
|
||||
|
||||
// BindingsAgain is what the last re-read of one module's bindings found: the seat's events read, the local names whose
|
||||
// binding it kept (Kept), those it found already kept as said (Already), those a dropped took away (Dropped), the oldest
|
||||
// event the stream still holds, and in plain words what it means.
|
||||
type BindingsAgain struct {
|
||||
V int `json:"v"`
|
||||
Module string `json:"module"`
|
||||
By string `json:"by"`
|
||||
At time.Time `json:"at"`
|
||||
Read int `json:"read"`
|
||||
Kept []string `json:"kept"`
|
||||
Already []string `json:"already"`
|
||||
Dropped []string `json:"dropped"`
|
||||
Oldest time.Time `json:"oldest,omitzero"`
|
||||
Says string `json:"says"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
// bindingsAgain reads module's bindings again from the seat's events the stream holds, keeps each, and writes what it
|
||||
// found to its own state; an error is said in the pass's errors and in the kept result.
|
||||
func (p *Part) bindingsAgain(res *Result, module, by string) {
|
||||
out := BindingsAgain{V: 1, Module: module, By: by, At: p.now(), Kept: []string{}, Already: []string{}, Dropped: []string{}}
|
||||
events, err := p.History.Said()
|
||||
if err != nil {
|
||||
// A read that failed partway applies nothing: a created whose dropped lies past the failure would be kept again.
|
||||
out.Error = "the shared events stream could not be read, so nothing was kept: " + err.Error()
|
||||
events = nil
|
||||
}
|
||||
heardCreate := false
|
||||
for _, e := range events {
|
||||
if out.Oldest.IsZero() || (!e.At.IsZero() && e.At.Before(out.Oldest)) {
|
||||
out.Oldest = e.At
|
||||
}
|
||||
name, ok := brokerEvent(e.Subject, p.Brokers())
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
var said BrokerEvent
|
||||
if json.Unmarshal(e.Data, &said) != nil || said.Module != module {
|
||||
continue
|
||||
}
|
||||
out.Read++
|
||||
local := firstOf(said.LocalName, said.Local, said.Binding.Local)
|
||||
changed, err := p.HearBroker(e.Subject, e.Data, p.Brokers())
|
||||
if err != nil {
|
||||
out.Error = fmt.Sprintf("the %s of %s could not be kept: %v", name, local, err)
|
||||
break
|
||||
}
|
||||
switch {
|
||||
case name == BrokerDropped && changed:
|
||||
out.Dropped = appendOnce(out.Dropped, local)
|
||||
out.Kept = slices.DeleteFunc(out.Kept, func(l string) bool { return l == local })
|
||||
case name != BrokerDropped:
|
||||
heardCreate = true
|
||||
out.Dropped = slices.DeleteFunc(out.Dropped, func(l string) bool { return l == local })
|
||||
if changed {
|
||||
out.Kept = appendOnce(out.Kept, local)
|
||||
out.Already = slices.DeleteFunc(out.Already, func(l string) bool { return l == local })
|
||||
} else if !slices.Contains(out.Kept, local) {
|
||||
out.Already = appendOnce(out.Already, local)
|
||||
}
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case out.Error != "":
|
||||
out.Says = "the re-read of " + module + " stopped: " + out.Error
|
||||
res.Errors = append(res.Errors, out.Says)
|
||||
case !heardCreate:
|
||||
out.Says = fmt.Sprintf("the shared events stream holds no created or recreated of %s (its oldest event: %s): a created older "+
|
||||
"than the stream keeps is not there, and its binding comes back only by the seat mesh-broker's list answering each "+
|
||||
"entry's binding (a record extending ADR 0595 §7), not by this tool", module, out.Oldest.Format(time.RFC3339))
|
||||
default:
|
||||
out.Says = fmt.Sprintf("read %d of the seat mesh-broker's events of %s again: kept %v, already kept %v, dropped %v; the pass "+
|
||||
"writes them into its memberships", out.Read, module, out.Kept, out.Already, out.Dropped)
|
||||
}
|
||||
if p.Logf != nil {
|
||||
p.Logf("the bindings of %s read again by hand, for %s: %s", module, by, out.Says)
|
||||
}
|
||||
raw, _ := json.Marshal(out)
|
||||
p.changed(res, StateUserList, KeyBindingsAgain, raw)
|
||||
}
|
||||
|
||||
func appendOnce(list []string, s string) []string {
|
||||
if slices.Contains(list, s) {
|
||||
return list
|
||||
}
|
||||
return append(list, s)
|
||||
}
|
||||
|
||||
// AskBindingsAgain answers the tool grants_bindings_again: it asks the part's process to read one module's bindings
|
||||
// again from the seat mesh-broker's events the shared events stream holds, and records the hand act through record (the
|
||||
// seat mesh-hand-acts) first; a module that is no module's name, or an ask without why, is refused before anything is
|
||||
// asked. grants_bindings shows what the process found.
|
||||
func (p *Part) AskBindingsAgain(caller, module, why string, record func(what, why string) error) (any, error) {
|
||||
if module == "" || strings.ContainsAny(module, ".*> ") {
|
||||
return nil, fmt.Errorf("grants_bindings_again needs one module, by its name: %q is none", module)
|
||||
}
|
||||
if strings.TrimSpace(why) == "" {
|
||||
return nil, errors.New("grants_bindings_again needs why: it is recorded as a hand act")
|
||||
}
|
||||
if record != nil {
|
||||
if err := record(ToolBindingsAgain+" "+module, why); err != nil {
|
||||
return nil, fmt.Errorf("the hand act could not be recorded, so nothing was asked: %w", err)
|
||||
}
|
||||
}
|
||||
ask, _ := json.Marshal(Ask{By: caller, At: p.now(), Bindings: module})
|
||||
rev, err := p.Store.Put(StateUserList, KeyAsked, ask)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the ask could not be written: %w", err)
|
||||
}
|
||||
return map[string]any{"asked": rev, "module": module,
|
||||
"says": "the part's process reads the seat mesh-broker's created, recreated and dropped of " + module + " again from the shared " +
|
||||
"events stream at this ask, and keeps each as if heard now; grants_bindings shows what it kept, what was already kept, " +
|
||||
"and whether the stream still holds its created"}, nil
|
||||
}
|
||||
|
||||
// lastBindingsAgain is the last re-read kept, when it was of module (or any, for an empty module).
|
||||
func (p *Part) lastBindingsAgain(module string) (*BindingsAgain, error) {
|
||||
raw, _, found, err := p.Store.Get(StateUserList, KeyBindingsAgain)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("its own %s %s cannot be read: %w", StateUserList, KeyBindingsAgain, err)
|
||||
}
|
||||
if !found {
|
||||
return nil, nil
|
||||
}
|
||||
var b BindingsAgain
|
||||
if err := json.Unmarshal(raw, &b); err != nil {
|
||||
return nil, fmt.Errorf("its own %s %s does not read: %w", StateUserList, KeyBindingsAgain, err)
|
||||
}
|
||||
if module != "" && b.Module != module {
|
||||
return nil, nil
|
||||
}
|
||||
return &b, nil
|
||||
}
|
||||
|
||||
func sameBinding(a, b membership.BusBinding) bool {
|
||||
ra, _ := json.Marshal(a)
|
||||
rb, _ := json.Marshal(b)
|
||||
|
||||
@@ -35,8 +35,7 @@ type Follower struct {
|
||||
// Hear hears the keeper's condition events and the vault's account-issued (Consumes) and hands each, by event name,
|
||||
// with its body, to handle; nil hears none. A condition of the kind root-not-free raised or cleared wakes a pass:
|
||||
// a machine's root-free word changed; and so does an account the vault minted (ADR 0561 §1), whose hash the list carries.
|
||||
// handle answers an event it could not apply, which the hearer then leaves unacknowledged, so it is said again (issue 1966).
|
||||
Hear func(handle func(event string, body []byte) error) error
|
||||
Hear func(handle func(event string, body []byte)) error
|
||||
|
||||
wake chan struct{}
|
||||
once sync.Once
|
||||
@@ -128,27 +127,25 @@ func (f *Follower) open() {
|
||||
}
|
||||
}
|
||||
if f.Hear != nil && !f.heard_ {
|
||||
err := f.Hear(func(event string, body []byte) error {
|
||||
err := f.Hear(func(event string, body []byte) {
|
||||
// The bus module's created, recreated or dropped: the binding it carries is kept, and a pass composes the
|
||||
// memberships with it (bindings.go). One not applied is answered, so the hearer leaves it unacknowledged and the
|
||||
// bus says it again (issue 1966: acknowledged, it was lost while this line said it would come again).
|
||||
// memberships with it (bindings.go).
|
||||
if changed, err := f.Part.HearBroker(event, body, f.Part.Brokers()); err != nil {
|
||||
f.logf("%v: not applied, said again at its redelivery", err)
|
||||
return err
|
||||
return
|
||||
} else if changed {
|
||||
f.Wake()
|
||||
return nil
|
||||
return
|
||||
}
|
||||
// An account the vault minted.
|
||||
if IsAccountIssued(event) {
|
||||
f.Wake()
|
||||
return nil
|
||||
return
|
||||
}
|
||||
// A condition event is heard from the keeper alone (ADR 0590 §3).
|
||||
if FromKeeper(event) && IsRootNotFree(body) {
|
||||
f.Wake()
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
f.logf("the keeper's condition events cannot be heard yet (%v): tried again at the next change", err)
|
||||
|
||||
@@ -57,10 +57,6 @@ type Store interface {
|
||||
// Keys and Delete are for the wanted accounts (accounts.go) and the grants (grants.go): every other state is only put.
|
||||
Keys(state string) ([]string, error)
|
||||
Delete(state, key string) error
|
||||
// Update writes a key only while it is still at revision (0: only while it holds nothing), and refuses one changed
|
||||
// meanwhile with an error wrapping stdio.ErrChanged: the kept bindings are written so, never lost to a second writer
|
||||
// (issue 1966).
|
||||
Update(state, key string, value []byte, revision uint64) (uint64, error)
|
||||
}
|
||||
|
||||
// Emitter says one of the part's events.
|
||||
@@ -174,9 +170,6 @@ type Part struct {
|
||||
// Say says a condition to the operator: the process's; nil in the tools.
|
||||
Say func(oc.Condition) error
|
||||
Now func() time.Time
|
||||
// History reads the seat mesh-broker's events again from the shared events stream (bindings.go): the process's
|
||||
// connection; nil in the tools, which ask the process.
|
||||
History BrokerHistory
|
||||
// SeatList is the seat list reader, which gates a module whose seats the list has not named (inputs.go, SeatGate); nil
|
||||
// gates none.
|
||||
SeatList SeatGate
|
||||
@@ -261,9 +254,6 @@ type Ask struct {
|
||||
Consumer string `json:"consumer,omitempty"`
|
||||
// FirstView is, from grants_first_view_again, one first view to ask once more by hand (its state), or AllFirstViews.
|
||||
FirstView string `json:"first-view,omitempty"`
|
||||
// Bindings is, from grants_bindings_again, one module whose bindings the part reads again from the seat mesh-broker's
|
||||
// events the shared events stream still holds (bindings.go, issue 1966).
|
||||
Bindings string `json:"bindings,omitempty"`
|
||||
}
|
||||
|
||||
// AllFirstViews asks every first view once more.
|
||||
@@ -304,14 +294,6 @@ func (p *Part) Compose() Result {
|
||||
p.Views.SayUnsaid()
|
||||
in := p.inputs()
|
||||
p.keepBrokers(in)
|
||||
var ask Ask
|
||||
if p.Create != nil || p.Consumers != nil {
|
||||
ask = p.asked()
|
||||
}
|
||||
// A module's bindings read again by hand are kept before the pass reads the kept bindings, so it writes them now.
|
||||
if ask.Bindings != "" && p.History != nil {
|
||||
p.bindingsAgain(&res, ask.Bindings, ask.By)
|
||||
}
|
||||
if err := p.withBound(&in); err != nil {
|
||||
res.Errors = append(res.Errors, err.Error())
|
||||
}
|
||||
@@ -507,6 +489,10 @@ func (p *Part) Compose() Result {
|
||||
}
|
||||
p.clearFull()
|
||||
held, heldErr := HeldStreams(p.Call)
|
||||
var ask Ask
|
||||
if p.Create != nil || p.Consumers != nil {
|
||||
ask = p.asked()
|
||||
}
|
||||
forced := ask.Assert
|
||||
for _, module := range sortedKeys(in.Manifests) {
|
||||
if !assigned(in, module) {
|
||||
|
||||
@@ -31,15 +31,11 @@ const (
|
||||
// ToolSeatList and ToolSeatListAgain read and read again the seat list the part composes from (mesh-sdk seatlist.Tools).
|
||||
ToolSeatList = "grants_seat_list"
|
||||
ToolSeatListAgain = "grants_seat_list_again"
|
||||
// ToolBindings reads the bindings the part keeps from the bus module's events (bindings.go, issue 1966).
|
||||
ToolBindings = "grants_bindings"
|
||||
// ToolBindingsAgain reads one module's bindings again from the seat mesh-broker's events the shared events stream holds.
|
||||
ToolBindingsAgain = "grants_bindings_again"
|
||||
)
|
||||
|
||||
// Tools is every tool, in the manifest's order.
|
||||
var Tools = []string{ToolGrants, ToolMembership, ToolUserList, ToolComposeNow, ToolAssert, ToolAccounts, ToolConsumerMatch,
|
||||
ToolAcceptHashes, ToolFirstViews, ToolFirstViewAgain, ToolSeatList, ToolSeatListAgain, ToolBindings, ToolBindingsAgain}
|
||||
ToolAcceptHashes, ToolFirstViews, ToolFirstViewAgain, ToolSeatList, ToolSeatListAgain}
|
||||
|
||||
// WritesGrants is what grants says of the part's own grants: written once the list the bus loaded carries a build's
|
||||
// accounts, and the one key the assigner reads (ADR 0571 clause 9, which changes ADR 0547 clause 5).
|
||||
|
||||
@@ -61,9 +61,7 @@
|
||||
"grants_first_views",
|
||||
"grants_first_view_again",
|
||||
"grants_seat_list",
|
||||
"grants_seat_list_again",
|
||||
"grants_bindings",
|
||||
"grants_bindings_again"
|
||||
"grants_seat_list_again"
|
||||
],
|
||||
"state": [
|
||||
{
|
||||
|
||||
+54
-4
@@ -1,8 +1,12 @@
|
||||
package buildstep
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"path"
|
||||
"slices"
|
||||
"sort"
|
||||
@@ -281,13 +285,55 @@ type CheckJob struct {
|
||||
|
||||
// ComposeJob asks one compose-candidate build.
|
||||
type ComposeJob struct {
|
||||
Fetch Fetch `json:"fetch"`
|
||||
Dir string `json:"dir"`
|
||||
Package string `json:"package"`
|
||||
Fetch Fetch `json:"fetch"`
|
||||
Dir string `json:"dir"`
|
||||
Package string `json:"package"`
|
||||
// Request is the composer's request: a JSON object, or a JSON string PackedRequestPrefix + the base64 of the object
|
||||
// gzip-compressed (issue 1966; the checker packs every request it sends, and
|
||||
// the part's `request` describes both forms). The part types it `any`, so both fit it; a
|
||||
// build module that does not know the packed form refuses the ask naming that it holds no request object.
|
||||
Request json.RawMessage `json:"request"`
|
||||
Reader string `json:"reader,omitempty"`
|
||||
}
|
||||
|
||||
// PackedRequestPrefix opens a compose job's request that travels gzip-compressed.
|
||||
const PackedRequestPrefix = "gzip+base64:"
|
||||
|
||||
// PackRequest sets the job's request from the JSON object raw, packed.
|
||||
func (c *ComposeJob) PackRequest(raw []byte) error {
|
||||
var b bytes.Buffer
|
||||
zw := gzip.NewWriter(&b)
|
||||
if _, err := zw.Write(raw); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := zw.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
packed, err := json.Marshal(PackedRequestPrefix + base64.StdEncoding.EncodeToString(b.Bytes()))
|
||||
c.Request = packed
|
||||
return err
|
||||
}
|
||||
|
||||
// RequestBytes is the job's request as a JSON object, unpacked when it travels packed.
|
||||
func (c ComposeJob) RequestBytes() ([]byte, error) {
|
||||
var packed string
|
||||
if len(c.Request) == 0 || c.Request[0] != '"' {
|
||||
return c.Request, nil
|
||||
}
|
||||
if err := json.Unmarshal(c.Request, &packed); err != nil || !strings.HasPrefix(packed, PackedRequestPrefix) {
|
||||
return nil, fmt.Errorf("buildstep: a compose job's request is a string that is no packed request")
|
||||
}
|
||||
raw, err := base64.StdEncoding.DecodeString(strings.TrimPrefix(packed, PackedRequestPrefix))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("buildstep: the packed request of a compose job is no base64: %w", err)
|
||||
}
|
||||
zr, err := gzip.NewReader(bytes.NewReader(raw))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("buildstep: the packed request of a compose job is no gzip: %w", err)
|
||||
}
|
||||
return io.ReadAll(zr)
|
||||
}
|
||||
|
||||
// AskOf is any seat's ask (or this form) as the neutral StepAsk: the one conversion, since the three seats' types
|
||||
// are the part's one layout. A value that does not fit the layout is refused.
|
||||
func AskOf[T any](seatAsk T) (StepAsk, error) {
|
||||
@@ -378,7 +424,11 @@ func (c ComposeJob) check(id string) error {
|
||||
if c.Package == "" {
|
||||
return fmt.Errorf("buildstep: the compose ask %s names no package to build", id)
|
||||
}
|
||||
if !json.Valid(c.Request) || len(c.Request) == 0 || c.Request[0] != '{' {
|
||||
req, err := c.RequestBytes()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !json.Valid(req) || len(req) == 0 || req[0] != '{' {
|
||||
return fmt.Errorf("buildstep: the compose ask %s holds no request object", id)
|
||||
}
|
||||
return nil
|
||||
|
||||
@@ -144,6 +144,28 @@ func TestAskCheckAndRoundTrip(t *testing.T) {
|
||||
if _, err := DecodeStepAsk([]byte(`{"v":2}`)); !errors.Is(err, ErrUnknownVersion) {
|
||||
t.Errorf("another version: %v", err)
|
||||
}
|
||||
// Issue 1966: a compose job's request travels packed when it passes the bus's message limit, and reads back as sent.
|
||||
t.Run("1966: a compose request travels packed", func(t *testing.T) {
|
||||
obj := []byte(`{"v":1,"input":{"nodes":[]}}`)
|
||||
job := ComposeJob{Fetch: Fetch{Repository: "novox/mesh-host", Commit: strings.Repeat("a", 40)}, Dir: ".", Package: "p"}
|
||||
if err := job.PackRequest(obj); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got, err := job.RequestBytes(); err != nil || string(got) != string(obj) {
|
||||
t.Errorf("read back %s, %v", got, err)
|
||||
}
|
||||
if err := job.check("ask"); err != nil {
|
||||
t.Errorf("a packed request is refused: %v", err)
|
||||
}
|
||||
job.Request = json.RawMessage(`"not packed"`)
|
||||
if err := job.check("ask"); err == nil {
|
||||
t.Errorf("a string that is no packed request is accepted")
|
||||
}
|
||||
job.Request = obj
|
||||
if got, err := job.RequestBytes(); err != nil || string(got) != string(obj) {
|
||||
t.Errorf("a whole request read back %s, %v", got, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// The seat's value and the neutral form are the same JSON: AskOf takes any seat's type (here its shape, with a
|
||||
|
||||
Reference in New Issue
Block a user