Compare commits
3
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
83f7a7612d | ||
|
|
1fe1beee58 | ||
|
|
40da5e924e |
@@ -3,7 +3,7 @@ module mesh-delivery-assigner
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102191058-f02f142af128
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103120252-42bd7b802ac2
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102191058-f02f142af128 h1:aI1DhZAuqVBXbybSqhLNOGX5rUOrn9T0cHNCDUQF7IQ=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102191058-f02f142af128/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103120252-42bd7b802ac2 h1:MOuutmbKBL1GtTZU9W1+ja/K7TL05EFU6hEt3Swf/oE=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103120252-42bd7b802ac2/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
|
||||
@@ -3,7 +3,7 @@ module mesh-delivery-builder
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102191058-f02f142af128
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103120252-42bd7b802ac2
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep v0.0.0-20261102193506-d221c633799a h1:oxgksCiq272/afXbUION/dxtAS5MPGnmyq6+AZQxmXg=
|
||||
git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep v0.0.0-20261102193506-d221c633799a/go.mod h1:kpsX4Pgss5EeuBvel1zWa2ogj7MkR3jZso6oa/J0YUQ=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102191058-f02f142af128 h1:aI1DhZAuqVBXbybSqhLNOGX5rUOrn9T0cHNCDUQF7IQ=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102191058-f02f142af128/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103120252-42bd7b802ac2 h1:MOuutmbKBL1GtTZU9W1+ja/K7TL05EFU6hEt3Swf/oE=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103120252-42bd7b802ac2/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
|
||||
@@ -128,7 +128,9 @@ func (m *manifestVerdicts) put(module string, v *delivery.ManifestVerdict) {
|
||||
delete(m.by, module)
|
||||
return
|
||||
}
|
||||
m.by[module] = *v
|
||||
// The sandbox's composer may be the node-engine of an older delivery, which does not know the verdict waits: it is handed a
|
||||
// wait as the refusal it is held as.
|
||||
m.by[module] = v.AsRefused()
|
||||
}
|
||||
|
||||
func (m *manifestVerdicts) all() map[string]delivery.ManifestVerdict {
|
||||
|
||||
@@ -170,4 +170,10 @@ func TestAManifestFallingBackToAPredefinedKeyIsRead(t *testing.T) {
|
||||
if _, err := parseManifest(manifest); err != nil {
|
||||
t.Errorf("manifestReader: %v", err)
|
||||
}
|
||||
// The sandbox's composer may be an older node-engine, which knows no verdict waits: it is handed a wait as a refusal.
|
||||
var held manifestVerdicts
|
||||
held.put("nats", &delivery.ManifestVerdict{Module: "nats", State: delivery.VerdictManifestWaits, Unknown: []string{"core-credentials"}, Judge: "j"})
|
||||
if v := held.all()["nats"]; v.State != delivery.VerdictRefused || len(v.Problems) != 1 || len(v.Unknown) != 0 {
|
||||
t.Errorf("a waiting verdict is handed to the sandbox as %+v", v)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,7 +3,7 @@ module mesh-delivery-checker
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102191058-f02f142af128
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103120252-42bd7b802ac2
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep v0.0.0-20261102193506-d221c633799a h1:oxgksCiq272/afXbUION/dxtAS5MPGnmyq6+AZQxmXg=
|
||||
git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep v0.0.0-20261102193506-d221c633799a/go.mod h1:kpsX4Pgss5EeuBvel1zWa2ogj7MkR3jZso6oa/J0YUQ=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102191058-f02f142af128 h1:aI1DhZAuqVBXbybSqhLNOGX5rUOrn9T0cHNCDUQF7IQ=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102191058-f02f142af128/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103120252-42bd7b802ac2 h1:MOuutmbKBL1GtTZU9W1+ja/K7TL05EFU6hEt3Swf/oE=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103120252-42bd7b802ac2/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
|
||||
@@ -3,7 +3,7 @@ module mesh-delivery-forge-bridge
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102191058-f02f142af128
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103120252-42bd7b802ac2
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102191058-f02f142af128 h1:aI1DhZAuqVBXbybSqhLNOGX5rUOrn9T0cHNCDUQF7IQ=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102191058-f02f142af128/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103120252-42bd7b802ac2 h1:MOuutmbKBL1GtTZU9W1+ja/K7TL05EFU6hEt3Swf/oE=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103120252-42bd7b802ac2/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
// Command mesh-delivery-indexer-tools serves the indexer's doing tool (novox/hq issue 1945): the runtime launches it for a
|
||||
// call. indexer_take_in writes one value of the indexer's takeins state and waits for the indexer's answer there. The
|
||||
// indexer is the other program: it serves no request subject (ADR 0437 §4). The read tools of the indexer's states are
|
||||
// indexer is the other program: it serves no request subject (ADR 0437 §4); indexer_judge asks the same way, and
|
||||
// indexer_verdicts reads the indexer's verdicts state. The read tools of the indexer's states are
|
||||
// mesh-delivery-view's (indexer_module, indexer_modules, indexer_candidate, indexer_candidates).
|
||||
package main
|
||||
|
||||
@@ -21,8 +22,8 @@ import (
|
||||
// asker writes a takeins value and waits for the indexer's answer to it.
|
||||
type asker func(key string, value []byte) (index.TakeInAsk, error)
|
||||
|
||||
// Tools are the indexer's tools.
|
||||
func Tools(ask asker, now func() time.Time) []stdio.Tool {
|
||||
// Tools are the indexer's tools. read is the indexer's state of verdicts, which indexer_verdicts reads.
|
||||
func Tools(ask asker, now func() time.Time, read stdio.KeptState) []stdio.Tool {
|
||||
str := func(args map[string]any, k string) string { s, _ := args[k].(string); return strings.TrimSpace(s) }
|
||||
return []stdio.Tool{
|
||||
{Name: index.TakeInTool,
|
||||
@@ -49,6 +50,50 @@ func Tools(ask asker, now func() time.Time) []stdio.Tool {
|
||||
}
|
||||
return a.Result, nil
|
||||
}},
|
||||
{Name: index.VerdictsTool,
|
||||
Description: "The indexer's verdicts on manifests that wait or are refused (all: every verdict), each as its verdicts state holds it: " +
|
||||
"the module, the digest, the state (waits: the manifest carries a field the indexer's judge does not know yet, named in " +
|
||||
"unknown, and every reader holds the module as a refused one and keeps what runs; refused: the judge's problems), the " +
|
||||
"judge that made it and when. A module's name narrows it to that module. A waiting manifest is judged again at each " +
|
||||
"new indexer and refused only once the newest build of the indexer on the trunk still does not know the field; " +
|
||||
"indexer_judge judges now.",
|
||||
Input: map[string]any{"type": "object", "properties": map[string]any{
|
||||
"module": map[string]any{"type": "string", "description": "only this module's verdicts"},
|
||||
"all": map[string]any{"type": "boolean", "description": "list accepted verdicts too"}}},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
keys, err := read.Keys()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the verdicts state cannot be listed: %w", err)
|
||||
}
|
||||
all, _ := args["all"].(bool)
|
||||
rows, unread := index.ListVerdicts(keys, func(key string) ([]byte, error) {
|
||||
e, err := read.Get(key)
|
||||
if err != nil || e == nil {
|
||||
return nil, err
|
||||
}
|
||||
return e.Value, nil
|
||||
}, str(args, "module"), all)
|
||||
return map[string]any{"verdicts": rows, "unread": unread}, nil
|
||||
}},
|
||||
{Name: index.JudgeAgainTool,
|
||||
Description: "Judge the manifests again now, as the indexer does at its start and after each merge, and answer how the module's " +
|
||||
"verdict stands (waits, refused or accepted, and why). For a manifest that waits for a judge that knows its fields, or " +
|
||||
"after a manifest was mended where the indexer has not heard it. A waiting manifest becomes refused here only when the " +
|
||||
"running indexer is the newest build of itself the trunk holds. indexer_verdicts reads the result.",
|
||||
Input: map[string]any{"type": "object", "required": []string{"module"}, "properties": map[string]any{
|
||||
"module": map[string]any{"type": "string", "description": "the module's name, as the catalogue registers it"},
|
||||
"why": map[string]any{"type": "string", "description": "why it is judged again now, in plain words"}}},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
key, value, err := index.NewJudgeAgain(str(args, "module"), str(args, "why"), stdio.CallerOf(args), now())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
a, err := ask(key, value)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return a.Result, nil
|
||||
}},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -106,7 +151,7 @@ func askTakeIn(s stdio.KeptState, bound time.Duration) asker {
|
||||
}
|
||||
|
||||
func main() {
|
||||
if err := stdio.Serve("", Tools(askTakeIn(stdio.State(index.TakeInsState), 25*time.Second), time.Now)); err != nil {
|
||||
if err := stdio.Serve("", Tools(askTakeIn(stdio.State(index.TakeInsState), 25*time.Second), time.Now, stdio.State(index.VerdictsState))); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "[mesh-delivery-indexer-tools] %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
@@ -300,7 +300,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
Verdicts: awaitedStore{verdicts}, SayRefused: sayRefused(js, cred, logger.Printf),
|
||||
Predefined: awaitedStore{predefined}, ConfigCandidates: awaitedStore{configCandidates}, Logf: logger.Printf,
|
||||
Named: namedVia(calls.Call), Trunk: trunkVia(calls.Call), Contains: containsVia(calls.Call), Read: readVia(calls.Call),
|
||||
ReadOpen: openVia(calls.Call), Packages: awaitedStore{packages}, Files: filesVia(calls.Call), GoSum: goSumVia(calls.Call), GoMod: goModVia(calls.Call)}
|
||||
ReadOpen: openVia(calls.Call), Packages: awaitedStore{packages}, Files: filesVia(calls.Call), GoSum: goSumVia(calls.Call), GoMod: goModVia(calls.Call),
|
||||
OwnBuild: ownBuildVia(calls.Call, cred.Node)}
|
||||
// The log's entries at the start are read before any fingerprint is computed or registered module indexed (below); each
|
||||
// new one indexes the module it names when that module is registered and the bucket holds none (registered.go).
|
||||
var started atomic.Bool
|
||||
@@ -421,8 +422,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
logger.Printf("judging the manifests at start: %v; judged again at the next message heard", err)
|
||||
return
|
||||
}
|
||||
logger.Printf("judged %d manifests at start: %d verdicts written, %d with problems, of which outgrown and kept accepted: %v",
|
||||
j.Judged, j.Written, len(j.Refused), j.Outgrown)
|
||||
logger.Printf("judged %d manifests at start: %d verdicts written, %d with problems, of which outgrown and kept accepted: %v, waiting for a judge that knows their fields: %v",
|
||||
j.Judged, j.Written, len(j.Refused), j.Outgrown, j.Waiting)
|
||||
})
|
||||
// A build module says each time it wrote its toolchains key: its stream, and the indexer's durable consumer on it, are
|
||||
// made by the controller as the forge's are. Where the stream is not there yet the indexer says so and is started again
|
||||
|
||||
@@ -50,6 +50,19 @@ func TestTheTrunkIsTheDirectoriesTheForgeListsTrees(t *testing.T) { // 1900/AC3
|
||||
if _, has := gotArgs.(map[string]any)["paths"]; has {
|
||||
t.Error("the trunk asked for paths")
|
||||
}
|
||||
// The build the indexer runs is the engine's word (Runs.FP of its composition verb), asked of its own machine.
|
||||
var asked string
|
||||
own := func(key string, body any) (json.RawMessage, error) {
|
||||
asked = key
|
||||
return json.RawMessage(`{"runs":{"fp":"src2:abc"}}`), nil
|
||||
}
|
||||
if fp, err := ownBuildVia(own, "n1")(context.Background()); err != nil || fp != "src2:abc" || asked != "seat:node-engine.composition@n1" {
|
||||
t.Errorf("own build = %q %v asked %q", fp, err, asked)
|
||||
}
|
||||
none := func(string, any) (json.RawMessage, error) { return json.RawMessage(`{}`), nil }
|
||||
if _, err := ownBuildVia(none, "n1")(context.Background()); err == nil {
|
||||
t.Error("an engine that says nothing runs gave a build")
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadViaReadsManifestsAndLocks(t *testing.T) {
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// compositionVerb is the node-engine seat's read verb that says what runs of a module on a machine (its Runs.FP), asked of
|
||||
// that machine's own engine as `seat:node-engine.composition@<machine>`. The indexer learns the build it runs from it, never
|
||||
// from the trunk's value of itself or the haves bucket.
|
||||
const compositionVerb = "seat:node-engine.composition"
|
||||
|
||||
// ownBuildVia asks the engine of node which fingerprint of the indexer runs there.
|
||||
func ownBuildVia(call toolCaller, node string) func(ctx context.Context) (string, error) {
|
||||
return func(context.Context) (string, error) {
|
||||
raw, err := call(compositionVerb+"@"+node, map[string]any{"module": delivery.IndexerModule})
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("%s: %w", compositionVerb, err)
|
||||
}
|
||||
var a struct {
|
||||
Runs *struct {
|
||||
FP string `json:"fp"`
|
||||
} `json:"runs"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &a); err != nil {
|
||||
return "", fmt.Errorf("the answer of %s is not a composition: %w", compositionVerb, err)
|
||||
}
|
||||
if a.Runs == nil || a.Runs.FP == "" {
|
||||
return "", errors.New("the engine says no fingerprint of the indexer runs")
|
||||
}
|
||||
return a.Runs.FP, nil
|
||||
}
|
||||
}
|
||||
@@ -14,6 +14,7 @@ import (
|
||||
// takerIn is the part of the indexer a takeins value asks through.
|
||||
type takerIn interface {
|
||||
TakeIn(ctx context.Context, module string) index.TakenIn
|
||||
JudgeAgain(ctx context.Context, module string) index.TakenIn
|
||||
}
|
||||
|
||||
// takenIn answers one value of the takeins state (the tool indexer_take_in, issue 1945): a value not yet answered takes the
|
||||
@@ -31,9 +32,14 @@ func takenIn(ctx context.Context, ix takerIn, kv jetstream.KeyValue, e jetstream
|
||||
if !a.Answered.IsZero() {
|
||||
return
|
||||
}
|
||||
r := ix.TakeIn(ctx, a.Module)
|
||||
asked, r := "take-in", index.TakenIn{}
|
||||
if a.Judge {
|
||||
asked, r = "judging again", ix.JudgeAgain(ctx, a.Module)
|
||||
} else {
|
||||
r = ix.TakeIn(ctx, a.Module)
|
||||
}
|
||||
a.Result, a.Answered = &r, time.Now().UTC()
|
||||
logf("%s: a take-in asked by %s (%s): %s", a.Module, a.By, a.Why, r.Answer)
|
||||
logf("%s: a %s asked by %s (%s): %s", a.Module, asked, a.By, a.Why, r.Answer)
|
||||
data, err := json.Marshal(a)
|
||||
if err != nil {
|
||||
return
|
||||
|
||||
@@ -3,7 +3,7 @@ module mesh-delivery-indexer
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102191058-f02f142af128
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103120252-42bd7b802ac2
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102191058-f02f142af128 h1:aI1DhZAuqVBXbybSqhLNOGX5rUOrn9T0cHNCDUQF7IQ=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102191058-f02f142af128/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103120252-42bd7b802ac2 h1:MOuutmbKBL1GtTZU9W1+ja/K7TL05EFU6hEt3Swf/oE=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103120252-42bd7b802ac2/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
|
||||
@@ -92,22 +92,27 @@ type Indexer struct {
|
||||
// closed message the indexer missed heals at the next one (issue 1126); nil asks none.
|
||||
ReadOpen func(ctx context.Context) ([]delivery.OpenPull, error)
|
||||
|
||||
mu sync.Mutex // guards source, toolchains and previews, which the watches and the forge messages all reach
|
||||
source sourceNow
|
||||
previews map[string]delivery.Preview // by previews key (previews.go)
|
||||
toolchains map[string]map[string]delivery.Toolchain // language, then node (toolchains.go)
|
||||
tried map[string]bool // repository, commit and directory a registered module's manifest was tried at (registered.go)
|
||||
unread map[string]bool // registered modules whose last read the forge could not answer, read again by RetryUnread
|
||||
moving map[string]delivery.Forge // by modules key: a merge naming a module another repository holds, waiting for that trunk to drop it (moves.go)
|
||||
retrying atomic.Bool // a RetryUnread runs in the background (kickRetry)
|
||||
retryAgain atomic.Bool // an event asked for one more RetryUnread after the running one
|
||||
retryWG sync.WaitGroup // the background RetryUnread, for tests to wait on
|
||||
recompute sync.Mutex // one Recompute at a time
|
||||
untaken map[string]string // repository to the latest merge applied while the taken state was missing (taken.go)
|
||||
filling atomic.Bool // a FillTaken runs in the background (KickFill)
|
||||
fillAgain atomic.Bool // a kick arrived while it ran
|
||||
atHeads map[string]map[string]delivery.PublishedPackages // a member head's published Go modules, by repository@head (candidatePublished)
|
||||
judgePending atomic.Bool // the start's judge pass failed: judged again at the next message (verdicts.go)
|
||||
mu sync.Mutex // guards source, toolchains and previews, which the watches and the forge messages all reach
|
||||
source sourceNow
|
||||
previews map[string]delivery.Preview // by previews key (previews.go)
|
||||
toolchains map[string]map[string]delivery.Toolchain // language, then node (toolchains.go)
|
||||
tried map[string]bool // repository, commit and directory a registered module's manifest was tried at (registered.go)
|
||||
unread map[string]bool // registered modules whose last read the forge could not answer, read again by RetryUnread
|
||||
moving map[string]delivery.Forge // by modules key: a merge naming a module another repository holds, waiting for that trunk to drop it (moves.go)
|
||||
retrying atomic.Bool // a RetryUnread runs in the background (kickRetry)
|
||||
retryAgain atomic.Bool // an event asked for one more RetryUnread after the running one
|
||||
retryWG sync.WaitGroup // the background RetryUnread, for tests to wait on
|
||||
recompute sync.Mutex // one Recompute at a time
|
||||
untaken map[string]string // repository to the latest merge applied while the taken state was missing (taken.go)
|
||||
filling atomic.Bool // a FillTaken runs in the background (KickFill)
|
||||
fillAgain atomic.Bool // a kick arrived while it ran
|
||||
atHeads map[string]map[string]delivery.PublishedPackages // a member head's published Go modules, by repository@head (candidatePublished)
|
||||
// OwnBuild answers the fingerprint of the build this program runs, from the node-engine's seat; nil knows none.
|
||||
OwnBuild func(ctx context.Context) (string, error)
|
||||
judgeMu sync.Mutex // one judge pass at a time (verdicts.go)
|
||||
ownMu sync.Mutex // guards ownFP
|
||||
ownFP string // the fingerprint of the build this program runs, asked at the start pass (learnOwnBuild)
|
||||
judgePending atomic.Bool // the start's judge pass failed: judged again at the next message (verdicts.go)
|
||||
}
|
||||
|
||||
func (ix *Indexer) logf(format string, args ...any) {
|
||||
|
||||
@@ -25,9 +25,11 @@ const (
|
||||
// TakeInAsk is a value of the takeins state: which module a person asked to be taken in, why, who and when, and, once the
|
||||
// indexer read it, its answer.
|
||||
type TakeInAsk struct {
|
||||
V int `json:"v"`
|
||||
Module string `json:"module,omitempty"`
|
||||
Why string `json:"why,omitempty"`
|
||||
V int `json:"v"`
|
||||
Module string `json:"module,omitempty"`
|
||||
Why string `json:"why,omitempty"`
|
||||
// Judge asks the manifests to be judged again now (indexer_judge_again) instead of the module to be taken in.
|
||||
Judge bool `json:"judge,omitempty"`
|
||||
By string `json:"by,omitempty"`
|
||||
At time.Time `json:"at"`
|
||||
Result *TakenIn `json:"result,omitempty"`
|
||||
|
||||
@@ -353,7 +353,7 @@ func (ix *Indexer) writeOverlay(ctx context.Context, overlay, finals map[string]
|
||||
if held && cur.Manifest == next.Manifest {
|
||||
// The judge's verdict stands while the digest does (ADR 0578 §3, ADR 0579 §1): a recompute never writes a
|
||||
// refused or outgrown module as accepted; only the judge changes it.
|
||||
next.Verdict, next.Problems, next.Warnings = cur.Verdict, cur.Problems, cur.Warnings
|
||||
next.Verdict, next.Problems, next.Unknown, next.Warnings = cur.Verdict, cur.Problems, cur.Unknown, cur.Warnings
|
||||
}
|
||||
if held && sameButWhere(cur, next) {
|
||||
return nil, false, nil
|
||||
@@ -380,7 +380,7 @@ func (ix *Indexer) writeOverlay(ctx context.Context, overlay, finals map[string]
|
||||
func sameButWhere(cur, next delivery.Module) bool {
|
||||
next = withV(next)
|
||||
next.Commit, next.Src = cur.Commit, cur.Src
|
||||
next.Verdict, next.Problems, next.Warnings = cur.Verdict, cur.Problems, cur.Warnings
|
||||
next.Verdict, next.Problems, next.Unknown, next.Warnings = cur.Verdict, cur.Problems, cur.Unknown, cur.Warnings
|
||||
return reflect.DeepEqual(cur, next)
|
||||
}
|
||||
|
||||
|
||||
@@ -7,6 +7,8 @@ import (
|
||||
"fmt"
|
||||
"maps"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
@@ -18,19 +20,23 @@ import (
|
||||
// The indexer judges every manifest it holds, once, with its own judge (novox/hq ADR 0578 §1): it writes each verdict in
|
||||
// its state verdicts (delivery.VerdictsBucket), one key per manifest digest, only when the verdict differs from the one
|
||||
// held; a module's value in the modules state carries a refusal (Verdict, Problems), so a reader of modules takes it
|
||||
// without a second read; a new refusal is said as the event manifest-refused. It judges at its start, so a judge that
|
||||
// without a second read; a new refusal is said as the event manifest-refused; a manifest with a field the judge does not know waits instead (ownBuildIsNewest). It judges at its start, so a judge that
|
||||
// changed (a new indexer) judges every manifest again, and after every merge it takes in, since a rule between manifests
|
||||
// can turn on any of them. Nothing polls.
|
||||
|
||||
// VerdictsBucket is the indexer's state of verdicts.
|
||||
// VerdictsBucket is the indexer's state of verdicts; VerdictsState is its local name, as the tools bundle reads it.
|
||||
var VerdictsBucket = delivery.VerdictsBucket
|
||||
|
||||
const VerdictsState = "verdicts"
|
||||
|
||||
// Judged is what one pass of the judge did: the manifests judged, the verdicts written, and the modules refused.
|
||||
type Judged struct {
|
||||
Judged, Written int
|
||||
Refused map[string][]string
|
||||
// Outgrown are the modules accepted before whose problems are now warnings (ADR 0579 §1).
|
||||
Outgrown []string
|
||||
// Waiting are the modules whose manifest carries a field this judge does not know and waits.
|
||||
Waiting []string
|
||||
}
|
||||
|
||||
// JudgeAll judges every manifest the modules state names, as one catalogue with the predefined list the indexer keeps,
|
||||
@@ -43,6 +49,10 @@ type Judged struct {
|
||||
// watchman raises as manifest-outgrown. A manifest in fresh (the digests a merge just took in) with no verdict yet, and
|
||||
// one already refused, is judged in full.
|
||||
func (ix *Indexer) JudgeAll(ctx context.Context, fresh map[string]bool) (Judged, error) {
|
||||
// A merge, the start pass and indexer_judge may all judge at once: one pass at a time, so none writes a verdict from
|
||||
// modules another is rewriting.
|
||||
ix.judgeMu.Lock()
|
||||
defer ix.judgeMu.Unlock()
|
||||
var out Judged
|
||||
if ix.Verdicts == nil || ix.Modules == nil || ix.Manifests == nil {
|
||||
return out, nil
|
||||
@@ -74,11 +84,15 @@ func (ix *Indexer) JudgeAll(ctx context.Context, fresh map[string]bool) (Judged,
|
||||
raws[name] = []byte(kept.JSON)
|
||||
digests[name] = digest
|
||||
}
|
||||
refused, unnamed := judge.Shelf(raws, ix.predefinedList(ctx), 0)
|
||||
refused, waits, unnamed := judge.ShelfWaiting(raws, ix.predefinedList(ctx), 0)
|
||||
for _, p := range unnamed {
|
||||
ix.logf("judge: a problem between the manifests that no one module's removal ends, so it refuses none: %s", p)
|
||||
}
|
||||
out.Refused = refused
|
||||
// A manifest with a field this judge does not know waits for a judge that may know it (clause 3 of the record on a
|
||||
// release's Approve; ADR 0578 §1): it is refused only when this program is the newest build of the indexer the trunk
|
||||
// holds, so no judge that could know the field is still to come.
|
||||
newest := ix.ownBuildIsNewest(held)
|
||||
id := ix.JudgeID
|
||||
if id == "" {
|
||||
id = judge.Identity()
|
||||
@@ -86,7 +100,21 @@ func (ix *Indexer) JudgeAll(ctx context.Context, fresh map[string]bool) (Judged,
|
||||
for _, name := range slices.Sorted(maps.Keys(digests)) {
|
||||
out.Judged++
|
||||
v := delivery.ManifestVerdict{Module: name, State: delivery.VerdictAccepted, Judge: id, At: ix.now()}
|
||||
if problems := refused[name]; len(problems) > 0 {
|
||||
if fields := waits[name]; len(fields) > 0 {
|
||||
accepted := ix.acceptedBefore(ctx, digests[name], fresh)
|
||||
if !newest && !accepted {
|
||||
v.State, v.Unknown = delivery.VerdictManifestWaits, fields
|
||||
out.Waiting = append(out.Waiting, name)
|
||||
} else {
|
||||
_, problems := judge.Manifest(raws[name], ix.predefinedList(ctx))
|
||||
if accepted {
|
||||
v.Warnings = problems
|
||||
out.Outgrown = append(out.Outgrown, name)
|
||||
} else {
|
||||
v.State, v.Problems = delivery.VerdictRefused, problems
|
||||
}
|
||||
}
|
||||
} else if problems := refused[name]; len(problems) > 0 {
|
||||
if ix.acceptedBefore(ctx, digests[name], fresh) {
|
||||
v.Warnings = problems
|
||||
out.Outgrown = append(out.Outgrown, name)
|
||||
@@ -104,6 +132,9 @@ func (ix *Indexer) JudgeAll(ctx context.Context, fresh map[string]bool) (Judged,
|
||||
}
|
||||
if wrote {
|
||||
out.Written++
|
||||
if v.Waits() {
|
||||
ix.logf("judge: %s", v.Why())
|
||||
}
|
||||
if v.Refused() {
|
||||
ix.logf("judge: %s", v.Why())
|
||||
if ix.SayRefused != nil {
|
||||
@@ -121,6 +152,7 @@ func (ix *Indexer) JudgeAll(ctx context.Context, fresh map[string]bool) (Judged,
|
||||
// JudgeAtStart is JudgeAll of every manifest held, none fresh, as the indexer starts (a judge that changed judges every
|
||||
// one again); one that fails is judged again at the next message or source revision heard (kickJudge).
|
||||
func (ix *Indexer) JudgeAtStart(ctx context.Context) (Judged, error) {
|
||||
ix.learnOwnBuild(ctx)
|
||||
j, err := ix.JudgeAll(ctx, nil)
|
||||
ix.judgePending.Store(err != nil)
|
||||
return j, err
|
||||
@@ -149,7 +181,7 @@ func (ix *Indexer) acceptedBefore(ctx context.Context, digest string, fresh map[
|
||||
return !fresh[digest]
|
||||
}
|
||||
prev, err := delivery.DecodeManifestVerdict(value)
|
||||
return err == nil && !prev.Refused()
|
||||
return err == nil && prev.State == delivery.VerdictAccepted
|
||||
}
|
||||
|
||||
// writeVerdict writes v under the digest's key unless the verdict held there says the same (ManifestVerdict.Same); wrote
|
||||
@@ -173,6 +205,36 @@ func (ix *Indexer) writeVerdict(ctx context.Context, digest string, v delivery.M
|
||||
return wrote, err
|
||||
}
|
||||
|
||||
// learnOwnBuild asks where the indexer's program is told which build it runs (OwnBuild: the node-engine's seat, never the
|
||||
// trunk's value of the indexer, which names a build that may be merged and not delivered). Asked at the start pass; where
|
||||
// it is not answered the build stays unknown and every unknown field waits, which is the safe side.
|
||||
func (ix *Indexer) learnOwnBuild(ctx context.Context) {
|
||||
if ix.OwnBuild == nil {
|
||||
return
|
||||
}
|
||||
fp, err := ix.OwnBuild(ctx)
|
||||
if err != nil || fp == "" {
|
||||
ix.logf("judge: the build this program runs is not known (%v), so a manifest with a field it does not know waits and is not refused", err)
|
||||
return
|
||||
}
|
||||
ix.ownMu.Lock()
|
||||
ix.ownFP = fp
|
||||
ix.ownMu.Unlock()
|
||||
}
|
||||
|
||||
// ownBuildIsNewest says whether the build of the indexer this program runs (ownFP, from the node-engine) is the newest the
|
||||
// trunk holds: the indexer's own modules value names it. A newer build merged and not yet delivered ends it at the trunk's
|
||||
// merge, as one that pins a changed mesh-sdk always is; false when either is unknown.
|
||||
func (ix *Indexer) ownBuildIsNewest(held map[string]delivery.Module) bool {
|
||||
fp := held[delivery.IndexerModule].FP2
|
||||
if fp == "" {
|
||||
return false
|
||||
}
|
||||
ix.ownMu.Lock()
|
||||
defer ix.ownMu.Unlock()
|
||||
return ix.ownFP != "" && ix.ownFP == fp
|
||||
}
|
||||
|
||||
// carryVerdict puts a refusal into the module's value, and takes one out once the manifest is accepted, while the value
|
||||
// still names the digest judged; an accepted module's value carries nothing, so judging writes no value of a module the
|
||||
// judge accepts (a reader takes a value without a verdict as accepted, delivery.TakeManifestVerdict).
|
||||
@@ -189,14 +251,18 @@ func (ix *Indexer) carryVerdict(ctx context.Context, name, digest string, v deli
|
||||
if err != nil || cur.Manifest != digest {
|
||||
return nil, false, nil // gone, unreadable, or moved to another manifest since: the next pass judges it
|
||||
}
|
||||
state := ""
|
||||
if v.Refused() {
|
||||
// A wait is written as a refusal naming the fields, so a reader pinned before the verdict waits holds it as refused.
|
||||
state, problems := "", v.Problems
|
||||
switch {
|
||||
case v.Waits():
|
||||
state, problems = delivery.VerdictRefused, []string{v.WaitsOn()}
|
||||
case v.Refused():
|
||||
state = delivery.VerdictRefused
|
||||
}
|
||||
if cur.Verdict == state && slices.Equal(cur.Problems, v.Problems) && slices.Equal(cur.Warnings, v.Warnings) {
|
||||
if cur.Verdict == state && slices.Equal(cur.Problems, problems) && slices.Equal(cur.Unknown, v.Unknown) && slices.Equal(cur.Warnings, v.Warnings) {
|
||||
return nil, false, nil
|
||||
}
|
||||
cur.Verdict, cur.Problems, cur.Warnings = state, v.Problems, v.Warnings
|
||||
cur.Verdict, cur.Problems, cur.Unknown, cur.Warnings = state, problems, v.Unknown, v.Warnings
|
||||
if state == "" {
|
||||
cur.Problems = nil
|
||||
}
|
||||
@@ -226,3 +292,111 @@ func (ix *Indexer) predefinedList(ctx context.Context) catalogue.PredefinedList
|
||||
}
|
||||
return list
|
||||
}
|
||||
|
||||
// The tools that read and steer the verdicts (the instruction "tools are how a module is spoken to"): indexer_verdicts
|
||||
// lists the manifests that wait or are refused, and indexer_judge judges one module again now.
|
||||
const (
|
||||
VerdictsTool = "indexer_verdicts"
|
||||
JudgeAgainTool = "indexer_judge"
|
||||
judgeAgainPrefix = "judge."
|
||||
)
|
||||
|
||||
// VerdictRow is one verdict as indexer_verdicts lists it: the manifest's digest, the module, its state, the fields it waits
|
||||
// on or the problems that refuse it, and the judge that made it.
|
||||
type VerdictRow struct {
|
||||
Module string `json:"module"`
|
||||
Digest string `json:"digest"`
|
||||
State string `json:"state"`
|
||||
Words string `json:"words,omitempty"`
|
||||
Unknown []string `json:"unknown,omitempty"`
|
||||
Problems []string `json:"problems,omitempty"`
|
||||
Warnings []string `json:"warnings,omitempty"`
|
||||
Judge string `json:"judge"`
|
||||
At time.Time `json:"at"`
|
||||
}
|
||||
|
||||
// ListVerdicts reads the verdicts under keys with get and lists those of module (every module when empty) that wait or are
|
||||
// refused, or every one with all, sorted by module and digest. A value that does not read is said in unread by its key.
|
||||
func ListVerdicts(keys []string, get func(key string) ([]byte, error), module string, all bool) (rows []VerdictRow, unread map[string]string) {
|
||||
unread = map[string]string{}
|
||||
for _, key := range keys {
|
||||
value, err := get(key)
|
||||
if err != nil || value == nil {
|
||||
if err != nil {
|
||||
unread[key] = err.Error()
|
||||
}
|
||||
continue
|
||||
}
|
||||
v, err := delivery.DecodeManifestVerdict(value)
|
||||
if err != nil {
|
||||
unread[key] = err.Error()
|
||||
continue
|
||||
}
|
||||
if (module != "" && v.Module != module) || (!all && !v.Held()) {
|
||||
continue
|
||||
}
|
||||
rows = append(rows, VerdictRow{Module: v.Module, Digest: key, State: v.State, Words: v.Why(), Unknown: v.Unknown,
|
||||
Problems: v.Problems, Warnings: v.Warnings, Judge: v.Judge, At: v.At})
|
||||
}
|
||||
slices.SortFunc(rows, func(a, b VerdictRow) int {
|
||||
if c := strings.Compare(a.Module, b.Module); c != 0 {
|
||||
return c
|
||||
}
|
||||
return strings.Compare(a.Digest, b.Digest)
|
||||
})
|
||||
return rows, unread
|
||||
}
|
||||
|
||||
// JudgeAgain judges the manifests again now, as at the start, and answers how the module's stands: the tool
|
||||
// indexer_judge. A module the modules state does not hold, or whose manifest is not in the manifests state, is
|
||||
// answered so.
|
||||
func (ix *Indexer) JudgeAgain(ctx context.Context, module string) TakenIn {
|
||||
out := TakenIn{Module: module}
|
||||
held, err := ix.heldModules(ctx)
|
||||
if err != nil {
|
||||
out.Answer = "the modules state cannot be read: " + err.Error()
|
||||
return out
|
||||
}
|
||||
m, ok := held[module]
|
||||
if !ok || m.Manifest == "" {
|
||||
out.Answer = "the indexer holds no manifest of " + module
|
||||
return out
|
||||
}
|
||||
// The build this program runs is asked again, so a start pass that missed the engine's answer is recovered without a restart.
|
||||
ix.learnOwnBuild(ctx)
|
||||
if _, err := ix.JudgeAll(ctx, nil); err != nil {
|
||||
out.Answer = "the manifests could not be judged: " + err.Error()
|
||||
return out
|
||||
}
|
||||
value, _, found, err := ix.Verdicts.Get(ctx, m.Manifest)
|
||||
if err != nil || !found {
|
||||
out.Answer = "judged, but the verdict on " + module + " cannot be read"
|
||||
return out
|
||||
}
|
||||
v, err := delivery.DecodeManifestVerdict(value)
|
||||
if err != nil {
|
||||
out.Answer = "judged, but the verdict on " + module + " cannot be read: " + err.Error()
|
||||
return out
|
||||
}
|
||||
out.Stored = true
|
||||
out.Answer = "judged again by " + v.Judge + ": " + v.State
|
||||
if v.Held() {
|
||||
out.Answer += " (" + v.Why() + ")"
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// NewJudgeAgain is the value indexer_judge writes for a module: its key in the takeins state and the value.
|
||||
func NewJudgeAgain(module, why, by string, at time.Time) (string, []byte, error) {
|
||||
key, data, err := NewTakeIn(module, why, by, at)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
var a TakeInAsk
|
||||
if err := json.Unmarshal(data, &a); err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
a.Judge = true
|
||||
data, err = json.Marshal(a)
|
||||
return judgeAgainPrefix + key, data, err
|
||||
}
|
||||
|
||||
@@ -2,6 +2,7 @@ package index
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -118,6 +119,85 @@ func TestTheIndexerJudgesEveryManifestOnce(t *testing.T) {
|
||||
if err := ix.writeOverlay(ctx, map[string]delivery.Module{"old": same}, nil, w); err != nil || !w.fresh[other] {
|
||||
t.Fatalf("a changed digest is not fresh: %v %v", w.fresh, err)
|
||||
}
|
||||
// Clause 3 of the record on a release's Approve: a manifest with a field this judge does not know waits (no refusal, no
|
||||
// event) while a newer build of the indexer is on the trunk, and is judged again at the next judge; once this program's
|
||||
// build is the trunk's newest it is refused; a wait is written into the module's value as a refusal naming the fields.
|
||||
ownFP := "src2:" + strings.Repeat("e", 64)
|
||||
hold(delivery.IndexerModule, `{"module":"mesh-delivery-indexer","version":"1"}`)
|
||||
ix.ownFP = ownFP // this program started when the trunk held another build of the indexer than the one it holds now
|
||||
later := hold("later", `{"module":"later","version":"1","a-field-from-later":1,"another":{"x":1}}`)
|
||||
said = nil
|
||||
before := len(said)
|
||||
j, err = ix.JudgeAll(ctx, map[string]bool{later: true})
|
||||
if err != nil || len(j.Waiting) != 1 || j.Waiting[0] != "later" || len(said) != before {
|
||||
t.Fatalf("a manifest with fields from later: %+v %v, said %d", j, err, len(said))
|
||||
}
|
||||
if v := verdict(later); !v.Waits() || v.Refused() || strings.Join(v.Unknown, ",") != "a-field-from-later,another" {
|
||||
t.Fatalf("later: %+v", v)
|
||||
}
|
||||
if m := module("later"); m.Verdict != delivery.VerdictRefused || len(m.Unknown) != 2 || !m.VerdictOf("later").Waits() ||
|
||||
!strings.Contains(m.Problems[0], "does not know yet") {
|
||||
t.Fatalf("modules.later carries %+v", m)
|
||||
}
|
||||
// The build this program runs is asked of the engine, and the trunk moving before the start pass changes nothing: the
|
||||
// trunk's value of the indexer names a newer build than the one the engine says runs, so the field still waits.
|
||||
ix.ownFP = ""
|
||||
ix.OwnBuild = func(context.Context) (string, error) { return ownFP, nil }
|
||||
if _, err := ix.JudgeAtStart(ctx); err != nil || !verdict(later).Waits() {
|
||||
t.Fatalf("the trunk moved before the start pass: %v %+v", err, verdict(later))
|
||||
}
|
||||
ix.OwnBuild = func(context.Context) (string, error) { return "", errors.New("the engine does not answer") }
|
||||
ix.ownFP = ""
|
||||
if _, err := ix.JudgeAtStart(ctx); err != nil || !verdict(later).Waits() {
|
||||
t.Fatalf("an unknown build refused: %v %+v", err, verdict(later))
|
||||
}
|
||||
ix.ownFP = ownFP
|
||||
// A renamed field is a known mistake: refused at once.
|
||||
renamed := hold("renamed", `{"module":"renamed","version":"1","needs":{"x":"y"}}`)
|
||||
if j, err := ix.JudgeAll(ctx, map[string]bool{renamed: true}); err != nil || len(j.Waiting) != 1 || !verdict(renamed).Refused() {
|
||||
t.Fatalf("a renamed field: %+v %v", j, err)
|
||||
}
|
||||
// The trunk's newest build of the indexer is this program's: the field is one no judge to come knows.
|
||||
ix.ownFP = "src2:" + strings.Repeat("a", 64)
|
||||
if j, err := ix.JudgeAll(ctx, nil); err != nil || len(j.Waiting) != 0 || !verdict(later).Refused() || !strings.Contains(verdict(later).Problems[0], "a-field-from-later") {
|
||||
t.Fatalf("the newest build refuses: %+v %v %+v", j, err, verdict(later))
|
||||
}
|
||||
if m := module("later"); m.Verdict != delivery.VerdictRefused || m.VerdictOf("later").Waits() || len(m.Unknown) != 0 {
|
||||
t.Fatalf("modules.later after the refusal carries %+v", m)
|
||||
}
|
||||
// The tools: indexer_verdicts lists the verdicts that wait or are refused (and every one with all), indexer_judge
|
||||
// judges now and says how the module's verdict stands.
|
||||
keys, _ := ix.Verdicts.Keys(ctx)
|
||||
get := func(key string) ([]byte, error) { v, _, _, err := ix.Verdicts.Get(ctx, key); return v, err }
|
||||
if rows, unread := ListVerdicts(keys, get, "later", false); len(rows) != 1 || rows[0].State != delivery.VerdictRefused || len(unread) != 0 {
|
||||
t.Fatalf("indexer_verdicts of later: %+v %v", rows, unread)
|
||||
}
|
||||
if rows, _ := ListVerdicts(keys, get, "", true); len(rows) <= len(keys)-3 || rows[0].Module > rows[len(rows)-1].Module {
|
||||
t.Fatalf("indexer_verdicts, all: %d rows of %d verdicts", len(rows), len(keys))
|
||||
}
|
||||
// indexer_judge asks the engine for the running build again: a start pass that missed it is recovered (the trunk's value of
|
||||
// the indexer is the newer one, so the field waits; asked and answered with that same build it is refused).
|
||||
ix.ownFP = ""
|
||||
ix.OwnBuild = func(context.Context) (string, error) { return "src2:" + strings.Repeat("a", 64), nil }
|
||||
if got := ix.JudgeAgain(ctx, "later"); !got.Stored || !strings.Contains(got.Answer, "refused") || ix.ownFP == "" {
|
||||
t.Fatalf("indexer_judge did not learn the running build: %+v", got)
|
||||
}
|
||||
ix.ownFP = ownFP
|
||||
ix.OwnBuild = nil
|
||||
if got := ix.JudgeAgain(ctx, "later"); !got.Stored || !strings.Contains(got.Answer, "waits") || !strings.Contains(got.Answer, "a-field-from-later") {
|
||||
t.Fatalf("indexer_judge: %+v", got)
|
||||
}
|
||||
if rows, _ := ListVerdicts(keys, get, "later", false); len(rows) != 1 {
|
||||
t.Fatalf("indexer_verdicts of later after: %+v", rows)
|
||||
}
|
||||
if got := ix.JudgeAgain(ctx, "nobody"); got.Stored || !strings.Contains(got.Answer, "holds no manifest") {
|
||||
t.Fatalf("indexer_judge of nobody: %+v", got)
|
||||
}
|
||||
if key, value, err := NewJudgeAgain("later", "why", "me", time.Now()); err != nil || key != "judge.later" {
|
||||
t.Fatalf("the ask: %q %v", key, err)
|
||||
} else if a, err := DecodeTakeIn(value); err != nil || !a.Judge || a.Module != "later" {
|
||||
t.Fatalf("the ask's value: %+v %v", a, err)
|
||||
}
|
||||
// Without the verdicts state the indexer judges nothing.
|
||||
ix.Verdicts = nil
|
||||
if j, err := ix.JudgeAll(ctx, nil); err != nil || j.Judged != 0 {
|
||||
|
||||
@@ -69,9 +69,27 @@ func problemsOf(err error) []string {
|
||||
// whose removal from the catalogue ends it (a rule two modules break together refuses both); one no removal ends is in
|
||||
// unnamed, for the caller to say. A module not in the answer is accepted.
|
||||
func Shelf(raws map[string][]byte, predefined catalogue.PredefinedList, longestMachine int) (refused map[string][]string, unnamed []string) {
|
||||
refused = map[string][]string{}
|
||||
refused, waits, unnamed := ShelfWaiting(raws, predefined, longestMachine)
|
||||
for name := range waits {
|
||||
// Strictly, as the checker judges a change: the judge that runs there is the one built at the change's head, which
|
||||
// knows the fields the change brings, so a field it does not know is a mistake.
|
||||
_, refused[name] = Manifest(raws[name], predefined)
|
||||
}
|
||||
return refused, unnamed
|
||||
}
|
||||
|
||||
// ShelfWaiting is Shelf where a manifest that carries a field this judge does not know (UnknownFields) is not refused: it is
|
||||
// in waits, by module, naming each field, and judged no further, because a newer judge may know the field (novox/hq ADR
|
||||
// 0578 §1, amended). The indexer judges so; the checker, which runs the judge built at the head of the change, judges with
|
||||
// Shelf.
|
||||
func ShelfWaiting(raws map[string][]byte, predefined catalogue.PredefinedList, longestMachine int) (refused, waits map[string][]string, unnamed []string) {
|
||||
refused, waits = map[string][]string{}, map[string][]string{}
|
||||
shelf := catalogue.Shelf{}
|
||||
for name, raw := range raws {
|
||||
if unknown := UnknownFields(raw); len(unknown) > 0 {
|
||||
waits[name] = unknown
|
||||
continue
|
||||
}
|
||||
m, problems := Manifest(raw, predefined)
|
||||
if len(problems) > 0 {
|
||||
refused[name] = problems
|
||||
@@ -122,7 +140,7 @@ func Shelf(raws map[string][]byte, predefined catalogue.PredefinedList, longestM
|
||||
for name := range refused {
|
||||
sort.Strings(refused[name])
|
||||
}
|
||||
return refused, unnamed
|
||||
return refused, waits, unnamed
|
||||
}
|
||||
|
||||
// Between is every problem between the manifests of shelf, sorted: catalogue.CatalogueProblems, IdentityProblems over
|
||||
|
||||
@@ -54,6 +54,21 @@ func TestTheJudge(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
// A field the judge does not know waits in the indexer's judge and stays a refusal in the checker's (clause 3 of the
|
||||
// record on a release's Approve); a renamed field and any other problem are refused by both.
|
||||
waitRaws := map[string][]byte{
|
||||
"later": []byte(`{"module":"later","version":"1","core-credentials":true,"other":1}`),
|
||||
"renamed": []byte(`{"module":"renamed","version":"1","needs":{"x":"y"}}`),
|
||||
"bad": []byte(`{"module":"bad","version":"1","provides":[{"name":"amqp"}]}`),
|
||||
"ok": []byte(`{"module":"ok","version":"1"}`),
|
||||
}
|
||||
refused, waits, _ := ShelfWaiting(waitRaws, nil, 0)
|
||||
if !reflect.DeepEqual(waits, map[string][]string{"later": {"core-credentials", "other"}}) || len(refused) != 2 || refused["renamed"] == nil || refused["bad"] == nil {
|
||||
t.Errorf("ShelfWaiting: waits %v, refused %v", waits, refused)
|
||||
}
|
||||
if strict, _ := Shelf(waitRaws, nil, 0); len(strict) != 3 || !strings.Contains(strings.Join(strict["later"], ""), "core-credentials") {
|
||||
t.Errorf("Shelf refuses as before: %v", strict)
|
||||
}
|
||||
// The private network's manifest, which the mesh provides itself, is accepted (moved from mesh-sdk overlay's test).
|
||||
if raw, err := json.Marshal(overlay.Manifest()); err != nil {
|
||||
t.Fatal(err)
|
||||
|
||||
@@ -2388,6 +2388,44 @@ func ownSecretPromptProblems(m catalogue.Manifest, name, prompt string, family b
|
||||
return problems
|
||||
}
|
||||
|
||||
// unknownFieldText is how the lenient decoder words the first field its shape does not have, and readWithout the list of
|
||||
// every field it read the manifest without (catalogue.Manifest.SetAside).
|
||||
var (
|
||||
unknownFieldText = regexp.MustCompile(`json: unknown field "([^"]+)"`)
|
||||
readWithout = regexp.MustCompile(`\(read without ([^)]*)\)`)
|
||||
)
|
||||
|
||||
// UnknownFields are the fields of a manifest the judge does not know, as the lenient decoder sets them aside (catalogue.Decode,
|
||||
// each once, the path it names them with), or none: a manifest this judge reads whole has none, and a field that used to
|
||||
// mean something (renamed) is a known mistake, not a field a later judge may know, so a manifest with one has none
|
||||
// either.
|
||||
func UnknownFields(raw []byte) []string {
|
||||
m, err := catalogue.Decode(raw)
|
||||
aside := m.SetAside()
|
||||
if err != nil || aside == "" {
|
||||
return nil
|
||||
}
|
||||
first := unknownFieldText.FindStringSubmatch(aside)
|
||||
if first == nil {
|
||||
return nil // set aside for another reason: the judge's refusal as it is
|
||||
}
|
||||
fields := []string{first[1]}
|
||||
if list := readWithout.FindStringSubmatch(aside); list != nil {
|
||||
fields = fields[:0]
|
||||
for _, f := range strings.Split(list[1], ", ") {
|
||||
if f = strings.TrimSpace(f); f != "" && !slices.Contains(fields, f) {
|
||||
fields = append(fields, f)
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, f := range fields {
|
||||
if _, was := renamed[f[strings.LastIndex(f, ".")+1:]]; was {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return fields
|
||||
}
|
||||
|
||||
func parseManifest(raw []byte, predefined catalogue.PredefinedList) (catalogue.Manifest, error) {
|
||||
var m catalogue.Manifest
|
||||
// Strictly. **An unknown key is refused**, which is the discipline the host's declaration
|
||||
|
||||
@@ -109,11 +109,13 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build
|
||||
// Three invokes: the file of a registered module's manifest and lock files (registered.go), the forge's trees (the named paths' objects, since the indexer is the one reader of a manifest's paths:
|
||||
// ADR 0499 clauses 2 and 3, ADR 0495 clause 1; and the module directories of the trunk, issue 1900) and whether a
|
||||
// commit is in another (issue 1900).
|
||||
if want := []string{"gitea.gitea_commit_trees", "gitea.gitea_contains", "gitea.gitea_get_file", "gitea.gitea_open_pulls"}; !reflect.DeepEqual(m.Invokes, want) {
|
||||
if want := []string{"gitea.gitea_commit_trees", "gitea.gitea_contains", "gitea.gitea_get_file", "gitea.gitea_open_pulls", "seat:node-engine.composition"}; !reflect.DeepEqual(m.Invokes, want) {
|
||||
t.Errorf("invokes = %v, want %v", m.Invokes, want)
|
||||
}
|
||||
// 1945: indexer_take_in, served by its tools bundle through the takeins state; the process serves none.
|
||||
if want := []string{"indexer_take_in"}; !reflect.DeepEqual(m.Tools, want) {
|
||||
// 1945: indexer_take_in, served by its tools bundle through the takeins state; the process serves none. The same bundle
|
||||
// reads the verdicts state (indexer_verdicts) and asks the indexer to judge again through the takeins state
|
||||
// (indexer_judge; issue 774, clause 3).
|
||||
if want := []string{"indexer_take_in", "indexer_verdicts", "indexer_judge"}; !reflect.DeepEqual(m.Tools, want) {
|
||||
t.Errorf("tools = %v, want %v", m.Tools, want)
|
||||
}
|
||||
// No streams, no bus, no seat (ADR 0437 §4); it says no event.
|
||||
|
||||
@@ -3,7 +3,9 @@
|
||||
"version": "1",
|
||||
"slug": "indexer",
|
||||
"tools": [
|
||||
"indexer_take_in"
|
||||
"indexer_take_in",
|
||||
"indexer_verdicts",
|
||||
"indexer_judge"
|
||||
],
|
||||
"consumes": [
|
||||
"mesh-build-go.toolchain.reported",
|
||||
@@ -38,7 +40,8 @@
|
||||
"gitea.gitea_commit_trees",
|
||||
"gitea.gitea_contains",
|
||||
"gitea.gitea_get_file",
|
||||
"gitea.gitea_open_pulls"
|
||||
"gitea.gitea_open_pulls",
|
||||
"seat:node-engine.composition"
|
||||
],
|
||||
"own-secrets": {
|
||||
"broker": "${dir:state}/broker"
|
||||
|
||||
@@ -3,7 +3,7 @@ module mesh-delivery-teller
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102191058-f02f142af128
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103120252-42bd7b802ac2
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102191058-f02f142af128 h1:aI1DhZAuqVBXbybSqhLNOGX5rUOrn9T0cHNCDUQF7IQ=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102191058-f02f142af128/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103120252-42bd7b802ac2 h1:MOuutmbKBL1GtTZU9W1+ja/K7TL05EFU6hEt3Swf/oE=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103120252-42bd7b802ac2/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
|
||||
@@ -3,7 +3,7 @@ module mesh-delivery-view
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102191058-f02f142af128
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103120252-42bd7b802ac2
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102191058-f02f142af128 h1:aI1DhZAuqVBXbybSqhLNOGX5rUOrn9T0cHNCDUQF7IQ=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102191058-f02f142af128/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103120252-42bd7b802ac2 h1:MOuutmbKBL1GtTZU9W1+ja/K7TL05EFU6hEt3Swf/oE=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103120252-42bd7b802ac2/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
|
||||
@@ -3,7 +3,7 @@ module mesh-delivery-watchman
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102191058-f02f142af128
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103120252-42bd7b802ac2
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102191058-f02f142af128 h1:aI1DhZAuqVBXbybSqhLNOGX5rUOrn9T0cHNCDUQF7IQ=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102191058-f02f142af128/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103120252-42bd7b802ac2 h1:MOuutmbKBL1GtTZU9W1+ja/K7TL05EFU6hEt3Swf/oE=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103120252-42bd7b802ac2/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op h1:1BOWQJweNyvZMlpAHXGLiZQn9S+QXGcz3xh94lC0w6E=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
|
||||
@@ -59,6 +59,18 @@ func TestTheWatchmansFindings(t *testing.T) {
|
||||
t.Fatalf("findings = %v", o.findings)
|
||||
}
|
||||
w.ModuleVerdict("node-tools", value(""), Put)
|
||||
// Clause 3 of the record on a release's Approve: a manifest that waits for a judge that knows its fields is its own
|
||||
// warning (a refusal in the value, naming the fields), not manifest-refused, and goes when it is accepted.
|
||||
waiting, _ := delivery.EncodeModule(delivery.Module{FP2: "src2:" + strings.Repeat("a", 64), Manifest: strings.Repeat("b", 64),
|
||||
Repository: "novox/mesh-catalog", Commit: "c", Verdict: delivery.VerdictRefused, Problems: []string{"waits"}, Unknown: []string{"core-credentials"}})
|
||||
w.ModuleVerdict("nats", waiting, Put)
|
||||
if f := o.findings[FindingKey(ScopeModule, "nats", KindManifestWaits)]; f.Severity != Warning || !strings.Contains(f.Summary, "core-credentials") {
|
||||
t.Fatalf("findings = %v", o.findings)
|
||||
}
|
||||
if _, refused := o.findings[FindingKey(ScopeModule, "nats", KindManifestRefused)]; refused {
|
||||
t.Fatalf("a waiting manifest was found refused: %v", o.findings)
|
||||
}
|
||||
w.ModuleVerdict("nats", value(""), Put)
|
||||
}, map[string]string{}},
|
||||
{"WM4: a stuck key not moved is its finding, and its move deletes it", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) {
|
||||
w.Step("go", "n1.ask1", runningStep("ask1", t0), Put)
|
||||
|
||||
@@ -14,6 +14,11 @@ const KindManifestRefused = "manifest-refused"
|
||||
// (novox/hq ADR 0579 §1): it stays accepted and runs; the warning stands until the manifest is mended.
|
||||
const KindManifestOutgrown = delivery.ManifestOutgrown
|
||||
|
||||
// KindManifestWaits is a module whose manifest carries a field the indexer's judge does not know yet (novox/hq ADR 0578
|
||||
// §1, amended): every reader holds it as a refused one and keeps what runs, and the warning stands until a judge knows the
|
||||
// field or the manifest is mended.
|
||||
const KindManifestWaits = delivery.ManifestWaits
|
||||
|
||||
// ModuleVerdict is a change of the indexer's value of a module (delivery.ModulesBucket): a value carrying a refusal is a
|
||||
// finding, one that does not (accepted, or not judged yet) or a deleted key ends it.
|
||||
func (w *Watchman) ModuleVerdict(module string, value []byte, op Op) {
|
||||
@@ -21,9 +26,11 @@ func (w *Watchman) ModuleVerdict(module string, value []byte, op Op) {
|
||||
defer w.mu.Unlock()
|
||||
key := FindingKey(ScopeModule, module, KindManifestRefused)
|
||||
outgrown := FindingKey(ScopeModule, module, KindManifestOutgrown)
|
||||
waits := FindingKey(ScopeModule, module, KindManifestWaits)
|
||||
if op != Put {
|
||||
w.unfind(key)
|
||||
w.unfind(outgrown)
|
||||
w.unfind(waits)
|
||||
return
|
||||
}
|
||||
m, err := delivery.DecodeModule(value)
|
||||
@@ -32,6 +39,20 @@ func (w *Watchman) ModuleVerdict(module string, value []byte, op Op) {
|
||||
return
|
||||
}
|
||||
v := m.VerdictOf(module)
|
||||
if v != nil && v.Waits() {
|
||||
w.unfind(key)
|
||||
w.unfind(outgrown)
|
||||
w.find(Finding{Scope: ScopeModule, ID: module, Kind: KindManifestWaits, Severity: Warning, Source: delivery.ModulesBucket,
|
||||
Summary: v.Why(),
|
||||
Said: strings.Join(v.Unknown, "; "),
|
||||
Headline: "The manifest of " + module + " waits for a judge that knows its fields",
|
||||
Explanation: "The manifest carries " + strings.Join(v.Unknown, ", ") + ", which the indexer's judge does not know yet, so every " +
|
||||
"reader holds " + module + " as a refused one and keeps what runs; an indexer that knows the fields judges it again " +
|
||||
"at its start, and the newest build of the indexer refuses a field it does not know.",
|
||||
Needs: "nothing while a newer indexer is on its way; else a pull request that mends the manifest of " + module})
|
||||
return
|
||||
}
|
||||
w.unfind(waits)
|
||||
if refused, _ := delivery.TakeManifestVerdict(v); !refused {
|
||||
w.unfind(key)
|
||||
if v == nil || len(v.Warnings) == 0 {
|
||||
|
||||
@@ -3,7 +3,7 @@ module mesh-enrolment
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102191058-f02f142af128
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103120252-42bd7b802ac2
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102191058-f02f142af128 h1:aI1DhZAuqVBXbybSqhLNOGX5rUOrn9T0cHNCDUQF7IQ=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102191058-f02f142af128/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103120252-42bd7b802ac2 h1:MOuutmbKBL1GtTZU9W1+ja/K7TL05EFU6hEt3Swf/oE=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103120252-42bd7b802ac2/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
|
||||
@@ -3,7 +3,7 @@ module mesh-grants
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102191058-f02f142af128
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103120252-42bd7b802ac2
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102191058-f02f142af128 h1:aI1DhZAuqVBXbybSqhLNOGX5rUOrn9T0cHNCDUQF7IQ=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102191058-f02f142af128/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103120252-42bd7b802ac2 h1:MOuutmbKBL1GtTZU9W1+ja/K7TL05EFU6hEt3Swf/oE=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103120252-42bd7b802ac2/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
|
||||
@@ -680,6 +680,22 @@ func TestGrants(t *testing.T) {
|
||||
if slices.Contains(web.Publish, "$JS.API.DIRECT.GET.KV_mesh-knowledge_sections.>") {
|
||||
t.Error("the object bucket's read is granted as a state's")
|
||||
}
|
||||
// ADR 0601 clause 3: the owner's trunk manifest now waits for a field; its running build's manifest still declares the
|
||||
// bucket, so web keeps its object grant.
|
||||
fp := "src2:" + strings.Repeat("cd", 32)
|
||||
b.put(grants.ReadModules, "mesh-knowledge", delivery.Module{V: delivery.Version, FP2: "src2:" + strings.Repeat("ab", 32),
|
||||
Manifest: delivery.ManifestDigest([]byte(`{"module":"mesh-knowledge","version":"2"}`)), Repository: "novox/x", Commit: "c",
|
||||
Verdict: delivery.VerdictRefused, Problems: []string{"it waits"}, Unknown: []string{"core-credentials"}})
|
||||
b.put(grants.ReadHave, "a.mesh-knowledge", delivery.Have{V: delivery.Version, Phase: delivery.PhaseRunning, Health: delivery.HealthOK, FP: fp})
|
||||
heldKey, _ := delivery.BuilderBuildKey("mesh-knowledge", fp)
|
||||
b.put(grants.ReadBuilds, heldKey, map[string]any{"v": delivery.Version, "state": "done", "manifest": delivery.ManifestDigest([]byte(knowledge))})
|
||||
if heldWeb := (func() grants.User {
|
||||
us := shadowOf(t, b)
|
||||
return us[slices.IndexFunc(us, func(u grants.User) bool { return u.User == "a.web" })]
|
||||
})(); !slices.Contains(heldWeb.Publish, read) {
|
||||
t.Errorf("a reader of a held module's object store lost its grant: %v", heldWeb.Publish)
|
||||
}
|
||||
b.manifestOf("mesh-knowledge", knowledge) // accepted again for the rest of the row
|
||||
// ADR 0592 §6, §25: the bucket past 75 percent of its cap is said, in one warning, and cleared once below; the history
|
||||
// stream at 10 percent is not.
|
||||
var said []asks.Condition
|
||||
@@ -739,6 +755,53 @@ func TestGrants(t *testing.T) {
|
||||
}
|
||||
})
|
||||
|
||||
// ADR 0601 clause 3: a held module's seats, hearings and reads are composed from the manifest its running build was judged
|
||||
// on, so a principal that claims a seat it defines keeps its grants while its trunk manifest is refused or waits.
|
||||
t.Run("a held module's seat is composed from the manifest its running build was judged on", func(t *testing.T) {
|
||||
const router = `{"module":"messenger","version":"1","runs-as":"messenger","resources":[{"id":"account","type":"user","name":"messenger","shell":"/usr/bin/nologin","home":"/var/lib/messenger"}],"own-secrets":{"broker":"/var/lib/m/broker"},"secrets-owner":"messenger",` +
|
||||
`"seats":[{"name":"operator-channel","scope":"mesh","serves":["open"]},{"name":"channel","scope":"mesh","kinded":true,"accepts":["show"]}],` +
|
||||
`"claims":[{"name":"operator-channel","scope":"mesh","serves":["open"]}]}`
|
||||
const telegram = `{"module":"telegram","version":"1","runs-as":"telegram","resources":[{"id":"account","type":"user","name":"telegram","shell":"/usr/bin/nologin","home":"/var/lib/telegram"}],"own-secrets":{"broker":"/var/lib/t/broker"},"secrets-owner":"telegram",` +
|
||||
`"claims":[{"name":"channel","scope":"mesh","kind":"telegram","capabilities":["deliver","verified-sender"]}]}`
|
||||
setup := func(held, runs bool) *fakeBus {
|
||||
b := fixture()
|
||||
b.manifestOf("messenger", router)
|
||||
b.manifestOf("telegram", telegram)
|
||||
b.assign("a", "messenger")
|
||||
b.assign("b", "telegram")
|
||||
if held {
|
||||
// The trunk's manifest waits for a field; the build that runs was judged on the manifest before.
|
||||
b.put(grants.ReadModules, "messenger", delivery.Module{V: delivery.Version, FP2: "src2:" + strings.Repeat("ab", 32),
|
||||
Manifest: delivery.ManifestDigest([]byte(`{"module":"messenger","version":"2"}`)), Repository: "novox/x", Commit: "c",
|
||||
Verdict: delivery.VerdictRefused, Problems: []string{"it waits"}, Unknown: []string{"core-credentials"}})
|
||||
if runs {
|
||||
fp := "src2:" + strings.Repeat("cd", 32)
|
||||
digest := delivery.ManifestDigest([]byte(router))
|
||||
b.put(grants.ReadManifests, digest, delivery.Manifest{V: delivery.Version, Module: "messenger", JSON: router})
|
||||
b.put(grants.ReadHave, "a.messenger", delivery.Have{V: delivery.Version, Phase: delivery.PhaseRunning, Health: delivery.HealthOK, FP: fp})
|
||||
key, _ := delivery.BuilderBuildKey("messenger", fp)
|
||||
b.put(grants.ReadBuilds, key, map[string]any{"v": delivery.Version, "state": "done", "manifest": digest})
|
||||
}
|
||||
}
|
||||
return b
|
||||
}
|
||||
grantsOf := func(b *fakeBus) string {
|
||||
for _, u := range shadowOf(t, b) {
|
||||
if u.User == "b.telegram" {
|
||||
return strings.Join(u.Publish, ",") + "|" + strings.Join(u.Subscribe, ",")
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
accepted, held, unseen := grantsOf(setup(false, false)), grantsOf(setup(true, true)), grantsOf(setup(true, false))
|
||||
if accepted == "" || held != accepted {
|
||||
t.Errorf("a caller of a held module's seat lost grants:\n%s\nwas\n%s", held, accepted)
|
||||
}
|
||||
if unseen == accepted {
|
||||
t.Errorf("the row proves nothing: the grants are the same without the held module's running manifest")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("root-free: each engine's word fills the composition, failing closed (issue 1945)", func(t *testing.T) {
|
||||
const router = `{"module":"messenger","version":"1","runs-as":"messenger","resources":[{"id":"account","type":"user","name":"messenger","shell":"/usr/bin/nologin","home":"/var/lib/messenger"}],"own-secrets":{"broker":"/var/lib/m/broker"},"secrets-owner":"messenger",` +
|
||||
`"seats":[{"name":"operator-channel","scope":"mesh","serves":["open"]},{"name":"channel","scope":"mesh","kinded":true,"accepts":["show"]}],` +
|
||||
@@ -1487,7 +1550,7 @@ func TestGrants(t *testing.T) {
|
||||
fp := partOn(b)
|
||||
fp.Objects = p.Objects
|
||||
fp.Say = func(c asks.Condition) error {
|
||||
if c.Key != grants.GrantMissingCondition && c.Key != grants.RefusedCondition { // their own rows
|
||||
if c.Key != grants.GrantMissingCondition && c.Key != grants.RefusedCondition && c.Key != grants.HeldModuleCondition { // their own rows
|
||||
fresh = append(fresh, c)
|
||||
}
|
||||
return nil
|
||||
@@ -2369,7 +2432,7 @@ func TestGrants(t *testing.T) {
|
||||
var said, behind []asks.Condition
|
||||
p.Say = func(c asks.Condition) error {
|
||||
switch c.Key {
|
||||
case grants.GrantMissingCondition, grants.RefusedCondition, grants.FullCondition: // their own rows
|
||||
case grants.GrantMissingCondition, grants.RefusedCondition, grants.HeldModuleCondition, grants.FullCondition: // their own rows
|
||||
case grants.ConsumerBehindCondition:
|
||||
behind = append(behind, c)
|
||||
default:
|
||||
@@ -2668,6 +2731,91 @@ func TestGrants(t *testing.T) {
|
||||
t.Errorf("the refusal was not cleared once it composes: %v, said %+v", res.PrincipalsRefused, said)
|
||||
}
|
||||
})
|
||||
|
||||
// Clause 3 of the record on a release's Approve (issue 774): a module whose manifest waits for a judge that knows a field
|
||||
// of it, or is refused, and whose want names no build, keeps its users of the list in force and is said in a condition;
|
||||
// the rest of the list is written as ever, and the users go the day the manifest is accepted and the module unassigned.
|
||||
t.Run("a module whose manifest waits or is refused keeps its users of the list in force", func(t *testing.T) {
|
||||
b := fixture()
|
||||
const ctl = `{"module":"mod-r","version":"1","own-secrets":{"broker":"/var/lib/r/broker","broker-mod-r-tools":"${dir:tools-secret}/broker-mod-r-tools"},` +
|
||||
`"resources":[{"id":"tools-secret","type":"directory","path":"/var/lib/mesh/mod-r-tools","mode":"0711"}],"build":{"artifacts":[{"name":"tools",` +
|
||||
`"kind":"bundle","language":"go","system":"arch","from":"cmd/mod-r-tools","binary":"mod-r-tools","loads":["mod-r-tools"],"bus":{}}]}}`
|
||||
b.manifestOf("mod-r", ctl)
|
||||
b.assign("b", "mod-r")
|
||||
for _, u := range shadowOf(t, b) {
|
||||
b.put(grants.ReadAccounts, u.User, map[string]any{"user": u.User, "hash": "$2a$10$first", "generation": 1, "fingerprint": "sha256:1"})
|
||||
}
|
||||
b.put(grants.ReadAccounts, "b.mod-r.mod-r-tools", map[string]any{"user": "b.mod-r.mod-r-tools", "hash": "$2a$10$first", "generation": 1, "fingerprint": "sha256:1"})
|
||||
busKey, err := seal.GenerateKeyPair()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b.put(grants.ReadHave, "a.nats", delivery.Have{V: delivery.Version, Phase: delivery.PhaseRunning, Health: delivery.HealthOK,
|
||||
UserList: &delivery.UserListWritten{Source: "a.nats.src2", Revision: 7, Digest: "sha256:aa", Users: 5, Key: busKey.PublicKey}})
|
||||
// mod-r runs a build whose manifest is the accepted one: the held module's seats and streams are composed from it.
|
||||
runFP := "src2:" + strings.Repeat("cd", 32)
|
||||
b.put(grants.ReadHave, "b.mod-r", delivery.Have{V: delivery.Version, Phase: delivery.PhaseRunning, Health: delivery.HealthOK, FP: runFP})
|
||||
runKey, _ := delivery.BuilderBuildKey("mod-r", runFP)
|
||||
b.put(grants.ReadBuilds, runKey, map[string]any{"v": delivery.Version, "state": "done", "manifest": delivery.ManifestDigest([]byte(ctl))})
|
||||
p := partOn(b)
|
||||
p.Objects = &fakeObjects{}
|
||||
var said []asks.Condition
|
||||
p.Say = func(c asks.Condition) error {
|
||||
if c.Key == grants.HeldModuleCondition {
|
||||
said = append(said, c)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
p.Compose()
|
||||
inForce, _, _ := p.Sealed()
|
||||
before, _, ok, _ := b.ownGet(grants.StateUserList, grants.SealedUserKey(inForce.Digest, "b.mod-r.mod-r-tools"))
|
||||
if !ok || len(said) != 1 || said[0].State != asks.StateCleared {
|
||||
t.Fatalf("the list in force keeps no grants of b.mod-r.mod-r-tools (%v), or a held module was said before: %+v", ok, said)
|
||||
}
|
||||
for gen, held := range []struct{ name, unknown string }{{"waits", "core-credentials"}, {"refused", ""}} {
|
||||
// Another user's hash changes in the same pass, so a list that is written shows the held module did not hold it.
|
||||
b.put(grants.ReadAccounts, "a.mod-x", map[string]any{"user": "a.mod-x", "hash": fmt.Sprintf("$2a$10$gen%d", gen), "generation": gen + 2, "fingerprint": fmt.Sprintf("sha256:%d", gen+2)})
|
||||
was, _, _ := p.Sealed()
|
||||
var unknown []string
|
||||
if held.unknown != "" {
|
||||
unknown = []string{held.unknown}
|
||||
}
|
||||
b.put(grants.ReadModules, "mod-r", delivery.Module{V: delivery.Version, FP2: "src2:" + strings.Repeat("ab", 32),
|
||||
Manifest: delivery.ManifestDigest([]byte(ctl)), Repository: "novox/x", Commit: "c", Verdict: delivery.VerdictRefused,
|
||||
Problems: []string{"it is held"}, Unknown: unknown})
|
||||
b.put(grants.ReadModules, "mod-q", delivery.Module{V: delivery.Version, FP2: "src2:" + strings.Repeat("ab", 32),
|
||||
Manifest: delivery.ManifestDigest([]byte(ctl)), Repository: "novox/x", Commit: "c"})
|
||||
res := p.Compose()
|
||||
now, _, _ := p.Sealed()
|
||||
after, _, ok, _ := b.ownGet(grants.StateUserList, grants.SealedUserKey(now.Digest, "b.mod-r.mod-r-tools"))
|
||||
if now.Digest == was.Digest || slices.ContainsFunc(res.Errors, func(e string) bool { return strings.Contains(e, "composed whole") }) {
|
||||
t.Errorf("%s: a held module held the whole list: sealed %+v", held.name, res.Sealed)
|
||||
}
|
||||
if !ok || string(after) != string(before) || len(res.PrincipalsRefused) != 0 {
|
||||
t.Errorf("%s: b.mod-r.mod-r-tools did not keep its grants of the list in force: %s, was %s", held.name, after, before)
|
||||
}
|
||||
if last := said[len(said)-1]; last.State != asks.StateRaised || !strings.Contains(last.Summary, "b.mod-r.mod-r-tools (stays in the list") {
|
||||
t.Errorf("%s: the kept user was not said: %+v", held.name, last)
|
||||
}
|
||||
}
|
||||
// A held module that runs no build the part can read declares nothing to anyone: the list in force is kept, said.
|
||||
delete(b.states[grants.ReadHave], "b.mod-r")
|
||||
b.put(grants.ReadModules, "mod-r", delivery.Module{V: delivery.Version, FP2: "src2:" + strings.Repeat("ab", 32),
|
||||
Manifest: delivery.ManifestDigest([]byte(ctl)), Repository: "novox/x", Commit: "c", Verdict: delivery.VerdictRefused,
|
||||
Problems: []string{"it is held"}, Unknown: []string{"core-credentials"}})
|
||||
b.put(grants.ReadAccounts, "a.mod-x", map[string]any{"user": "a.mod-x", "hash": "$2a$10$gen9", "generation": 9, "fingerprint": "sha256:9"})
|
||||
wasList, _, _ := p.Sealed()
|
||||
res := p.Compose()
|
||||
nowList, _, _ := p.Sealed()
|
||||
if nowList.Digest != wasList.Digest || res.Sealed == nil || !strings.Contains(res.Sealed.Why, "running manifest of mod-r") {
|
||||
t.Errorf("a held module with no running manifest did not keep the list in force: %+v", res.Sealed)
|
||||
}
|
||||
// Accepted again: the users compose from the manifest and the condition clears.
|
||||
b.manifestOf("mod-r", ctl)
|
||||
if p.Compose(); said[len(said)-1].State != asks.StateCleared {
|
||||
t.Errorf("the condition was not cleared once the manifest is accepted: %+v", said[len(said)-1])
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// fakeObjects is the part's object bucket of sealed lists.
|
||||
|
||||
@@ -0,0 +1,120 @@
|
||||
package grants
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
)
|
||||
|
||||
// **A module whose manifest waits or is refused keeps its bus users** (clause 3 of the record on a release's Approve; issue
|
||||
// 774). The indexer holds a manifest it refuses, or that waits for a judge that knows a field of it, out of the manifests
|
||||
// the part composes from (inputs.go, Refused), so a module with no want naming a build had no manifest to derive its users
|
||||
// from and its users left the list: the bus module's accounts among them, which locked its tools out. Now such a module's
|
||||
// users, the ones the list in force carries, stay in the list as it holds them, the same way a refused principal's do
|
||||
// (refused.go), and the part says so as the warning mesh-grants.module-held until the manifest is accepted. One module's
|
||||
// manifest never holds the list: every other principal is composed as ever. A module whose want names a build is composed
|
||||
// from the manifest of that build (Inputs.Running), not kept.
|
||||
|
||||
// HeldModuleCondition is the key of the warning raised while a module's manifest is held and its users are kept.
|
||||
const HeldModuleCondition = "mesh-grants.module-held"
|
||||
|
||||
// heldModuleUsers are, by user, why it is kept: the users of candidates (the ones the list in force and the bus carry) that
|
||||
// belong to a module placed on a node whose manifest the indexer holds (in.Refused) and whose want names no build. A user of
|
||||
// a module on a node is `<node>.<module>` or `<node>.<module>.<process>` (buscompose.Principal.Username). Pure.
|
||||
func heldModuleUsers(in Inputs, candidates []string) map[string]string {
|
||||
held := map[string]string{}
|
||||
for _, g := range in.Refused {
|
||||
if module, ok := strings.CutPrefix(g.Input, "manifest of "); ok {
|
||||
held[module] = g.Why
|
||||
}
|
||||
}
|
||||
out := map[string]string{}
|
||||
if len(held) == 0 {
|
||||
return out
|
||||
}
|
||||
for node, modules := range in.Assigned {
|
||||
for _, module := range modules {
|
||||
why, ok := held[module]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if _, runs := in.Running[node+"."+module]; runs {
|
||||
continue
|
||||
}
|
||||
prefix := node + "." + module
|
||||
for _, user := range candidates {
|
||||
if user == prefix || strings.HasPrefix(user, prefix+".") {
|
||||
out[user] = fmt.Sprintf("the manifest of %s is held: %s", module, why)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// inForceGrants are the grants of each user of the list in force as the part sealed them, or nil where it kept none.
|
||||
func (p *Part) inForceGrants() map[string]User {
|
||||
if g, _, had := p.Sealed(); had && g.Digest != "" && p.Store != nil {
|
||||
inForce, _ := p.sealedGrantsOf(g.Digest)
|
||||
return inForce
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// mergedRefusals is refused and held in one map, for the parts that keep a user's grants of the list in force.
|
||||
func mergedRefusals(refused, held map[string]string) map[string]string {
|
||||
if len(held) == 0 {
|
||||
return refused
|
||||
}
|
||||
out := make(map[string]string, len(refused)+len(held))
|
||||
for u, why := range held {
|
||||
out[u] = why
|
||||
}
|
||||
for u, why := range refused {
|
||||
out[u] = why
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// sayHeldModules raises HeldModuleCondition while a held module's users are kept, naming each user with what became of it,
|
||||
// and clears it once none is; said once per set, again when it changes, and at the first pass after a start.
|
||||
func (p *Part) sayHeldModules(held map[string]string, kept map[string]User) {
|
||||
if p.Say == nil {
|
||||
return
|
||||
}
|
||||
var lines []string
|
||||
users := make([]string, 0, len(held))
|
||||
for u := range held {
|
||||
users = append(users, u)
|
||||
}
|
||||
sort.Strings(users)
|
||||
for _, user := range users {
|
||||
became := "not in the list in force and left out"
|
||||
if _, ok := kept[user]; ok {
|
||||
became = "stays in the list as the list in force holds it"
|
||||
}
|
||||
lines = append(lines, user+" ("+became+"): "+held[user])
|
||||
}
|
||||
why := strings.Join(lines, "; ")
|
||||
p.holdMu.Lock()
|
||||
defer p.holdMu.Unlock()
|
||||
if p.heldModulesKnown && why == p.heldModulesSaid {
|
||||
return
|
||||
}
|
||||
c := asks.Condition{Key: HeldModuleCondition, Subject: asks.Scope{Scope: "module", ID: "mesh-grants"}}
|
||||
if why == "" {
|
||||
c.State = asks.StateCleared
|
||||
} else {
|
||||
c.Severity, c.State = asks.SeverityWarning, asks.StateRaised
|
||||
c.Summary = fmt.Sprintf("%d bus user(s) of a module whose manifest is held are kept in the list: %s", len(users), why)
|
||||
c.Headline, c.Needs = "A module's manifest is held, its bus users are kept", "a manifest the indexer accepts, or a judge that knows its fields"
|
||||
c.Explanation = asks.NeedsYou + "a manifest the indexer accepts (a refused one mended, or a waiting one judged by an indexer that knows its fields). " +
|
||||
"The rest of the user list is written as ever; a held module's users stay as the list in force holds them: " + why
|
||||
c.Resolved = "every module's manifest is accepted"
|
||||
}
|
||||
if p.Say(c) == nil {
|
||||
p.heldModulesSaid, p.heldModulesKnown = why, true
|
||||
}
|
||||
}
|
||||
@@ -131,7 +131,11 @@ type Inputs struct {
|
||||
// Refused is every manifest the indexer holds that the part cannot read or mesh-sdk's reader refuses, assigned or
|
||||
// not, each with why: its module counts as placed nowhere, so it is said loudly, never passed over (issue 1945).
|
||||
Refused []Gap
|
||||
Gaps []Gap
|
||||
// Held is, by module, the manifest a held module's running build was judged on (its have's fingerprint, the builder's
|
||||
// build of it): what the seats it defines, the streams others hear from it and its reads are composed from while its
|
||||
// trunk manifest is refused or waits, so no other principal loses a grant to it (ADR 0601 clause 3).
|
||||
Held map[string]catalogue.Manifest
|
||||
Gaps []Gap
|
||||
}
|
||||
|
||||
// HeldHash is one key of the vault's bus-accounts: the hash, and what it is of. The hash is never said by the part, and
|
||||
@@ -353,8 +357,8 @@ func ReadInputs(r Reader, cache *Manifests) Inputs {
|
||||
}
|
||||
var m catalogue.Manifest
|
||||
if err == nil {
|
||||
// The indexer's refusal, carried in its value (ADR 0578 §3): held and named as a refused manifest.
|
||||
if verdict := v.VerdictOf(module); verdict != nil && verdict.Refused() {
|
||||
// The indexer's refusal, or its wait for a judge that knows a field, carried in its value (ADR 0578 §3): held and named as a refused manifest.
|
||||
if verdict := v.VerdictOf(module); verdict != nil && verdict.Held() {
|
||||
err = errors.New(verdict.Why())
|
||||
}
|
||||
}
|
||||
@@ -440,6 +444,23 @@ func ReadInputs(r Reader, cache *Manifests) Inputs {
|
||||
sort.Strings(fromTrunk)
|
||||
in.Gaps = append(in.Gaps, Gap{Input: "wanted manifests", Why: "no build record names the manifest at the wanted fingerprint, so the trunk's is used: " + strings.Join(fromTrunk, ", ")})
|
||||
}
|
||||
in.Held = heldManifests(r, in, cache)
|
||||
// A held module placed on a node whose running manifest is not found declares no seat, stream or object bucket to
|
||||
// anyone: composing past it would drop the grants of those who use them, so the list in force is kept (ADR 0601
|
||||
// clause 3), and the gap says which module and why.
|
||||
for _, g := range in.Refused {
|
||||
module, ok := strings.CutPrefix(g.Input, "manifest of ")
|
||||
if _, found := in.Held[module]; !ok || found {
|
||||
continue
|
||||
}
|
||||
for _, assigned := range in.Assigned {
|
||||
if slices.Contains(assigned, module) {
|
||||
in.Gaps = append(in.Gaps, Gap{Input: "running manifest of " + module, Failed: true,
|
||||
Why: "its manifest is held and no running build's manifest is found, so the seats and streams it declares cannot be composed: the list in force is kept"})
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
in.Hashes = readHashes(r, &in.Gaps)
|
||||
// The seat mesh-enrolment's keys are every token since the controller's end (issue 1126).
|
||||
in.Enrolling, _ = readEnrolling(r, &in.Gaps)
|
||||
@@ -551,16 +572,81 @@ func sortGaps(g []Gap) {
|
||||
sort.SliceStable(g, func(i, j int) bool { return strings.Compare(g[i].Input, g[j].Input) < 0 })
|
||||
}
|
||||
|
||||
// ReadFailures names every read this pass could not make and every manifest refused: what a list composed now may lack.
|
||||
func (in Inputs) ReadFailures() []string {
|
||||
// ReadFailed names every read this pass could not make: what a list composed now may lack. A manifest the indexer holds
|
||||
// (refused, or waiting for a judge that knows a field of it) is not one: it holds its module's users in the list as the
|
||||
// list in force has them (held.go), and never holds the list, so one manifest never freezes the bus's users (issue 774).
|
||||
func (in Inputs) ReadFailed() []string {
|
||||
var out []string
|
||||
for _, g := range in.Gaps {
|
||||
if g.Failed {
|
||||
out = append(out, g.Input+": "+g.Why)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// ReadFailures is ReadFailed and every manifest refused or waiting: what the part retires nothing on (the accounts it
|
||||
// keeps, the hearings it withholds).
|
||||
func (in Inputs) ReadFailures() []string {
|
||||
out := in.ReadFailed()
|
||||
for _, g := range in.Refused {
|
||||
out = append(out, g.Input+": "+g.Why)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// heldManifests are the manifests the running builds of the held modules (in.Refused) were judged on, by module: the have
|
||||
// of the module on a node names the fingerprint that runs, and the builder's build of it the manifest. A module that runs
|
||||
// nowhere, or whose build or manifest cannot be read, has none, and that is said as a gap.
|
||||
func heldManifests(r Reader, in Inputs, cache *Manifests) map[string]catalogue.Manifest {
|
||||
out := map[string]catalogue.Manifest{}
|
||||
for _, g := range in.Refused {
|
||||
module, ok := strings.CutPrefix(g.Input, "manifest of ")
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
for _, node := range in.Nodes {
|
||||
raw, found, err := r.Get(ReadHave, node+"."+module)
|
||||
if err != nil || !found {
|
||||
continue
|
||||
}
|
||||
h, err := delivery.DecodeHave(raw)
|
||||
if err != nil || h.FP == "" {
|
||||
continue
|
||||
}
|
||||
bk, err := delivery.BuilderBuildKey(module, h.FP)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
braw, found, err := r.Get(ReadBuilds, bk)
|
||||
if err != nil || !found {
|
||||
continue
|
||||
}
|
||||
b, err := delivery.DecodeBuilderBuild(braw)
|
||||
if err != nil || b.Manifest == "" {
|
||||
continue
|
||||
}
|
||||
if m, err := cache.get(r, b.Manifest); err == nil {
|
||||
out[module] = m
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// declared are the manifests the seats, hearings and reads are composed from: the trunk's, and for a held module the
|
||||
// manifest its running build was judged on (Held).
|
||||
func (in Inputs) declared() map[string]catalogue.Manifest {
|
||||
if len(in.Held) == 0 {
|
||||
return in.Manifests
|
||||
}
|
||||
out := make(map[string]catalogue.Manifest, len(in.Manifests)+len(in.Held))
|
||||
for m, man := range in.Held {
|
||||
out[m] = man
|
||||
}
|
||||
for m, man := range in.Manifests {
|
||||
out[m] = man
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
@@ -189,6 +189,9 @@ type Part struct {
|
||||
// refusedSaid is the refused principals last said as a condition, once refusedKnown (refused.go).
|
||||
refusedSaid string
|
||||
refusedKnown bool
|
||||
// heldModulesSaid is the held modules' kept users last said as a condition, once heldModulesKnown (held.go).
|
||||
heldModulesSaid string
|
||||
heldModulesKnown bool
|
||||
// missingSaid is the modules without a grant last said as a condition, once missingKnown (grants.go).
|
||||
missingSaid string
|
||||
missingKnown bool
|
||||
@@ -259,14 +262,24 @@ func (p *Part) Compose() Result {
|
||||
res.Errors = append(res.Errors, "the shadow list could not be composed: "+err.Error())
|
||||
}
|
||||
// A refused principal keeps its grants of the list in force, said once per change (refused.go).
|
||||
kept, _ := p.keptOf(refused)
|
||||
// And a module whose manifest is held (refused, or waiting for a judge that knows a field of it) and whose want names no
|
||||
// build keeps its users of the list in force (held.go).
|
||||
loaded, loadErr := LoadedUsers(p.Call)
|
||||
inForce := p.inForceGrants()
|
||||
candidates := names(loaded)
|
||||
for user := range inForce {
|
||||
candidates = append(candidates, user)
|
||||
}
|
||||
heldUsers := heldModuleUsers(in, candidates)
|
||||
allRefused := mergedRefusals(refused, heldUsers)
|
||||
kept, _ := p.keptOf(allRefused)
|
||||
for _, user := range sortedKeys(refused) {
|
||||
res.PrincipalsRefused = append(res.PrincipalsRefused, user+": "+refused[user])
|
||||
}
|
||||
p.sayPrincipalsRefused(refused, kept)
|
||||
p.sayHeldModules(heldUsers, kept)
|
||||
users = WithHashes(withKept(users, kept), in.Hashes)
|
||||
st.Digest, st.Users = Digest(users), len(users)
|
||||
loaded, loadErr := LoadedUsers(p.Call)
|
||||
st.Compare = Compare(users, loaded, Placed(in)).Excusing(ExcusedBy(MultiHeld(listRecs)))
|
||||
if loadErr != nil {
|
||||
st.Compare.Unread = loadErr.Error()
|
||||
@@ -298,7 +311,7 @@ func (p *Part) Compose() Result {
|
||||
if p.Store != nil {
|
||||
p.writeWithheld(&res, in, recs)
|
||||
}
|
||||
list, n, leftOut, dropped, err := ComposeList(listRecs, in.Hashes, refused, kept)
|
||||
list, n, leftOut, dropped, err := ComposeList(listRecs, in.Hashes, allRefused, kept)
|
||||
// The grants the assigner reads (ADR 0547 §4, §5): one that adds a permission only once the bus loaded a list that
|
||||
// carries it (issue 774).
|
||||
if p.Store != nil {
|
||||
|
||||
@@ -27,7 +27,8 @@ func records(in Inputs, union bool) (buscompose.Records, []Gap) {
|
||||
var gaps []Gap
|
||||
seats := map[string]catalogue.SeatDeclaration{}
|
||||
declarers := map[string]string{}
|
||||
for _, m := range in.Manifests {
|
||||
declared := in.declared()
|
||||
for _, m := range declared {
|
||||
for _, s := range m.DefinesSeats {
|
||||
seats[s.Name] = s
|
||||
declarers[s.Name] = m.Module
|
||||
@@ -79,8 +80,8 @@ func records(in Inputs, union bool) (buscompose.Records, []Gap) {
|
||||
for _, sr := range catalogue.ReadsGranted(m, here, node) {
|
||||
d.KeyedReads = append(d.KeyedReads, buscompose.KeyedRead{Read: sr.Read, Key: sr.Key})
|
||||
}
|
||||
d.Hears, d.HearsAStream, d.Consumes = hearsOf(m, in.Manifests, node, values, d.Consumes)
|
||||
d.Reads, d.ReadsObjects, d.ReadsRefused = readsOfObjects(m.Module, node, d.Reads, d.ReadsRefused, in.Manifests, values)
|
||||
d.Hears, d.HearsAStream, d.Consumes = hearsOf(m, declared, node, values, d.Consumes)
|
||||
d.Reads, d.ReadsObjects, d.ReadsRefused = readsOfObjects(m.Module, node, d.Reads, d.ReadsRefused, declared, values)
|
||||
return d
|
||||
}
|
||||
d := declare(m)
|
||||
@@ -101,7 +102,7 @@ func records(in Inputs, union bool) (buscompose.Records, []Gap) {
|
||||
}
|
||||
// The node-engine's own manifest, placed on every node (ADR 0563): its state, logs and streams alone, created by this
|
||||
// part; it adds no principal, since each node's engine speaks as the node's account.
|
||||
if m, ok := in.Manifests[buscompose.EngineModule]; ok {
|
||||
if m, ok := declared[buscompose.EngineModule]; ok {
|
||||
out.Engine = &buscompose.Declared{Module: m.Module, State: bucketsFor(m, in.Nodes), Logs: logsOf(m), Streams: streamsOf(m)}
|
||||
}
|
||||
return out, gaps
|
||||
|
||||
@@ -299,7 +299,7 @@ func listUsers(list []byte) []string {
|
||||
// they could not be read). Empty when it may.
|
||||
func (p *Part) holdsBack(in Inputs, leftOut, dropped, bus, lost []string, digest string) string {
|
||||
p.hashHold = nil
|
||||
if f := in.ReadFailures(); len(f) > 0 {
|
||||
if f := in.ReadFailed(); len(f) > 0 {
|
||||
return "held: the list in force is kept while a read of this pass failed (" + strings.Join(f, "; ") + ")"
|
||||
}
|
||||
if bus == nil {
|
||||
|
||||
Reference in New Issue
Block a user