Compare commits
57
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
dae8e288d2 | ||
|
|
fe11db9293 | ||
|
|
22bc4dc8ef | ||
|
|
a7a1fbd34e | ||
|
|
1888d1d230 | ||
|
|
960d6c34d2 | ||
|
|
ff81bdf8cc | ||
|
|
65ee43f075 | ||
|
|
c3e8fc4ba6 | ||
|
|
75be8acbad | ||
|
|
d103ed88ab | ||
|
|
3ac644baef | ||
|
|
fd7d93333e | ||
|
|
d515c71e1c | ||
|
|
36061b5c56 | ||
|
|
4ef49d0391 | ||
|
|
17b7cc480d | ||
|
|
6ac8c5c229 | ||
|
|
21252194d5 | ||
|
|
545a0c9cb4 | ||
|
|
527d87613e | ||
|
|
36c92861d2 | ||
|
|
852ba81c5a | ||
|
|
3203358c4c | ||
|
|
9b01c94a9e | ||
|
|
33395f68b3 | ||
|
|
45564e69a6 | ||
|
|
d96b9fd660 | ||
|
|
2aae6f5580 | ||
|
|
6b84973142 | ||
|
|
e7f0dc612e | ||
|
|
76deab6c67 | ||
|
|
e66c6fea62 | ||
|
|
6b1c7a5613 | ||
|
|
04902ff24a | ||
|
|
bb71310337 | ||
|
|
18e0a01c54 | ||
|
|
ddafe1b1aa | ||
|
|
b4cd95e7c4 | ||
|
|
89c5838176 | ||
|
|
c2a410bb63 | ||
|
|
af66adc180 | ||
|
|
9c61918ac3 | ||
|
|
83ee7769d8 | ||
|
|
175a5567b8 | ||
|
|
1747899419 | ||
|
|
f5c233a928 | ||
|
|
0abe480dba | ||
|
|
85fa8511c4 | ||
|
|
79649c3fa9 | ||
|
|
c3544b6a17 | ||
|
|
136fa8b2a8 | ||
|
|
8f6ee2fd18 | ||
|
|
095d41fa3e | ||
|
|
7ac4493ce9 | ||
|
|
3db051be57 | ||
|
|
e0d1de0475 |
@@ -46,6 +46,10 @@ func Key(module string) (string, error) {
|
||||
type State struct {
|
||||
Name string `json:"name"`
|
||||
Readable bool `json:"readable,omitempty"`
|
||||
// Readers, when named, are the only modules that may read the state (novox/hq ADR 0547 clause 9, issue 1945): a
|
||||
// state readable by some and not all, such as the vault's bus account hashes. Only ReadableBy grants them; Readable,
|
||||
// which a composer that knows no readers asks, stays false for it, so such a composer grants nobody.
|
||||
Readers []string `json:"readers,omitempty"`
|
||||
}
|
||||
|
||||
// States is an announcement: the module, the machine whose instance said it, and every state the module keeps, sorted
|
||||
@@ -67,6 +71,26 @@ func (s States) Keeps(state string) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// ReadableBy says whether the module announced the state readable by reader: readable, and reader among the readers
|
||||
// when it names any.
|
||||
func (s States) ReadableBy(state, reader string) bool {
|
||||
for _, k := range s.States {
|
||||
if k.Name != state {
|
||||
continue
|
||||
}
|
||||
if len(k.Readers) > 0 {
|
||||
for _, r := range k.Readers {
|
||||
if r == reader {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
return k.Readable
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Readable says whether the module announced the state readable by others.
|
||||
func (s States) Readable(state string) bool {
|
||||
for _, k := range s.States {
|
||||
@@ -93,6 +117,11 @@ func (s *States) check() error {
|
||||
return fmt.Errorf("announce: %s announces the state %q twice", s.Module, k.Name)
|
||||
}
|
||||
seen[k.Name] = true
|
||||
for _, r := range k.Readers {
|
||||
if !name.MatchString(r) {
|
||||
return fmt.Errorf("announce: %s names %q as a reader of %s, and a reader is a module: lower-case letters, digits and hyphens", s.Module, r, k.Name)
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -9,7 +9,8 @@ import (
|
||||
// own event subject, in a versioned value the controller keeps by module.
|
||||
func TestAnAnnouncementRoundTripsSortedAndSaysWhatIsReadable(t *testing.T) {
|
||||
data, err := Encode(States{Module: "mesh-controller", Node: "novox", States: []State{
|
||||
{Name: "plans-building", Readable: true}, {Name: "assignments", Readable: true}, {Name: "own"}}})
|
||||
{Name: "plans-building", Readable: true}, {Name: "assignments", Readable: true}, {Name: "own"},
|
||||
{Name: "hashes", Readers: []string{"mesh-grants"}}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -24,12 +25,17 @@ func TestAnAnnouncementRoundTripsSortedAndSaysWhatIsReadable(t *testing.T) {
|
||||
for _, s := range back.States {
|
||||
names = append(names, s.Name)
|
||||
}
|
||||
if strings.Join(names, ",") != "assignments,own,plans-building" {
|
||||
if strings.Join(names, ",") != "assignments,hashes,own,plans-building" {
|
||||
t.Fatalf("the states are not sorted: %v", names)
|
||||
}
|
||||
if !back.Readable("plans-building") || back.Readable("own") || back.Readable("absent") {
|
||||
t.Fatal("Readable does not say what was announced readable")
|
||||
}
|
||||
// Named readers (ADR 0547 clause 9): readable by them alone, and not readable to a composer that knows no readers.
|
||||
if back.Readable("hashes") || !back.ReadableBy("hashes", "mesh-grants") || back.ReadableBy("hashes", "spy") ||
|
||||
!back.ReadableBy("plans-building", "spy") || back.ReadableBy("own", "spy") {
|
||||
t.Fatal("ReadableBy does not say who was named")
|
||||
}
|
||||
if !back.Keeps("own") || back.Keeps("absent") {
|
||||
t.Fatal("Keeps does not say what was announced")
|
||||
}
|
||||
@@ -57,6 +63,7 @@ func TestAnAnnouncementRefusesWhatNoBucketCouldBe(t *testing.T) {
|
||||
{States{Module: "a", States: []State{{Name: "x_y"}}}, `"x_y"`},
|
||||
{States{Module: "a", States: []State{{Name: ""}}}, `""`},
|
||||
{States{Module: "a", States: []State{{Name: "x"}, {Name: "x", Readable: true}}}, "twice"},
|
||||
{States{Module: "a", States: []State{{Name: "x", Readers: []string{"b.c"}}}}, `"b.c"`},
|
||||
} {
|
||||
if _, err := Encode(c.s); err == nil || !strings.Contains(err.Error(), c.says) {
|
||||
t.Errorf("%+v: %v, wanted a refusal saying %s", c.s, err, c.says)
|
||||
|
||||
@@ -0,0 +1,215 @@
|
||||
// Package buildcpu is how much of a build node's processors its builds may take, and how they yield to everything else
|
||||
// there (novox/hq ADR 0553, issue 1806; the operator, 2026-10-26: "limit CPU usage on our build nodes so the total cpu
|
||||
// usage never impacts the normal workings of our nodes ... We have 3 build agents so they should share the cpu power").
|
||||
//
|
||||
// Three parts, each with one owner:
|
||||
//
|
||||
// - **Priority.** A build seat's holder (catalogue.Seat.Yields) runs at the lowest priority there is: the composer
|
||||
// gives its runtime process declaration.Yields, and the node-engine places every yielding process of the node in
|
||||
// one slice (declaration.YieldSlice) whose CPU weight is idle, with Nice=19 for every process it starts.
|
||||
// Interactive and service work always comes first; builds take the cycles nothing else wants.
|
||||
// - **One cap per node, shared.** The node's share of its processors is one value, the node-engine's own config
|
||||
// `build-cpu-share` (ShareSetting), set per node through the node's settings layer. The node-engine writes it as the
|
||||
// slice's CPUQuota: one bound the kernel holds for all the node's build modules together, never one per module.
|
||||
// - **Threads.** The quota bounds the total; threads only size each step. A step gets the share divided by its own
|
||||
// module's capacity (Threads), from the share the composer hands as a word (ShareWord): GOMAXPROCS sets every go
|
||||
// command's -p, the compiler's concurrency, a test binary's processors and -parallel. A module given no word sets
|
||||
// no thread limit at all: without the slice there is nothing to size a step to, and a limit then would only slow
|
||||
// builds.
|
||||
package buildcpu
|
||||
|
||||
import (
|
||||
"os"
|
||||
"runtime"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// ShareSetting is the node-engine's config value of the node's processor share for its builds, in percent of the
|
||||
// machine's processors, 1 to 100: one value per node, set through the node's settings layer (novox/hq ADR 0553 §3).
|
||||
const ShareSetting = "build-cpu-share"
|
||||
|
||||
// DefaultPercent is the share where nothing sets one: half the machine, so a desktop keeps half its processors for
|
||||
// itself whatever builds.
|
||||
const DefaultPercent = 50
|
||||
|
||||
// ShareWord is the word the composer gives every yielding module (novox/hq ADR 0553 §4): the node's share, in percent.
|
||||
const ShareWord = "MESH_BUILD_CPU_SHARE"
|
||||
|
||||
// Quota is a CPUQuota for a share of a machine with cores processors: systemd counts a quota in percent of one
|
||||
// processor, so half of 24 is 1200%. At least 100%, one processor, so a share on a small machine never stops a step.
|
||||
func Quota(percent, cores int) int {
|
||||
q := percent * cores
|
||||
if q < 100 {
|
||||
q = 100
|
||||
}
|
||||
return q
|
||||
}
|
||||
|
||||
// Threads is how many threads one step may run at once: its part of the share when its own module runs slots steps at
|
||||
// once, at least one. The slice's quota bounds the steps of all the node's build modules together; this only sizes one.
|
||||
func Threads(percent, cores, slots int) int {
|
||||
if slots < 1 {
|
||||
slots = 1
|
||||
}
|
||||
if percent < 1 || percent > 100 {
|
||||
percent = DefaultPercent
|
||||
}
|
||||
t := cores * percent / 100 / slots
|
||||
if t < 1 {
|
||||
t = 1
|
||||
}
|
||||
return t
|
||||
}
|
||||
|
||||
// Limits is what one step is held to on this node, as a status tool says it.
|
||||
type Limits struct {
|
||||
// Percent is the node's share in percent of its processors, and From where it was read: ShareWord, or "none" when
|
||||
// the composer gave no share and a step runs without a thread limit.
|
||||
Percent int `json:"share-percent,omitempty"`
|
||||
From string `json:"share-from"`
|
||||
// Cores is the machine's processor count.
|
||||
Cores int `json:"cores"`
|
||||
// Slots is this module's own capacity on the node: steps at once.
|
||||
Slots int `json:"slots"`
|
||||
// Threads is what each step is given (Threads); 0 is no limit.
|
||||
Threads int `json:"threads-per-step"`
|
||||
}
|
||||
|
||||
// FromWords is the limits of a step from the share the composer gave this process (ShareWord) and the module's own
|
||||
// capacity. Without a usable share there is no limit: Threads is 0.
|
||||
func FromWords(getenv func(string) string, own int) Limits {
|
||||
if own < 1 {
|
||||
own = 1
|
||||
}
|
||||
l := Limits{From: "none", Cores: runtime.NumCPU(), Slots: own}
|
||||
if n, err := strconv.Atoi(strings.TrimSpace(getenv(ShareWord))); err == nil && n >= 1 && n <= 100 {
|
||||
l.Percent, l.From = n, ShareWord
|
||||
l.Threads = Threads(n, l.Cores, own)
|
||||
}
|
||||
return l
|
||||
}
|
||||
|
||||
// Env is the environment a step's commands are given so they start no more threads than they may: GOMAXPROCS for every
|
||||
// Go program (the compiler, a test binary, esbuild, buildah), UV_THREADPOOL_SIZE for Node's worker pool. Returned as
|
||||
// KEY=value words to append to a command's environment; nothing for 0 threads, no limit.
|
||||
func Env(threads int) []string {
|
||||
if threads < 1 {
|
||||
return nil
|
||||
}
|
||||
t := strconv.Itoa(threads)
|
||||
return []string{"GOMAXPROCS=" + t, "UV_THREADPOOL_SIZE=" + t}
|
||||
}
|
||||
|
||||
// InForce is what the kernel holds this process to, read rather than assumed (evidence over assertion): its nice
|
||||
// level, its cgroup (the unit) and the cgroup's parent (the slice the node-engine placed it in), each one's CPU weight,
|
||||
// idle mark and quota. A field the machine does not answer is left empty and Unread says which.
|
||||
//
|
||||
// The IO weight is read but binds only where the disk's scheduler honours it (bfq, or the io.cost controller); under
|
||||
// the scheduler `none`, the usual one for NVMe, it does nothing, and the CPU bound is what holds.
|
||||
type InForce struct {
|
||||
Nice int `json:"nice"`
|
||||
Cgroup string `json:"cgroup,omitempty"`
|
||||
Unit Group `json:"unit"`
|
||||
Slice Group `json:"slice"`
|
||||
// Quota is the tighter of the two groups' cpu.max as a percent of one processor, 0 with none.
|
||||
Quota int `json:"quota-percent,omitempty"`
|
||||
Unread []string `json:"unread,omitempty"`
|
||||
}
|
||||
|
||||
// Group is one cgroup's controls.
|
||||
type Group struct {
|
||||
Path string `json:"path,omitempty"`
|
||||
CPUIdle string `json:"cpu-idle,omitempty"`
|
||||
CPUWeight string `json:"cpu-weight,omitempty"`
|
||||
// CPUMax is the cgroup's cpu.max: "<quota µs> <period µs>", or "max" with no quota.
|
||||
CPUMax string `json:"cpu-max,omitempty"`
|
||||
IOWeight string `json:"io-weight,omitempty"`
|
||||
}
|
||||
|
||||
// cgroupRoot is where the unified hierarchy is mounted. A variable so a test can point it elsewhere.
|
||||
var cgroupRoot = "/sys/fs/cgroup"
|
||||
|
||||
// procSelfCgroup is this process's cgroup file. A variable so a test can point it elsewhere.
|
||||
var procSelfCgroup = "/proc/self/cgroup"
|
||||
|
||||
// procSelfStat is this process's stat file. A variable so a test can point it elsewhere.
|
||||
var procSelfStat = "/proc/self/stat"
|
||||
|
||||
// ReadInForce reads what the kernel holds this process to.
|
||||
func ReadInForce() InForce {
|
||||
var f InForce
|
||||
f.Nice = nice()
|
||||
raw, err := os.ReadFile(procSelfCgroup)
|
||||
if err != nil {
|
||||
f.Unread = append(f.Unread, "cgroup: "+err.Error())
|
||||
return f
|
||||
}
|
||||
for _, line := range strings.Split(strings.TrimSpace(string(raw)), "\n") {
|
||||
if strings.HasPrefix(line, "0::") {
|
||||
f.Cgroup = strings.TrimPrefix(line, "0::")
|
||||
}
|
||||
}
|
||||
if f.Cgroup == "" {
|
||||
f.Unread = append(f.Unread, "cgroup: not on the unified hierarchy")
|
||||
return f
|
||||
}
|
||||
read := func(path string) Group {
|
||||
g := Group{Path: path}
|
||||
value := func(name string) string {
|
||||
b, err := os.ReadFile(cgroupRoot + path + "/" + name)
|
||||
if err != nil {
|
||||
f.Unread = append(f.Unread, path+"/"+name+": "+err.Error())
|
||||
return ""
|
||||
}
|
||||
return strings.TrimSpace(string(b))
|
||||
}
|
||||
g.CPUIdle, g.CPUWeight, g.CPUMax, g.IOWeight = value("cpu.idle"), value("cpu.weight"), value("cpu.max"), value("io.weight")
|
||||
return g
|
||||
}
|
||||
f.Unit = read(f.Cgroup)
|
||||
if i := strings.LastIndex(f.Cgroup, "/"); i > 0 {
|
||||
f.Slice = read(f.Cgroup[:i])
|
||||
}
|
||||
for _, q := range []int{quotaOf(f.Unit.CPUMax), quotaOf(f.Slice.CPUMax)} {
|
||||
if q > 0 && (f.Quota == 0 || q < f.Quota) {
|
||||
f.Quota = q
|
||||
}
|
||||
}
|
||||
return f
|
||||
}
|
||||
|
||||
// quotaOf is a cpu.max value as a percent of one processor: "1200000 100000" is 1200; "max ..." or nothing is 0.
|
||||
func quotaOf(cpuMax string) int {
|
||||
fields := strings.Fields(cpuMax)
|
||||
if len(fields) != 2 || fields[0] == "max" {
|
||||
return 0
|
||||
}
|
||||
q, err1 := strconv.Atoi(fields[0])
|
||||
p, err2 := strconv.Atoi(fields[1])
|
||||
if err1 != nil || err2 != nil || p <= 0 {
|
||||
return 0
|
||||
}
|
||||
return q * 100 / p
|
||||
}
|
||||
|
||||
// nice is this process's nice level from its stat file (the 19th field, counted after the command's closing
|
||||
// parenthesis, which may itself hold spaces); 0 when it cannot be read.
|
||||
func nice() int {
|
||||
raw, err := os.ReadFile(procSelfStat)
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
s := string(raw)
|
||||
i := strings.LastIndex(s, ")")
|
||||
if i < 0 {
|
||||
return 0
|
||||
}
|
||||
fields := strings.Fields(s[i+1:])
|
||||
// After the command: state is field 3, so nice (field 19) is index 16.
|
||||
if len(fields) < 17 {
|
||||
return 0
|
||||
}
|
||||
n, _ := strconv.Atoi(fields[16])
|
||||
return n
|
||||
}
|
||||
@@ -0,0 +1,90 @@
|
||||
package buildcpu
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"runtime"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// The node's share as a quota, the threads one step is given for its own module's capacity, and the word the composer
|
||||
// hands, in one table (ADR 0553, issue 1806).
|
||||
func TestShareQuotaAndThreads(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
percent, cores, slots int
|
||||
quota, threads int
|
||||
}{
|
||||
{"shanks: half of 24, three slots", 50, 24, 3, 1200, 4},
|
||||
{"ace: three quarters of 24, two slots", 75, 24, 2, 1800, 9},
|
||||
{"g14: half of 16, one slot", 50, 16, 1, 800, 8},
|
||||
{"one processor at least", 10, 2, 4, 100, 1},
|
||||
{"no slots counts as one", 100, 4, 0, 400, 4},
|
||||
{"a share out of range is the default", 0, 8, 1, 0, 4},
|
||||
} {
|
||||
if c.quota != 0 {
|
||||
if q := Quota(c.percent, c.cores); q != c.quota {
|
||||
t.Errorf("%s: Quota = %d, want %d", c.name, q, c.quota)
|
||||
}
|
||||
}
|
||||
if th := Threads(c.percent, c.cores, c.slots); th != c.threads {
|
||||
t.Errorf("%s: Threads = %d, want %d", c.name, th, c.threads)
|
||||
}
|
||||
}
|
||||
l := FromWords(func(k string) string { return map[string]string{ShareWord: "75"}[k] }, 2)
|
||||
if l.Percent != 75 || l.From != ShareWord || l.Slots != 2 || l.Threads != Threads(75, runtime.NumCPU(), 2) {
|
||||
t.Errorf("from the word: %+v", l)
|
||||
}
|
||||
// Told nothing usable, a step has no thread limit: nothing in its environment.
|
||||
for _, word := range []string{"", "250", "lots"} {
|
||||
l := FromWords(func(string) string { return word }, 3)
|
||||
if l.Threads != 0 || l.From != "none" || Env(l.Threads) != nil {
|
||||
t.Errorf("share %q: %+v, env %v", word, l, Env(l.Threads))
|
||||
}
|
||||
}
|
||||
if got := Env(3); !reflect.DeepEqual(got, []string{"GOMAXPROCS=3", "UV_THREADPOOL_SIZE=3"}) {
|
||||
t.Errorf("Env = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
// What the kernel holds a process to is read from its unit's and its slice's cgroup and its stat file, not assumed.
|
||||
func TestReadInForce(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
cg := filepath.Join(dir, "cg")
|
||||
slice := filepath.Join(cg, "meshyield.slice")
|
||||
unit := filepath.Join(slice, "mesh-build-go-runtime.service")
|
||||
if err := os.MkdirAll(unit, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for at, files := range map[string]map[string]string{
|
||||
slice: {"cpu.idle": "1\n", "cpu.weight": "0\n", "cpu.max": "1200000 100000\n", "io.weight": "default 1\n"},
|
||||
unit: {"cpu.idle": "0\n", "cpu.weight": "100\n", "cpu.max": "max 100000\n", "io.weight": "default 100\n"},
|
||||
} {
|
||||
for name, v := range files {
|
||||
if err := os.WriteFile(filepath.Join(at, name), []byte(v), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
self := filepath.Join(dir, "cgroup")
|
||||
stat := filepath.Join(dir, "stat")
|
||||
_ = os.WriteFile(self, []byte("0::/meshyield.slice/mesh-build-go-runtime.service\n"), 0o644)
|
||||
// A command with a space and a parenthesis in it, then nice 19 in field 19.
|
||||
_ = os.WriteFile(stat, []byte("42 (mesh (build) go) S 1 42 42 0 -1 4194560 1 0 0 0 0 0 0 0 39 19 8 0 1 2 3\n"), 0o644)
|
||||
cgroupRoot, procSelfCgroup, procSelfStat = cg, self, stat
|
||||
t.Cleanup(func() {
|
||||
cgroupRoot, procSelfCgroup, procSelfStat = "/sys/fs/cgroup", "/proc/self/cgroup", "/proc/self/stat"
|
||||
})
|
||||
|
||||
got := ReadInForce()
|
||||
want := InForce{Nice: 19, Cgroup: "/meshyield.slice/mesh-build-go-runtime.service",
|
||||
Unit: Group{Path: "/meshyield.slice/mesh-build-go-runtime.service", CPUIdle: "0", CPUWeight: "100",
|
||||
CPUMax: "max 100000", IOWeight: "default 100"},
|
||||
Slice: Group{Path: "/meshyield.slice", CPUIdle: "1", CPUWeight: "0", CPUMax: "1200000 100000",
|
||||
IOWeight: "default 1"},
|
||||
Quota: 1200}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("ReadInForce = %+v\nwant %+v", got, want)
|
||||
}
|
||||
}
|
||||
+31
-10
@@ -34,6 +34,9 @@ type AskSteps struct {
|
||||
// the module's sweep.
|
||||
Release func(workspace string) error
|
||||
Logf func(format string, args ...any)
|
||||
// Has says whether this node has a need a package's tests declare (delivery.AskNeeds, issue 890); a package needing
|
||||
// what it lacks is not tested here but named in the level's Unmet, for the asker to place. Nil has nothing.
|
||||
Has func(need string) bool
|
||||
}
|
||||
|
||||
// RunAsk runs ask's job with steps and answers a delivery.BuildAnswer or a delivery.CheckAnswer.
|
||||
@@ -208,6 +211,9 @@ func (s AskSteps) level(ctx context.Context, job delivery.CheckJob, level string
|
||||
_ = json.Unmarshal(r, &t)
|
||||
return t.Passed, nil
|
||||
}
|
||||
// Every test step of the level runs, also after one that did not pass, so one check names every failing test of the
|
||||
// level and not only the first (issue 1921); the level as a whole passes only when each of its steps did.
|
||||
all := true
|
||||
if s.Digest == nil {
|
||||
// A module with no digest (Node) tests each module directory of the tree, those whose tests changed first.
|
||||
for _, priority := range []bool{true, false} {
|
||||
@@ -219,23 +225,24 @@ func (s AskSteps) level(ctx context.Context, job delivery.CheckJob, level string
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("the %s level of %s: %w", level, dir, err)
|
||||
}
|
||||
if !passed {
|
||||
return false, nil
|
||||
}
|
||||
all = all && passed
|
||||
}
|
||||
}
|
||||
return true, nil
|
||||
return all, nil
|
||||
}
|
||||
todo, all, keepRunner := s.plan(ctx, job, level, f, runner, withRunner, la)
|
||||
todo, whole, keepRunner := s.plan(ctx, job, level, f, runner, withRunner, la)
|
||||
if !keepRunner {
|
||||
runner = "" // an installed module that ignores the runner judges the level as it judges any other (ADR 0535 §2)
|
||||
}
|
||||
if whole && len(job.Packages) > 0 {
|
||||
todo, whole = job.Packages, false // a placed ask tests its packages alone, whatever the digest said (ADR 0552 §4)
|
||||
}
|
||||
for _, priority := range []bool{true, false} {
|
||||
args := withRunner(map[string]any{"workspace": f.Workspace, "level": level})
|
||||
if job.Run != "" {
|
||||
args["run"] = job.Run
|
||||
}
|
||||
if all {
|
||||
if whole {
|
||||
if priority {
|
||||
continue // nothing to put first: the level runs whole after
|
||||
}
|
||||
@@ -255,11 +262,9 @@ func (s AskSteps) level(ctx context.Context, job delivery.CheckJob, level string
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("the %s level: %w", level, err)
|
||||
}
|
||||
if !passed {
|
||||
return false, nil
|
||||
}
|
||||
all = all && passed // the packages left still run (issue 1921)
|
||||
}
|
||||
return true, nil
|
||||
return all, nil
|
||||
}
|
||||
|
||||
// release removes an ask's workspace, saying a failure: the module's sweep removes it later.
|
||||
@@ -308,9 +313,16 @@ func (s AskSteps) plan(ctx context.Context, job delivery.CheckJob, level string,
|
||||
return nil, true, runner != "" // a digest that names nothing or not the siblings keys no verdict: the level runs whole
|
||||
}
|
||||
la.Digest = &d
|
||||
only := map[string]bool{}
|
||||
for _, p := range job.Packages {
|
||||
only[p] = true
|
||||
}
|
||||
var todo []string
|
||||
for _, pd := range d.Packages {
|
||||
p := pd.Package
|
||||
if len(only) > 0 && !only[p] {
|
||||
continue // a placed ask's other packages were judged by the first ask (ADR 0552 §4)
|
||||
}
|
||||
if job.Kept && s.Kept != nil {
|
||||
tests, found, err := s.Kept(ctx, level, pd.Digest)
|
||||
if err != nil && s.Logf != nil {
|
||||
@@ -321,6 +333,15 @@ func (s AskSteps) plan(ctx context.Context, job delivery.CheckJob, level string,
|
||||
continue
|
||||
}
|
||||
}
|
||||
if needs := d.NeedsOf(p); len(delivery.Lacks(needs, s.Has)) > 0 {
|
||||
// Its tests would only skip here (issue 890): it is left for a node that has what they need.
|
||||
la.Unmet = append(la.Unmet, delivery.PackageNeeds{Package: p, Needs: needs})
|
||||
if s.Logf != nil {
|
||||
s.Logf("the %s level of %s is left to a node with %s: this node lacks %s", level, p,
|
||||
strings.Join(needs, ", "), strings.Join(delivery.Lacks(needs, s.Has), ", "))
|
||||
}
|
||||
continue
|
||||
}
|
||||
todo = append(todo, p)
|
||||
}
|
||||
return todo, false, false
|
||||
|
||||
@@ -20,6 +20,7 @@ type recorder struct {
|
||||
fetch map[string]any
|
||||
tests map[string]bool // package or module directory -> passed
|
||||
digests map[string]string
|
||||
needs []delivery.PackageNeeds
|
||||
}
|
||||
|
||||
func (r *recorder) steps() AskSteps {
|
||||
@@ -35,8 +36,10 @@ func (r *recorder) steps() AskSteps {
|
||||
Build: call("build", func(args map[string]any) any {
|
||||
return map[string]any{"digest": "sha256:" + args["artifact"].(string), "reference": "artifact-store://" + args["artifact"].(string)}
|
||||
}),
|
||||
Copy: call("copy", func(args map[string]any) any { return map[string]any{"digest": "sha256:copied"} }),
|
||||
Digest: call("digest", func(map[string]any) any { return map[string]any{"level": "unit", "packages": r.digests} }),
|
||||
Copy: call("copy", func(args map[string]any) any { return map[string]any{"digest": "sha256:copied"} }),
|
||||
Digest: call("digest", func(map[string]any) any {
|
||||
return map[string]any{"level": "unit", "packages": r.digests, "needs": r.needs}
|
||||
}),
|
||||
Test: call("test", func(args map[string]any) any {
|
||||
passed := true
|
||||
if pkgs, ok := args["packages"].([]string); ok {
|
||||
@@ -109,9 +112,9 @@ func TestRunAskRefusesAnotherToolchain(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// 1887/A3, A5a, issue 1887 design limit 1: a check ask runs the levels of one check after one fetch: it digests the unit
|
||||
// level, takes the kept verdicts, tests the packages whose tests changed first, and stops at the first failure, running
|
||||
// no later level.
|
||||
// 1887/A3, A5a: a check ask runs the levels of one check after one fetch: it digests the unit level, takes the kept
|
||||
// verdicts, tests the packages whose tests changed first, then the rest also after a failure (issue 1921), and runs no
|
||||
// level after one that did not pass (ADR 0484 clause 13).
|
||||
func TestRunAskChecksTheLevelsOfOneCheck(t *testing.T) {
|
||||
r := &recorder{
|
||||
fetch: map[string]any{"workspace": "/w", "changed": []string{"pkg/b/b_test.go"}},
|
||||
@@ -126,7 +129,8 @@ func TestRunAskChecksTheLevelsOfOneCheck(t *testing.T) {
|
||||
got := strings.Join(r.calls, "\n")
|
||||
want := `fetch {"base":"` + runCommit + `","commit":"` + runCommit + `","repository":"novox/x"}
|
||||
digest {"level":"unit","workspace":"/w"}
|
||||
test {"level":"unit","packages":["x/pkg/b"],"workspace":"/w"}`
|
||||
test {"level":"unit","packages":["x/pkg/b"],"workspace":"/w"}
|
||||
test {"level":"unit","packages":["x/pkg/a"],"workspace":"/w"}`
|
||||
if got != want {
|
||||
t.Fatalf("calls:\n%s\nwant:\n%s", got, want)
|
||||
}
|
||||
@@ -135,9 +139,46 @@ test {"level":"unit","packages":["x/pkg/b"],"workspace":"/w"}`
|
||||
t.Fatalf("levels %+v, want the unit level alone", a.Levels)
|
||||
}
|
||||
u := a.Levels[0]
|
||||
if u.Level != delivery.LevelUnit || len(u.Kept) != 1 || u.Kept[0].Package != "x/pkg/k" || u.Kept[0].Tests != 7 || len(u.Tests) != 1 || u.Digest == nil {
|
||||
if u.Level != delivery.LevelUnit || len(u.Kept) != 1 || u.Kept[0].Package != "x/pkg/k" || u.Kept[0].Tests != 7 || len(u.Tests) != 2 || u.Digest == nil {
|
||||
t.Fatalf("the unit level %+v", u)
|
||||
}
|
||||
|
||||
// Issue 890/AC2: a package declaring node-build-node is not tested on a node that lacks it but named unmet, and is
|
||||
// tested on a node that has it.
|
||||
for _, has := range []bool{false, true} {
|
||||
r.calls, r.tests["x/pkg/b"] = nil, true
|
||||
r.needs = []delivery.PackageNeeds{{Package: "x/pkg/a", Needs: []string{delivery.NeedNodeBuildNode}}}
|
||||
steps := r.steps()
|
||||
if has {
|
||||
steps.Has = func(n string) bool { return n == delivery.NeedNodeBuildNode }
|
||||
}
|
||||
out, err := RunAsk(context.Background(), steps, ask(delivery.CheckJob{Levels: []string{delivery.LevelUnit}, Kept: true,
|
||||
Fetch: map[string]any{"repository": "novox/x", "commit": runCommit}}))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
u := out.(delivery.CheckAnswer).Levels[0]
|
||||
tested := strings.Contains(strings.Join(r.calls, "\n"), `"x/pkg/a"`)
|
||||
if tested != has || (len(u.Unmet) == 0) == !has || (!has && (u.Unmet[0].Package != "x/pkg/a" || delivery.UnmetNeeds([]delivery.LevelAnswer{u})[0] != delivery.NeedNodeBuildNode)) {
|
||||
t.Fatalf("has %v: tested x/pkg/a %v, unmet %+v, calls %v", has, tested, u.Unmet, r.calls)
|
||||
}
|
||||
}
|
||||
// A placed ask naming its packages tests those alone and takes no other's kept verdict, also when the level would
|
||||
// run whole (ADR 0552 §4).
|
||||
for _, digests := range []map[string]string{r.digests, nil} {
|
||||
r.calls, r.digests = nil, digests
|
||||
steps := r.steps()
|
||||
steps.Has = func(string) bool { return true }
|
||||
out, err := RunAsk(context.Background(), steps, ask(delivery.CheckJob{Levels: []string{delivery.LevelUnit}, Kept: true, Packages: []string{"x/pkg/a"},
|
||||
Fetch: map[string]any{"repository": "novox/x", "commit": runCommit}}))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
u := out.(delivery.CheckAnswer).Levels[0]
|
||||
if last := r.calls[len(r.calls)-1]; !strings.HasSuffix(last, `"packages":["x/pkg/a"],"workspace":"/w"}`) || len(u.Kept) != 0 || len(u.Tests) != 1 {
|
||||
t.Fatalf("digests %v: calls %v, kept %+v: want x/pkg/a alone tested, nothing taken kept", digests != nil, r.calls, u.Kept)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ADR 0535 §2, §4: a merge that changes the build module's own code is judged by the candidate's runner: its digest says
|
||||
@@ -203,3 +244,42 @@ func TestRunAskChecksNodeModulesAndConflicts(t *testing.T) {
|
||||
t.Fatalf("a conflict: %v %v", err, r.calls)
|
||||
}
|
||||
}
|
||||
|
||||
// Issue 1921: a failure in the packages whose tests changed does not stop the level: the other packages still run, so two
|
||||
// failing packages, one changed and one not, are two test results of the one level, and the level does not pass. A Node
|
||||
// check runs every module directory after one fails too.
|
||||
func TestRunAskRunsEveryPackageAfterAFailure(t *testing.T) {
|
||||
r := &recorder{
|
||||
fetch: map[string]any{"workspace": "/w", "changed": []string{"pkg/b/b_test.go"}},
|
||||
digests: map[string]string{"x/pkg/a": "d-a", "x/pkg/b": "d-b"},
|
||||
tests: map[string]bool{"x/pkg/a": false, "x/pkg/b": false},
|
||||
}
|
||||
out, err := RunAsk(context.Background(), r.steps(), ask(delivery.CheckJob{Levels: []string{delivery.LevelIntegration, delivery.LevelUnit},
|
||||
Fetch: map[string]any{"repository": "novox/x", "commit": runCommit}}))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
a := out.(delivery.CheckAnswer)
|
||||
if len(a.Levels) != 1 || len(a.Levels[0].Tests) != 2 {
|
||||
t.Fatalf("levels %+v, want the one level with both packages' results", a.Levels)
|
||||
}
|
||||
for i, raw := range a.Levels[0].Tests {
|
||||
var res struct{ Passed bool }
|
||||
if err := json.Unmarshal(raw, &res); err != nil || res.Passed {
|
||||
t.Errorf("result %d %s, want a failure", i, raw)
|
||||
}
|
||||
}
|
||||
|
||||
r = &recorder{fetch: map[string]any{"workspace": "/w", "modules": []string{"a", "b"}, "changed": []string{"b/test/x.test.ts"}},
|
||||
tests: map[string]bool{"a": true, "b": false}}
|
||||
s := r.steps()
|
||||
s.Digest = nil
|
||||
out, err = RunAsk(context.Background(), s, ask(delivery.CheckJob{Levels: []string{delivery.LevelUnit, delivery.LevelIntegration},
|
||||
Fetch: map[string]any{"repository": "novox/x", "commit": runCommit}}))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if a := out.(delivery.CheckAnswer); len(r.calls) != 3 || len(a.Levels) != 1 || len(a.Levels[0].Tests) != 2 {
|
||||
t.Fatalf("calls %v, levels %+v: want both directories of the unit level and no integration level", r.calls, a.Levels)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -36,3 +36,15 @@ func TestNodeCapacity(t *testing.T) {
|
||||
t.Errorf("a missing file = %d, want 0", got)
|
||||
}
|
||||
}
|
||||
|
||||
// ADR 0553 §4: a step's threads are the composer's share over the module's own capacity; without the word, none.
|
||||
func TestNodeLimits(t *testing.T) {
|
||||
t.Setenv("MESH_BUILD_CPU_SHARE", "")
|
||||
if l := NodeLimits(3); l.Threads != 0 || l.Slots != 3 {
|
||||
t.Errorf("without the word: %+v", l)
|
||||
}
|
||||
t.Setenv("MESH_BUILD_CPU_SHARE", "75")
|
||||
if l := NodeLimits(3); l.Percent != 75 || l.Slots != 3 || l.Threads < 1 {
|
||||
t.Errorf("from the word: %+v", l)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,14 @@
|
||||
package buildstep
|
||||
|
||||
import (
|
||||
"os"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buildcpu"
|
||||
)
|
||||
|
||||
// NodeLimits is what one step of a build module is held to (novox/hq ADR 0553 §4): the node's processor share from the
|
||||
// word the composer gives the module (buildcpu.ShareWord), divided by the module's own capacity, own, over the machine's
|
||||
// processors. Without the word, no thread limit: Threads is 0.
|
||||
func NodeLimits(own int) buildcpu.Limits {
|
||||
return buildcpu.FromWords(os.Getenv, own)
|
||||
}
|
||||
+6
-11
@@ -71,7 +71,7 @@ func FetchCommit(ctx context.Context, run forgekey.Runner, dir, remote, commit,
|
||||
full := append([]string{"-c", "credential.helper=", "-C", dir}, args...)
|
||||
r := run(ctx, env, "git", full...)
|
||||
if r.Status != 0 || r.Err != "" {
|
||||
return "", fmt.Errorf("git %s: %s", args[0], firstLine(r.Err+" "+r.Stderr))
|
||||
return "", fmt.Errorf("git %s: %s", args[0], forgekey.Said(r.Err+"\n"+r.Stderr))
|
||||
}
|
||||
return r.Stdout, nil
|
||||
}
|
||||
@@ -128,7 +128,11 @@ func UpdateMirror(ctx context.Context, run forgekey.Runner, mirror, remote, comm
|
||||
git := func(args ...string) error {
|
||||
r := run(ctx, env, "git", append([]string{"-c", "credential.helper="}, args...)...)
|
||||
if r.Status != 0 || r.Err != "" {
|
||||
return fmt.Errorf("git %s: %s", args[0], firstLine(r.Err+" "+r.Stderr))
|
||||
verb := args[0]
|
||||
if verb == "-C" && len(args) > 2 { // the mirror's commands are named by what they do, not by -C
|
||||
verb = args[2]
|
||||
}
|
||||
return fmt.Errorf("git %s: %s", verb, forgekey.Said(r.Err+"\n"+r.Stderr))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -190,12 +194,3 @@ func checkFetch(o fetchOptions, dir, remote, commit, ref string) error {
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func firstLine(s string) string {
|
||||
for _, l := range strings.Split(s, "\n") {
|
||||
if l = strings.TrimSpace(l); l != "" {
|
||||
return l
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
+14
-1
@@ -15,6 +15,9 @@ type MergeResult struct {
|
||||
Base, Head, Merge string
|
||||
Conflicts []string
|
||||
Changed []string
|
||||
// Programs are the changed files that are compiled programs (ExecutableKind of their first bytes, read in the workspace
|
||||
// at the merge); never nil for a clean merge, so a caller tells none from a module that reports none (issue 1577).
|
||||
Programs []Program
|
||||
}
|
||||
|
||||
// MergeInto makes, in the workspace dir that FetchCommit left at commit, the merge of commit into the trunk commit
|
||||
@@ -52,7 +55,7 @@ func MergeInto(ctx context.Context, run forgekey.Runner, dir, remote, base, comm
|
||||
full := append([]string{"-c", "credential.helper=", "-C", dir}, args...)
|
||||
r := run(ctx, env, "git", full...)
|
||||
if r.Status != 0 || r.Err != "" {
|
||||
return "", fmt.Errorf("git %s: %s", args[0], firstLine(r.Err+" "+r.Stderr))
|
||||
return "", fmt.Errorf("git %s: %s", args[0], forgekey.Said(r.Err+"\n"+r.Stderr))
|
||||
}
|
||||
return r.Stdout, nil
|
||||
}
|
||||
@@ -98,6 +101,16 @@ func MergeInto(ctx context.Context, run forgekey.Runner, dir, remote, base, comm
|
||||
return res, fmt.Errorf("the paths the merge changes cannot be read: %w", err)
|
||||
}
|
||||
res.Changed = strings.Fields(changed)
|
||||
raw, err := git("diff", "--raw", "-z", "--no-renames", "--diff-filter=AM", base, res.Merge)
|
||||
if err != nil {
|
||||
return res, fmt.Errorf("the files the merge adds or modifies cannot be read: %w", err)
|
||||
}
|
||||
res.Programs = programsIn(dir, raw)
|
||||
history, err := historyPrograms(git, base, commit, res.Programs)
|
||||
if err != nil {
|
||||
return res, fmt.Errorf("the commits of the branch cannot be read for compiled programs: %w", err)
|
||||
}
|
||||
res.Programs = append(res.Programs, history...)
|
||||
return res, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -79,6 +79,30 @@ func (r *mergeRig) head(files map[string]string) string {
|
||||
return h
|
||||
}
|
||||
|
||||
// headThatAddsAndRemoves is head with an earlier commit on the branch that adds a program and a later one that removes it: the
|
||||
// branch's tree is clean, its history is not. It answers the head and the earlier commit.
|
||||
func (r *mergeRig) headThatAddsAndRemoves(files map[string]string) (head, added string) {
|
||||
realGit(r.t, r.work, "checkout", "-q", "-B", "pr", r.fork)
|
||||
writeFile(r.t, r.work, "gone/prog", "\x7fELF\x02\x01")
|
||||
realGit(r.t, r.work, "add", ".")
|
||||
realGit(r.t, r.work, "commit", "-q", "-m", "oops")
|
||||
added = realGit(r.t, r.work, "rev-parse", "HEAD")
|
||||
realGit(r.t, r.work, "rm", "-q", "gone/prog")
|
||||
return r.head2(files), added
|
||||
}
|
||||
|
||||
// head2 commits files on the current branch and pushes it as the pull request's head.
|
||||
func (r *mergeRig) head2(files map[string]string) string {
|
||||
for n, c := range files {
|
||||
writeFile(r.t, r.work, n, c)
|
||||
}
|
||||
realGit(r.t, r.work, "add", ".")
|
||||
realGit(r.t, r.work, "commit", "-q", "-m", "head")
|
||||
h := realGit(r.t, r.work, "rev-parse", "HEAD")
|
||||
realGit(r.t, r.work, "push", "-q", r.bare, "HEAD:refs/pull/1/head")
|
||||
return h
|
||||
}
|
||||
|
||||
// advance moves the trunk past the fork point with a commit changing files, pushes it and answers its sha.
|
||||
func (r *mergeRig) advance(files map[string]string) string {
|
||||
realGit(r.t, r.work, "checkout", "-q", "-B", "main", r.fork)
|
||||
@@ -102,7 +126,7 @@ func (r *mergeRig) fetch(head string) {
|
||||
// holds both sides' changes.
|
||||
func TestMergeIntoACleanMergeAnswersTheMergeCommitAndTheChangedFiles(t *testing.T) {
|
||||
r := newMergeRig(t)
|
||||
head := r.head(map[string]string{"b.txt": "head\n"})
|
||||
head, added := r.headThatAddsAndRemoves(map[string]string{"b.txt": "head\n", "prog": "\x7fELF\x02\x01"})
|
||||
base := r.advance(map[string]string{"a.txt": "one\ntwo\nthree\nfour\n"})
|
||||
r.fetch(head)
|
||||
res, err := MergeInto(context.Background(), forgekey.ExecRunner, r.ws, r.remote, base, head, AllowFileRemote())
|
||||
@@ -112,8 +136,11 @@ func TestMergeIntoACleanMergeAnswersTheMergeCommitAndTheChangedFiles(t *testing.
|
||||
if res.Base != base || res.Head != head || len(res.Merge) != 40 || res.Merge == base || res.Merge == head || len(res.Conflicts) != 0 {
|
||||
t.Fatalf("result = %+v", res)
|
||||
}
|
||||
if !reflect.DeepEqual(res.Changed, []string{"b.txt"}) {
|
||||
t.Errorf("changed = %v, want only what the head adds to the trunk: b.txt", res.Changed)
|
||||
if !reflect.DeepEqual(res.Changed, []string{"b.txt", "prog"}) {
|
||||
t.Errorf("changed = %v, want only what the head adds to the trunk: b.txt, prog", res.Changed)
|
||||
}
|
||||
if !reflect.DeepEqual(res.Programs, []Program{{"prog", "ELF", ""}, {"gone/prog", "ELF", added}}) { // issue 1577
|
||||
t.Errorf("programs = %v, want the ELF the head adds and the one an earlier commit added and a later one removed", res.Programs)
|
||||
}
|
||||
if at := realGit(t, r.ws, "rev-parse", "HEAD"); at != res.Merge {
|
||||
t.Errorf("the workspace is at %s, not at the merge %s", at, res.Merge)
|
||||
|
||||
@@ -104,3 +104,21 @@ func TestUpdateMirrorRefusesBeforeRunning(t *testing.T) {
|
||||
t.Errorf("ran %v", f.calls)
|
||||
}
|
||||
}
|
||||
|
||||
// The forge's refusal reaches the reason in git's own words: the mirror of a repository its account cannot read said
|
||||
// only "git -C: error:" (2026-10-26), since git writes a bare "error:" line before the forge's message.
|
||||
func TestUpdateMirrorSaysTheForgesRefusal(t *testing.T) {
|
||||
f := &fakeGit{}
|
||||
f.rule = func(args string) (r forgekey.Ran) {
|
||||
if strings.Contains(args, "fetch") {
|
||||
r.Status = 128
|
||||
r.Stderr = "error: \nerror: User: 22:nox-builder with Key: 30:mesh:ace.mesh-build-go is not authorized to read o/r.\nerror: \nfatal: Could not read from remote repository.\n"
|
||||
}
|
||||
return
|
||||
}
|
||||
err := UpdateMirror(context.Background(), f.run, filepath.Join(t.TempDir(), "m.git"), "ssh://mesh-forge/o/r.git", sha, "")
|
||||
if err == nil || !strings.Contains(err.Error(), "nox-builder with Key: 30:mesh:ace.mesh-build-go is not authorized to read o/r.") ||
|
||||
!strings.Contains(err.Error(), "Could not read from remote repository") || !strings.HasPrefix(err.Error(), "git fetch: ") {
|
||||
t.Errorf("err = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,115 @@
|
||||
package buildstep
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Program is a changed file whose first bytes are an executable binary: a compiled program, which is build output and not
|
||||
// source (novox/hq issue 1577: `go build` in a module's folder writes one beside go.mod, and `git add -A` takes it).
|
||||
type Program struct {
|
||||
Path string `json:"path"`
|
||||
Kind string `json:"kind"` // ELF, Mach-O or PE
|
||||
// Commit is set for a program a commit of the branch added and a later one removed (or replaced): the merge's tree is clean,
|
||||
// but the merge commit carries the blob into the trunk's history for good. Empty for a program the merge's tree holds.
|
||||
Commit string `json:"commit,omitempty"`
|
||||
}
|
||||
|
||||
// ExecutableKind says what executable format the first bytes of a file are, or "" when they are none. A PE needs its header
|
||||
// ("PE\0\0" at the offset in bytes 0x3c) so a text starting with "MZ" is not one; a fat Mach-O (0xCAFEBABE) is told from a
|
||||
// Java class by its architecture count (bytes 4 to 7, fewer than 0x20; a class has its version there).
|
||||
func ExecutableKind(head []byte) string {
|
||||
switch {
|
||||
case bytes.HasPrefix(head, []byte("\x7fELF")):
|
||||
return "ELF"
|
||||
case bytes.HasPrefix(head, []byte("MZ")):
|
||||
if len(head) >= 0x40 {
|
||||
off := int(head[0x3c]) | int(head[0x3d])<<8 | int(head[0x3e])<<16 | int(head[0x3f])<<24
|
||||
if off >= 0 && off+4 <= len(head) && bytes.Equal(head[off:off+4], []byte("PE\x00\x00")) {
|
||||
return "PE"
|
||||
}
|
||||
}
|
||||
case bytes.HasPrefix(head, []byte{0xcf, 0xfa, 0xed, 0xfe}), bytes.HasPrefix(head, []byte{0xce, 0xfa, 0xed, 0xfe}),
|
||||
bytes.HasPrefix(head, []byte{0xfe, 0xed, 0xfa, 0xcf}), bytes.HasPrefix(head, []byte{0xfe, 0xed, 0xfa, 0xce}),
|
||||
bytes.HasPrefix(head, []byte{0xca, 0xfe, 0xba, 0xbe}) && len(head) >= 8 && head[4] == 0 && head[5] == 0 && head[6] == 0 && head[7] < 0x20:
|
||||
return "Mach-O"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// programHeadSize is how many bytes of a file are read to tell its format: enough for a PE header at its usual offsets.
|
||||
const programHeadSize = 0x100
|
||||
|
||||
// historyPrograms names the compiled programs that a commit of base..head added or modified and the merge's tree no longer
|
||||
// holds at that path (those it holds are in final already): each with the commit that added it. git is the merge's git runner.
|
||||
func historyPrograms(git func(args ...string) (string, error), base, head string, final []Program) ([]Program, error) {
|
||||
revs, err := git("rev-list", "--no-merges", "--reverse", base+".."+head)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
held := map[string]bool{}
|
||||
for _, p := range final {
|
||||
held[p.Path] = true
|
||||
}
|
||||
out, seen := []Program{}, map[string]bool{}
|
||||
for _, c := range strings.Fields(revs) {
|
||||
raw, err := git("diff-tree", "-r", "-z", "--no-commit-id", "--no-renames", "--root", "--diff-filter=AM", c)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
f := strings.Split(raw, "\x00")
|
||||
for i := 0; i+1 < len(f); i += 2 {
|
||||
meta := strings.Fields(strings.TrimPrefix(f[i], ":"))
|
||||
if len(meta) < 4 || (meta[1] != "100644" && meta[1] != "100755") || held[f[i+1]] || seen[meta[3]] {
|
||||
continue
|
||||
}
|
||||
seen[meta[3]] = true
|
||||
blob, err := git("cat-file", "blob", meta[3])
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
head := []byte(blob)
|
||||
if len(head) > programHeadSize {
|
||||
head = head[:programHeadSize]
|
||||
}
|
||||
if k := ExecutableKind(head); k != "" {
|
||||
out = append(out, Program{Path: f[i+1], Kind: k, Commit: c})
|
||||
}
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// programsIn reads the output of `git diff --raw -z --no-renames --diff-filter=AM` (":<old mode> <new mode> <old sha> <new
|
||||
// sha> <status>\0<path>\0" per file) and names each added or modified regular file of the tree dir that is a compiled
|
||||
// program, from its first bytes read locally. A symlink (mode 120000) and a submodule (160000) are not read. It never
|
||||
// answers nil, so a caller can tell no programs from none reported.
|
||||
func programsIn(dir, raw string) []Program {
|
||||
out := []Program{}
|
||||
f := strings.Split(raw, "\x00")
|
||||
for i := 0; i+1 < len(f); i += 2 {
|
||||
meta := strings.Fields(strings.TrimPrefix(f[i], ":"))
|
||||
if len(meta) < 2 || (meta[1] != "100644" && meta[1] != "100755") {
|
||||
continue
|
||||
}
|
||||
p := f[i+1]
|
||||
full := filepath.Join(dir, filepath.FromSlash(p))
|
||||
if st, err := os.Lstat(full); err != nil || !st.Mode().IsRegular() {
|
||||
continue
|
||||
}
|
||||
fh, err := os.Open(full)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
head := make([]byte, programHeadSize)
|
||||
n, _ := io.ReadFull(fh, head)
|
||||
fh.Close()
|
||||
if k := ExecutableKind(head[:n]); k != "" {
|
||||
out = append(out, Program{Path: p, Kind: k})
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
package buildstep
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func pe(offset int, magic string) []byte {
|
||||
b := make([]byte, 0x100)
|
||||
copy(b, "MZ")
|
||||
b[0x3c] = byte(offset)
|
||||
copy(b[offset:], magic)
|
||||
return b
|
||||
}
|
||||
|
||||
func TestExecutableKindTellsProgramsFromText(t *testing.T) { // issue 1577
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
head []byte
|
||||
want string
|
||||
}{
|
||||
{"elf", []byte("\x7fELF\x02\x01\x01"), "ELF"},
|
||||
{"mach-o 64", []byte{0xcf, 0xfa, 0xed, 0xfe, 7, 0, 0, 1}, "Mach-O"},
|
||||
{"mach-o 32 big", []byte{0xfe, 0xed, 0xfa, 0xce, 0, 0, 0, 7}, "Mach-O"},
|
||||
{"fat mach-o", []byte{0xca, 0xfe, 0xba, 0xbe, 0, 0, 0, 2}, "Mach-O"},
|
||||
{"java class", []byte{0xca, 0xfe, 0xba, 0xbe, 0, 0, 0, 0x34}, ""},
|
||||
{"pe", pe(0x80, "PE\x00\x00"), "PE"},
|
||||
{"mz text", []byte("MZ is a text file starting with two letters"), ""},
|
||||
{"mz without a pe header", pe(0x80, "XX\x00\x00"), ""},
|
||||
{"text", []byte("package main"), ""},
|
||||
{"empty", nil, ""},
|
||||
} {
|
||||
if got := ExecutableKind(c.head); got != c.want {
|
||||
t.Errorf("%s: kind = %q, want %q", c.name, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestProgramsInNamesTheAddedAndModifiedRegularFilesThatArePrograms(t *testing.T) { // issue 1577
|
||||
dir := t.TempDir()
|
||||
write := func(name string, b []byte) {
|
||||
if err := os.MkdirAll(filepath.Dir(filepath.Join(dir, name)), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(dir, name), b, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
write("m/prog", []byte("\x7fELF\x02\x01"))
|
||||
write("m/notes", []byte("plain text"))
|
||||
write("m/mac", []byte{0xcf, 0xfa, 0xed, 0xfe, 7, 0, 0, 1})
|
||||
if err := os.Symlink("prog", filepath.Join(dir, "m/link")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
line := func(mode, path string) string { return ":000000 " + mode + " 0000 1111 A\x00" + path + "\x00" }
|
||||
for _, c := range []struct {
|
||||
name, raw string
|
||||
want []Program
|
||||
}{
|
||||
{"none", "", []Program{}},
|
||||
{"a program and a text", line("100755", "m/prog") + line("100644", "m/notes"), []Program{{"m/prog", "ELF", ""}}},
|
||||
{"two programs", line("100644", "m/prog") + line("100755", "m/mac"), []Program{{"m/prog", "ELF", ""}, {"m/mac", "Mach-O", ""}}},
|
||||
{"a symlink and a submodule are not read", line("120000", "m/link") + line("160000", "m/sub"), []Program{}},
|
||||
{"a path the tree lacks", line("100644", "m/gone"), []Program{}},
|
||||
} {
|
||||
if got := programsIn(dir, c.raw); !reflect.DeepEqual(got, c.want) {
|
||||
t.Errorf("%s: programs = %v, want %v", c.name, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
+56
-1
@@ -6,6 +6,7 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
@@ -76,6 +77,11 @@ type Puller struct {
|
||||
// for this node, and wakes the puller on each change (watchCapacity): WatchConfigServed in a module, a test's own
|
||||
// double. Until it hands a value the puller judges by Capacity.
|
||||
WatchConfig func(key string, handle func(stdio.ProvisionChange) error) error
|
||||
// Has says whether this node has a need an ask names (delivery.AskNeeds, issue 890). An ask needing what it lacks is
|
||||
// declined: its step ends refused as declined (delivery.StepDeclined), the message is dropped for its asker to put
|
||||
// again, and the worker it came from is not looked at again until the next pull's wait, so a node lacking the need
|
||||
// does not take the asker's next copy at once. Nil has nothing.
|
||||
Has func(need string) bool
|
||||
|
||||
mu sync.Mutex
|
||||
inHand int
|
||||
@@ -86,6 +92,9 @@ type Puller struct {
|
||||
// watched is the capacity the config watch handed last, used in place of Capacity once hasWatched.
|
||||
watched int
|
||||
hasWatched bool
|
||||
// declined are the kinds whose worker handed an ask this node declined, left out of the pulls without a wait until
|
||||
// the next pull's wait ends.
|
||||
declined map[string]bool
|
||||
}
|
||||
|
||||
func (p *Puller) init() {
|
||||
@@ -283,17 +292,59 @@ func (p *Puller) Pull(ctx context.Context) error {
|
||||
|
||||
// next is one ask from the workers in their order: one already waiting on delivery, check or recheck, and when none
|
||||
// waits, a pull of the delivery worker that waits for one.
|
||||
//
|
||||
// A kind whose worker handed an ask this node declined is left out until that wait ended (issue 890): the pull's own
|
||||
// wait, no timer of its own.
|
||||
func (p *Puller) next(ctx context.Context, wait time.Duration) (*Pulled, error) {
|
||||
p.mu.Lock()
|
||||
declined := p.declined
|
||||
p.mu.Unlock()
|
||||
for _, kind := range delivery.AskKinds {
|
||||
if declined[kind] {
|
||||
continue
|
||||
}
|
||||
msg, err := p.Source.Next(ctx, kind, 0)
|
||||
if err != nil || msg != nil {
|
||||
return withKind(msg, kind), err
|
||||
}
|
||||
}
|
||||
msg, err := p.Source.Next(ctx, delivery.AskDelivery, wait)
|
||||
p.mu.Lock()
|
||||
p.declined = nil
|
||||
p.mu.Unlock()
|
||||
return withKind(msg, delivery.AskDelivery), err
|
||||
}
|
||||
|
||||
// decline ends the step of an ask this node lacks a need of, refused as declined with what it lacks, and drops the
|
||||
// message: the asker puts the ask again under a new id (issue 890). A message whose decline cannot be written is handed
|
||||
// back, as one whose start cannot be.
|
||||
func (p *Puller) decline(msg *Pulled, ask delivery.StepAsk, lacks []string) {
|
||||
step, key, err := begin(p.Put, p.Node, ask.Kind+" ask", ask.ID, AskWhat(ask))
|
||||
if err == nil {
|
||||
why := Refuse("the ask %s needs %s, and %s lacks %s: it is left to a node that has it", ask.ID,
|
||||
strings.Join(ask.Needs, ", "), p.Node, strings.Join(lacks, ", "))
|
||||
step.Ended = now().UTC()
|
||||
step.State, step.Reason = delivery.StepRefused, why.Error()
|
||||
step.Result, err = json.Marshal(delivery.StepDeclined{Declined: true, Needs: ask.Needs, Lacks: lacks})
|
||||
if err == nil {
|
||||
err = writeEnd(p.Put, key, step)
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
p.logf("the ask %s cannot be declined on %s, so it is given back: %v", ask.ID, p.Node, err)
|
||||
_ = msg.Nak()
|
||||
return
|
||||
}
|
||||
p.logf("the ask %s is declined on %s: it needs %s, and this node lacks %s", ask.ID, p.Node, strings.Join(ask.Needs, ", "), strings.Join(lacks, ", "))
|
||||
_ = msg.Term()
|
||||
p.mu.Lock()
|
||||
if p.declined == nil {
|
||||
p.declined = map[string]bool{}
|
||||
}
|
||||
p.declined[msg.Kind] = true
|
||||
p.mu.Unlock()
|
||||
}
|
||||
|
||||
func withKind(msg *Pulled, kind string) *Pulled {
|
||||
if msg != nil {
|
||||
msg.Kind = kind
|
||||
@@ -314,7 +365,11 @@ func (p *Puller) take(ctx context.Context, msg *Pulled) {
|
||||
_ = msg.Term()
|
||||
return
|
||||
}
|
||||
step, key, err := begin(p.Put, p.Node, ask.Kind+" ask", ask.ID)
|
||||
if lacks := delivery.Lacks(ask.Needs, p.Has); len(lacks) > 0 {
|
||||
p.decline(msg, ask, lacks)
|
||||
return
|
||||
}
|
||||
step, key, err := begin(p.Put, p.Node, ask.Kind+" ask", ask.ID, AskWhat(ask))
|
||||
if err != nil {
|
||||
p.logf("the ask %s cannot start on %s: %v", ask.ID, p.Node, err)
|
||||
_ = msg.Nak()
|
||||
|
||||
@@ -137,23 +137,42 @@ func TestPullHoldsAtMostItsCapacity(t *testing.T) {
|
||||
waitFor(t, "all acked", func() bool { a, _, _ := src.Fates(); return len(a) == 3 })
|
||||
}
|
||||
|
||||
// An ask that is not one, or not this seat's, is dropped and runs nowhere.
|
||||
// An ask that is not one, or not this seat's, is dropped and runs nowhere. And issue 890/AC1: a check ask needing
|
||||
// node-build-node is declined on a node that lacks it (its step refused as declined, naming what it lacks, the message
|
||||
// dropped for the asker to put again), and runs on a node that has it.
|
||||
func TestPullDropsWhatIsNoAsk(t *testing.T) {
|
||||
src, st := NewFakeAsks(), newSteps()
|
||||
src.Publish(delivery.AskDelivery, []byte(`{"v":1}`))
|
||||
other := testAsk(delivery.AskDelivery)
|
||||
other.Language = delivery.AskNode
|
||||
_ = src.PublishAsk(other)
|
||||
placed := testAsk(delivery.AskCheck)
|
||||
placed.Needs = []string{delivery.NeedNodeBuildNode}
|
||||
if err := src.PublishAsk(placed); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ran := false
|
||||
p := &Puller{Source: src, Language: delivery.AskGo, Node: "shanks", Put: st.put, Capacity: func() int { return 1 },
|
||||
p := &Puller{Source: src, Language: delivery.AskGo, Node: "ace", Put: st.put, Capacity: func() int { return 1 },
|
||||
PullWait: 10 * time.Millisecond, Run: func(ctx context.Context, a delivery.StepAsk) (any, error) { ran = true; return nil, nil }}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
go p.Pull(ctx)
|
||||
waitFor(t, "two termed", func() bool { _, tm, _ := src.Fates(); return len(tm) == 2 })
|
||||
waitFor(t, "three termed", func() bool { _, tm, _ := src.Fates(); return len(tm) == 3 })
|
||||
if ran {
|
||||
t.Fatal("a wrong ask ran")
|
||||
t.Fatal("a wrong or a declined ask ran")
|
||||
}
|
||||
key, _ := delivery.AskStepKey("ace", placed.ID)
|
||||
s, _ := st.get(key)
|
||||
if d, ok := delivery.DeclinedOf(s); !ok || len(d.Lacks) != 1 || d.Lacks[0] != delivery.NeedNodeBuildNode || !strings.Contains(s.Reason, "lacks node-build-node") {
|
||||
t.Fatalf("the declined step = %+v", s)
|
||||
}
|
||||
has, hasSteps := NewFakeAsks(), newSteps()
|
||||
_ = has.PublishAsk(placed)
|
||||
q := &Puller{Source: has, Language: delivery.AskGo, Node: "g14", Put: hasSteps.put, Capacity: func() int { return 1 },
|
||||
PullWait: 10 * time.Millisecond, Has: func(n string) bool { return n == delivery.NeedNodeBuildNode },
|
||||
Run: func(ctx context.Context, a delivery.StepAsk) (any, error) { return nil, nil }}
|
||||
go q.Pull(ctx)
|
||||
waitFor(t, "the placed ask acked where its need is met", func() bool { a, _, _ := has.Fates(); return len(a) == 1 })
|
||||
}
|
||||
|
||||
// load reads the recorded steps state's raw values: the pause key is not a step, so it is kept apart.
|
||||
|
||||
@@ -0,0 +1,272 @@
|
||||
package buildstep
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// **A step left running** (novox/hq issue 1890, note 5). A build module that stops while a step runs leaves its key in
|
||||
// state running for good: nothing else writes that key's end, and the stream's redelivery only rewrites it when the same
|
||||
// ask comes back. At its start a module ends the steps its node left running before the process began (EndStale), and
|
||||
// its step-end tool ends one by hand (EndOne).
|
||||
|
||||
// RestartReason is why a step a previous process left running is ended.
|
||||
const RestartReason = "the build module restarted while it ran"
|
||||
|
||||
// Boot is when this process began. A step started at or after it belongs to this process, which may still run it, so
|
||||
// EndOne refuses it.
|
||||
var Boot = time.Now()
|
||||
|
||||
// Scan reads and writes a steps state with its revisions: its keys, a key's stored value and revision (nil and 0 for
|
||||
// none), and a write that lands only while the key is still at the revision read (KeptState.Update), failing with
|
||||
// stdio.ErrChanged when another writer was first.
|
||||
type Scan struct {
|
||||
Keys func() ([]string, error)
|
||||
Get func(key string) (json.RawMessage, uint64, error)
|
||||
Update func(key string, value any, revision uint64) (uint64, error)
|
||||
}
|
||||
|
||||
// ScanOf is the Scan of a state the process keeps.
|
||||
func ScanOf(s stdio.KeptState) Scan {
|
||||
return Scan{Keys: s.Keys, Update: s.Update, Get: func(key string) (json.RawMessage, uint64, error) {
|
||||
e, err := s.Get(key)
|
||||
if err != nil || e == nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
return e.Value, e.Revision, nil
|
||||
}}
|
||||
}
|
||||
|
||||
// isAsk says a step is a pulled ask's: its key is <node>.<ask id> and its tool is "<kind> ask" (a hand-started step
|
||||
// has the tool's name). An ask's key is the stream's to settle, never a restart's: the puller leaves an ask it was
|
||||
// running unacknowledged so the stream redelivers it, and the asker's wait takes the first end it reads on the key.
|
||||
// Only once the asker's bound has passed (delivery.StepBound) is the key nobody's, and then it is ended.
|
||||
func isAsk(key string, s delivery.Step) bool {
|
||||
_, _, err := delivery.ParseAskStepKey(key)
|
||||
return err == nil && strings.HasSuffix(s.Tool, " ask")
|
||||
}
|
||||
|
||||
// AskWhat says what an ask is, in one line: its kind, and its repository, head and levels, or its module and
|
||||
// fingerprint.
|
||||
func AskWhat(a delivery.StepAsk) string {
|
||||
short := func(s string) string {
|
||||
if len(s) > 12 {
|
||||
return s[:12]
|
||||
}
|
||||
return s
|
||||
}
|
||||
str := func(m map[string]any, k string) string { v, _ := m[k].(string); return v }
|
||||
parts := []string{a.Kind + " ask"}
|
||||
switch {
|
||||
case a.Check != nil:
|
||||
parts = append(parts, str(a.Check.Fetch, "repository")+"@"+short(str(a.Check.Fetch, "commit")), "levels "+strings.Join(a.Check.Levels, ","))
|
||||
if a.Check.Run != "" {
|
||||
parts = append(parts, "run "+a.Check.Run)
|
||||
}
|
||||
case a.Build != nil:
|
||||
parts = append(parts, "module "+a.Build.Module)
|
||||
if r := str(a.Build.Fetch, "repository"); r != "" {
|
||||
parts = append(parts, r+"@"+short(str(a.Build.Fetch, "commit")))
|
||||
}
|
||||
if a.Build.FP != "" {
|
||||
parts = append(parts, "fp "+short(a.Build.FP))
|
||||
}
|
||||
case a.Compose != nil:
|
||||
parts = append(parts, "compose", str(a.Compose.Fetch, "repository")+"@"+short(str(a.Compose.Fetch, "commit")))
|
||||
}
|
||||
return strings.Join(parts, ", ")
|
||||
}
|
||||
|
||||
// endRunning writes step ended failed with why, only while its key is still at revision.
|
||||
func endRunning(scan Scan, key string, step delivery.Step, revision uint64, why string) error {
|
||||
step.State, step.Reason, step.Ended, step.Result = delivery.StepFailed, why, now().UTC(), nil
|
||||
raw, err := delivery.EncodeStep(step)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = scan.Update(key, json.RawMessage(raw), revision)
|
||||
return err
|
||||
}
|
||||
|
||||
// EndStale ends this node's steps that are running and started before `before` (the start of this process) as failed
|
||||
// with RestartReason, and answers their ids. Only keys of node are read; a value that is no step is left alone. A step
|
||||
// of a pulled ask is left to the stream's redelivery (isAsk) until its start is older than delivery.StepBound: its asker
|
||||
// stopped waiting then, whether the stream gave it to another node or gave it up, and it is ended as failed, "past the
|
||||
// step bound: its asker stopped waiting at <time>". A key written by another writer since it was read
|
||||
// (a redelivered ask's start, a real end) is skipped.
|
||||
func EndStale(scan Scan, node string, before time.Time, logf func(string, ...any)) ([]string, error) {
|
||||
keys, err := scan.Keys()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("buildstep: listing the steps of %s: %w", node, err)
|
||||
}
|
||||
var ended []string
|
||||
for _, k := range keys {
|
||||
n, _, perr := delivery.ParseStepKey(k)
|
||||
if perr != nil || n != node {
|
||||
continue
|
||||
}
|
||||
raw, rev, err := scan.Get(k)
|
||||
if err != nil {
|
||||
return ended, fmt.Errorf("buildstep: reading the step %s: %w", k, err)
|
||||
}
|
||||
if len(raw) == 0 {
|
||||
continue
|
||||
}
|
||||
step, err := delivery.DecodeStep(raw)
|
||||
if err != nil || step.State != delivery.StepRunning {
|
||||
continue
|
||||
}
|
||||
why := RestartReason
|
||||
if isAsk(k, step) {
|
||||
if !step.Started.Add(delivery.StepBound).Before(now()) {
|
||||
continue
|
||||
}
|
||||
why = "past the step bound: its asker stopped waiting at " + step.Started.Add(delivery.StepBound).UTC().Format("2006-01-02T15:04:05Z")
|
||||
} else if !step.Started.Before(before) {
|
||||
continue
|
||||
}
|
||||
if err := endRunning(scan, k, step, rev, why); err != nil {
|
||||
if errors.Is(err, stdio.ErrChanged) {
|
||||
continue
|
||||
}
|
||||
return ended, fmt.Errorf("buildstep: ending the step %s: %w", k, err)
|
||||
}
|
||||
if logf != nil {
|
||||
logf("the step %s (%s) was left running and is ended: %s", k, step.Tool, why)
|
||||
}
|
||||
ended = append(ended, step.ID)
|
||||
}
|
||||
return ended, nil
|
||||
}
|
||||
|
||||
// EndOne ends this node's running step id (or the key <node>.<id>) as failed with why. A step that is not running is
|
||||
// refused, and so is one started since this process began (Boot: it may still run, whether a pulled ask or a step a
|
||||
// tool started), one live reports as held, and one whose key changed since it was read.
|
||||
func EndOne(scan Scan, node, id, why string, live func(id string) bool) (delivery.Step, error) {
|
||||
id = strings.TrimSpace(id)
|
||||
if n, rest, ok := strings.Cut(id, "."); ok {
|
||||
if n != node {
|
||||
return delivery.Step{}, fmt.Errorf("the step %q is of node %q, this module runs on %q", id, n, node)
|
||||
}
|
||||
id = rest
|
||||
}
|
||||
if strings.TrimSpace(why) == "" {
|
||||
return delivery.Step{}, fmt.Errorf("say why the step is ended (why)")
|
||||
}
|
||||
key, err := delivery.StepKey(node, id)
|
||||
if err != nil {
|
||||
return delivery.Step{}, err
|
||||
}
|
||||
raw, rev, err := scan.Get(key)
|
||||
if err != nil {
|
||||
return delivery.Step{}, err
|
||||
}
|
||||
if len(raw) == 0 {
|
||||
return delivery.Step{}, fmt.Errorf("no step %s in the steps state", key)
|
||||
}
|
||||
step, err := delivery.DecodeStep(raw)
|
||||
if err != nil {
|
||||
return delivery.Step{}, err
|
||||
}
|
||||
if step.State != delivery.StepRunning {
|
||||
return delivery.Step{}, fmt.Errorf("the step %s is %s, not running", key, step.State)
|
||||
}
|
||||
if !step.Started.Before(Boot) || (live != nil && live(id)) {
|
||||
return delivery.Step{}, fmt.Errorf("the step %s was started by this process and may run now; only a step a previous process left is ended", key)
|
||||
}
|
||||
if err := endRunning(scan, key, step, rev, why); err != nil {
|
||||
if errors.Is(err, stdio.ErrChanged) {
|
||||
return delivery.Step{}, fmt.Errorf("the step %s changed while it was being ended; read it again", key)
|
||||
}
|
||||
return delivery.Step{}, err
|
||||
}
|
||||
step.State, step.Reason = delivery.StepFailed, why
|
||||
return step, nil
|
||||
}
|
||||
|
||||
// StepEndTool is the module's step-end tool, named name (build_go_step_end): it ends one step left running, with a
|
||||
// reason, refusing one this process started. p may be nil.
|
||||
func StepEndTool(name string, node string, scan Scan, p *Puller) stdio.Tool {
|
||||
return stdio.Tool{Name: name,
|
||||
Description: "End one step of this machine that is stuck in state running (for instance left by a restart of the module) " +
|
||||
"as failed, with the reason given; its asker then sees it over, so ending an ask still in its stream fails its asker. " +
|
||||
"A step not running, or one this process started, is refused. " +
|
||||
"id is the step id or the key <node>.<id>.",
|
||||
Input: map[string]any{"type": "object", "required": []string{"id", "why"}, "properties": map[string]any{
|
||||
"id": map[string]any{"type": "string", "description": "the step id, or the key <node>.<id>"},
|
||||
"why": map[string]any{"type": "string", "description": "why it is ended; it becomes the step's reason"}}},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
id, _ := args["id"].(string)
|
||||
why, _ := args["why"].(string)
|
||||
var live func(string) bool
|
||||
if p != nil {
|
||||
live = func(id string) bool {
|
||||
for _, r := range p.Running() {
|
||||
if r.ID == id {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
}
|
||||
return EndOne(scan, node, id, why, live)
|
||||
}}
|
||||
}
|
||||
|
||||
// StaleAttempts and StaleRetry bound EndStaleAtStart's tries: the steps state is readable once the runtime serves.
|
||||
var (
|
||||
StaleAttempts = 6
|
||||
StaleRetry = 5 * time.Second
|
||||
)
|
||||
|
||||
// EndStaleAtStart is what a build module runs once at its start, in a goroutine: it waits for the runtime to serve,
|
||||
// then ends this node's steps left running before `before` (EndStale), trying again when the state cannot be read yet.
|
||||
// A step this process starts is never one of them: it starts after `before`.
|
||||
func EndStaleAtStart(ctx context.Context, scan Scan, node string, before time.Time, logf func(string, ...any)) {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-stdio.Served():
|
||||
}
|
||||
for attempt := 1; ; attempt++ {
|
||||
ended, err := EndStale(scan, node, before, logf)
|
||||
if err == nil {
|
||||
if logf != nil {
|
||||
logf("%d steps of %s left running by a previous process ended", len(ended), node)
|
||||
}
|
||||
return
|
||||
}
|
||||
if logf != nil {
|
||||
logf("ending the steps left running (try %d of %d): %v", attempt, StaleAttempts, err)
|
||||
}
|
||||
if attempt >= StaleAttempts {
|
||||
return
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-time.After(StaleRetry):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// EndStaleRound is EndStale as the module's periodic round runs it: at once, and only when the runtime serves (a round
|
||||
// before that is skipped, the start's EndStaleAtStart and the next round cover it). It ends the steps a previous
|
||||
// process left and the asks past delivery.StepBound.
|
||||
func EndStaleRound(scan Scan, node string, logf func(string, ...any)) {
|
||||
select {
|
||||
case <-stdio.Served():
|
||||
default:
|
||||
return
|
||||
}
|
||||
if _, err := EndStale(scan, node, Boot, logf); err != nil && logf != nil {
|
||||
logf("ending the steps left running: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,131 @@
|
||||
package buildstep
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// scanOf is the steps a keys double holds, last value per key, its revision the count of its puts; a write through
|
||||
// Update lands only at the revision read, as the state's does. race, when set, runs once before the next Update's check.
|
||||
func scanOf(k *keys, race func(key string)) Scan {
|
||||
return Scan{
|
||||
Keys: func() ([]string, error) {
|
||||
k.mu.Lock()
|
||||
defer k.mu.Unlock()
|
||||
var out []string
|
||||
for key := range k.puts {
|
||||
out = append(out, key)
|
||||
}
|
||||
return out, nil
|
||||
},
|
||||
Get: func(key string) (json.RawMessage, uint64, error) {
|
||||
k.mu.Lock()
|
||||
defer k.mu.Unlock()
|
||||
s, ok := k.puts[key]
|
||||
if !ok {
|
||||
return nil, 0, nil
|
||||
}
|
||||
raw, err := delivery.EncodeStep(s[len(s)-1])
|
||||
return raw, uint64(len(s)), err
|
||||
},
|
||||
Update: func(key string, value any, revision uint64) (uint64, error) {
|
||||
if race != nil {
|
||||
race(key)
|
||||
}
|
||||
k.mu.Lock()
|
||||
n := uint64(len(k.puts[key]))
|
||||
k.mu.Unlock()
|
||||
if n != revision {
|
||||
return 0, stdio.ErrChanged
|
||||
}
|
||||
return k.put(key, value)
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// 1890: EndStale ends only this node's hand-started steps left running before the process began and the asks past the
|
||||
// step bound (a younger ask is the stream's to redeliver), and never a key written meanwhile; EndOne ends one by hand, never one this process started.
|
||||
func TestEndStaleAndEndOne(t *testing.T) {
|
||||
k := newKeys()
|
||||
boot := time.Now()
|
||||
oldBoot := Boot
|
||||
Boot = boot
|
||||
defer func() { Boot = oldBoot }()
|
||||
put := func(node, id, tool, state string, started time.Time) {
|
||||
key, _ := delivery.StepKey(node, id)
|
||||
_, _ = k.put(key, json.RawMessage(mustStep(t, delivery.Step{ID: id, Tool: tool, Node: node, State: state, Started: started})))
|
||||
}
|
||||
put("ace", "old1", "build_go_fetch", delivery.StepRunning, boot.Add(-time.Hour))
|
||||
put("ace", "mjk3l9x0abcdef", "check ask", delivery.StepRunning, boot.Add(-time.Hour))
|
||||
put("ace", "mjk3l9x0abcdee", "check ask", delivery.StepRunning, time.Now().Add(-5*time.Minute))
|
||||
put("ace", "raced1", "build_go_test", delivery.StepRunning, boot.Add(-time.Hour))
|
||||
put("ace", "new1", "build_go_fetch", delivery.StepRunning, boot.Add(time.Second))
|
||||
put("ace", "done1", "build_go_fetch", delivery.StepPassed, boot.Add(-time.Hour))
|
||||
put("g14", "other1", "build_go_fetch", delivery.StepRunning, boot.Add(-time.Hour))
|
||||
// raced1 is ended by its own run between the read and the write.
|
||||
raced := false
|
||||
scan := scanOf(k, func(key string) {
|
||||
if key == "ace.raced1" && !raced {
|
||||
raced = true
|
||||
put("ace", "raced1", "build_go_test", delivery.StepPassed, boot.Add(-time.Hour))
|
||||
}
|
||||
})
|
||||
ended, err := EndStale(scan, "ace", boot, nil)
|
||||
if err != nil || len(ended) != 2 || ended[0] == ended[1] {
|
||||
t.Fatalf("ended %v, %v", ended, err)
|
||||
}
|
||||
if s := k.last("ace.old1"); s.State != delivery.StepFailed || s.Reason != RestartReason || s.Ended.IsZero() {
|
||||
t.Errorf("old1 = %+v", s)
|
||||
}
|
||||
if s := k.last("ace.mjk3l9x0abcdef"); s.State != delivery.StepFailed || !strings.HasPrefix(s.Reason, "past the step bound: its asker stopped waiting at ") {
|
||||
t.Errorf("the old ask = %+v", s)
|
||||
}
|
||||
for key, state := range map[string]string{"ace.mjk3l9x0abcdee": delivery.StepRunning, "ace.raced1": delivery.StepPassed, "ace.new1": delivery.StepRunning, "ace.done1": delivery.StepPassed, "g14.other1": delivery.StepRunning} {
|
||||
if s := k.last(key); s.State != state {
|
||||
t.Errorf("%s = %s, want %s", key, s.State, state)
|
||||
}
|
||||
}
|
||||
// EndOne: refuses without why, a held step, one started by this process (a pulled ask or a step tool's), a step not
|
||||
// running and another node's; ends a step a previous process left.
|
||||
for _, c := range []struct{ id, why string }{{"old1", ""}, {"new1", "stuck"}, {"done1", "x"}, {"g14.other1", "x"}} {
|
||||
if _, err := EndOne(scan, "ace", c.id, c.why, nil); err == nil {
|
||||
t.Errorf("ended %s with why %q", c.id, c.why)
|
||||
}
|
||||
}
|
||||
put("ace", "old2", "build_go_fetch", delivery.StepRunning, boot.Add(-time.Hour))
|
||||
if _, err := EndOne(scan, "ace", "old2", "x", func(string) bool { return true }); err == nil {
|
||||
t.Error("ended a held step")
|
||||
}
|
||||
if _, err := EndOne(scan, "ace", "ace.old2", "stuck for good", nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if s := k.last("ace.old2"); s.State != delivery.StepFailed || s.Reason != "stuck for good" {
|
||||
t.Errorf("old2 = %+v", s)
|
||||
}
|
||||
}
|
||||
|
||||
func mustStep(t *testing.T, s delivery.Step) []byte {
|
||||
t.Helper()
|
||||
raw, err := delivery.EncodeStep(s)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return raw
|
||||
}
|
||||
|
||||
// 1890: AskWhat names the repository, head and levels of a check ask and the module of a build ask.
|
||||
func TestAskWhat(t *testing.T) {
|
||||
check := delivery.StepAsk{Kind: "check", Check: &delivery.CheckJob{Fetch: map[string]any{"repository": "novox/mesh-sdk", "commit": "0123456789abcdef0123"}, Levels: []string{"unit", "integration"}}}
|
||||
if got, want := AskWhat(check), "check ask, novox/mesh-sdk@0123456789ab, levels unit,integration"; got != want {
|
||||
t.Errorf("check = %q", got)
|
||||
}
|
||||
build := delivery.StepAsk{Kind: "delivery", Build: &delivery.BuildJob{Module: "mesh-x", FP: "abcdef0123456789"}}
|
||||
if got, want := AskWhat(build), "delivery ask, module mesh-x, fp abcdef012345"; got != want {
|
||||
t.Errorf("build = %q", got)
|
||||
}
|
||||
}
|
||||
@@ -43,7 +43,7 @@ func NewID() string {
|
||||
// the error's text, or refused when the error is a *Refusal; a panic in run ends the step failed, naming it, and
|
||||
// the process goes on. If the running write fails, nothing was started and Start answers the error.
|
||||
func Start(ctx context.Context, put Put, node, tool string, run func(ctx context.Context) (result any, err error)) (string, error) {
|
||||
step, key, err := begin(put, node, tool, NewID())
|
||||
step, key, err := begin(put, node, tool, NewID(), "")
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
@@ -52,8 +52,8 @@ func Start(ctx context.Context, put Put, node, tool string, run func(ctx context
|
||||
}
|
||||
|
||||
// begin writes the step running under id.
|
||||
func begin(put Put, node, tool, id string) (delivery.Step, string, error) {
|
||||
step := delivery.Step{ID: id, Tool: tool, Node: node, State: delivery.StepRunning, Started: now().UTC()}
|
||||
func begin(put Put, node, tool, id, what string) (delivery.Step, string, error) {
|
||||
step := delivery.Step{ID: id, Tool: tool, Node: node, State: delivery.StepRunning, Started: now().UTC(), What: what}
|
||||
key, err := delivery.StepKey(node, step.ID)
|
||||
if err != nil {
|
||||
return delivery.Step{}, "", err
|
||||
|
||||
@@ -36,7 +36,7 @@ var lockFiles = []string{"go.sum", "package-lock.json"}
|
||||
func ModuleTrees(ctx context.Context, run forgekey.Runner, env []string, dir, commit string) (map[string]ModuleTree, error) {
|
||||
r := run(ctx, env, "git", "-C", dir, "ls-files", "--", "module.json", "*/module.json")
|
||||
if r.Status != 0 || r.Err != "" {
|
||||
return nil, fmt.Errorf("git ls-files: %s", firstLine(r.Err+" "+r.Stderr))
|
||||
return nil, fmt.Errorf("git ls-files: %s", forgekey.Said(r.Err+"\n"+r.Stderr))
|
||||
}
|
||||
trees := map[string]ModuleTree{}
|
||||
for _, l := range strings.Split(r.Stdout, "\n") {
|
||||
@@ -50,7 +50,7 @@ func ModuleTrees(ctx context.Context, run forgekey.Runner, env []string, dir, co
|
||||
}
|
||||
t := run(ctx, env, "git", "-C", dir, "rev-parse", spec)
|
||||
if t.Status != 0 || t.Err != "" {
|
||||
return nil, fmt.Errorf("git rev-parse %s: %s", spec, firstLine(t.Err+" "+t.Stderr))
|
||||
return nil, fmt.Errorf("git rev-parse %s: %s", spec, forgekey.Said(t.Err+"\n"+t.Stderr))
|
||||
}
|
||||
mt := ModuleTree{Tree: strings.TrimSpace(t.Stdout)}
|
||||
raw, err := os.ReadFile(filepath.Join(dir, filepath.FromSlash(l)))
|
||||
@@ -93,7 +93,7 @@ func narrow(ctx context.Context, run forgekey.Runner, env []string, dir, commit,
|
||||
}
|
||||
r := run(ctx, env, "git", append([]string{"-C", dir, "ls-tree", "-z", commit, "--"}, full...)...)
|
||||
if r.Status != 0 || r.Err != "" {
|
||||
return fmt.Errorf("git ls-tree %s of the module %s: %s", commit, mdir, firstLine(r.Err+" "+r.Stderr))
|
||||
return fmt.Errorf("git ls-tree %s of the module %s: %s", commit, mdir, forgekey.Said(r.Err+"\n"+r.Stderr))
|
||||
}
|
||||
objects := map[string]string{}
|
||||
for _, l := range strings.Split(r.Stdout, "\x00") {
|
||||
|
||||
@@ -45,10 +45,25 @@ func ControllerAnnouncement(node string) announce.States {
|
||||
// announced the state readable, or announced nothing yet (the transition); refused when the owner announced and did
|
||||
// not announce that state readable.
|
||||
func ReadsAnnounced(reads []string, announced map[string]announce.States) (granted, refused []string) {
|
||||
return ReadsAnnouncedBy("", reads, announced)
|
||||
}
|
||||
|
||||
// ReadsAnnouncedBy is ReadsAnnounced for one reader module, which also honours the readers an owner named for a state
|
||||
// (announce.State.Readers) and the composer's named readers (NamedReaders, ADR 0547 clause 9): a read named for other
|
||||
// modules is refused. An empty reader is judged by the readable mark alone.
|
||||
func ReadsAnnouncedBy(reader string, reads []string, announced map[string]announce.States) (granted, refused []string) {
|
||||
for _, r := range reads {
|
||||
owner, name, ok := strings.Cut(r, ".")
|
||||
s, said := announced[owner]
|
||||
switch {
|
||||
case reader != "" && !ReaderMay(reader, r):
|
||||
refused = append(refused, r)
|
||||
case reader != "" && ok && said && s.Keeps(name):
|
||||
if s.ReadableBy(name, reader) {
|
||||
granted = append(granted, r)
|
||||
} else {
|
||||
refused = append(refused, r)
|
||||
}
|
||||
case !ok || !said || s.Readable(name):
|
||||
granted = append(granted, r)
|
||||
default:
|
||||
|
||||
@@ -5,15 +5,19 @@ import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"regexp"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// Defends issue 1377 and novox/hq ADR 0547 §5: this is the controller's bus composer copied, and it composes byte for
|
||||
// byte what the controller's composes. testdata/controller-user-lists.json was written by the controller's own
|
||||
// internal/broker (mesh-controller branch fix/1945-mesh-grants-publishes-memberships), lists and memberships, from fixtures_test.go compiled there unchanged; composed.conf is the
|
||||
// internal/broker (mesh-controller branch fix/1945-mesh-grants-publishes-memberships, and fix/1790-grants-create-buckets for the grants part creating declared streams), lists and memberships, from fixtures_test.go compiled there unchanged; composed.conf is the
|
||||
// controller's golden of the whole server configuration. A row that differs is the copy drifting from the controller,
|
||||
// which is a fault while both compose (until the controller loses its lease, ADR 0547 §13).
|
||||
// createdAdvisory is one subscription to a bucket's creation in a composed list, as stateCreatedHeard composes it.
|
||||
var createdAdvisory = regexp.MustCompile(`"\$JS\.EVENT\.ADVISORY\.STREAM\.CREATED\.KV_[^"]*", `)
|
||||
|
||||
func TestTheCopiedComposerComposesAsTheControllers(t *testing.T) {
|
||||
raw, err := os.ReadFile(filepath.Join("testdata", "controller-user-lists.json"))
|
||||
if err != nil {
|
||||
@@ -48,6 +52,11 @@ func TestTheCopiedComposerComposesAsTheControllers(t *testing.T) {
|
||||
if strings.Join(missing, ",") != strings.Join(want[i].Missing, ",") {
|
||||
t.Errorf("left out %v, the controller %v", missing, want[i].Missing)
|
||||
}
|
||||
// The one row the copy composes beyond the controller's, on purpose (novox/hq issue 1790, 2026-10-26, after
|
||||
// the take-over of ADR 0547 milestone 2): each principal hears its own and read buckets created, so a state
|
||||
// missing at its start is bound when it appears. The controller's composer is frozen and its list no longer
|
||||
// loaded; while it is not, this row is left out of the comparison and nothing else.
|
||||
list = createdAdvisory.ReplaceAllString(list, "")
|
||||
if list != want[i].List {
|
||||
t.Errorf("the list differs from the controller's:\n%s", list)
|
||||
}
|
||||
@@ -69,6 +78,37 @@ func TestTheCopiedComposerComposesAsTheControllers(t *testing.T) {
|
||||
}
|
||||
})
|
||||
}
|
||||
// An asker on a seat pulled by kind hears what that seat's workers gave up on, and no other stream's dead letters
|
||||
// (novox/hq issue 734: a check ask given up during a bus reload left its try running for ever).
|
||||
t.Run("an asker reads its seat's dead letters", func(t *testing.T) {
|
||||
build := Seat{Name: "node-build-go", Scope: "node", Accepts: []string{"delivery", "check", "recheck"}, PulledByKind: true}
|
||||
perms, err := PermissionsFor(Principal{Kind: KindModule, Node: "one", Module: "mesh-delivery-checker", Uses: []Seat{build}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for subject, want := range map[string]bool{
|
||||
"$JS.API.CONSUMER.CREATE.DEAD_LETTERS.x1.mesh.events.dead.SEAT_NODE_BUILD_GO.>": true,
|
||||
"$JS.API.CONSUMER.CREATE.DEAD_LETTERS.x1.mesh.events.dead.SEAT_NODE_BUILD_GO.w.next": true,
|
||||
"$JS.API.CONSUMER.DELETE.DEAD_LETTERS.x1": true,
|
||||
"$JS.API.CONSUMER.MSG.NEXT.DEAD_LETTERS.x1": true,
|
||||
"$JS.API.CONSUMER.CREATE.DEAD_LETTERS.x1.mesh.events.dead.EVENTS.>": false,
|
||||
"$JS.API.CONSUMER.CREATE.DEAD_LETTERS.x1": false,
|
||||
"$JS.API.STREAM.PURGE.DEAD_LETTERS": false,
|
||||
"$JS.FC.DEAD_LETTERS.x1": false,
|
||||
} {
|
||||
if got := MayPublish(perms, subject); got != want {
|
||||
t.Errorf("may publish %s: %v, want %v", subject, got, want)
|
||||
}
|
||||
}
|
||||
plain, err := PermissionsFor(Principal{Kind: KindModule, Node: "one", Module: "shop",
|
||||
Uses: []Seat{{Name: "telegram-sender", Accepts: []string{"send"}}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if MayPublish(plain, "$JS.API.CONSUMER.DELETE.DEAD_LETTERS.x1") {
|
||||
t.Error("a user of a seat not pulled by kind reads DEAD_LETTERS")
|
||||
}
|
||||
})
|
||||
t.Run("the controller's golden server configuration", func(t *testing.T) {
|
||||
seat := Seat{Name: "telegram-sender", Accepts: []string{"send"}, Emits: []string{"delivered", "failed"}}
|
||||
got, err := Compose(
|
||||
|
||||
@@ -0,0 +1,88 @@
|
||||
package buscompose
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
)
|
||||
|
||||
// The streams a manifest declares, as the server is asked to create them (novox/hq ADR 0547 §5, issue 1790): the
|
||||
// grants part creates each one the bus does not hold yet, with what the controller asserts at its start, so the two
|
||||
// make the same stream. Pure. Only a create is ever asked with these: a stream the bus holds is never updated by the
|
||||
// grants part (CreateStreamGrants grants no update).
|
||||
|
||||
// BucketStreamConfig is a module's bucket as the stream the controller's CreateOrUpdateKeyValue makes of its options
|
||||
// (mesh-controller internal/broker BucketConfig, with no marker age held): the client library's key-value shape
|
||||
// (rollup, no delete, direct reads, discard new, a duplicate window of two minutes or the age when shorter) with the
|
||||
// mesh's caps, file storage and one replica.
|
||||
func BucketStreamConfig(b Bucket) jetstream.StreamConfig {
|
||||
history := b.History
|
||||
if history < 1 {
|
||||
history = 1
|
||||
}
|
||||
age := time.Duration(b.TTLSeconds) * time.Second
|
||||
dupes := 2 * time.Minute
|
||||
if age > 0 && age < dupes {
|
||||
dupes = age
|
||||
}
|
||||
cfg := jetstream.StreamConfig{
|
||||
Name: "KV_" + b.Bucket(),
|
||||
Description: b.Why(),
|
||||
Subjects: []string{"$KV." + b.Bucket() + ".>"},
|
||||
MaxMsgsPerSubject: int64(history),
|
||||
MaxBytes: StateMaxBytes,
|
||||
MaxAge: age,
|
||||
MaxMsgSize: StateMaxValueBytes,
|
||||
Storage: jetstream.FileStorage,
|
||||
Replicas: 1,
|
||||
AllowRollup: true,
|
||||
DenyDelete: true,
|
||||
Duplicates: dupes,
|
||||
MaxMsgs: -1,
|
||||
MaxConsumers: -1,
|
||||
AllowDirect: true,
|
||||
Discard: jetstream.DiscardNew,
|
||||
}
|
||||
if b.KeyTTL {
|
||||
cfg.AllowMsgTTL, cfg.SubjectDeleteMarkerTTL = true, KeyTTLMarkerTTL
|
||||
}
|
||||
return cfg
|
||||
}
|
||||
|
||||
// ModuleStreamConfig is a module's own stream as the controller asserts it (mesh-controller internal/broker
|
||||
// StreamConfigFor of ModuleStream.Stream).
|
||||
func ModuleStreamConfig(m ModuleStream) jetstream.StreamConfig {
|
||||
s := m.Stream()
|
||||
cfg := jetstream.StreamConfig{
|
||||
Name: s.Name,
|
||||
Subjects: s.Subjects,
|
||||
Retention: jetstream.LimitsPolicy,
|
||||
MaxAge: time.Duration(s.MaxAge) * time.Second,
|
||||
MaxMsgsPerSubject: int64(s.MaxMsgsPerSubject),
|
||||
Description: s.Why,
|
||||
AllowDirect: s.Direct,
|
||||
AllowMsgTTL: s.AllowMsgTTL,
|
||||
}
|
||||
if s.Retention == RetentionWorkQueue {
|
||||
cfg.Retention = jetstream.WorkQueuePolicy
|
||||
}
|
||||
if s.Restricted || s.WorkQueue {
|
||||
cfg.Metadata = map[string]string{}
|
||||
if s.Restricted {
|
||||
cfg.Metadata[RestrictedMetadata] = "true"
|
||||
}
|
||||
if s.WorkQueue {
|
||||
cfg.Metadata[WorkQueueMetadata] = "true"
|
||||
}
|
||||
}
|
||||
if s.MaxBytes > 0 {
|
||||
cfg.MaxBytes = s.MaxBytes
|
||||
}
|
||||
if s.DiscardNew {
|
||||
cfg.Discard = jetstream.DiscardNew
|
||||
}
|
||||
if s.DuplicatesSeconds > 0 {
|
||||
cfg.Duplicates = time.Duration(s.DuplicatesSeconds) * time.Second
|
||||
}
|
||||
return cfg
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
package buscompose
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
)
|
||||
|
||||
// Defends issue 1790 (ADR 0547 §5): the grants part creates a declared bucket and stream as the controller asserts them.
|
||||
func TestDeclaredStreamsAreCreatedAsTheControllerAssertsThem(t *testing.T) {
|
||||
plain := BucketStreamConfig(Bucket{Module: "mesh-vault", Name: "bus-accounts"})
|
||||
keyed := BucketStreamConfig(Bucket{Module: "m", Name: "presence", History: 5, TTLSeconds: 60, KeyTTL: true})
|
||||
queue := ModuleStreamConfig(ModuleStream{Module: "b", Name: "ASKS", Events: []string{"ask"}, MaxAgeDays: 7, MaxMiB: 32,
|
||||
WorkQueue: true, Duplicates: 1})
|
||||
for _, row := range []struct {
|
||||
what string
|
||||
got, want any
|
||||
}{
|
||||
{"bucket name", plain.Name, "KV_mesh-vault_bus-accounts"},
|
||||
{"bucket subjects", plain.Subjects[0], "$KV.mesh-vault_bus-accounts.>"},
|
||||
{"bucket history defaults to one", plain.MaxMsgsPerSubject, int64(1)},
|
||||
{"bucket cap", plain.MaxBytes, int64(StateMaxBytes)},
|
||||
{"bucket value cap", plain.MaxMsgSize, int32(StateMaxValueBytes)},
|
||||
{"bucket duplicate window", plain.Duplicates, 2 * time.Minute},
|
||||
{"bucket discards new", plain.Discard, jetstream.DiscardNew},
|
||||
{"bucket direct and rollup, no delete", plain.AllowDirect && plain.AllowRollup && plain.DenyDelete, true},
|
||||
{"bucket without key ttl", plain.AllowMsgTTL, false},
|
||||
{"history", keyed.MaxMsgsPerSubject, int64(5)},
|
||||
{"age", keyed.MaxAge, time.Minute},
|
||||
{"duplicate window under the age", keyed.Duplicates, time.Minute},
|
||||
{"key ttl markers", keyed.SubjectDeleteMarkerTTL, KeyTTLMarkerTTL},
|
||||
{"key ttl", keyed.AllowMsgTTL, true},
|
||||
{"queue retention", queue.Retention, jetstream.WorkQueuePolicy},
|
||||
{"queue subjects", queue.Subjects[0], "mesh.stream.b.event.ask"},
|
||||
{"queue age", queue.MaxAge, 7 * 24 * time.Hour},
|
||||
{"queue cap", queue.MaxBytes, int64(32 << 20)},
|
||||
{"queue discards new", queue.Discard, jetstream.DiscardNew},
|
||||
{"queue duplicates", queue.Duplicates, time.Hour},
|
||||
{"queue metadata", queue.Metadata[WorkQueueMetadata] + queue.Metadata[RestrictedMetadata], "truetrue"},
|
||||
} {
|
||||
if row.got != row.want {
|
||||
t.Errorf("%s: %v, want %v", row.what, row.got, row.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,130 @@
|
||||
package buscompose
|
||||
|
||||
import "testing"
|
||||
|
||||
// Defends issue 1790 and novox/hq ADR 0547 §5 (the operator, 2026-10-26 08:54): the grants part may create the stream
|
||||
// of each bucket, log and stream a placed module's manifest declares, by its own name, and read it back; nothing
|
||||
// wider, nothing updated or deleted, and no other principal so. Held byte-equal in the controller's copy. And ADR 0547
|
||||
// clause 7 (issue 1945, the operator's second exception): the grants part writes its own object bucket of sealed user
|
||||
// lists, and the node-engine of the node holding mesh-broker alone reads its one key and an object of it by name.
|
||||
func TestTheGrantsPartCreatesOnlyDeclaredStreams(t *testing.T) {
|
||||
records := Records{
|
||||
Nodes: []string{"novox", "ace"},
|
||||
Assigned: map[string][]Declared{"ace": {{Module: "plex"}}, "novox": {
|
||||
{Module: "nats", SnapshotsTheBus: true},
|
||||
// mesh-sdk's copy alone (ADR 0547 clause 9, named readers): the vault's hashes are read by the grants part
|
||||
// alone, whatever another module's reads say; the controller's copy grants any module that declares it.
|
||||
{Module: "mesh-vault", State: []Bucket{{Module: "mesh-vault", Name: "bus-accounts"}}},
|
||||
{Module: "spy", Serves: []string{"spy_x"}, Reads: []string{VaultBusAccounts}},
|
||||
{Module: "mesh-delivery-builder",
|
||||
State: []Bucket{{Module: "mesh-delivery-builder", Name: "steps", History: 5}},
|
||||
Logs: []Log{{Module: "mesh-delivery-builder", Name: "builds"}},
|
||||
Streams: []ModuleStream{{Module: "mesh-delivery-builder", Name: "BUILDASKS", Events: []string{"ask"}}}},
|
||||
{Module: GrantsModule, State: []Bucket{{Module: GrantsModule, Name: "assertions"}}, Reads: []string{VaultBusAccounts}},
|
||||
{Module: RuntimeModule},
|
||||
}},
|
||||
}
|
||||
users, err := Users(records)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
perms := map[string]Permissions{}
|
||||
for _, u := range users {
|
||||
if u.Kind == KindNode || u.Kind == KindNodeTools {
|
||||
p, err := PermissionsFor(u)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", u.Username(), err)
|
||||
}
|
||||
perms[u.Username()] = p
|
||||
continue
|
||||
}
|
||||
if u.Kind != KindModule {
|
||||
continue
|
||||
}
|
||||
p, err := PermissionsFor(u)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", u.Username(), err)
|
||||
}
|
||||
perms[u.Module] = p
|
||||
}
|
||||
for _, row := range []struct {
|
||||
who, subject string
|
||||
may bool
|
||||
}{
|
||||
{GrantsModule, "$JS.API.STREAM.CREATE.KV_mesh-delivery-builder_steps", true},
|
||||
{GrantsModule, "$JS.API.STREAM.INFO.KV_mesh-delivery-builder_steps", true},
|
||||
{GrantsModule, "$JS.API.STREAM.CREATE.LOG_mesh-delivery-builder_builds", true},
|
||||
{GrantsModule, "$JS.API.STREAM.CREATE.BUILDASKS", true},
|
||||
{GrantsModule, "$JS.API.STREAM.CREATE.KV_mesh-grants_assertions", true},
|
||||
{GrantsModule, "$JS.API.STREAM.UPDATE.KV_mesh-delivery-builder_steps", false},
|
||||
{GrantsModule, "$JS.API.STREAM.DELETE.KV_mesh-delivery-builder_steps", false},
|
||||
{GrantsModule, "$JS.API.STREAM.PURGE.KV_mesh-delivery-builder_steps", false},
|
||||
{GrantsModule, "$JS.API.STREAM.CREATE.KV_mesh-delivery-builder_undeclared", false},
|
||||
{GrantsModule, "$JS.API.STREAM.CREATE.EVENTS", false},
|
||||
{GrantsModule, "$JS.API.STREAM.CREATE.ASSIGNMENTS", false},
|
||||
{GrantsModule, "$JS.API.STREAM.INFO.EVENTS", false},
|
||||
{"mesh-delivery-builder", "$JS.API.STREAM.CREATE.KV_mesh-delivery-builder_steps", false},
|
||||
// The sealed user list (ADR 0547 clause 7): the grants part writes its own bucket and no other.
|
||||
{GrantsModule, "$JS.API.STREAM.CREATE.OBJ_mesh-grants-user-list", true},
|
||||
{GrantsModule, "$O.mesh-grants-user-list.C.abc", true},
|
||||
{GrantsModule, "$JS.API.STREAM.PURGE.OBJ_mesh-grants-user-list", true},
|
||||
{GrantsModule, "$O.declaration-files-novox.C.abc", false},
|
||||
{GrantsModule, "$JS.API.STREAM.DELETE.OBJ_mesh-grants-user-list", false},
|
||||
{"mesh-delivery-builder", "$O.mesh-grants-user-list.C.abc", false},
|
||||
// The bus node's engine reads the one key and an object by name; nothing of the bucket's list, no write.
|
||||
{"node.novox", "$JS.API.DIRECT.GET.KV_mesh-grants_user-list.$KV.mesh-grants_user-list.sealed", true},
|
||||
{"node.novox", "$JS.API.CONSUMER.CREATE.KV_mesh-grants_user-list.abc.$KV.mesh-grants_user-list.sealed", true},
|
||||
{"node.novox", "$JS.API.DIRECT.GET.OBJ_mesh-grants-user-list.$O.mesh-grants-user-list.M.abc", true},
|
||||
{"node.novox", "$JS.API.CONSUMER.CREATE.OBJ_mesh-grants-user-list.abc.$O.mesh-grants-user-list.C.abc", true},
|
||||
{"node.novox", "$JS.API.DIRECT.GET.KV_mesh-grants_user-list.$KV.mesh-grants_user-list.list", false},
|
||||
{"node.novox", "$JS.API.STREAM.INFO.OBJ_mesh-grants-user-list", false},
|
||||
{"node.novox", "$JS.API.STREAM.MSG.GET.OBJ_mesh-grants-user-list", false},
|
||||
{"node.novox", "$O.mesh-grants-user-list.C.abc", false},
|
||||
{"node.ace", "$JS.API.DIRECT.GET.KV_mesh-grants_user-list.$KV.mesh-grants_user-list.sealed", false},
|
||||
{"node.ace", "$JS.API.DIRECT.GET.OBJ_mesh-grants-user-list.$O.mesh-grants-user-list.M.abc", false},
|
||||
{GrantsModule, "$JS.API.DIRECT.GET.KV_mesh-vault_bus-accounts.$KV.mesh-vault_bus-accounts.x", true},
|
||||
{"spy", "$JS.API.DIRECT.GET.KV_mesh-vault_bus-accounts.$KV.mesh-vault_bus-accounts.x", false},
|
||||
{"novox.node-tools", "$JS.API.DIRECT.GET.KV_mesh-vault_bus-accounts.$KV.mesh-vault_bus-accounts.x", false},
|
||||
} {
|
||||
if got := MayPublish(perms[row.who], row.subject); got != row.may {
|
||||
t.Errorf("%s may publish %s: %v, want %v", row.who, row.subject, got, row.may)
|
||||
}
|
||||
}
|
||||
// Issue 1790 (2026-10-26): a module hears the bus create each bucket it may bind, its own and those it reads, so a
|
||||
// state missing at its start is bound when it appears; no other bucket's, and no other advisory.
|
||||
for _, row := range []struct {
|
||||
who, subject string
|
||||
may bool
|
||||
}{
|
||||
{"mesh-delivery-builder", "$JS.EVENT.ADVISORY.STREAM.CREATED.KV_mesh-delivery-builder_steps", true},
|
||||
{GrantsModule, "$JS.EVENT.ADVISORY.STREAM.CREATED.KV_mesh-vault_bus-accounts", true},
|
||||
{"mesh-delivery-builder", "$JS.EVENT.ADVISORY.STREAM.CREATED.KV_mesh-grants_assertions", false},
|
||||
{"mesh-delivery-builder", "$JS.EVENT.ADVISORY.STREAM.CREATED.LOG_mesh-delivery-builder_builds", false},
|
||||
{"mesh-delivery-builder", "$JS.EVENT.ADVISORY.STREAM.DELETED.KV_mesh-delivery-builder_steps", false},
|
||||
} {
|
||||
if got := MaySubscribe(perms[row.who], row.subject); got != row.may {
|
||||
t.Errorf("%s may hear %s: %v, want %v", row.who, row.subject, got, row.may)
|
||||
}
|
||||
}
|
||||
// A name that is no plain token is never granted, and a module other than the grants part is never granted one.
|
||||
if got := CreateStreamGrants([]string{"*", ">", "KV_a.b", "KV_a_b"}); len(got) != 2 {
|
||||
t.Errorf("grants for wildcard names: %v", got)
|
||||
}
|
||||
other, err := PermissionsFor(Principal{Kind: KindModule, Node: "novox", Module: "shop", CreatesStreams: []string{"KV_shop_x"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if MayPublish(other, "$JS.API.STREAM.CREATE.KV_shop_x") {
|
||||
t.Error("a module other than the grants part was granted a create")
|
||||
}
|
||||
// The writers table refuses the grants part an update, and a create of a stream it was not given.
|
||||
g := Principal{Kind: KindModule, Node: "novox", Module: GrantsModule, CreatesStreams: []string{"KV_a_b"}}
|
||||
if err := CheckWriters(g, []string{"$JS.API.STREAM.CREATE.KV_a_b"}); err != nil {
|
||||
t.Errorf("the declared create refused: %v", err)
|
||||
}
|
||||
for _, refused := range []string{"$JS.API.STREAM.UPDATE.KV_a_b", "$JS.API.STREAM.CREATE.KV_a_c", "$JS.API.STREAM.CREATE.*"} {
|
||||
if CheckWriters(g, []string{refused}) == nil {
|
||||
t.Errorf("%s was not refused", refused)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -3,7 +3,7 @@
|
||||
// streams, state and logs it composes them from.
|
||||
//
|
||||
// **A copy of the controller's internal/broker, composer half** (issue 1377, novox/hq ADR 0547 §5), taken at
|
||||
// mesh-controller 7cd4d18b with the grants part's membership grant of f67d1553 (ADR 0547 §5): the bus client (jetstream.go), the broker's address from the environment (broker.go, onnats.go
|
||||
// mesh-controller 7cd4d18b with the grants part's membership grant of f67d1553 (ADR 0547 §5) and its grant to create the streams manifests declare (issue 1790): the bus client (jetstream.go), the broker's address from the environment (broker.go, onnats.go
|
||||
// but its one constant) and the two methods on that client are left out; the lease's two subjects are copied in
|
||||
// (nats.go). The controller keeps its own copy, unchanged, until it loses its lease at ADR 0547's third milestone, so
|
||||
// for that time two composers state who may say what and must say the same: TestTheCopiedComposerComposesAsTheControllers
|
||||
|
||||
@@ -62,6 +62,9 @@ func UserListFixtures() []UserListFixture {
|
||||
},
|
||||
{Module: "mesh-delivery-view", Reads: []string{"mesh-delivery-builder.steps"},
|
||||
KeyedReads: []KeyedRead{{Read: "mesh-delivery-builder.presence", Key: "novox"}}},
|
||||
// The grants part, which may create every stream declared here (ADR 0547 §5, issue 1790).
|
||||
{Module: "mesh-grants", Emits: []string{"membership-changed"},
|
||||
State: []Bucket{{Module: "mesh-grants", Name: "assertions", History: 5}}},
|
||||
},
|
||||
"ace": {
|
||||
{Module: "plex", Serves: []string{"plex_status"}, Checks: []string{"plex.plex_status"}},
|
||||
|
||||
+38
-2
@@ -213,9 +213,17 @@ type Principal struct {
|
||||
// StatesItsKeyToTheVault is a machine principal whose node-engine may ask the vault's `node-key` in its own name
|
||||
// (issue 786, novox/hq ADR 0372): set while the vault's holder serves it. That one verb, and nothing else new.
|
||||
StatesItsKeyToTheVault bool
|
||||
// FollowsGrantsUserList is the machine principal of the node holding mesh-broker: its node-engine reads the grants
|
||||
// part's sealed user list, one key and an object by name (novox/hq ADR 0547 clause 7, GrantsUserListFollows).
|
||||
FollowsGrantsUserList bool
|
||||
// BoundBuckets are the providers' buckets of entries, `<module>_bound`, a provisioner client reads its own
|
||||
// machine's keys of (mesh-sdk go/provider, issue 679): every bucket named bound of a module the mesh places.
|
||||
BoundBuckets []string
|
||||
// CreatesStreams are the streams the grants part's principal alone may create (novox/hq ADR 0547 §5; the operator,
|
||||
// 2026-10-26 08:54): the stream of every bucket, log and stream a manifest of a module placed anywhere declares,
|
||||
// by name. Created only, never updated or deleted: a new module state is on the bus when the grants part hears its
|
||||
// manifest, not at the controller's next start (issue 1790).
|
||||
CreatesStreams []string
|
||||
|
||||
// PasswordHash is the bcrypt hash the mesh minted. The plaintext is sealed to the principal
|
||||
// and never appears here: this file is written to a node's disk and read by a server, and a
|
||||
@@ -907,6 +915,11 @@ func PermissionsFor(p Principal) (Permissions, error) {
|
||||
if p.StatesItsKeyToTheVault {
|
||||
pub = append(pub, "mesh.seat."+VaultNodeKey.Seat+"."+CallKind+"."+VaultNodeKey.Verb+"."+CallerToken(p.Username()))
|
||||
}
|
||||
// And, on the node holding mesh-broker, the grants part's sealed user list it follows from take-over (novox/hq ADR
|
||||
// 0547 clause 7): one key, and an object by its name.
|
||||
if p.FollowsGrantsUserList {
|
||||
pub = append(pub, GrantsUserListFollows()...)
|
||||
}
|
||||
|
||||
case KindModule:
|
||||
// 0. The watchman alone hears who connects and disconnects, and asks who is connected (ADR 0431 §2).
|
||||
@@ -919,6 +932,11 @@ func PermissionsFor(p Principal) (Permissions, error) {
|
||||
// first so it publishes only what changed, and no stream it defines.
|
||||
if p.Module == GrantsModule {
|
||||
pub = append(pub, "mesh.assignment.*.*", "$JS.API.DIRECT.GET."+AssignmentsStream+".mesh.assignment.*.*")
|
||||
// And creating each stream a manifest declares, by its own name (the operator, 2026-10-26 08:54; issue
|
||||
// 1790): the create and the read of that one stream, never an update, a delete or a stream not declared.
|
||||
pub = append(pub, CreateStreamGrants(p.CreatesStreams)...)
|
||||
// And its own object bucket, where it writes the user list sealed to the bus node (ADR 0547 clause 7).
|
||||
pub = append(pub, GrantsUserListWrites()...)
|
||||
}
|
||||
// The view alone reads the build seats' workers for the backlog of asks (issue 1887 A8a).
|
||||
if p.Module == ViewModule {
|
||||
@@ -1067,8 +1085,12 @@ func PermissionsFor(p Principal) (Permissions, error) {
|
||||
|
||||
// 5. Its state, and the state of others it reads (novox/hq ADR 0201): every one read and
|
||||
// watched, its own written too.
|
||||
pub = append(pub, stateGrants(stateAccess{Module: p.Module, Node: p.Node, Keeps: p.State,
|
||||
PerMachine: p.PerMachine, Reads: p.Reads, KeyedReads: p.KeyedReads})...)
|
||||
kept := stateAccess{Module: p.Module, Node: p.Node, Keeps: p.State, PerMachine: p.PerMachine, Reads: p.Reads,
|
||||
KeyedReads: p.KeyedReads}
|
||||
pub = append(pub, stateGrants(kept)...)
|
||||
// And it hears each of those buckets created, so one not on the bus at its start is bound when it appears
|
||||
// (novox/hq issue 1790; mesh-sdk process.AwaitBucket).
|
||||
sub = append(sub, stateCreatedHeard(kept)...)
|
||||
// And its logs (novox/hq ADR 0297): appended to and read directly, its own alone.
|
||||
pub = append(pub, logGrants(p.Module, p.Logs)...)
|
||||
|
||||
@@ -1586,6 +1608,20 @@ func ComposeAccounts(principals []Principal) (string, error) {
|
||||
// from them. No other principal is granted the subjects below, and nothing is a wildcard over them.
|
||||
const WatchmanModule = "mesh-delivery-watchman"
|
||||
|
||||
// CreateStreamGrants is what creating each named stream needs, and nothing wider (novox/hq ADR 0547 §5; the operator,
|
||||
// 2026-10-26 08:54): its create, and its information to read back what it holds. Each by its own name: a name that
|
||||
// is not one plain subject token is left out, since a wildcard there would be every stream.
|
||||
func CreateStreamGrants(streams []string) []string {
|
||||
var out []string
|
||||
for _, s := range streams {
|
||||
if !safeSubject.MatchString(s) {
|
||||
continue
|
||||
}
|
||||
out = append(out, "$JS.API.STREAM.CREATE."+s, "$JS.API.STREAM.INFO."+s)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// GrantsModule is the one module whose principal publishes the modules' memberships (novox/hq ADR 0547 §5): the grants
|
||||
// part, which composes them from the manifests and placement. It defines no stream and is granted no other subject here.
|
||||
const GrantsModule = "mesh-grants"
|
||||
|
||||
@@ -30,7 +30,11 @@ func processGrants(p Principal) (pub, sub []string, err error) {
|
||||
|
||||
// 3. Its module's own state, by the local names its bus names alone — never the whole of what the
|
||||
// module keeps (novox/hq ADR 0436 §1), read and written as stateGrants gives any module's own bucket.
|
||||
pub = append(pub, stateGrants(stateAccess{Module: p.Module, Node: p.Node, Keeps: p.State})...)
|
||||
kept := stateAccess{Module: p.Module, Node: p.Node, Keeps: p.State}
|
||||
pub = append(pub, stateGrants(kept)...)
|
||||
// And it hears each of them created, so one not on the bus at its start is bound when it appears (novox/hq issue
|
||||
// 1790; process.AwaitBucket).
|
||||
sub = append(sub, stateCreatedHeard(kept)...)
|
||||
|
||||
// 4. The router's records of its module's own asks (novox/hq ADR 0436 §1, ADR 0259 §3): the same
|
||||
// direct-get subjects a module's own principal reads for the seats its Uses names that keep them —
|
||||
|
||||
@@ -24,8 +24,9 @@ import (
|
||||
// subjects cover it, so what travels there — a code typed by the operator — is never persisted. A kinded
|
||||
// holder asks with its own kind; the modules that watch the seat answer.
|
||||
//
|
||||
// And one read: **a holder's records**, a bucket the seat names, read by each user under its own name only
|
||||
// (`$KV.<bucket>.<module>.>`), so an asker reads the state of its own asks and no other asker's.
|
||||
// And two reads: **a holder's records**, a bucket the seat names, read by each user under its own name only
|
||||
// (`$KV.<bucket>.<module>.>`), so an asker reads the state of its own asks and no other asker's; and on a seat pulled by
|
||||
// kind, **what its workers gave up on**, the copies DEAD_LETTERS keeps of the seat's stream (DeadLetterReads, issue 734).
|
||||
|
||||
// BuildAskMaxDeliver bounds the deliveries of a build seat's ask (issue 1907): mesh-sdk delivery.AskMaxDeliver, written
|
||||
// here as the broker mirrors the catalogue rather than importing it.
|
||||
@@ -129,6 +130,9 @@ func SeatTrafficOf(module string, holds, uses, watches []Seat) SeatTraffic {
|
||||
}
|
||||
t.Records = append(t.Records, "$JS.API.DIRECT.GET.KV_"+b+".$KV."+b+"."+module+".>")
|
||||
}
|
||||
if s.PulledByKind && len(s.Accepts) > 0 && safeSubject.MatchString(s.Name) {
|
||||
t.Records = append(t.Records, DeadLetterReads(seatStreamName(s.Name))...)
|
||||
}
|
||||
}
|
||||
for _, w := range watches {
|
||||
if w.Kinded {
|
||||
@@ -151,6 +155,20 @@ func SeatTrafficOf(module string, holds, uses, watches []Seat) SeatTraffic {
|
||||
return t
|
||||
}
|
||||
|
||||
// DeadLetterReads is what a user of a seat pulled by kind is granted to hear which of the asks it put the seat's workers
|
||||
// gave up on (novox/hq issue 734): an ordered consumer on DEAD_LETTERS filtered to the copies the controller keeps of that
|
||||
// seat's stream (DeadLetterSubject(stream, *)), pulled, and its deletion. A copy carries the ask's own message id
|
||||
// (Mesh-Dead-Msg-Id), so the asker ends the step it waits for instead of waiting for ever; it reads the seat's dead
|
||||
// letters and no other stream's, and changes none.
|
||||
func DeadLetterReads(stream string) []string {
|
||||
return []string{
|
||||
"$JS.API.STREAM.INFO." + DeadLettersStream,
|
||||
"$JS.API.CONSUMER.CREATE." + DeadLettersStream + ".*." + DeadLetterSubject(stream, ">"),
|
||||
"$JS.API.CONSUMER.MSG.NEXT." + DeadLettersStream + ".*",
|
||||
"$JS.API.CONSUMER.DELETE." + DeadLettersStream + ".*",
|
||||
}
|
||||
}
|
||||
|
||||
// seatTrafficGrants is the bus permissions seat traffic needs: the subjects themselves, and the JetStream API a
|
||||
// worker is pulled and acknowledged through and a record is read through.
|
||||
func seatTrafficGrants(t SeatTraffic) (pub, sub []string) {
|
||||
|
||||
@@ -166,6 +166,49 @@ func stateGrants(a stateAccess) []string {
|
||||
return out
|
||||
}
|
||||
|
||||
// StreamCreatedAdvisory is the subject the bus says a stream was created on (the server's own advisory), here for a
|
||||
// bucket's stream KV_<bucket>.
|
||||
func StreamCreatedAdvisory(bucket string) string {
|
||||
return "$JS.EVENT.ADVISORY.STREAM.CREATED.KV_" + bucket
|
||||
}
|
||||
|
||||
// stateCreatedHeard is what a principal subscribes to so that a bucket it may bind and that is not on the bus yet is
|
||||
// bound when it appears (novox/hq issue 1790, 2026-10-26: since the grants part creates a manifest's buckets, ADR 0547,
|
||||
// a module's delivery can start it before its new state exists): the bus's advisory that the bucket's stream was
|
||||
// created, for each bucket stateGrants lets it bind, and no other. Read-only: the server publishes it in the mesh's own
|
||||
// account, and hearing it changes nothing.
|
||||
func stateCreatedHeard(a stateAccess) []string {
|
||||
var out []string
|
||||
seen := map[string]bool{}
|
||||
add := func(bucket string) {
|
||||
if !seen[bucket] {
|
||||
seen[bucket] = true
|
||||
out = append(out, StreamCreatedAdvisory(bucket))
|
||||
}
|
||||
}
|
||||
perMachine := map[string]bool{}
|
||||
for _, n := range a.PerMachine {
|
||||
perMachine[n] = true
|
||||
}
|
||||
for _, name := range a.Keeps {
|
||||
if !safeSubject.MatchString(name) || (perMachine[name] && !safeSubject.MatchString(a.Node)) {
|
||||
continue
|
||||
}
|
||||
add(BucketName(a.Module, name))
|
||||
}
|
||||
for _, r := range a.Reads {
|
||||
if bucket, ok := bucketOfRead(r); ok {
|
||||
add(bucket)
|
||||
}
|
||||
}
|
||||
for _, k := range a.KeyedReads {
|
||||
if bucket, ok := bucketOfRead(k.Read); ok && safeSubject.MatchString(k.Key) {
|
||||
add(bucket)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// KeyedRead is one key of another module's state a principal reads, and no other key of it.
|
||||
type KeyedRead struct {
|
||||
Read string `json:"read"`
|
||||
|
||||
+24
-2
File diff suppressed because one or more lines are too long
@@ -0,0 +1,111 @@
|
||||
package buscompose
|
||||
|
||||
import (
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// **The grants part's user list, sealed to the bus node** (novox/hq ADR 0547 clause 7, issue 1945). The grants part writes
|
||||
// the list it composed as one object of its own object bucket, sealed to the key of the node holding mesh-broker, and
|
||||
// names it in one key of its own state `user-list`. The node-engine of that node follows the key when the switch says
|
||||
// so (clause 8), opens the object and writes the bus's user-list file.
|
||||
//
|
||||
// Who may do what, and nothing more:
|
||||
//
|
||||
// - the grants part writes its object bucket (create, put, purge of an object's old chunks, delete) and its own state
|
||||
// as before;
|
||||
// - the node-engine of the node holding mesh-broker reads that one key of the state, watched, and an object of the
|
||||
// bucket by its name: never the bucket's list of names, never a write, never another key of the grants part's state;
|
||||
// - no other principal touches either.
|
||||
|
||||
// GrantsUserListBucket, GrantsUserListKey, GrantsUserListObjects and GrantsUserListObjectStream are delivery's names,
|
||||
// here where the grants are composed.
|
||||
const (
|
||||
GrantsUserListBucket = delivery.GrantsUserListBucket
|
||||
GrantsUserListKey = delivery.GrantsUserListKey
|
||||
GrantsUserListObjects = delivery.GrantsUserListObjects
|
||||
GrantsUserListObjectStream = delivery.GrantsUserListObjectStream
|
||||
)
|
||||
|
||||
// GrantsUserListFollows is what the node-engine of the node holding mesh-broker may publish to follow the grants part's
|
||||
// list: the one key `user-list.sealed` read and watched (its own ordered consumer, filtered on that key alone), and an
|
||||
// object of the bucket by its name, as a node reads its declaration's files (ADR 0486 §2: never STREAM.INFO of the object
|
||||
// stream, which lists every name, and never STREAM.MSG.GET).
|
||||
func GrantsUserListFollows() []string {
|
||||
kv := "KV_" + GrantsUserListBucket
|
||||
key := "$KV." + GrantsUserListBucket + "." + GrantsUserListKey
|
||||
obj, s := GrantsUserListObjects, GrantsUserListObjectStream
|
||||
return []string{
|
||||
"$JS.API.DIRECT.GET." + kv + "." + key,
|
||||
"$JS.API.CONSUMER.CREATE." + kv + ".*." + key,
|
||||
"$JS.API.CONSUMER.DELETE." + kv + ".*",
|
||||
"$JS.API.STREAM.INFO." + kv,
|
||||
"$JS.FC." + kv + ".>",
|
||||
"$JS.API.DIRECT.GET." + s + ".$O." + obj + ".M.*",
|
||||
"$JS.API.CONSUMER.CREATE." + s + ".*.$O." + obj + ".C.*",
|
||||
"$JS.API.CONSUMER.DELETE." + s + ".*",
|
||||
"$JS.FC." + s + ".>",
|
||||
}
|
||||
}
|
||||
|
||||
// GrantsUserListWrites is what the grants part may publish to keep its object bucket: create it and read its info back,
|
||||
// put an object (its chunks and its metadata, the old chunks of a name purged), read an object's metadata, and delete
|
||||
// one. Its own state `user-list` it writes already, as every module writes its own.
|
||||
func GrantsUserListWrites() []string {
|
||||
obj, s := GrantsUserListObjects, GrantsUserListObjectStream
|
||||
return []string{
|
||||
"$JS.API.STREAM.CREATE." + s,
|
||||
"$JS.API.STREAM.INFO." + s,
|
||||
"$JS.API.STREAM.PURGE." + s,
|
||||
"$JS.API.DIRECT.GET." + s + ".>",
|
||||
"$JS.API.STREAM.MSG.GET." + s,
|
||||
"$O." + obj + ".>",
|
||||
}
|
||||
}
|
||||
|
||||
// grantsOwnObjects is the grants part writing its own object bucket, and nothing of another.
|
||||
func grantsOwnObjects(p Principal, pattern string) bool {
|
||||
return p.Kind == KindModule && p.Module == GrantsModule &&
|
||||
(pattern == "$O."+GrantsUserListObjects+".>" || pattern == "$JS.API.STREAM.CREATE."+GrantsUserListObjectStream)
|
||||
}
|
||||
|
||||
// **Named readers** (novox/hq ADR 0547 clause 9, progressive insight of 2026-10-26; issue 1945). A state named here is
|
||||
// read by the modules named and by no other, whatever another module's `reads` says: the vault's `bus-accounts` holds
|
||||
// every bus account's bcrypt hash, and from take-over the grants part alone may read it. A module's announcement may name
|
||||
// readers too (announce.State.Readers); both are honoured.
|
||||
const VaultBusAccounts = "mesh-vault.bus-accounts"
|
||||
|
||||
// NamedReaders are the states only the modules named may read, by `<owner>.<name>`.
|
||||
var NamedReaders = map[string][]string{VaultBusAccounts: {GrantsModule}}
|
||||
|
||||
// ReaderMay says whether module may read the state read (`<owner>.<name>`) by the named readers: always, for a state
|
||||
// that names none.
|
||||
func ReaderMay(module, read string) bool {
|
||||
readers, named := NamedReaders[read]
|
||||
return !named || slices.Contains(readers, module)
|
||||
}
|
||||
|
||||
// readsNamed splits a module's reads by the named readers: those it may read, and those named for others.
|
||||
func readsNamed(module string, reads []string) (granted, refused []string) {
|
||||
for _, r := range reads {
|
||||
if ReaderMay(module, strings.TrimSpace(r)) {
|
||||
granted = append(granted, r)
|
||||
} else {
|
||||
refused = append(refused, r)
|
||||
}
|
||||
}
|
||||
return granted, refused
|
||||
}
|
||||
|
||||
// withNamedReaders is d with the reads named for other modules moved to its refused reads.
|
||||
func withNamedReaders(d Declared) Declared {
|
||||
granted, refused := readsNamed(d.Module, d.Reads)
|
||||
if len(refused) == 0 {
|
||||
return d
|
||||
}
|
||||
d.Reads = granted
|
||||
d.ReadsRefused = append(append([]string(nil), d.ReadsRefused...), refused...)
|
||||
return d
|
||||
}
|
||||
@@ -0,0 +1,213 @@
|
||||
//go:build integration
|
||||
|
||||
package buscompose
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/seal"
|
||||
"github.com/nats-io/nats-server/v2/server"
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
)
|
||||
|
||||
// Defends novox/hq ADR 0547 clause 7 on a real server (issue 1945): with the list this composer writes, the grants part
|
||||
// creates its object bucket and its state, puts the sealed list and names it in `user-list.sealed`; the node-engine of
|
||||
// the node holding mesh-broker watches that one key, reads the object by its name and opens it with its key, and may
|
||||
// neither list the bucket, nor read another key of the state, nor write; another node reads nothing of either.
|
||||
func TestTheBusNodeFollowsTheGrantsPartsSealedListUnderTheComposedGrants(t *testing.T) {
|
||||
records := Records{
|
||||
Nodes: []string{"novox", "ace"},
|
||||
Assigned: map[string][]Declared{
|
||||
"novox": {{Module: "nats", SnapshotsTheBus: true},
|
||||
{Module: GrantsModule, State: []Bucket{{Module: GrantsModule, Name: "user-list"}}}},
|
||||
"ace": {{Module: "plex"}},
|
||||
},
|
||||
}
|
||||
users, err := Users(records)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for i := range users {
|
||||
users[i].PasswordHash = "pw-" + users[i].Username()
|
||||
}
|
||||
accounts, err := ComposeAccounts(users)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
dir := t.TempDir()
|
||||
conf := filepath.Join(dir, "server.conf")
|
||||
if err := os.WriteFile(conf, []byte("listen: 127.0.0.1:-1\njetstream { store_dir: \""+dir+"\" }\n"+accounts), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
opts, err := server.ProcessConfigFile(conf)
|
||||
if err != nil {
|
||||
t.Fatalf("the composed list is not the server's: %v", err)
|
||||
}
|
||||
opts.NoLog, opts.NoSigs = true, true
|
||||
s, err := server.NewServer(opts)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
go s.Start()
|
||||
if !s.ReadyForConnections(30 * time.Second) {
|
||||
t.Fatal("the bus did not come up")
|
||||
}
|
||||
t.Cleanup(s.Shutdown)
|
||||
dial := func(user string) *nats.Conn {
|
||||
nc, err := nats.Connect(s.ClientURL(), nats.UserInfo(user, "pw-"+user), nats.CustomInboxPrefix("_INBOX."+user))
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", user, err)
|
||||
}
|
||||
t.Cleanup(nc.Close)
|
||||
return nc
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
// The grants part: its state and its object bucket, the sealed list put and named.
|
||||
grants, err := jetstream.New(dial("novox." + GrantsModule))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Its state is made as the grants part makes a declared state, by its stream's name (CreateKeyValue would ask the
|
||||
// account's info, which no module is granted).
|
||||
if _, err := grants.CreateStream(ctx, jetstream.StreamConfig{Name: "KV_" + GrantsUserListBucket,
|
||||
Subjects: []string{"$KV." + GrantsUserListBucket + ".>"}, MaxMsgsPerSubject: 1, AllowDirect: true,
|
||||
AllowRollup: true, DenyDelete: true, Discard: jetstream.DiscardNew}); err != nil {
|
||||
t.Fatalf("the grants part's state: %v", err)
|
||||
}
|
||||
kv, err := grants.KeyValue(ctx, GrantsUserListBucket)
|
||||
if err != nil {
|
||||
t.Fatalf("the grants part binds its state: %v", err)
|
||||
}
|
||||
objects, err := grants.CreateObjectStore(ctx, jetstream.ObjectStoreConfig{Bucket: GrantsUserListObjects})
|
||||
if err != nil {
|
||||
t.Fatalf("the grants part's object bucket: %v", err)
|
||||
}
|
||||
node, err := seal.GenerateKeyPair()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
list := []byte(accounts)
|
||||
digest := delivery.UserListDigest(list)
|
||||
object, err := delivery.SealUserList(list, node.PublicKey)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
name := delivery.UserListObjectName(digest)
|
||||
for range 2 { // put twice: a second put of a name purges its old chunks
|
||||
if _, err := objects.PutBytes(ctx, name, object); err != nil {
|
||||
t.Fatalf("the grants part put its object: %v", err)
|
||||
}
|
||||
}
|
||||
value, err := delivery.EncodeGrantsUserList(delivery.GrantsUserList{Follow: delivery.FollowGrants, Digest: digest,
|
||||
Object: name, SealedTo: delivery.SealingKeyID(node.PublicKey), Node: "novox", Revision: 1, Users: len(users)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := kv.Put(ctx, GrantsUserListKey, value); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := kv.Put(ctx, "list", []byte(`{"v":1}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// The bus node's engine: the key watched, the object by name (as mesh-host's link reads a declaration's file).
|
||||
busNode := dial("node.novox")
|
||||
js, err := busNode.JetStream(nats.MaxWait(5 * time.Second))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
keys, err := js.KeyValue(GrantsUserListBucket)
|
||||
if err != nil {
|
||||
t.Fatalf("the bus node binds the grants part's state: %v", err)
|
||||
}
|
||||
w, err := keys.Watch(GrantsUserListKey)
|
||||
if err != nil {
|
||||
t.Fatalf("the bus node watches the key: %v", err)
|
||||
}
|
||||
defer w.Stop()
|
||||
var got delivery.GrantsUserList
|
||||
select {
|
||||
case e := <-w.Updates():
|
||||
if e == nil {
|
||||
t.Fatal("the watch found no value")
|
||||
}
|
||||
if got, err = delivery.DecodeGrantsUserList(e.Value()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
case <-ctx.Done():
|
||||
t.Fatal("the watch said nothing")
|
||||
}
|
||||
read := func(js nats.JetStreamContext, name string) ([]byte, error) {
|
||||
meta, err := js.GetLastMsg(GrantsUserListObjectStream, "$O."+GrantsUserListObjects+".M."+
|
||||
base64.URLEncoding.EncodeToString([]byte(name)), nats.DirectGet())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var info struct {
|
||||
NUID string `json:"nuid"`
|
||||
Size int `json:"size"`
|
||||
}
|
||||
if err := json.Unmarshal(meta.Data, &info); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sub, err := js.SubscribeSync("$O."+GrantsUserListObjects+".C."+info.NUID, nats.OrderedConsumer(),
|
||||
nats.BindStream(GrantsUserListObjectStream))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer sub.Unsubscribe()
|
||||
var body []byte
|
||||
for len(body) < info.Size {
|
||||
m, err := sub.NextMsg(5 * time.Second)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
body = append(body, m.Data...)
|
||||
}
|
||||
return body, nil
|
||||
}
|
||||
body, err := read(js, got.Object)
|
||||
if err != nil {
|
||||
t.Fatalf("the bus node reads the object by its name: %v", err)
|
||||
}
|
||||
opened, err := delivery.OpenUserList(body, got.Digest, func(b seal.Box) ([]byte, error) {
|
||||
s, err := seal.Open(b, node.PrivateKey)
|
||||
return []byte(s), err
|
||||
})
|
||||
if err != nil || !bytes.Equal(opened, list) {
|
||||
t.Fatalf("the bus node opened %d bytes, %v", len(opened), err)
|
||||
}
|
||||
|
||||
// And nothing beyond: no other key, no list of the bucket, no write.
|
||||
if _, err := keys.Get("list"); err == nil {
|
||||
t.Error("the bus node read another key of the grants part's state")
|
||||
}
|
||||
if _, err := js.StreamInfo(GrantsUserListObjectStream); err == nil {
|
||||
t.Error("the bus node read the object stream's info, which lists every name")
|
||||
}
|
||||
if _, err := js.Publish("$O."+GrantsUserListObjects+".M.x", []byte("x")); err == nil {
|
||||
t.Error("the bus node wrote the grants part's bucket")
|
||||
}
|
||||
// Another node reads nothing of it.
|
||||
other, err := dial("node.ace").JetStream(nats.MaxWait(2 * time.Second))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := read(other, got.Object); err == nil {
|
||||
t.Error("another node read the object")
|
||||
}
|
||||
if _, err := other.GetLastMsg("KV_"+GrantsUserListBucket, "$KV."+GrantsUserListBucket+"."+GrantsUserListKey, nats.DirectGet()); err == nil {
|
||||
t.Errorf("another node read the key: %v", err)
|
||||
}
|
||||
}
|
||||
+41
-1
@@ -147,14 +147,19 @@ func Users(r Records) ([]Principal, error) {
|
||||
for _, node := range sortedCopy(r.Nodes) {
|
||||
witness := false
|
||||
var checks []string
|
||||
bus := false
|
||||
for _, d := range r.Assigned[node] {
|
||||
if d.Module == controllerModule {
|
||||
witness = true
|
||||
}
|
||||
if d.SnapshotsTheBus {
|
||||
bus = true
|
||||
}
|
||||
checks = append(checks, d.Checks...)
|
||||
}
|
||||
out = append(out, Principal{Kind: KindNode, Node: node, WitnessesController: witness, Checks: checks,
|
||||
MovesIntoTheVault: servesMoveIn(r), StatesItsKeyToTheVault: servesVerb(r, VaultNodeKey)})
|
||||
MovesIntoTheVault: servesMoveIn(r), StatesItsKeyToTheVault: servesVerb(r, VaultNodeKey),
|
||||
FollowsGrantsUserList: bus})
|
||||
// **And the machine's provisioner client, on an account of its own** (novox/hq ADR 0349 clause 11): one
|
||||
// per machine that runs it, delivered to the module running it.
|
||||
for _, d := range r.Assigned[node] {
|
||||
@@ -176,6 +181,9 @@ func Users(r Records) ([]Principal, error) {
|
||||
}
|
||||
}
|
||||
for _, d := range r.Assigned[node] {
|
||||
// A state with named readers is read by them alone (ADR 0547 clause 9): a read named for another module is
|
||||
// granted nothing, here and in the runtime carrying the module.
|
||||
d = withNamedReaders(d)
|
||||
// **A process account is its own, delivered in its own secret** (broker-<process id>, novox/hq
|
||||
// ADR 0436 §1, §2), never the module's `broker` — so it is composed whether or not the module
|
||||
// itself has an account, and is never carried by the machine's runtime, which runs as the
|
||||
@@ -207,6 +215,11 @@ func Users(r Records) ([]Principal, error) {
|
||||
PerMachine: perMachineNames(d.State), KeyedReads: d.KeyedReads,
|
||||
Logs: logNames(d.Logs), Streams: d.Streams, Hears: ComposedHears(d), Queues: d.Queues,
|
||||
})
|
||||
// **The grants part creates what a manifest declares** (novox/hq ADR 0547 §5; the operator, 2026-10-26
|
||||
// 08:54): every placed module's buckets, logs and streams, each by its stream's name.
|
||||
if d.Module == GrantsModule {
|
||||
out[len(out)-1].CreatesStreams = declaredStreamsOf(r)
|
||||
}
|
||||
// **The vault's holder asks the desks and the issuing providers itself** (issue 723), under the
|
||||
// account its own runtime runs on (ADR 0259 §8's runs-as): the one module granted it.
|
||||
if holdsSeat(d, VaultSeat) {
|
||||
@@ -218,6 +231,7 @@ func Users(r Records) ([]Principal, error) {
|
||||
var carried []Declared
|
||||
for _, d := range r.Assigned[node] {
|
||||
if d.RunsAs == "" && runnerServes(d) {
|
||||
d = withNamedReaders(d)
|
||||
// Never a restricted stream for what the tool runner carries (novox/hq ADR 0376 §1).
|
||||
d.Hears = ComposedHears(d)
|
||||
carried = append(carried, d)
|
||||
@@ -418,3 +432,29 @@ func boundBucketsOf(r Records) []string {
|
||||
func runnerServes(d Declared) bool {
|
||||
return d.Module == RuntimeModule || d.NoAccount || d.Loads || len(d.Serves) > 0 || len(d.Holds) > 0 || len(d.Waits) > 0 || len(d.Checks) > 0
|
||||
}
|
||||
|
||||
// declaredStreamsOf is the stream of every bucket, log and stream a module placed anywhere declares, once each, in name
|
||||
// order (novox/hq ADR 0547 §5, issue 1790): `KV_<module>_<state>`, `LOG_<module>_<log>` and each module stream's own
|
||||
// name. What the grants part may create, and nothing beside.
|
||||
func declaredStreamsOf(r Records) []string {
|
||||
seen := map[string]bool{}
|
||||
for _, mods := range r.Assigned {
|
||||
for _, d := range mods {
|
||||
for _, b := range d.State {
|
||||
seen["KV_"+b.Bucket()] = true
|
||||
}
|
||||
for _, l := range d.Logs {
|
||||
seen[l.Stream()] = true
|
||||
}
|
||||
for _, s := range d.Streams {
|
||||
seen[s.Name] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
out := make([]string, 0, len(seen))
|
||||
for s := range seen {
|
||||
out = append(out, s)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
@@ -37,6 +37,20 @@ type WriterRow struct {
|
||||
// isController, and the other writers' tests, are who a row's writer is as a principal.
|
||||
func isController(p Principal, _ string) bool { return p.Kind == KindController }
|
||||
|
||||
// createsDeclared is the controller, or the grants part creating one stream a manifest declares, by its own name: the
|
||||
// create alone, never an update or a delete (novox/hq ADR 0547 §5; the operator, 2026-10-26 08:54).
|
||||
func createsDeclared(p Principal, pattern string) bool {
|
||||
if isController(p, pattern) {
|
||||
return true
|
||||
}
|
||||
name, ok := strings.CutPrefix(pattern, "$JS.API.STREAM.CREATE.")
|
||||
if ok && p.Kind == KindModule && p.Module == GrantsModule && name == GrantsUserListObjectStream {
|
||||
return true
|
||||
}
|
||||
return ok && p.Kind == KindModule && p.Module == GrantsModule && slices.Contains(p.CreatesStreams, name) &&
|
||||
safeSubject.MatchString(name)
|
||||
}
|
||||
|
||||
// ownMachine is a node writing a subject whose third token is its own name, and no wildcard there.
|
||||
func ownMachine(p Principal, pattern string) bool {
|
||||
tokens := strings.Split(pattern, ".")
|
||||
@@ -170,7 +184,8 @@ var WritersTable = []WriterRow{
|
||||
// bucket's stream (ADR 0201), which defines nothing the mesh keeps.
|
||||
Subjects: []string{"$JS.API.STREAM.CREATE.>", "$JS.API.STREAM.UPDATE.>", "$JS.API.STREAM.DELETE.>",
|
||||
"$JS.API.CONSUMER.DURABLE.CREATE.>"},
|
||||
Writes: isController},
|
||||
Writes: createsDeclared,
|
||||
Shared: "the grants part creates a stream a manifest declares, by its name, and never updates or deletes one (novox/hq ADR 0547 §5; the operator, 2026-10-26 08:54)"},
|
||||
{State: "builds and their outcomes", Writer: "the build seat's holder", KeptIn: "its own state",
|
||||
Others: "the controller asks",
|
||||
Subjects: []string{"mesh.seat.node-build-agent.event.built", "mesh.seat.mesh-build-machine.event.built"},
|
||||
@@ -237,7 +252,10 @@ var WritersTable = []WriterRow{
|
||||
KeptIn: "the object bucket " + DeclarationFilesBucket + "-<node> of each node, an object removed a day after the " +
|
||||
"node's latest declaration no longer names it",
|
||||
Others: "a node-engine reads an object of its own bucket by its name and may neither list the bucket nor write it",
|
||||
Subjects: []string{"$O.*.>"}, Writes: isControllerOrTheHandPath},
|
||||
Subjects: []string{"$O.*.>"}, Writes: func(p Principal, pattern string) bool {
|
||||
return isControllerOrTheHandPath(p, pattern) || grantsOwnObjects(p, pattern)
|
||||
},
|
||||
Shared: "the grants part writes its own object bucket " + GrantsUserListObjects + ", the user list sealed to the bus node, and no other (novox/hq ADR 0547 clause 7)"},
|
||||
{State: "the declared-config mirror", Writer: "controller (lease holder), from its store, only on a change",
|
||||
KeptIn: "key-value " + DeclaredConfigBucket, Others: "read through `reads` as mesh-controller." + MirrorDeclaredConfig,
|
||||
Subjects: kvOf(DeclaredConfigBucket), Writes: isController},
|
||||
|
||||
@@ -172,7 +172,7 @@ func TestAStoreRowWithoutAProtocolKeepsTheCompiledOne(t *testing.T) {
|
||||
t.Cleanup(func() { UseSeats(was) })
|
||||
UseSeats([]Seat{{Name: "node-build-go", Scope: ScopeNode, Decision: "row"}})
|
||||
got, ok := SeatNamed("node-build-go")
|
||||
if !ok || len(got.Accepts) == 0 || !got.PulledByKind {
|
||||
if !ok || len(got.Accepts) == 0 || !got.PulledByKind || !got.Yields {
|
||||
t.Fatalf("the Go build seat lost what it accepts when loaded from the store: %+v", got)
|
||||
}
|
||||
if got.Decision != "row" {
|
||||
|
||||
@@ -311,6 +311,17 @@ func (r Resolution) ownRuntimes(with Rendering) ([]map[string]any, error) {
|
||||
if words, err = r.readsWords(m, words, with); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// A build seat's holder gives way on its node, and its steps are told their part of the node's share
|
||||
// (novox/hq ADR 0553).
|
||||
yielding, yieldWords := r.yieldsOf(m, with)
|
||||
if len(yieldWords) > 0 {
|
||||
if words == nil {
|
||||
words = map[string]string{}
|
||||
}
|
||||
for k, v := range yieldWords {
|
||||
words[k] = v
|
||||
}
|
||||
}
|
||||
if len(words) > 0 {
|
||||
body, err := json.Marshal(map[string]map[string]string{m.Module: words})
|
||||
if err != nil {
|
||||
@@ -329,6 +340,9 @@ func (r Resolution) ownRuntimes(with Rendering) ([]map[string]any, error) {
|
||||
process[HealthField] = m.RuntimeHealth
|
||||
healthInto(process, m, with)
|
||||
}
|
||||
if yielding != nil {
|
||||
process["yields"] = yielding
|
||||
}
|
||||
if err := artifactsInto(process, RuntimeModule, with); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -498,6 +498,54 @@ func TestAModuleOfItsOwnAccountIsServedByARuntimeOfItsOwn(t *testing.T) {
|
||||
if _, told := env[RuntimeOperatorAccount]; told {
|
||||
t.Error("a runtime of a module's own account is told the operator's account")
|
||||
}
|
||||
if _, yields := own["yields"]; yields {
|
||||
t.Errorf("telegram holds no build seat and yields: %v", own["yields"])
|
||||
}
|
||||
// The build seats' holders yield (ADR 0553): idle, under the node's one share from the node-engine's build-cpu-share,
|
||||
// and their steps are told it.
|
||||
builder := func(name, seat string) Manifest {
|
||||
b := aToolsModule(t, name, "tools/index.js")
|
||||
b.RunsAs, b.SecretsOwner = name, name
|
||||
b.OwnSecrets = OwnSecrets{"broker": {Path: "/var/lib/" + name + "/broker"}}
|
||||
b.Claims = []Claim{{Name: seat}}
|
||||
return b
|
||||
}
|
||||
gob, img := builder("gob", "node-build-go"), builder("img", "node-build-image")
|
||||
with.Needed["gob"], with.Needed["img"] = map[string]string{"broker": "own"}, map[string]string{"broker": "own"}
|
||||
for _, c := range []struct {
|
||||
settings SettingsBy
|
||||
share string
|
||||
}{
|
||||
{nil, "50"},
|
||||
{SettingsBy{"mesh-host": {{From: "anchor", Values: map[string]any{"build-cpu-share": float64(75)}}}}, "75"},
|
||||
{SettingsBy{"mesh-host": {{From: "anchor", Values: map[string]any{"build-cpu-share": float64(0)}}}}, "50"},
|
||||
} {
|
||||
with.Settings = c.settings
|
||||
runtimes, err := Resolution{Node: "anchor", Modules: []Manifest{gob, img, telegram, goRuntime}}.ownRuntimes(with)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(runtimes) != 3 {
|
||||
t.Fatalf("%d runtimes of their own, want gob's, img's and telegram's", len(runtimes))
|
||||
}
|
||||
for _, p := range runtimes {
|
||||
name := fmt.Sprint(p["name"])
|
||||
words := map[string]map[string]string{}
|
||||
_ = json.Unmarshal([]byte(p["env"].(map[string]string)[RuntimeToolEnv]), &words)
|
||||
mod := strings.TrimSuffix(name, "-runtime")
|
||||
if mod == "telegram" {
|
||||
if p["yields"] != nil || words[mod]["MESH_BUILD_CPU_SHARE"] != "" {
|
||||
t.Errorf("telegram yields under %v: %v %v", c.settings, p["yields"], words)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if got := fmt.Sprint(p["yields"]); got != "map[cpu-share:"+c.share+" priority:idle]" ||
|
||||
words[mod]["MESH_BUILD_CPU_SHARE"] != c.share {
|
||||
t.Errorf("%s under %v yields %s with words %v; want share %s", name, c.settings, got, words[mod], c.share)
|
||||
}
|
||||
}
|
||||
}
|
||||
with.Settings = nil
|
||||
// Without the machine's runtime to run it from, it is refused in words.
|
||||
if _, err := (Resolution{Node: "anchor", Modules: []Manifest{telegram}}).ownRuntimes(with); err == nil {
|
||||
t.Error("a module of its own account composed without a runtime program")
|
||||
|
||||
@@ -49,6 +49,11 @@ type Seat struct {
|
||||
// order Accepts lists them (novox/hq ADR 0525 §2, ADR 0536 §4): the build seats'. Compiled, never stored, like
|
||||
// Receives.
|
||||
PulledByKind bool
|
||||
// Yields says the seat's holder gives way to everything else on its node (novox/hq ADR 0553): its runtime runs at
|
||||
// idle priority in the node's one yielding slice, under the node's one processor share (buildcpu.ShareSetting) that
|
||||
// every yielding holder of the node shares. The build seats': a build must never slow the operator's desktop.
|
||||
// Compiled, never stored, like Receives.
|
||||
Yields bool
|
||||
// Serves carries each verb in full — name, description, schema — because a role's tools are the
|
||||
// mesh's to define and an agent's to call (novox/hq ADR 0132, design 33 §2).
|
||||
Serves []Verb
|
||||
@@ -256,11 +261,11 @@ var defaultSeats = append([]Seat{
|
||||
// its holder taking nothing, kept on the module's own pause key (issue 1887's design limit 2). They replace
|
||||
// node-build-agent, retired with build-agent (ADR 0524 §5).
|
||||
{Name: "node-build-go", Scope: ScopeNode, Accepts: buildAskKinds(), PulledByKind: true, Serves: buildSeatVerbs("Go"),
|
||||
Decision: "novox/hq ADR 0525, ADR 0536"},
|
||||
Yields: true, Decision: "novox/hq ADR 0525, ADR 0536, ADR 0553"},
|
||||
{Name: "node-build-node", Scope: ScopeNode, Accepts: buildAskKinds(), PulledByKind: true, Serves: buildSeatVerbs("Node"),
|
||||
Decision: "novox/hq ADR 0525, ADR 0536"},
|
||||
Yields: true, Decision: "novox/hq ADR 0525, ADR 0536, ADR 0553"},
|
||||
{Name: "node-build-image", Scope: ScopeNode, Accepts: buildAskKinds(), PulledByKind: true, Serves: buildSeatVerbs("image"),
|
||||
Decision: "novox/hq ADR 0525, ADR 0536"},
|
||||
Yields: true, Decision: "novox/hq ADR 0525, ADR 0536, ADR 0553"},
|
||||
// **The mesh's resolvers** (novox/hq ADR 0194, 0196, 0223): every node's internal domain, and
|
||||
// every node and container asks them and nothing else. Replicated since ADR 0223: held on more
|
||||
// than one machine, each answering the same names from the same roster, and every machine's
|
||||
@@ -497,6 +502,7 @@ func UseSeats(s []Seat) {
|
||||
row.Paths = d.Paths
|
||||
row.Replicated = d.Replicated
|
||||
row.PulledByKind = d.PulledByKind
|
||||
row.Yields = d.Yields // never stored (ADR 0553)
|
||||
row.Serves = optionalAsCompiled(row.Serves, d.Serves)
|
||||
}
|
||||
merged = append(merged, row)
|
||||
|
||||
@@ -324,6 +324,9 @@ func TestNodeBuildAgentIsRetiredForTheBuildSeats(t *testing.T) {
|
||||
if !ok || s.Scope != ScopeNode || !s.PulledByKind || strings.Join(s.Accepts, ",") != "delivery,check,recheck" {
|
||||
t.Fatalf("%s = %+v", name, s)
|
||||
}
|
||||
if !s.Yields {
|
||||
t.Errorf("%s does not yield (ADR 0553)", name)
|
||||
}
|
||||
var verbs []string
|
||||
for _, v := range s.Serves {
|
||||
verbs = append(verbs, v.Name)
|
||||
@@ -332,4 +335,14 @@ func TestNodeBuildAgentIsRetiredForTheBuildSeats(t *testing.T) {
|
||||
t.Errorf("%s serves %v", name, verbs)
|
||||
}
|
||||
}
|
||||
// The build seats are the seats that yield, and no other (ADR 0553 §1).
|
||||
var yielding []string
|
||||
for _, s := range Seats() {
|
||||
if s.Yields {
|
||||
yielding = append(yielding, s.Name)
|
||||
}
|
||||
}
|
||||
if strings.Join(yielding, ",") != "node-build-go,node-build-node,node-build-image" {
|
||||
t.Errorf("the seats that yield are %v", yielding)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,69 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"strconv"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buildcpu"
|
||||
)
|
||||
|
||||
// yieldsOf is how the runtime of its own a module is served by gives way on this node (novox/hq ADR 0553), or nil when
|
||||
// it does not, and the word its steps size their threads from:
|
||||
//
|
||||
// - a module holding a seat that yields (Seat.Yields, the build seats) runs at idle priority under the node's one share
|
||||
// of its processors: the node-engine's own config build-cpu-share (buildcpu.ShareSetting) as this node's layers give
|
||||
// it, its default where none does. Every yielding module of the node carries the same share, so the node-engine's
|
||||
// one yielding slice has one quota (ADR 0553 §2, §3).
|
||||
// - its word: that share (buildcpu.ShareWord), which a step divides by its own module's capacity (ADR 0553 §4).
|
||||
//
|
||||
// Never a refusal: a share out of range is the default, so a mistyped value bounds the builds at the default and never
|
||||
// stops the node's whole composition.
|
||||
func (r Resolution) yieldsOf(m Manifest, with Rendering) (map[string]any, map[string]string) {
|
||||
if !yields(m) {
|
||||
return nil, nil
|
||||
}
|
||||
engine := Manifest{Module: engineModule}
|
||||
for _, e := range r.Modules {
|
||||
if e.Module == engineModule {
|
||||
engine = e
|
||||
}
|
||||
}
|
||||
share := buildcpu.DefaultPercent
|
||||
for _, s := range Effective(engine, with.Settings[engineModule]) {
|
||||
if s.Key == buildcpu.ShareSetting {
|
||||
if n, ok := percentOf(s.Value); ok {
|
||||
share = n
|
||||
}
|
||||
}
|
||||
}
|
||||
return map[string]any{"priority": "idle", "cpu-share": share},
|
||||
map[string]string{buildcpu.ShareWord: strconv.Itoa(share)}
|
||||
}
|
||||
|
||||
// yields is whether a module claims a seat that yields.
|
||||
func yields(m Manifest) bool {
|
||||
for _, c := range m.Claims {
|
||||
if s, ok := SeatNamed(c.Name); ok && s.Yields {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// percentOf is a setting's value as a percent from 1 to 100: a number, or a string holding one.
|
||||
func percentOf(v any) (int, bool) {
|
||||
var n int
|
||||
switch x := v.(type) {
|
||||
case float64:
|
||||
n = int(x)
|
||||
case int:
|
||||
n = x
|
||||
case string:
|
||||
var err error
|
||||
if n, err = strconv.Atoi(x); err != nil {
|
||||
return 0, false
|
||||
}
|
||||
default:
|
||||
return 0, false
|
||||
}
|
||||
return n, n >= 1 && n <= 100
|
||||
}
|
||||
@@ -916,6 +916,56 @@ type Process struct {
|
||||
// Health is how this resource is ready (novox/hq ADR 0240 rule 2, Phase B): one kind and its
|
||||
// timing, judged by the node-engine beside liveness. Absent: judged alive or not, and nothing more.
|
||||
Health *Health `json:"health,omitempty"`
|
||||
|
||||
// Yields says this process gives way to everything else on its machine (novox/hq ADR 0553, issue 1806): the
|
||||
// runtime of a build seat's holder, whose builds must never slow the operator's desktop. Absent: the machine's
|
||||
// defaults. For a process that stays up.
|
||||
Yields *Yields `json:"yields,omitempty"`
|
||||
}
|
||||
|
||||
// YieldIdle is the one priority a process yields at: the lowest there is.
|
||||
const YieldIdle = "idle"
|
||||
|
||||
// YieldSlice is the one slice the node-engine places every yielding process of its node in (novox/hq ADR 0553 §2): a
|
||||
// slice at the top of the hierarchy, beside the operator's user slice and the system's, so its idle weight and its
|
||||
// quota bind against everything else on the machine. No dash in its name: a dash makes a slice a child of the slice
|
||||
// named before it, where its weight would count only against its siblings.
|
||||
const YieldSlice = "meshyield.slice"
|
||||
|
||||
// Yields is how a process gives way, written by the node-engine (novox/hq ADR 0553).
|
||||
//
|
||||
// **Priority** "idle": the process runs in YieldSlice, whose CPUWeight is idle (it runs only on cycles nothing else
|
||||
// wants) and whose IOWeight is 1, with Nice=19 and the lowest best-effort IO level for every process it starts. The IO
|
||||
// controls bind only where the disk's scheduler honours them (bfq, or io.cost for the weight); under `none`, the
|
||||
// usual one for NVMe, they do nothing, and the CPU bound is what holds. **CPUShare** is the most of the machine's
|
||||
// processors the slice may take, in percent, shared by every yielding process of the node together: the node-engine
|
||||
// writes it as the slice's CPUQuota for the processors the machine has, so the bound is the kernel's. Every yielding
|
||||
// process of one node carries the same share: the composer gives the node's one value (ADR 0553 §3). 0 is no cap.
|
||||
type Yields struct {
|
||||
Priority string `json:"priority,omitempty"`
|
||||
CPUShare int `json:"cpu-share,omitempty"`
|
||||
}
|
||||
|
||||
func (y *Yields) problems(where string, staysUp bool) []string {
|
||||
if y == nil {
|
||||
return nil
|
||||
}
|
||||
var problems []string
|
||||
if !staysUp {
|
||||
problems = append(problems, where+": yields is for a process that stays up")
|
||||
}
|
||||
switch y.Priority {
|
||||
case "", YieldIdle:
|
||||
default:
|
||||
problems = append(problems, fmt.Sprintf("%s: yields priority %q is not one this host knows: idle", where, y.Priority))
|
||||
}
|
||||
if y.CPUShare < 0 || y.CPUShare > 100 {
|
||||
problems = append(problems, fmt.Sprintf("%s: yields cpu-share %d is not a percent of the machine, 0 to 100", where, y.CPUShare))
|
||||
}
|
||||
if y.Priority == "" && y.CPUShare == 0 {
|
||||
problems = append(problems, where+": yields says neither a priority nor a cpu-share")
|
||||
}
|
||||
return problems
|
||||
}
|
||||
|
||||
func (d *Process) Identity() string { return d.ID }
|
||||
@@ -948,6 +998,7 @@ func ProcessNameProblem(name string) string {
|
||||
|
||||
func (d *Process) validate(where string, _ bool) []string {
|
||||
problems := d.Health.problems(where, false, !d.RunOnce && d.Schedule == "")
|
||||
problems = append(problems, d.Yields.problems(where, !d.RunOnce && d.Schedule == "")...)
|
||||
if problem := ProcessNameProblem(d.Name); problem != "" {
|
||||
problems = append(problems, where+": "+problem)
|
||||
}
|
||||
|
||||
@@ -92,6 +92,20 @@ func TestTheModesAreExclusive(t *testing.T) {
|
||||
if problems := watching.validate("a process", false); len(problems) == 0 {
|
||||
t.Fatal("a scheduled process was given something to restart on, and it is never running")
|
||||
}
|
||||
|
||||
// A process yields only while it stays up, at a priority the host knows, within the machine (issue 1806).
|
||||
for name, bad := range map[string]func(*Process){
|
||||
"a step that yields": func(p *Process) { p.RunOnce = true; p.Yields = &Yields{Priority: YieldIdle} },
|
||||
"an unknown priority": func(p *Process) { p.Yields = &Yields{Priority: "low"} },
|
||||
"a share over the whole": func(p *Process) { p.Yields = &Yields{CPUShare: 150} },
|
||||
"a yield saying nothing": func(p *Process) { p.Yields = &Yields{} },
|
||||
} {
|
||||
p := aProcess()
|
||||
bad(p)
|
||||
if problems := p.validate("a process", false); len(problems) == 0 {
|
||||
t.Errorf("%s was accepted", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A cadence that is not a cadence is refused near its author, rather than by a machine at the far
|
||||
@@ -118,6 +132,11 @@ func TestEachModeOnItsOwnIsAccepted(t *testing.T) {
|
||||
if problems := every.validate("a process", false); len(problems) != 0 {
|
||||
t.Fatalf("a scheduled process was refused: %v", problems)
|
||||
}
|
||||
yielding := aProcess()
|
||||
yielding.Yields = &Yields{Priority: YieldIdle, CPUShare: 50}
|
||||
if problems := yielding.validate("a process", false); len(problems) != 0 {
|
||||
t.Fatalf("a daemon yielding at idle under half the machine was refused: %v", problems)
|
||||
}
|
||||
}
|
||||
|
||||
// **The one that cannot be escaped, only refused.**
|
||||
|
||||
@@ -2,11 +2,15 @@ package buildrun
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/health"
|
||||
)
|
||||
|
||||
func goAsk(id string) delivery.StepAsk {
|
||||
@@ -48,6 +52,74 @@ func TestAnAskerPutsOnceAndReadsTheStep(t *testing.T) {
|
||||
if len(g.Asks()) != 0 {
|
||||
t.Fatal("a wait put an ask")
|
||||
}
|
||||
// Issue 734: an ask the build seat gave up on (DEAD_LETTERS keeps its copy with the ask's id) ends its wait as failed
|
||||
// with the words, though a node had taken it; under NotTakenBound one no node takes ends at the bound; a step written
|
||||
// first is answered as the node wrote it.
|
||||
dead := &DeadLetters{}
|
||||
lw := LostAskWaiter{Inner: g, Dead: dead}
|
||||
k = Asker{Publisher: g, Waiter: lw, Bound: time.Minute}
|
||||
g.Write(delivery.Step{ID: "mvl0frv18d1970", Node: "ace", State: delivery.StepRunning, Started: time.Now()})
|
||||
given, ok := GivenUpOf(map[string][]string{"Mesh-Dead-Msg-Id": {"mvl0frv18d1970"}, "Mesh-Dead-Deliveries": {"5"},
|
||||
"Mesh-Dead-Stream": {"SEAT_NODE_BUILD_GO"}, "Mesh-Dead-Gave-Up": {"2026-10-26T12:26:40Z"}})
|
||||
if !ok {
|
||||
t.Fatal("a dead letter's headers were not read")
|
||||
}
|
||||
go dead.Heard(given)
|
||||
step, err = k.Wait(context.Background(), goAsk("mvl0frv18d1970"))
|
||||
if err != nil || step.State != delivery.StepFailed || !IsLost(step) ||
|
||||
!strings.Contains(step.Reason, LostWords+" after 5 deliveries at 2026-10-26T12:26:40Z") {
|
||||
t.Fatalf("a given-up ask: %+v, %v", step, err)
|
||||
}
|
||||
if _, err := k.Passed(step, nil); err == nil || !strings.Contains(err.Error(), "on "+LostNode+" failed: "+LostWords) {
|
||||
t.Fatalf("a given-up ask's error: %v", err)
|
||||
}
|
||||
step, err = k.Wait(NotTakenBound(context.Background(), 20*time.Millisecond), goAsk("mvl0frv18d1971"))
|
||||
if err != nil || !IsNotTaken(step) || IsLost(step) {
|
||||
t.Fatalf("an ask no node took: %+v, %v", step, err)
|
||||
}
|
||||
g.Write(delivery.Step{ID: "mvl0frv18d1972", Node: "g14", State: delivery.StepPassed})
|
||||
if step, err = k.Wait(NotTakenBound(context.Background(), time.Minute), goAsk("mvl0frv18d1972")); err != nil || step.Node != "g14" {
|
||||
t.Fatalf("a step written: %+v, %v", step, err)
|
||||
}
|
||||
// Review round 1 of issue 734: a hearing the bus refuses at start is deaf and says so, never holds the start up, and
|
||||
// tries again on the next ask waited for; heard, it says so and hands the given-up asks on.
|
||||
fol := &refusingFollower{refuse: 1}
|
||||
var mu sync.Mutex
|
||||
var told []Hearing
|
||||
hearing := NewDeadLetterHearing(context.Background(), fol, []string{delivery.AskGo}, dead, nil,
|
||||
func(h Hearing) { mu.Lock(); told = append(told, h); mu.Unlock() })
|
||||
hearing.Wait()
|
||||
if why := hearing.Deaf(); !strings.Contains(why, "SEAT_NODE_BUILD_GO: permissions violation") {
|
||||
t.Fatalf("a refused hearing says %q", why)
|
||||
}
|
||||
k = Asker{Publisher: g, Waiter: LostAskWaiter{Inner: g, Dead: dead, Hearing: hearing}, Bound: time.Minute}
|
||||
g.Write(delivery.Step{ID: "mvl0frv18d1973", Node: "g14", State: delivery.StepPassed})
|
||||
if _, err := k.Wait(context.Background(), goAsk("mvl0frv18d1973")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
hearing.Wait()
|
||||
mu.Lock()
|
||||
if hearing.Deaf() != "" || len(told) != 2 || told[0].Heard || !told[1].Heard || fol.calls() != 2 {
|
||||
t.Fatalf("after an ask: deaf %q, told %+v, followed %d times", hearing.Deaf(), told, fol.calls())
|
||||
}
|
||||
mu.Unlock()
|
||||
fol.heard(GivenUp{AskID: "mvl0frv18d1974", Deliveries: 5})
|
||||
step, err = k.Wait(context.Background(), goAsk("mvl0frv18d1974"))
|
||||
if err != nil || !IsLost(step) {
|
||||
t.Fatalf("an ask heard given up after the hearing came back: %+v, %v", step, err)
|
||||
}
|
||||
for _, row := range []struct {
|
||||
h *Hearing
|
||||
want string
|
||||
}{{nil, health.StateUnhealthy}, {&told[0], health.StateUnhealthy}, {&told[1], health.StateHealthy}} {
|
||||
var raw []byte
|
||||
if row.h != nil {
|
||||
raw, _ = json.Marshal(row.h)
|
||||
}
|
||||
if a := HearingHealth(raw); a.State != row.want || a.Why == "" {
|
||||
t.Errorf("health of %+v = %+v, want %s", row.h, a, row.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Review (issue 734 path): an ask taken again by the same node (its build module replaced mid-run) is bounded from the
|
||||
@@ -69,3 +141,26 @@ func TestARedeliveryToTheSameNodeRestartsTheBound(t *testing.T) {
|
||||
t.Fatalf("step %+v, %v, want the redelivered run's pass", step, err)
|
||||
}
|
||||
}
|
||||
|
||||
// refusingFollower refuses the first `refuse` follows, as the bus does before the asker's grant is composed.
|
||||
type refusingFollower struct {
|
||||
mu sync.Mutex
|
||||
refuse int
|
||||
n int
|
||||
hand func(GivenUp)
|
||||
}
|
||||
|
||||
func (f *refusingFollower) Follow(_ context.Context, _ string, _ time.Time, heard func(GivenUp)) (func(), error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
f.n++
|
||||
if f.n <= f.refuse {
|
||||
return nil, errors.New("permissions violation")
|
||||
}
|
||||
f.hand = heard
|
||||
return func() {}, nil
|
||||
}
|
||||
|
||||
func (f *refusingFollower) calls() int { f.mu.Lock(); defer f.mu.Unlock(); return f.n }
|
||||
|
||||
func (f *refusingFollower) heard(g GivenUp) { f.mu.Lock(); h := f.hand; f.mu.Unlock(); h(g) }
|
||||
|
||||
@@ -0,0 +1,503 @@
|
||||
package buildrun
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/health"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
)
|
||||
|
||||
// **An ask the build seat gave up on ends its step** (novox/hq issue 734). A build seat's worker offers an ask
|
||||
// delivery.AskMaxDeliver times; one handed out that often without an answer (a bus reload that drops every hand-out, a node
|
||||
// that dies taking it each time) is given up by the server, and the controller keeps a copy of it in DEAD_LETTERS under
|
||||
// mesh.events.dead.<seat stream>.<worker>, with the ask's message id (its ask id), how often it was handed out and when it
|
||||
// was given up. No build node writes a step for it, so an asker that waits for the step waits for ever: a try stays
|
||||
// running and holds a try place, a check's verdict stays running and its pull request waits.
|
||||
//
|
||||
// An asker of the build seats (the checker, the builder) hears those copies (DeadLetterHearing: an ordered consumer on
|
||||
// DEAD_LETTERS filtered to the build seats' streams, granted to a user of a seat pulled by kind, buscompose
|
||||
// DeadLetterReads) into DeadLetters, and LostAskWaiter ends the waiting step at once as failed on the node LostNode, its
|
||||
// reason saying LostWords; the asker decides what follows (the checker ends a try as error and asks a check again, the
|
||||
// builder asks its part of a build again).
|
||||
//
|
||||
// Belt and braces, where the asker says so in the context (NotTakenBound; the checker's tries): an ask no build node took
|
||||
// within a bound ends the same way, saying NotTakenWords. An ask waits in the stream as long as the backlog is otherwise
|
||||
// (ADR 0525 §9): a backlog is no fault.
|
||||
|
||||
// LostNode is the node a step ended for a lost ask names: no build node wrote it.
|
||||
const LostNode = "bus"
|
||||
|
||||
// The words of a step ended for its ask.
|
||||
const (
|
||||
LostWords = "its ask was given up by the build seat"
|
||||
NotTakenWords = "no build node took its ask"
|
||||
)
|
||||
|
||||
// The headers the controller keeps a given-up message's copy with in DEAD_LETTERS (mesh-controller internal/link
|
||||
// deadletters.go): the original message id (an ask's is its ask id), how often it was handed out, when it was given up.
|
||||
const (
|
||||
DeadHeaderMsgID = "Mesh-Dead-Msg-Id"
|
||||
DeadHeaderDeliveries = "Mesh-Dead-Deliveries"
|
||||
DeadHeaderGaveUp = "Mesh-Dead-Gave-Up"
|
||||
DeadHeaderStream = "Mesh-Dead-Stream"
|
||||
)
|
||||
|
||||
// DeadLettersStream is the stream the controller keeps given-up messages in, and DeadLetterFilter the subjects of one seat
|
||||
// stream's copies there (buscompose DeadLetterSubject).
|
||||
const DeadLettersStream = "DEAD_LETTERS"
|
||||
|
||||
// DeadLetterFilter is the subjects DEAD_LETTERS keeps a seat stream's given-up messages under.
|
||||
func DeadLetterFilter(stream string) string { return "mesh.events.dead." + stream + ".>" }
|
||||
|
||||
// SeatStream is the stream of a build seat's language (buscompose seatStreamName of node-build-<language>).
|
||||
func SeatStream(language string) string { return "SEAT_NODE_BUILD_" + strings.ToUpper(language) }
|
||||
|
||||
// GivenUp is one ask a build seat gave up on, as DEAD_LETTERS keeps it.
|
||||
type GivenUp struct {
|
||||
AskID string
|
||||
Stream string
|
||||
Deliveries int
|
||||
At time.Time
|
||||
}
|
||||
|
||||
// GivenUpOf reads a DEAD_LETTERS copy's headers; false when it names no message id.
|
||||
func GivenUpOf(header map[string][]string) (GivenUp, bool) {
|
||||
get := func(k string) string {
|
||||
if v := header[k]; len(v) > 0 {
|
||||
return strings.TrimSpace(v[0])
|
||||
}
|
||||
return ""
|
||||
}
|
||||
g := GivenUp{AskID: get(DeadHeaderMsgID), Stream: get(DeadHeaderStream)}
|
||||
if g.AskID == "" {
|
||||
return GivenUp{}, false
|
||||
}
|
||||
g.Deliveries, _ = strconv.Atoi(get(DeadHeaderDeliveries))
|
||||
g.At, _ = time.Parse(time.RFC3339Nano, get(DeadHeaderGaveUp))
|
||||
return g, true
|
||||
}
|
||||
|
||||
// Words are what a step ended for the given-up ask says.
|
||||
func (g GivenUp) Words() string {
|
||||
at := "at an unknown time"
|
||||
if !g.At.IsZero() {
|
||||
at = "at " + g.At.UTC().Format(time.RFC3339)
|
||||
}
|
||||
return fmt.Sprintf("%s after %d deliveries %s (kept in %s); ask again", LostWords, g.Deliveries, at, DeadLettersStream)
|
||||
}
|
||||
|
||||
// keptGivenUp is how many given-up asks DeadLetters remembers for a waiter that starts after the copy was heard.
|
||||
const keptGivenUp = 1024
|
||||
|
||||
// DeadLetters holds the given-up asks heard, and hands each to the waiter of its ask. Safe for concurrent use.
|
||||
type DeadLetters struct {
|
||||
mu sync.Mutex
|
||||
heard map[string]GivenUp
|
||||
order []string
|
||||
watchers map[string][]chan GivenUp
|
||||
}
|
||||
|
||||
// Heard takes one given-up ask in.
|
||||
func (d *DeadLetters) Heard(g GivenUp) {
|
||||
d.mu.Lock()
|
||||
if d.heard == nil {
|
||||
d.heard, d.watchers = map[string]GivenUp{}, map[string][]chan GivenUp{}
|
||||
}
|
||||
if _, ok := d.heard[g.AskID]; !ok {
|
||||
d.order = append(d.order, g.AskID)
|
||||
if len(d.order) > keptGivenUp {
|
||||
delete(d.heard, d.order[0])
|
||||
d.order = d.order[1:]
|
||||
}
|
||||
}
|
||||
d.heard[g.AskID] = g
|
||||
ws := d.watchers[g.AskID]
|
||||
delete(d.watchers, g.AskID)
|
||||
d.mu.Unlock()
|
||||
for _, w := range ws {
|
||||
w <- g // buffered, one each
|
||||
}
|
||||
}
|
||||
|
||||
// watch answers a channel that gets the ask's given-up copy once (at once when it was heard already), and its cancel.
|
||||
func (d *DeadLetters) watch(askID string) (<-chan GivenUp, func()) {
|
||||
ch := make(chan GivenUp, 1)
|
||||
d.mu.Lock()
|
||||
defer d.mu.Unlock()
|
||||
if d.heard == nil {
|
||||
d.heard, d.watchers = map[string]GivenUp{}, map[string][]chan GivenUp{}
|
||||
}
|
||||
if g, ok := d.heard[askID]; ok {
|
||||
ch <- g
|
||||
return ch, func() {}
|
||||
}
|
||||
d.watchers[askID] = append(d.watchers[askID], ch)
|
||||
return ch, func() {
|
||||
d.mu.Lock()
|
||||
defer d.mu.Unlock()
|
||||
ws := d.watchers[askID]
|
||||
for i, w := range ws {
|
||||
if w == ch {
|
||||
d.watchers[askID] = append(ws[:i], ws[i+1:]...)
|
||||
break
|
||||
}
|
||||
}
|
||||
if len(d.watchers[askID]) == 0 {
|
||||
delete(d.watchers, askID)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type notTakenKey struct{}
|
||||
|
||||
// NotTakenBound says, in ctx, that an ask no build node took within bound ends its step (a try's, never a check's).
|
||||
func NotTakenBound(ctx context.Context, bound time.Duration) context.Context {
|
||||
return context.WithValue(ctx, notTakenKey{}, bound)
|
||||
}
|
||||
|
||||
func notTakenBoundOf(ctx context.Context) time.Duration {
|
||||
d, _ := ctx.Value(notTakenKey{}).(time.Duration)
|
||||
return d
|
||||
}
|
||||
|
||||
// LostAskWaiter is an AskWaiter that ends an ask's step when the build seat gave the ask up (DeadLetters), or, under
|
||||
// NotTakenBound, when no build node took it within the bound: it hands the asker a failed step whose reason says why, on
|
||||
// the node "bus". Every step the build module writes is handed on as it is.
|
||||
type LostAskWaiter struct {
|
||||
Inner AskWaiter
|
||||
Dead *DeadLetters
|
||||
// Hearing, when set, is asked to hear DEAD_LETTERS again each time an ask is waited for while it cannot: an ask put is
|
||||
// the event that makes hearing matter.
|
||||
Hearing interface {
|
||||
Deaf() string
|
||||
Again()
|
||||
}
|
||||
Now func() time.Time
|
||||
}
|
||||
|
||||
func (w LostAskWaiter) now() time.Time {
|
||||
if w.Now != nil {
|
||||
return w.Now()
|
||||
}
|
||||
return time.Now()
|
||||
}
|
||||
|
||||
// WatchAsk is AskWaiter's.
|
||||
func (w LostAskWaiter) WatchAsk(ctx context.Context, module, askID string) (<-chan delivery.Step, error) {
|
||||
if w.Hearing != nil && w.Hearing.Deaf() != "" {
|
||||
w.Hearing.Again()
|
||||
}
|
||||
inner, err := w.Inner.WatchAsk(ctx, module, askID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var dead <-chan GivenUp
|
||||
cancel := func() {}
|
||||
if w.Dead != nil {
|
||||
dead, cancel = w.Dead.watch(askID)
|
||||
}
|
||||
// The not-taken bound is a context's (ADR 0437 §8: no timer), ended when a node takes the ask.
|
||||
var notTaken <-chan struct{}
|
||||
bound := notTakenBoundOf(ctx)
|
||||
stopBound := func() {}
|
||||
if bound > 0 {
|
||||
var nt context.Context
|
||||
nt, stopBound = context.WithTimeout(context.Background(), bound)
|
||||
notTaken = nt.Done()
|
||||
}
|
||||
out := make(chan delivery.Step)
|
||||
go func() {
|
||||
defer close(out)
|
||||
defer cancel()
|
||||
defer func() { stopBound() }()
|
||||
send := func(s delivery.Step) bool {
|
||||
select {
|
||||
case out <- s:
|
||||
return true
|
||||
case <-ctx.Done():
|
||||
return false
|
||||
}
|
||||
}
|
||||
ended := func(reason string) delivery.Step {
|
||||
at := w.now().UTC()
|
||||
return delivery.Step{ID: askID, Node: LostNode, State: delivery.StepFailed, Reason: reason, Ended: at}
|
||||
}
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case s, ok := <-inner:
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if notTaken != nil {
|
||||
stopBound() // a node took it: its step is bounded by the asker from its start
|
||||
notTaken = nil
|
||||
}
|
||||
if !send(s) {
|
||||
return
|
||||
}
|
||||
case g := <-dead:
|
||||
dead = nil
|
||||
send(ended(g.Words()))
|
||||
return
|
||||
case <-notTaken:
|
||||
send(ended(fmt.Sprintf("%s within %s of its put; it was lost or waits behind a backlog; ask again", NotTakenWords, bound)))
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// IsLost says whether a step was ended by LostAskWaiter because the build seat gave its ask up (LostWords).
|
||||
func IsLost(s delivery.Step) bool { return s.Node == LostNode && strings.Contains(s.Reason, LostWords) }
|
||||
|
||||
// IsNotTaken says whether a step was ended by LostAskWaiter because no build node took its ask within the bound.
|
||||
func IsNotTaken(s delivery.Step) bool {
|
||||
return s.Node == LostNode && strings.Contains(s.Reason, NotTakenWords)
|
||||
}
|
||||
|
||||
// **The asker hears DEAD_LETTERS, tries again when it cannot, and says whether it hears** (issue 734). The read is
|
||||
// granted with the asker's seat traffic (buscompose DeadLetterReads); an asker started before its grant was composed is
|
||||
// refused, and an asker that only tried once would stay deaf until it starts again, silently. DeadLetterHearing tries in
|
||||
// the background, never holding the start up, and again on an event (Again): the asker's reconnect to the bus, and every
|
||||
// ask waited for while it is deaf (LostAskWaiter.Hearing); no timer. Each change of whether it hears is said (Said), so
|
||||
// the module's health tool says it: a deaf asker is not healthy.
|
||||
|
||||
// Hearing is what a DeadLetterHearing says: whether it hears every build seat stream it follows, and why not.
|
||||
type Hearing struct {
|
||||
Heard bool `json:"heard"`
|
||||
Streams []string `json:"streams"`
|
||||
Why string `json:"why,omitempty"`
|
||||
At time.Time `json:"at"`
|
||||
}
|
||||
|
||||
// HearingKey is the key a module keeps its Hearing under, in its own state (the checker's and the builder's hearing).
|
||||
const HearingKey = "dead-letters"
|
||||
|
||||
// DeadLetterFollower starts following one seat stream's copies in DEAD_LETTERS from since, handing each given-up ask to
|
||||
// heard; it answers the stop, or why it cannot.
|
||||
type DeadLetterFollower interface {
|
||||
Follow(ctx context.Context, stream string, since time.Time, heard func(GivenUp)) (func(), error)
|
||||
}
|
||||
|
||||
// JetStreamDeadLetters is the bus side of a DeadLetterFollower: an ordered consumer on DEAD_LETTERS filtered to the
|
||||
// stream's copies, which resumes where it was after a reconnect and, reset before it heard any, starts at since again.
|
||||
type JetStreamDeadLetters struct{ JS jetstream.JetStream }
|
||||
|
||||
// Follow is DeadLetterFollower's.
|
||||
func (j JetStreamDeadLetters) Follow(ctx context.Context, stream string, since time.Time, heard func(GivenUp)) (func(), error) {
|
||||
cons, err := j.JS.OrderedConsumer(ctx, DeadLettersStream, jetstream.OrderedConsumerConfig{
|
||||
FilterSubjects: []string{DeadLetterFilter(stream)}, DeliverPolicy: jetstream.DeliverByStartTimePolicy, OptStartTime: &since})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
cc, err := cons.Consume(func(m jetstream.Msg) {
|
||||
if g, ok := GivenUpOf(m.Headers()); ok {
|
||||
heard(g)
|
||||
}
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return cc.Stop, nil
|
||||
}
|
||||
|
||||
// DeadLetterHearing follows the build seats' given-up asks into Dead. Make it with NewDeadLetterHearing.
|
||||
type DeadLetterHearing struct {
|
||||
follower DeadLetterFollower
|
||||
streams []string
|
||||
dead *DeadLetters
|
||||
logf func(string, ...any)
|
||||
said func(Hearing)
|
||||
ctx context.Context
|
||||
since time.Time
|
||||
|
||||
mu sync.Mutex
|
||||
stops map[string]func()
|
||||
why map[string]string
|
||||
trying bool
|
||||
again bool
|
||||
told *Hearing
|
||||
wg sync.WaitGroup
|
||||
}
|
||||
|
||||
// NewDeadLetterHearing starts hearing the given-up asks of each language's build seat stream, in the background. said
|
||||
// is told each change of whether it hears (nil tells nothing); ctx ends it.
|
||||
func NewDeadLetterHearing(ctx context.Context, follower DeadLetterFollower, languages []string, dead *DeadLetters,
|
||||
logf func(string, ...any), said func(Hearing)) *DeadLetterHearing {
|
||||
h := &DeadLetterHearing{follower: follower, dead: dead, logf: logf, said: said, ctx: ctx, since: time.Now(),
|
||||
stops: map[string]func(){}, why: map[string]string{}}
|
||||
for _, l := range languages {
|
||||
s := SeatStream(l)
|
||||
h.streams = append(h.streams, s)
|
||||
h.why[s] = "not tried yet"
|
||||
}
|
||||
if h.logf == nil {
|
||||
h.logf = func(string, ...any) {}
|
||||
}
|
||||
h.Again()
|
||||
return h
|
||||
}
|
||||
|
||||
// Deaf says why a stream is not heard, or "" when every one is.
|
||||
func (h *DeadLetterHearing) Deaf() string {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
return h.deafLocked()
|
||||
}
|
||||
|
||||
func (h *DeadLetterHearing) deafLocked() string {
|
||||
var out []string
|
||||
for _, s := range h.streams {
|
||||
if why, deaf := h.why[s]; deaf {
|
||||
out = append(out, s+": "+why)
|
||||
}
|
||||
}
|
||||
if len(out) == 0 {
|
||||
return ""
|
||||
}
|
||||
return DeadLettersStream + " is not heard, so an ask a build seat gives up on is waited for until its bound: " +
|
||||
strings.Join(out, "; ")
|
||||
}
|
||||
|
||||
// Again tries, in the background, every stream not heard yet; a call while it tries makes it try once more after.
|
||||
func (h *DeadLetterHearing) Again() {
|
||||
h.mu.Lock()
|
||||
if h.trying {
|
||||
h.again = true
|
||||
h.mu.Unlock()
|
||||
return
|
||||
}
|
||||
h.trying = true
|
||||
h.mu.Unlock()
|
||||
h.wg.Add(1)
|
||||
go func() {
|
||||
defer h.wg.Done()
|
||||
for {
|
||||
h.try()
|
||||
h.mu.Lock()
|
||||
if !h.again || h.ctx.Err() != nil {
|
||||
h.trying = false
|
||||
h.mu.Unlock()
|
||||
return
|
||||
}
|
||||
h.again = false
|
||||
h.mu.Unlock()
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// Wait returns when no try runs.
|
||||
func (h *DeadLetterHearing) Wait() { h.wg.Wait() }
|
||||
|
||||
// Stop ends every stream's following.
|
||||
func (h *DeadLetterHearing) Stop() {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
for s, stop := range h.stops {
|
||||
stop()
|
||||
delete(h.stops, s)
|
||||
}
|
||||
}
|
||||
|
||||
func (h *DeadLetterHearing) try() {
|
||||
for _, s := range h.streams {
|
||||
h.mu.Lock()
|
||||
_, deaf := h.why[s]
|
||||
h.mu.Unlock()
|
||||
if !deaf || h.ctx.Err() != nil {
|
||||
continue
|
||||
}
|
||||
stream := s
|
||||
stop, err := h.follower.Follow(h.ctx, stream, h.since, func(g GivenUp) {
|
||||
h.logf("the ask %s on %s was given up after %d deliveries at %s: the wait for its step ends", g.AskID, g.Stream,
|
||||
g.Deliveries, g.At.Format(time.RFC3339))
|
||||
h.dead.Heard(g)
|
||||
})
|
||||
h.mu.Lock()
|
||||
if err != nil {
|
||||
h.why[stream] = err.Error()
|
||||
h.logf("the asks %s gives up on are not heard (%s of %s); tried again at the next reconnect or ask: %v",
|
||||
stream, DeadLetterFilter(stream), DeadLettersStream, err)
|
||||
} else {
|
||||
delete(h.why, stream)
|
||||
h.stops[stream] = stop
|
||||
h.logf("the asks %s gives up on are heard from %s", stream, DeadLettersStream)
|
||||
}
|
||||
h.mu.Unlock()
|
||||
}
|
||||
h.tell()
|
||||
}
|
||||
|
||||
// tell says the hearing when it changed.
|
||||
func (h *DeadLetterHearing) tell() {
|
||||
h.mu.Lock()
|
||||
why := h.deafLocked()
|
||||
now := Hearing{Heard: why == "", Streams: append([]string(nil), h.streams...), Why: why, At: time.Now().UTC()}
|
||||
same := h.told != nil && h.told.Heard == now.Heard && h.told.Why == now.Why
|
||||
if !same {
|
||||
h.told = &now
|
||||
}
|
||||
h.mu.Unlock()
|
||||
if !same && h.said != nil {
|
||||
h.said(now)
|
||||
}
|
||||
}
|
||||
|
||||
// HearingState is the name of the state an asker keeps its Hearing in (`<module>_hearing`, key HearingKey), which its
|
||||
// health tool reads.
|
||||
const HearingState = "hearing"
|
||||
|
||||
// HearingKeeper is where a Hearing is kept: a jetstream.KeyValue, or a process.AwaitedBucket for a state that may not be
|
||||
// on the bus at the part's start (issue 1790), which keeps the latest until it appears and answers process.ErrNotYet.
|
||||
type HearingKeeper interface {
|
||||
Put(ctx context.Context, key string, value []byte) (uint64, error)
|
||||
}
|
||||
|
||||
// SayHearing is a DeadLetterHearing's said that keeps each Hearing in kv under HearingKey; nil kv keeps nothing and
|
||||
// says so once. A write kv keeps for later (process.ErrNotYet) is not an error: the awaited bucket said it waits.
|
||||
func SayHearing(ctx context.Context, kv HearingKeeper, logf func(string, ...any)) func(Hearing) {
|
||||
if kv == nil {
|
||||
if logf != nil {
|
||||
logf("whether %s is heard is not kept: the state %s is not there, so the health tool cannot say it", DeadLettersStream, HearingState)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
return func(h Hearing) {
|
||||
data, err := json.Marshal(h)
|
||||
if err == nil {
|
||||
_, err = kv.Put(ctx, HearingKey, data)
|
||||
}
|
||||
if err != nil && !errors.Is(err, process.ErrNotYet) && logf != nil {
|
||||
logf("whether %s is heard could not be kept: %v", DeadLettersStream, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// HearingHealth is the health answer of an asker's kept Hearing (raw, empty when none is kept): healthy while it hears
|
||||
// DEAD_LETTERS, unhealthy saying why while it does not or has not said.
|
||||
func HearingHealth(raw []byte) health.Answer {
|
||||
if len(raw) == 0 {
|
||||
return health.Unhealthy("it has not said whether it hears " + DeadLettersStream + " (issue 734): it is starting, or its state " + HearingState + " is not there")
|
||||
}
|
||||
var h Hearing
|
||||
if err := json.Unmarshal(raw, &h); err != nil {
|
||||
return health.Unhealthy("what it keeps of hearing " + DeadLettersStream + " is not a hearing: " + err.Error())
|
||||
}
|
||||
if !h.Heard {
|
||||
return health.Unhealthy(h.Why + " (since " + h.At.Format(time.RFC3339) + "; tried again at its next reconnect or ask)")
|
||||
}
|
||||
return health.Healthy("it hears " + DeadLettersStream + " for " + strings.Join(h.Streams, ", "))
|
||||
}
|
||||
+60
-6
@@ -115,6 +115,10 @@ type Composition struct {
|
||||
Words string `json:"words,omitempty"`
|
||||
SDK string `json:"sdk,omitempty"`
|
||||
Machines []string `json:"machines,omitempty"`
|
||||
// FPSet is the candidate's fingerprint set (Candidate.FPSet) the composition judged: a carry keeps a composition that
|
||||
// passed while the candidate's set is the same, and composes again when a fingerprint changed (novox/hq ADR 0548 §5).
|
||||
// Absent in a value written before it, which a carry composes again.
|
||||
FPSet string `json:"fpset,omitempty"`
|
||||
}
|
||||
|
||||
// Took is the time of each part of a check in integer milliseconds (ADR 0447 clause 3): waiting in the queue, cloning,
|
||||
@@ -139,16 +143,25 @@ type FailedFirst struct {
|
||||
Words string `json:"words,omitempty"`
|
||||
}
|
||||
|
||||
// MaxFailures is the most entries a verdict's Failures holds (issue 1921): fifty tests named one by one, and an entry
|
||||
// after them saying how many more failed.
|
||||
const MaxFailures = 50
|
||||
|
||||
// Verdict is the value of a verdict key: the group and its heads hash, the trunk head of each repository it was judged
|
||||
// against (repository to full sha), the State (running, pass or fail), what failed first, the tests run and kept per
|
||||
// level, each part's time, the machines composed and the mesh-sdk version the composition ran with (ADR 0446 clause 5).
|
||||
type Verdict struct {
|
||||
V int `json:"v"`
|
||||
Group string `json:"group"`
|
||||
Heads string `json:"heads"`
|
||||
Trunk map[string]string `json:"trunk,omitempty"`
|
||||
State string `json:"state"`
|
||||
Failed FailedFirst `json:"failed,omitzero"`
|
||||
V int `json:"v"`
|
||||
Group string `json:"group"`
|
||||
Heads string `json:"heads"`
|
||||
Trunk map[string]string `json:"trunk,omitempty"`
|
||||
State string `json:"state"`
|
||||
Failed FailedFirst `json:"failed,omitzero"`
|
||||
// Failures names every test that failed, in the order judged, each with the first line of its words (issue 1921):
|
||||
// Failed keeps the first one's words whole, and a check that ran every test says all it found here. At most
|
||||
// MaxFailures entries, the last of a longer list saying how many more failed. Absent in a value written before it,
|
||||
// and from a verdict whose failure is no test.
|
||||
Failures []FailedFirst `json:"failures,omitempty"`
|
||||
Levels map[string]LevelResult `json:"levels,omitempty"`
|
||||
Took Took `json:"took,omitzero"`
|
||||
Machines []string `json:"machines,omitempty"`
|
||||
@@ -157,6 +170,14 @@ type Verdict struct {
|
||||
Composition *Composition `json:"composition,omitempty"`
|
||||
// OverBound names the levels whose running went over their bound; it does not change the State (ADR 0453 clause 9).
|
||||
OverBound []string `json:"over-bound,omitempty"`
|
||||
// Beside is the commit of each sibling repository the check read beside the repositories it merged (ADR 0484 §4), by
|
||||
// repository: a move of one puts the verdict out of date as a trunk move does, and it is carried (novox/hq ADR 0548 §6).
|
||||
// A repository is in Trunk or here, never both. Absent in a value written before it.
|
||||
Beside map[string]string `json:"beside,omitempty"`
|
||||
// CarriedFrom names, of a carried verdict, the trunk and sibling commits it was judged on before the moves it was
|
||||
// carried over, by repository (ADR 0548 §5): the verdict was rewritten for Trunk and Beside by running only the packages
|
||||
// whose digest the moves changed. Absent from a verdict judged whole, and from one still running.
|
||||
CarriedFrom map[string]string `json:"carried-from,omitempty"`
|
||||
}
|
||||
|
||||
func (c *Composition) check() error {
|
||||
@@ -194,6 +215,33 @@ func (v Verdict) check() error {
|
||||
return fmt.Errorf("delivery: the verdict's trunk head of %s is empty", repository)
|
||||
}
|
||||
}
|
||||
for repository, commit := range v.Beside {
|
||||
if commit == "" {
|
||||
return fmt.Errorf("delivery: the verdict's sibling commit of %s is empty", repository)
|
||||
}
|
||||
if _, ok := v.Trunk[repository]; ok {
|
||||
return fmt.Errorf("delivery: the verdict names %s as a trunk it merged and as a sibling it read", repository)
|
||||
}
|
||||
}
|
||||
if len(v.CarriedFrom) > 0 {
|
||||
if v.State == VerdictRunning {
|
||||
return fmt.Errorf("delivery: a running verdict is carried from nothing: carried-from is written with its end")
|
||||
}
|
||||
moved := false
|
||||
for repository, commit := range v.CarriedFrom {
|
||||
now, inTrunk := v.Trunk[repository]
|
||||
if !inTrunk {
|
||||
now = v.Beside[repository]
|
||||
}
|
||||
if commit == "" || now == "" {
|
||||
return fmt.Errorf("delivery: the verdict's carried-from names %s, which it judged on no commit of", repository)
|
||||
}
|
||||
moved = moved || commit != now
|
||||
}
|
||||
if !moved {
|
||||
return fmt.Errorf("delivery: the verdict's carried-from names the commits it is judged on: nothing moved, so nothing was carried")
|
||||
}
|
||||
}
|
||||
for level, r := range v.Levels {
|
||||
if err := checkLevel(level); err != nil {
|
||||
return err
|
||||
@@ -222,6 +270,12 @@ func (v Verdict) check() error {
|
||||
}
|
||||
}
|
||||
named := v.Failed.Test != "" || v.Failed.Machine != ""
|
||||
if len(v.Failures) > 0 && v.State != VerdictFail {
|
||||
return fmt.Errorf("delivery: a %s verdict names no failures", v.State)
|
||||
}
|
||||
if len(v.Failures) > MaxFailures+1 {
|
||||
return fmt.Errorf("delivery: the verdict names %d failures, more than %d and the count of the rest", len(v.Failures), MaxFailures)
|
||||
}
|
||||
switch v.State {
|
||||
case VerdictRunning:
|
||||
if v.Failed != (FailedFirst{}) {
|
||||
|
||||
@@ -3,6 +3,7 @@ package delivery
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -28,7 +29,12 @@ func verdictSamples() map[string]Verdict {
|
||||
failTest.Levels = map[string]LevelResult{LevelUnit: {Run: 1}}
|
||||
failMachine := passVerdict()
|
||||
failMachine.State, failMachine.Failed = VerdictFail, FailedFirst{Machine: "ace", Words: "path /etc/x is owned by two modules"}
|
||||
return map[string]Verdict{"running": running, "pass": passVerdict(), "fail by a test": failTest, "fail by a machine": failMachine}
|
||||
carried := passVerdict()
|
||||
carried.Beside = map[string]string{"novox/mesh-catalog": "cccc"}
|
||||
carried.CarriedFrom = map[string]string{"novox/mesh-sdk": "9999", "novox/mesh-control": "bbbb", "novox/mesh-catalog": "cccc"}
|
||||
carried.Composition = &Composition{State: CompositionPass, SDK: carried.SDK, Machines: []string{"ace"}, FPSet: "fps1:00"}
|
||||
return map[string]Verdict{"running": running, "pass": passVerdict(), "fail by a test": failTest, "fail by a machine": failMachine,
|
||||
"carried": carried}
|
||||
}
|
||||
|
||||
func keptSample() Kept {
|
||||
@@ -247,7 +253,14 @@ func TestVerdictCarriesSkippedCompositionAndOverBound(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, field := range []string{`"skipped":4`, `"composition":{"state":"pass"`, `"over-bound":["integration"]`} {
|
||||
v.Beside, v.CarriedFrom = map[string]string{"novox/mesh-catalog": "cccc"}, map[string]string{"novox/mesh-catalog": "cbcb"}
|
||||
v.Composition.FPSet = "fps1:ab"
|
||||
data, err = EncodeVerdict(v)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, field := range []string{`"skipped":4`, `"composition":{"state":"pass"`, `"over-bound":["integration"]`, `"beside":{"novox/mesh-catalog":"cccc"}`,
|
||||
`"carried-from":{"novox/mesh-catalog":"cbcb"}`, `"fpset":"fps1:ab"`} {
|
||||
if !strings.Contains(string(data), field) {
|
||||
t.Errorf("the value lacks %s: %s", field, data)
|
||||
}
|
||||
@@ -294,6 +307,13 @@ func TestVerdictNewFieldsChecked(t *testing.T) {
|
||||
"a not-run composition naming machines": {func(v *Verdict) { v.Composition = &Composition{State: CompositionNotRun, Machines: []string{"ace"}} }, []string{"composition", "not run"}},
|
||||
"an unknown over-bound level": {func(v *Verdict) { v.OverBound = []string{"e2e"} }, []string{"e2e"}},
|
||||
"an over-bound level named twice": {func(v *Verdict) { v.OverBound = []string{LevelUnit, LevelUnit} }, []string{"over-bound", "unit"}},
|
||||
// ADR 0548 §5, §6: the siblings read and the commits a verdict was carried from.
|
||||
"a sibling with no commit": {func(v *Verdict) { v.Beside = map[string]string{"novox/mesh-catalog": ""} }, []string{"sibling", "mesh-catalog"}},
|
||||
"a trunk named as a sibling": {func(v *Verdict) { v.Beside = map[string]string{"novox/mesh-sdk": "aaaa"} }, []string{"trunk", "sibling", "mesh-sdk"}},
|
||||
"carried from an unjudged repo": {func(v *Verdict) { v.CarriedFrom = map[string]string{"novox/other": "1111"} }, []string{"carried-from", "novox/other"}},
|
||||
"carried from an empty commit": {func(v *Verdict) { v.CarriedFrom = map[string]string{"novox/mesh-sdk": ""} }, []string{"carried-from", "mesh-sdk"}},
|
||||
"carried over no move": {func(v *Verdict) { v.CarriedFrom = map[string]string{"novox/mesh-sdk": "aaaa"} }, []string{"nothing moved"}},
|
||||
"a running verdict carried": {func(v *Verdict) { v.State, v.CarriedFrom = VerdictRunning, map[string]string{"novox/mesh-sdk": "9999"} }, []string{"running", "carried"}},
|
||||
}
|
||||
for name, c := range cases {
|
||||
v := passVerdict()
|
||||
@@ -318,7 +338,7 @@ func TestVerdictOldAndNewReadersAgree(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("the new reader of an old value: %v", err)
|
||||
}
|
||||
if v.Composition != nil || v.OverBound != nil || v.Levels[LevelUnit].Skipped != 0 || v.Levels[LevelUnit].Run != 12 {
|
||||
if v.Composition != nil || v.OverBound != nil || v.Beside != nil || v.CarriedFrom != nil || v.Levels[LevelUnit].Skipped != 0 || v.Levels[LevelUnit].Run != 12 {
|
||||
t.Errorf("an old value grew fields: %+v", v)
|
||||
}
|
||||
again, err := EncodeVerdict(v)
|
||||
@@ -329,6 +349,9 @@ func TestVerdictOldAndNewReadersAgree(t *testing.T) {
|
||||
n.Levels[LevelUnit] = LevelResult{Run: 12, Kept: 301, Skipped: 4}
|
||||
n.Composition = &Composition{State: CompositionNotRun}
|
||||
n.OverBound = []string{LevelUnit}
|
||||
n.Beside = map[string]string{"novox/mesh-catalog": "cccc"}
|
||||
n.CarriedFrom = map[string]string{"novox/mesh-sdk": "9999"}
|
||||
n.Composition.FPSet = "fps1:00"
|
||||
data, err := EncodeVerdict(n)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -600,3 +623,38 @@ func TestPreviewOfAHeadInConflict(t *testing.T) {
|
||||
t.Errorf("the old reader: %+v %v", old, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Issue 1921: a failed verdict names every failed test in Failures and reads back the same; an older reader's value
|
||||
// without it reads too; Failures on a verdict that is no fail, and more than MaxFailures and the count of the rest, are
|
||||
// refused.
|
||||
func TestVerdictNamesEveryFailure(t *testing.T) {
|
||||
v := passVerdict()
|
||||
v.State = VerdictFail
|
||||
v.Failed = FailedFirst{Test: "integration: x/a.TestA", Words: "a_test.go:9: want 1, got 2\nmore"}
|
||||
v.Failures = []FailedFirst{{Test: "integration: x/a.TestA", Words: "a_test.go:9: want 1, got 2"}, {Test: "integration: x/b.TestB", Words: "b_test.go:3: boom"}}
|
||||
data, err := EncodeVerdict(v)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, err := DecodeVerdict(data)
|
||||
v.V = Version
|
||||
if err != nil || !SameVerdict(got, v) || !strings.Contains(string(data), `"failures":[{"test":"integration: x/a.TestA"`) {
|
||||
t.Fatalf("%s: %+v %v", data, got, err)
|
||||
}
|
||||
old := strings.Replace(string(data), string(data[strings.Index(string(data), `,"failures":`):strings.Index(string(data), `,"levels":`)]), "", 1)
|
||||
if got, err := DecodeVerdict([]byte(old)); err != nil || len(got.Failures) != 0 || got.Failed != v.Failed {
|
||||
t.Fatalf("a value without failures: %+v %v (%s)", got, err, old)
|
||||
}
|
||||
p := passVerdict()
|
||||
p.Failures = v.Failures
|
||||
if _, err := EncodeVerdict(p); err == nil {
|
||||
t.Error("a pass with failures was written")
|
||||
}
|
||||
v.Failures = make([]FailedFirst, MaxFailures+2)
|
||||
for i := range v.Failures {
|
||||
v.Failures[i] = FailedFirst{Test: "unit: x.T" + strconv.Itoa(i)}
|
||||
}
|
||||
if _, err := EncodeVerdict(v); err == nil {
|
||||
t.Error("a verdict naming more than MaxFailures and the rest was written")
|
||||
}
|
||||
}
|
||||
|
||||
+19
-3
@@ -34,9 +34,12 @@
|
||||
// **The release 4 layouts** (issues 1356 to 1359; ADR 0453 clauses 4, 6 and 16): the new fingerprint form (fingerprint2.go),
|
||||
// the indexer's manifests by digest and the new fields of its modules and candidates (manifests.go, source.go), the
|
||||
// builder's build keys (builder.go) and the checker's verdicts and kept verdicts (checker.go). A manifests value is the
|
||||
// one exception to the size above: a manifest is up to about 21 KB, whole, byte for byte.
|
||||
// one exception to the size above: a manifest is up to about 21 KB, whole, byte for byte. The bus module's have on the
|
||||
// node holding mesh-broker is the other: its user list's handing carries a sealed box of every loaded user's hash, some
|
||||
// kilobytes (handed.go).
|
||||
//
|
||||
// Standard library only, and this module's own asks and declaration packages (a composition holds declaration resources).
|
||||
// Standard library only, and this module's own asks, declaration and seal packages (a composition holds declaration
|
||||
// resources; the adoption handing is a sealed box, handed.go).
|
||||
package delivery
|
||||
|
||||
import (
|
||||
@@ -352,6 +355,18 @@ type UserListWritten struct {
|
||||
Users int `json:"users"`
|
||||
At time.Time `json:"at"`
|
||||
Error string `json:"error,omitempty"`
|
||||
// Follows is which list the node follows now (FollowController or FollowGrants, novox/hq ADR 0547 clause 8), empty
|
||||
// from a node-engine before the switch.
|
||||
Follows string `json:"follows,omitempty"`
|
||||
// Key is the bus node's sealing key, its public half as SPKI PEM: the key the grants part seals its list to
|
||||
// (clause 7). Public, as the vault holds it.
|
||||
Key string `json:"key,omitempty"`
|
||||
// Unwatched says why the bus node does not watch the grants part's key now (its watch refused or ended, and tried
|
||||
// again), empty while it does or where it need not (ADR 0547 clause 8): it then keeps the last list it followed.
|
||||
Unwatched string `json:"unwatched,omitempty"`
|
||||
// Handed is the hashes of the users of the list on the machine, sealed to the vault, for its adoption of the
|
||||
// accounts it holds no value for (ADR 0547 clause 9, handed.go); nil while the vault's key is not known.
|
||||
Handed *AccountsHanded `json:"handed,omitempty"`
|
||||
}
|
||||
|
||||
// HaveNode is a node's own facts, written under HaveNodeKey. Each field is the node-engine's own JSON, carried whole:
|
||||
@@ -502,7 +517,8 @@ func SameHave(a, b Have) bool {
|
||||
}
|
||||
x, y := *ua, *ub
|
||||
x.At, y.At = time.Time{}, time.Time{}
|
||||
return x == y && ua.At.Equal(ub.At)
|
||||
x.Handed, y.Handed = nil, nil
|
||||
return x == y && ua.At.Equal(ub.At) && sameHanded(ua.Handed, ub.Handed)
|
||||
}
|
||||
|
||||
// SameHaveNode says whether two node facts are the same value, comparing each carried JSON without its spacing.
|
||||
|
||||
@@ -1,12 +1,15 @@
|
||||
package delivery
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/seal"
|
||||
)
|
||||
|
||||
// 1131/AC1: the have key forms and parses back; bad names are refused, naming them.
|
||||
@@ -167,6 +170,108 @@ func TestRoundTrip(t *testing.T) {
|
||||
if !reflect.DeepEqual(out, in) || !out.UserList.At.Equal(at) {
|
||||
t.Fatalf("got %+v want %+v", out, in)
|
||||
}
|
||||
// And with the hashes handed to the vault (ADR 0547 clause 9): sealed to its key, opened by it alone, every
|
||||
// listed user with a bcrypt hash once, and refused to another key.
|
||||
hash := "$2a$11$" + strings.Repeat("a", 53)
|
||||
list := []byte("{ user: \"controller\", password: \"" + hash + "\", permissions: {\n" +
|
||||
"{ user: \"node.ace\", password: \"" + hash + "\", permissions: {\n" +
|
||||
"{ user: \"controller\", password: \"" + hash + "\" }\n{ user: \"nkey\", password: \"plain\" }\n")
|
||||
accounts := HandedAccountsOf(list)
|
||||
if len(accounts) != 2 || accounts[0].User != "controller" || accounts[1].User != "node.ace" || accounts[1].Hash != hash {
|
||||
t.Fatalf("handed accounts: %+v", accounts)
|
||||
}
|
||||
vault, err := seal.GenerateKeyPair()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
other, _ := seal.GenerateKeyPair()
|
||||
handed, err := HandAccounts(accounts, vault.PublicKey, "sha256:ab", at)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
in.UserList.Handed = &handed
|
||||
b, err = EncodeHave(in)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Contains(string(b), "node.ace") || strings.Contains(string(b), hash) {
|
||||
t.Fatalf("a user or a hash is in the clear: %s", b)
|
||||
}
|
||||
out, err = DecodeHave(b)
|
||||
if err != nil || !SameHave(out, in) {
|
||||
t.Fatalf("handed round trip: %v %+v", err, out.UserList)
|
||||
}
|
||||
opened, err := OpenHanded(*out.UserList.Handed, vault.PublicKey, vault.PrivateKey)
|
||||
if err != nil || !reflect.DeepEqual(opened, accounts) {
|
||||
t.Fatalf("opened %+v, %v", opened, err)
|
||||
}
|
||||
if _, err := OpenHanded(*out.UserList.Handed, other.PublicKey, other.PrivateKey); err == nil {
|
||||
t.Fatal("a handing opened for another key")
|
||||
}
|
||||
if _, err := HandAccounts(accounts, "", "sha256:ab", at); err == nil {
|
||||
t.Fatal("handed with no vault key")
|
||||
}
|
||||
})
|
||||
// The grants part's list sealed to the bus node (novox/hq ADR 0547 clause 7): the key's value round-trips and refuses
|
||||
// what the bus node could not follow, and the object opens with the node's key alone, to the digest the key names.
|
||||
t.Run("the grants part's sealed user list", func(t *testing.T) {
|
||||
in := Have{FP: "f1", Phase: PhaseRunning, Health: HealthOK, UserList: &UserListWritten{Source: GrantsUserListSource,
|
||||
Revision: 3, Wanted: "sha256:ab", Digest: "sha256:ab", Users: 2, Follows: FollowGrants, Key: "PEM"}}
|
||||
b, err := EncodeHave(in)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if out, err := DecodeHave(b); err != nil || out.UserList.Follows != FollowGrants || out.UserList.Key != "PEM" {
|
||||
t.Fatalf("got %+v, %v", out.UserList, err)
|
||||
}
|
||||
list := []byte("accounts { MESH { users = [ { user: \"controller\" } ] } }\n")
|
||||
digest := UserListDigest(list)
|
||||
node, err := seal.GenerateKeyPair()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
other, _ := seal.GenerateKeyPair()
|
||||
g := GrantsUserList{Follow: FollowGrants, Digest: digest, Object: UserListObjectName(digest),
|
||||
SealedTo: SealingKeyID(node.PublicKey), Node: "novox", Revision: 1, Users: 1}
|
||||
raw, err := EncodeGrantsUserList(g)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
back, err := DecodeGrantsUserList(raw)
|
||||
if err != nil || back.Digest != digest || back.LeftOut == nil || back.Follow != FollowGrants {
|
||||
t.Fatalf("decoded %+v, %v", back, err)
|
||||
}
|
||||
for _, bad := range []GrantsUserList{{Follow: "both", Digest: digest, Object: g.Object},
|
||||
{Follow: FollowGrants, Digest: digest, Object: "other"}, {Follow: FollowGrants, Digest: "md5:x", Object: "md5:x"}} {
|
||||
if _, err := EncodeGrantsUserList(bad); err == nil {
|
||||
t.Errorf("%+v was written", bad)
|
||||
}
|
||||
}
|
||||
if _, err := EncodeGrantsUserList(GrantsUserList{Follow: FollowController}); err != nil {
|
||||
t.Errorf("a switch with no list yet refused: %v", err)
|
||||
}
|
||||
object, err := SealUserList(list, node.PublicKey)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if bytes.Contains(object, []byte("controller")) {
|
||||
t.Fatal("the object carries the list in the clear")
|
||||
}
|
||||
opener := func(priv string) func(seal.Box) ([]byte, error) {
|
||||
return func(b seal.Box) ([]byte, error) { s, err := seal.Open(b, priv); return []byte(s), err }
|
||||
}
|
||||
if got, err := OpenUserList(object, digest, opener(node.PrivateKey)); err != nil || !bytes.Equal(got, list) {
|
||||
t.Fatalf("opened %q, %v", got, err)
|
||||
}
|
||||
if _, err := OpenUserList(object, digest, opener(other.PrivateKey)); err == nil {
|
||||
t.Error("another key opened the list")
|
||||
}
|
||||
if _, err := OpenUserList(object, "sha256:00", opener(node.PrivateKey)); err == nil {
|
||||
t.Error("a list of another digest was taken")
|
||||
}
|
||||
if _, err := SealUserList(list, "not a key"); err == nil {
|
||||
t.Error("a list was sealed to no key")
|
||||
}
|
||||
})
|
||||
t.Run("have node", func(t *testing.T) {
|
||||
in := HaveNode{Windows: json.RawMessage(`{"a":1}`), Busy: json.RawMessage(`true`)}
|
||||
@@ -306,6 +411,20 @@ func TestSame(t *testing.T) {
|
||||
if SameHave(ua, ub) {
|
||||
t.Error("different user lists same")
|
||||
}
|
||||
// A handing is compared by what it says: one said and one not differ, as do two to other keys.
|
||||
ub.UserList.Revision = 1
|
||||
ua.UserList.Handed = &AccountsHanded{From: "sha256:a", To: "sha256:k", Users: 2, At: at}
|
||||
if SameHave(ua, ub) {
|
||||
t.Error("a handing said and one not are the same")
|
||||
}
|
||||
ub.UserList.Handed = &AccountsHanded{From: "sha256:a", To: "sha256:k", Users: 2, At: at.In(time.Local)}
|
||||
if !SameHave(ua, ub) {
|
||||
t.Error("equal handings differ")
|
||||
}
|
||||
ub.UserList.Handed.To = "sha256:j"
|
||||
if SameHave(ua, ub) {
|
||||
t.Error("handings to other keys same")
|
||||
}
|
||||
n := HaveNode{V: 1, Busy: json.RawMessage(`true`)}
|
||||
if !SameHaveNode(n, HaveNode{V: 1, Busy: json.RawMessage(`true`)}) || SameHaveNode(n, HaveNode{V: 1}) {
|
||||
t.Error("have node")
|
||||
|
||||
@@ -123,6 +123,15 @@ func sampleMerge() Merge {
|
||||
}}
|
||||
}
|
||||
|
||||
func sampleTurn() Turn {
|
||||
at := time.Date(2026, 10, 26, 9, 0, 0, 0, time.UTC)
|
||||
return Turn{Repository: "novox/mesh-sdk", Waiting: []TurnWait{
|
||||
{Group: "feat/x", Heads: mustHash(gateHeads()), Ask: "merge.feat/x." + mustHash(gateHeads()), Approved: at, Since: at,
|
||||
WaitsFor: "feat/y.0123456789ab", WaitsSince: at.Add(-time.Minute), Step: "the check ask of novox/mesh-sdk#40"},
|
||||
{Group: "fix/1.2", Heads: "0123456789ac", Approved: at.Add(time.Minute), Since: at.Add(time.Minute)},
|
||||
}}
|
||||
}
|
||||
|
||||
func sampleAsk() Ask {
|
||||
key, _ := MergeAskKey("feat/x", gateHeads())
|
||||
return Ask{Group: "feat/x", Heads: gateHeads(), Ask: asks.Ask{
|
||||
@@ -181,7 +190,24 @@ func TestGateValuesRoundTrip(t *testing.T) { // 1201/AC4
|
||||
t.Errorf("owed %s round trip: %+v %v", o.Act, got, err)
|
||||
}
|
||||
}
|
||||
tu := sampleTurn()
|
||||
data, err = EncodeTurn(tu)
|
||||
hasV("turn", data, err)
|
||||
tu.V = Version
|
||||
if got, err := DecodeTurn(data); err != nil || !SameTurn(got, tu) {
|
||||
t.Errorf("turn round trip: %+v %v", got, err)
|
||||
}
|
||||
for _, field := range []string{`"waits-for":"feat/y.0123456789ab"`, `"waits-since":`, `"approved":`, `"step":`} {
|
||||
if !strings.Contains(string(data), field) {
|
||||
t.Errorf("the turn lacks %s: %s", field, data)
|
||||
}
|
||||
}
|
||||
// ADR 0548 §3: a turn holds off every check but those of the heads that wait in it, whoever else waits there.
|
||||
if !TurnHolds(tu, "fedcba987654") || TurnHolds(tu, tu.Waiting[0].Heads) || TurnHolds(tu, tu.Waiting[1].Heads) || TurnHolds(Turn{}, "fedcba987654") {
|
||||
t.Error("TurnHolds: a turn holds other heads' checks and never the check of heads waiting in it")
|
||||
}
|
||||
for name, dec := range map[string]func([]byte) error{
|
||||
"turn": func(b []byte) error { _, err := DecodeTurn(b); return err },
|
||||
"merge": func(b []byte) error { _, err := DecodeMerge(b); return err },
|
||||
"ask": func(b []byte) error { _, err := DecodeAsk(b); return err },
|
||||
"answer": func(b []byte) error { _, err := DecodeAnswer(b); return err },
|
||||
@@ -265,6 +291,27 @@ func TestGateValuesRefused(t *testing.T) { // 1201/AC5
|
||||
}
|
||||
_, err = EncodeOwed(Owed{Act: OwedNote, Text: "line"})
|
||||
wantErr("note without ref", err, "ref")
|
||||
|
||||
for name, c := range map[string]struct {
|
||||
edit func(*Turn)
|
||||
words []string
|
||||
}{
|
||||
"no delivery": {func(t *Turn) { t.Waiting = nil }, []string{"no delivery", "deleted"}},
|
||||
"a repository in capitals": {func(t *Turn) { t.Repository = "novox/Mesh-SDK" }, []string{"lowercase"}},
|
||||
"no repository": {func(t *Turn) { t.Repository = "" }, []string{"repository"}},
|
||||
"no group": {func(t *Turn) { t.Waiting[0].Group = "" }, []string{"group"}},
|
||||
"a bad heads hash": {func(t *Turn) { t.Waiting[0].Heads = "xyz" }, []string{"xyz"}},
|
||||
"the same heads twice": {func(t *Turn) { t.Waiting[1].Heads = t.Waiting[0].Heads }, []string{"twice"}},
|
||||
"no Approve time": {func(t *Turn) { t.Waiting[1].Approved = time.Time{} }, []string{"Approve"}},
|
||||
"out of the Approve's order": {func(t *Turn) { t.Waiting[1].Approved = t.Waiting[0].Approved.Add(-time.Hour) }, []string{"order"}},
|
||||
"waits for no verdict key": {func(t *Turn) { t.Waiting[0].WaitsFor = "feat/y" }, []string{"verdict key"}},
|
||||
"a step and no check": {func(t *Turn) { t.Waiting[1].Step = "x" }, []string{"names no check"}},
|
||||
} {
|
||||
tu := sampleTurn()
|
||||
c.edit(&tu)
|
||||
_, err = EncodeTurn(tu)
|
||||
wantErr("turn with "+name, err, c.words...)
|
||||
}
|
||||
_, err = EncodeOwed(Owed{Act: OwedStatus, State: "success"})
|
||||
wantErr("status without context", err, "context")
|
||||
}
|
||||
@@ -297,7 +344,8 @@ func TestGateSame(t *testing.T) { // 1201/AC6
|
||||
func TestGateBuckets(t *testing.T) {
|
||||
if MergesBucket != "mesh-delivery-gate_merges" || AsksBucket != "mesh-delivery-gate_asks" ||
|
||||
OwedBucket != "mesh-delivery-gate_owed" || AnswersBucket != "mesh-delivery-teller_answers" ||
|
||||
MergesHistory != 10 || AsksHistory != 5 || OwedHistory != 5 || AnswersHistory != 5 {
|
||||
MergesHistory != 10 || AsksHistory != 5 || OwedHistory != 5 || AnswersHistory != 5 ||
|
||||
TurnsBucket != "mesh-delivery-gate_turns" || TurnsHistory != 5 {
|
||||
t.Error("a bucket or history differs from the decided layout")
|
||||
}
|
||||
}
|
||||
@@ -370,6 +418,21 @@ func TestKeysEscapeAndRoundTrip(t *testing.T) { // 1323/AC1
|
||||
if r, n, h, a, err := ParseOwedKey(ok); err != nil || r != repo || n != 3 || h != "aaaa" || a != "view" {
|
||||
t.Errorf("owed round trip %q = %q %d %q %q %v", repo, r, n, h, a, err)
|
||||
}
|
||||
// ADR 0548 §3: a repository's turn key is <owner>.<name>, lowercase, each part escaped.
|
||||
tk, err := TurnsKey(repo)
|
||||
if err != nil || strings.Count(tk, ".") != 1 {
|
||||
t.Errorf("turns key for repository %q = %q, %v", repo, tk, err)
|
||||
} else if r, err := ParseTurnsKey(tk); err != nil || r != strings.ToLower(repo) {
|
||||
t.Errorf("turns round trip %q = %q, %v", repo, r, err)
|
||||
}
|
||||
}
|
||||
if k, err := TurnsKey("novox/Mesh-SDK"); err != nil || k != "novox.mesh-sdk" {
|
||||
t.Errorf("turns key = %q, %v; want novox.mesh-sdk", k, err)
|
||||
}
|
||||
for _, bad := range []string{"", "novox", "novox.", ".x", "a.b.c", "Novox.mesh-sdk", "novox.a=zz"} {
|
||||
if _, err := ParseTurnsKey(bad); err == nil {
|
||||
t.Errorf("parse turns key %q accepted", bad)
|
||||
}
|
||||
}
|
||||
if EscapeName("a.b") != "a=2eb" || EscapeName("a=b") != "a=3db" || EscapeName("a+b") != "a=2bb" {
|
||||
t.Errorf("escapes: %q %q %q", EscapeName("a.b"), EscapeName("a=b"), EscapeName("a+b"))
|
||||
|
||||
@@ -0,0 +1,125 @@
|
||||
package delivery
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/seal"
|
||||
)
|
||||
|
||||
// The adoption handing of novox/hq ADR 0547 clause 9 (issue 1945): the vault adopts, once and before `take-over`, the
|
||||
// bcrypt hashes of the bus accounts it holds no value for (the controller's own, the node accounts, the enrolled
|
||||
// devices) from the list the bus has loaded. The node-engine of the node holding mesh-broker is the one part that has
|
||||
// that list, so it hands the hashes in the bus module's have, beside what it says of the list it wrote: sealed to the
|
||||
// vault's own key, which it learns from the vault's answer to `node-key`, so that of every reader of the have bucket
|
||||
// the vault alone can open them. The have says in the clear only how many users it hands, from which list and to which
|
||||
// key; the users' names stay sealed, as the list itself is (ADR 0547 clause 7: it names every account).
|
||||
//
|
||||
// It hands every user of the loaded list; the vault takes from it only the users it holds no value for, and never
|
||||
// replaces a hash it made from a value it holds, so the node-engine never needs to read the vault's state (which the
|
||||
// grants part alone is to read, clause 9).
|
||||
|
||||
// AccountsHanded is the hashes of the loaded list's users, sealed to the vault, in UserListWritten.
|
||||
type AccountsHanded struct {
|
||||
// From is the digest of the list on the machine the hashes are taken from, `sha256:<hex>` (UserListWritten.Digest).
|
||||
From string `json:"from"`
|
||||
// To is the fingerprint of the vault's key the box is sealed to (KeyFingerprint).
|
||||
To string `json:"to"`
|
||||
// Users is how many users the box holds.
|
||||
Users int `json:"users"`
|
||||
// Box is the JSON of []HandedAccount, sealed to the vault's key (ADR 0206).
|
||||
Box seal.Box `json:"box"`
|
||||
At time.Time `json:"at"`
|
||||
}
|
||||
|
||||
// HandedAccount is one user of an AccountsHanded box, opened: the bus user and the bcrypt hash the loaded list
|
||||
// carries for it.
|
||||
type HandedAccount struct {
|
||||
User string `json:"user"`
|
||||
Hash string `json:"hash"`
|
||||
}
|
||||
|
||||
// KeyFingerprint is a public key's fingerprint as a handing names it: `sha256:<hex>` of its PEM, without the space
|
||||
// around it.
|
||||
func KeyFingerprint(publicPEM string) string {
|
||||
sum := sha256.Sum256([]byte(strings.TrimSpace(publicPEM)))
|
||||
return "sha256:" + hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
// listedAccount is one user of a bus user list as the composer writes it (buscompose), with its password's hash.
|
||||
var listedAccount = regexp.MustCompile(`\{\s*user:\s*"([^"]+)",\s*password:\s*"([^"]+)"`)
|
||||
|
||||
// HandedAccountsOf is every user of a bus user list that carries a bcrypt hash, in the list's order. A user listed
|
||||
// twice is handed once, as the list first names it.
|
||||
func HandedAccountsOf(list []byte) []HandedAccount {
|
||||
var out []HandedAccount
|
||||
seen := map[string]bool{}
|
||||
for _, m := range listedAccount.FindAllSubmatch(list, -1) {
|
||||
user, hash := string(m[1]), string(m[2])
|
||||
if seen[user] || !IsBcrypt(hash) {
|
||||
continue
|
||||
}
|
||||
seen[user] = true
|
||||
out = append(out, HandedAccount{User: user, Hash: hash})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// bcryptHash is the form of a bcrypt hash: version, cost, and 53 characters of salt and digest.
|
||||
var bcryptHash = regexp.MustCompile(`^\$2[abxy]?\$[0-9]{2}\$[./A-Za-z0-9]{53}$`)
|
||||
|
||||
// IsBcrypt says whether s has the form of a bcrypt hash.
|
||||
func IsBcrypt(s string) bool { return bcryptHash.MatchString(s) }
|
||||
|
||||
// HandAccounts seals the accounts to the vault's public key (SPKI PEM) as the handing of the list whose digest is from.
|
||||
func HandAccounts(accounts []HandedAccount, vaultPEM, from string, at time.Time) (AccountsHanded, error) {
|
||||
if strings.TrimSpace(vaultPEM) == "" {
|
||||
return AccountsHanded{}, errors.New("delivery: no vault key to seal the handed accounts to")
|
||||
}
|
||||
body, err := json.Marshal(accounts)
|
||||
if err != nil {
|
||||
return AccountsHanded{}, err
|
||||
}
|
||||
box, err := seal.Seal(string(body), vaultPEM)
|
||||
if err != nil {
|
||||
return AccountsHanded{}, fmt.Errorf("delivery: the handed accounts could not be sealed to the vault's key: %w", err)
|
||||
}
|
||||
return AccountsHanded{From: from, To: KeyFingerprint(vaultPEM), Users: len(accounts), Box: box, At: at.UTC()}, nil
|
||||
}
|
||||
|
||||
// OpenHanded opens a handing with the vault's private key (PEM), refusing one sealed to another key than publicPEM's
|
||||
// and a box that does not hold as many users as it says.
|
||||
func OpenHanded(h AccountsHanded, publicPEM, privatePEM string) ([]HandedAccount, error) {
|
||||
if h.To != KeyFingerprint(publicPEM) {
|
||||
return nil, fmt.Errorf("delivery: the handing is sealed to the key %s, not to this one (%s)", h.To,
|
||||
KeyFingerprint(publicPEM))
|
||||
}
|
||||
plain, err := seal.Open(h.Box, privatePEM)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("delivery: the handing does not open: %w", err)
|
||||
}
|
||||
var out []HandedAccount
|
||||
if err := json.Unmarshal([]byte(plain), &out); err != nil {
|
||||
return nil, fmt.Errorf("delivery: the handing holds no accounts: %w", err)
|
||||
}
|
||||
if len(out) != h.Users {
|
||||
return nil, fmt.Errorf("delivery: the handing says %d users and holds %d", h.Users, len(out))
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// sameHanded says whether two handings say the same: the same list, key, count and box, whatever zone their time is in.
|
||||
func sameHanded(a, b *AccountsHanded) bool {
|
||||
if a == nil || b == nil {
|
||||
return a == b
|
||||
}
|
||||
x, y := *a, *b
|
||||
x.At, y.At = time.Time{}, time.Time{}
|
||||
return x == y && a.At.Equal(b.At)
|
||||
}
|
||||
+122
-1
@@ -45,6 +45,86 @@ const (
|
||||
// AskLanguages are the build seats' languages.
|
||||
var AskLanguages = []string{AskGo, AskNode, AskImage}
|
||||
|
||||
// **An ask may name what its node needs** (issue 890, novox/hq ADR 0525 §1 as the placement record amends it). The seat's
|
||||
// workers are shared, so the asker still chooses no node; a check ask whose packages' tests need something only some
|
||||
// nodes have names it in Needs, and a holder whose node lacks one of them takes no part of the ask: it ends the ask's step
|
||||
// refused as declined (StepDeclined) and drops the message, and the asker puts the ask again under a new id, so a node
|
||||
// that has the need takes it. A need is a node seat whose holder installs what the tests read on its node; a package
|
||||
// declares it beside its tests, and the build module's digest reports it (LevelDigest.Needs).
|
||||
|
||||
// NeedNodeBuildNode is the need of a package whose integration tests read the Node build module's installed toolchain
|
||||
// (ADR 0533 §1): the node holds the seat node-build-node, so its toolchain is installed where the Go build module's
|
||||
// integration level is given it.
|
||||
const NeedNodeBuildNode = "node-build-node"
|
||||
|
||||
// AskNeeds are the needs an ask may name.
|
||||
var AskNeeds = []string{NeedNodeBuildNode}
|
||||
|
||||
// CheckNeed refuses a need that is not one of AskNeeds.
|
||||
func CheckNeed(need string) error {
|
||||
for _, n := range AskNeeds {
|
||||
if need == n {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return fmt.Errorf("delivery: %q is not a need an ask may name (%s)", need, strings.Join(AskNeeds, ", "))
|
||||
}
|
||||
|
||||
// Lacks is what of needs has does not say the node has, in the order of needs; a nil has has nothing.
|
||||
func Lacks(needs []string, has func(need string) bool) []string {
|
||||
var out []string
|
||||
for _, n := range needs {
|
||||
if has == nil || !has(n) {
|
||||
out = append(out, n)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// PackageNeeds is one package and what its tests need of the node at a level.
|
||||
type PackageNeeds struct {
|
||||
Package string `json:"package"`
|
||||
Needs []string `json:"needs"`
|
||||
}
|
||||
|
||||
// StepDeclined is the result of the step of an ask a holder declined: the ask's needs, and those the holder's node lacks.
|
||||
// The step's state is refused; the asker puts the ask again under a new id (DeclinedOf).
|
||||
type StepDeclined struct {
|
||||
Declined bool `json:"declined"`
|
||||
Needs []string `json:"needs"`
|
||||
Lacks []string `json:"lacks"`
|
||||
}
|
||||
|
||||
// DeclinedOf says whether step is a declined ask's, and what its node lacked.
|
||||
func DeclinedOf(step Step) (StepDeclined, bool) {
|
||||
if step.State != StepRefused || len(step.Result) == 0 {
|
||||
return StepDeclined{}, false
|
||||
}
|
||||
var d StepDeclined
|
||||
if err := json.Unmarshal(step.Result, &d); err != nil || !d.Declined {
|
||||
return StepDeclined{}, false
|
||||
}
|
||||
return d, true
|
||||
}
|
||||
|
||||
// UnmetNeeds are the needs of the packages the levels left untested (LevelAnswer.Unmet), each once, sorted.
|
||||
func UnmetNeeds(levels []LevelAnswer) []string {
|
||||
seen := map[string]bool{}
|
||||
var out []string
|
||||
for _, l := range levels {
|
||||
for _, u := range l.Unmet {
|
||||
for _, n := range u.Needs {
|
||||
if !seen[n] {
|
||||
seen[n] = true
|
||||
out = append(out, n)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// AskRuns is how many times an ask is run before it is taken for poison (issue 1907): an ask whose run never ends (it
|
||||
// takes its node down, or its process dies with it) is delivered again by its stream to the next free node (ADR 0525
|
||||
// §6); the holder that takes it past AskRuns runs it no more, writes its step failed saying so, and ends it, so the
|
||||
@@ -171,6 +251,8 @@ type StepAsk struct {
|
||||
Build *BuildJob `json:"build,omitempty"`
|
||||
Check *CheckJob `json:"check,omitempty"`
|
||||
Compose *ComposeJob `json:"compose,omitempty"`
|
||||
// Needs are what the node that runs the ask must have (AskNeeds); a check ask's alone. Empty: any node.
|
||||
Needs []string `json:"needs,omitempty"`
|
||||
}
|
||||
|
||||
// ComposeJob is a compose ask's job (novox/hq ADR 0482 clause 4, issue 1622): fetch the commit, build the composer
|
||||
@@ -255,6 +337,9 @@ type CheckJob struct {
|
||||
RunnerWhen string `json:"runner_when,omitempty"`
|
||||
// Run is a try's test pattern (go test -run): each level runs it alone, digesting nothing.
|
||||
Run string `json:"run,omitempty"`
|
||||
// Packages, when set, are the only packages each level tests (issue 890, ADR 0552 §4): a placed ask names the
|
||||
// packages the first ask left unmet, and the others are neither tested nor taken kept, so nothing is judged twice.
|
||||
Packages []string `json:"packages,omitempty"`
|
||||
}
|
||||
|
||||
func (a StepAsk) check() error {
|
||||
@@ -290,6 +375,13 @@ func (a StepAsk) check() error {
|
||||
return fmt.Errorf("delivery: the ask %s is a check on the delivery stream, which carries trunk builds alone", a.ID)
|
||||
case a.Check != nil && a.Language == AskImage:
|
||||
return fmt.Errorf("delivery: the ask %s is a check of an image, which has no tests", a.ID)
|
||||
case len(a.Needs) > 0 && a.Check == nil:
|
||||
return fmt.Errorf("delivery: the ask %s names needs and is no check, whose tests alone need a node", a.ID)
|
||||
}
|
||||
for _, n := range a.Needs {
|
||||
if err := CheckNeed(n); err != nil {
|
||||
return fmt.Errorf("the ask %s: %w", a.ID, err)
|
||||
}
|
||||
}
|
||||
if a.Build != nil {
|
||||
return a.Build.check(a.ID)
|
||||
@@ -350,6 +442,11 @@ func (c CheckJob) check(id string) error {
|
||||
return err
|
||||
}
|
||||
}
|
||||
for _, p := range c.Packages {
|
||||
if p == "" || strings.ContainsAny(p, " \t\n") {
|
||||
return fmt.Errorf("delivery: the check ask %s names a package %q that is no import path", id, p)
|
||||
}
|
||||
}
|
||||
return checkFetchArgs(id, c.Fetch)
|
||||
}
|
||||
|
||||
@@ -433,6 +530,9 @@ type LevelAnswer struct {
|
||||
Digest *LevelDigest `json:"digest,omitempty"`
|
||||
Kept []KeptTaken `json:"kept,omitempty"`
|
||||
Tests []json.RawMessage `json:"tests,omitempty"`
|
||||
// Unmet are the packages the level did not test on this node, because their tests need what it lacks (issue 890): the
|
||||
// asker keeps what passed and asks them again of a node that has it (StepAsk.Needs).
|
||||
Unmet []PackageNeeds `json:"unmet,omitempty"`
|
||||
}
|
||||
|
||||
// LevelDigest is the digest's answer as a level keeps it in the ask's step result: its packages a list of
|
||||
@@ -446,6 +546,8 @@ type LevelDigest struct {
|
||||
Beside []string `json:"beside,omitempty"`
|
||||
WithoutTests []string `json:"without_tests,omitempty"`
|
||||
JudgedBy string `json:"judged_by,omitempty"`
|
||||
// Needs are the packages whose tests at this level need something of the node (issue 890), as they declare it.
|
||||
Needs []PackageNeeds `json:"needs,omitempty"`
|
||||
}
|
||||
|
||||
// PackageDigest is one package's digest at a level.
|
||||
@@ -463,6 +565,7 @@ func (d *LevelDigest) UnmarshalJSON(raw []byte) error {
|
||||
Beside []string `json:"beside"`
|
||||
WithoutTests []string `json:"without_tests"`
|
||||
JudgedBy string `json:"judged_by"`
|
||||
Needs []PackageNeeds `json:"needs"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &v); err != nil {
|
||||
return err
|
||||
@@ -482,7 +585,20 @@ func (d *LevelDigest) UnmarshalJSON(raw []byte) error {
|
||||
}
|
||||
}
|
||||
sort.Slice(pkgs, func(i, j int) bool { return pkgs[i].Package < pkgs[j].Package })
|
||||
*d = LevelDigest{Level: v.Level, Packages: pkgs, Beside: v.Beside, WithoutTests: v.WithoutTests, JudgedBy: v.JudgedBy}
|
||||
*d = LevelDigest{Level: v.Level, Packages: pkgs, Beside: v.Beside, WithoutTests: v.WithoutTests, JudgedBy: v.JudgedBy, Needs: v.Needs}
|
||||
return nil
|
||||
}
|
||||
|
||||
// NeedsOf is what the package's tests need of the node at the digest's level; nil when it declares nothing.
|
||||
func (d *LevelDigest) NeedsOf(pkg string) []string {
|
||||
if d == nil {
|
||||
return nil
|
||||
}
|
||||
for _, n := range d.Needs {
|
||||
if n.Package == pkg {
|
||||
return n.Needs
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -535,6 +651,11 @@ func ParseAskStepKey(key string) (node, askID string, err error) {
|
||||
return node, askID, nil
|
||||
}
|
||||
|
||||
// StepBound is how long an asker waits for one ask's step once a build node took it (30 minutes): past it the asker has
|
||||
// stopped waiting, so a key still running is nobody's. The build modules end such a key (buildstep.EndStale); the
|
||||
// checker's, the builder's and the watchman's own copies of the number are to read this one (issue 1916).
|
||||
const StepBound = 30 * time.Minute
|
||||
|
||||
// PausesState is the state a build module keeps its nodes' pause keys in. It has no age, unlike StepsState, whose keys
|
||||
// are per step and dropped after their age: a pause kept among the steps would end by itself.
|
||||
const PausesState = "pauses"
|
||||
|
||||
@@ -56,6 +56,7 @@ func TestAskRefusesUnknownKindAndLanguage(t *testing.T) {
|
||||
"recheck": func(a *StepAsk) { a.Kind = AskRecheck },
|
||||
"commit": func(a *StepAsk) { a.Build.Fetch = map[string]any{"repository": "novox/x", "commit": "main"} },
|
||||
"path": func(a *StepAsk) { a.Build.Path = "" },
|
||||
"needs": func(a *StepAsk) { a.Needs = []string{NeedNodeBuildNode} }, // issue 890: a build needs no node
|
||||
}
|
||||
for name, change := range cases {
|
||||
a := buildAsk()
|
||||
@@ -80,6 +81,23 @@ func TestAskRefusesUnknownKindAndLanguage(t *testing.T) {
|
||||
if _, err := EncodeStepAsk(c); err == nil {
|
||||
t.Error("a check of an image encoded")
|
||||
}
|
||||
// Issue 890: a check names a known need and round-trips it; an unknown need is refused.
|
||||
c = checkAsk()
|
||||
c.Check.Packages = []string{"a b"}
|
||||
if _, err := EncodeStepAsk(c); err == nil {
|
||||
t.Error("a package that is no import path encoded")
|
||||
}
|
||||
c.Check.Packages = []string{"x/pkg/a"}
|
||||
c.Needs = []string{"gpu"}
|
||||
if _, err := EncodeStepAsk(c); err == nil {
|
||||
t.Error("an unknown need encoded")
|
||||
}
|
||||
c.Needs = []string{NeedNodeBuildNode}
|
||||
if raw, err := EncodeStepAsk(c); err != nil {
|
||||
t.Errorf("a check needing %s: %v", NeedNodeBuildNode, err)
|
||||
} else if back, err := DecodeStepAsk(raw); err != nil || len(back.Needs) != 1 || back.Needs[0] != NeedNodeBuildNode {
|
||||
t.Errorf("the need did not round-trip: %+v %v", back.Needs, err)
|
||||
}
|
||||
raw, _ := EncodeStepAsk(buildAsk())
|
||||
for _, bad := range []string{
|
||||
strings.Replace(string(raw), `"kind":"delivery"`, `"kind":"queue"`, 1),
|
||||
|
||||
@@ -70,6 +70,9 @@ type Step struct {
|
||||
Ended time.Time `json:"ended,omitzero"`
|
||||
Reason string `json:"reason,omitempty"`
|
||||
Result json.RawMessage `json:"result,omitempty"`
|
||||
// What says what the step is, for a reader of the steps state: an ask's kind, repository, head and levels or
|
||||
// module (issue 1890). Empty for a step a tool started by hand.
|
||||
What string `json:"what,omitempty"`
|
||||
}
|
||||
|
||||
func (s Step) check() error {
|
||||
|
||||
@@ -0,0 +1,174 @@
|
||||
package delivery
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// **The merge turns** (novox/hq ADR 0548 clauses 3 and 4). An approved merge holds a turn on each repository it goes into:
|
||||
// before it reads the checker's verdicts for the merge, the gate writes one key per repository in its bucket turns, naming
|
||||
// the deliveries that wait in the order of their Approve, the verdict key each waits for and since when. The gate is the
|
||||
// key's one writer; the checker is granted to read the bucket, reads it absent as empty, and claims no check of a repository
|
||||
// a turn names, except the check of the waiting delivery's own heads (TurnHolds). The key goes when the last delivery in it
|
||||
// merged or left.
|
||||
const (
|
||||
// TurnsBucket is the gate's bucket of merge turns.
|
||||
TurnsBucket = GateModule + "_turns"
|
||||
// TurnsHistory is the revisions the turns bucket keeps per key.
|
||||
TurnsHistory = 5
|
||||
)
|
||||
|
||||
// TurnsKey is the key, in TurnsBucket, of a repository's turn: `<owner>.<name>`, lowercase, each part escaped (EscapeName),
|
||||
// so the key names the repository the checker's verdicts name as a trunk whatever its case.
|
||||
func TurnsKey(repository string) (string, error) {
|
||||
repository = strings.ToLower(repository)
|
||||
if _, err := escapeRepository(repository); err != nil {
|
||||
return "", err
|
||||
}
|
||||
owner, name, _ := strings.Cut(repository, "/")
|
||||
return EscapeName(owner) + "." + EscapeName(name), nil
|
||||
}
|
||||
|
||||
// ParseTurnsKey is the inverse of TurnsKey: the repository, lowercase.
|
||||
func ParseTurnsKey(key string) (string, error) {
|
||||
owner, name, ok := strings.Cut(key, ".")
|
||||
if !ok || owner == "" || name == "" || strings.Contains(name, ".") {
|
||||
return "", fmt.Errorf("delivery: %q is not a turns key, <owner>.<name>", key)
|
||||
}
|
||||
o, err := UnescapeName(owner)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("delivery: %q is not a turns key: %w", key, err)
|
||||
}
|
||||
n, err := UnescapeName(name)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("delivery: %q is not a turns key: %w", key, err)
|
||||
}
|
||||
repository := o + "/" + n
|
||||
if again, err := TurnsKey(repository); err != nil || again != key {
|
||||
return "", fmt.Errorf("delivery: %q is not a turns key: it does not name a lowercase <owner>/<name>", key)
|
||||
}
|
||||
return repository, nil
|
||||
}
|
||||
|
||||
// Turn is the value of a turns key: the repository and the approved deliveries that wait to merge into it, in the order of
|
||||
// their Approve. A turn with no delivery is no value: its key is deleted.
|
||||
type Turn struct {
|
||||
V int `json:"v"`
|
||||
Repository string `json:"repository"`
|
||||
Waiting []TurnWait `json:"waiting"`
|
||||
}
|
||||
|
||||
// TurnWait is one approved delivery in a turn: its group (a lone pull request's branch), the hash of the heads the Approve
|
||||
// covers (the checker's verdict key's hash, HeadsHash over its members), the Approve's ask key, when it was approved and since
|
||||
// when it waits; and, while a check of another delivery runs on the repository, that check's verdict key, when the gate saw
|
||||
// it start, and the step the checker says it is at.
|
||||
type TurnWait struct {
|
||||
Group string `json:"group"`
|
||||
Heads string `json:"heads"`
|
||||
Ask string `json:"ask,omitempty"`
|
||||
Approved time.Time `json:"approved"`
|
||||
Since time.Time `json:"since"`
|
||||
// WaitsFor is the verdict key (`<group>.<heads hash>`) of the running check the merge waits for; empty when none runs.
|
||||
WaitsFor string `json:"waits-for,omitempty"`
|
||||
WaitsSince time.Time `json:"waits-since,omitzero"`
|
||||
Step string `json:"step,omitempty"`
|
||||
}
|
||||
|
||||
func (t Turn) check() error {
|
||||
repository := strings.ToLower(t.Repository)
|
||||
if repository != t.Repository {
|
||||
return fmt.Errorf("delivery: the turn's repository %q is not lowercase", t.Repository)
|
||||
}
|
||||
if _, err := escapeRepository(repository); err != nil {
|
||||
return err
|
||||
}
|
||||
if len(t.Waiting) == 0 {
|
||||
return fmt.Errorf("delivery: the turn of %s names no delivery: its key is deleted instead", t.Repository)
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
for i, w := range t.Waiting {
|
||||
if w.Group == "" {
|
||||
return fmt.Errorf("delivery: a delivery in the turn of %s names no group", t.Repository)
|
||||
}
|
||||
if !isHex12(w.Heads) {
|
||||
return fmt.Errorf("delivery: the heads hash %q in the turn of %s is not 12 lowercase hex digits", w.Heads, t.Repository)
|
||||
}
|
||||
if seen[w.Heads] {
|
||||
return fmt.Errorf("delivery: the turn of %s names the heads %s twice", t.Repository, w.Heads)
|
||||
}
|
||||
seen[w.Heads] = true
|
||||
if w.Approved.IsZero() || w.Since.IsZero() {
|
||||
return fmt.Errorf("delivery: the delivery %s in the turn of %s says no time of its Approve or of its wait", w.Group, t.Repository)
|
||||
}
|
||||
if i > 0 && w.Approved.Before(t.Waiting[i-1].Approved) {
|
||||
return fmt.Errorf("delivery: the turn of %s is not in the order of the Approve: %s was approved before %s", t.Repository, w.Group, t.Waiting[i-1].Group)
|
||||
}
|
||||
if w.WaitsFor != "" {
|
||||
if _, _, err := ParseCheckerVerdictKey(w.WaitsFor); err != nil {
|
||||
return fmt.Errorf("delivery: the delivery %s in the turn of %s waits for what is not a verdict key: %w", w.Group, t.Repository, err)
|
||||
}
|
||||
} else if !w.WaitsSince.IsZero() || w.Step != "" {
|
||||
return fmt.Errorf("delivery: the delivery %s in the turn of %s says when and where a check runs, and names no check", w.Group, t.Repository)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// TurnHolds says whether a turn holds off the check of the heads: it names deliveries, and none of them of these heads. A
|
||||
// turn never holds the check of heads that wait in it (the carry a waiting merge needs): two approved merges in one turn whose
|
||||
// trunk moved would otherwise each hold the other's carry, and neither would ever merge (review of ADR 0548's build, round 1).
|
||||
func TurnHolds(t Turn, heads string) bool {
|
||||
if len(t.Waiting) == 0 {
|
||||
return false
|
||||
}
|
||||
for _, w := range t.Waiting {
|
||||
if w.Heads == heads {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// EncodeTurn is t as JSON at Version; it refuses a turn with no delivery, a repository that is not lowercase <owner>/<name>,
|
||||
// a delivery without its group, heads hash or times, the same heads twice, an order other than the Approve's, and a check
|
||||
// waited for that is not a verdict key.
|
||||
func EncodeTurn(t Turn) ([]byte, error) {
|
||||
if err := t.check(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
t.V = Version
|
||||
return json.Marshal(t)
|
||||
}
|
||||
|
||||
// DecodeTurn reads a turn; it refuses an unknown version (ErrUnknownVersion) and what EncodeTurn refuses.
|
||||
func DecodeTurn(data []byte) (Turn, error) {
|
||||
var t Turn
|
||||
if err := decode(data, &t); err != nil {
|
||||
return Turn{}, err
|
||||
}
|
||||
if err := t.check(); err != nil {
|
||||
return Turn{}, err
|
||||
}
|
||||
return t, nil
|
||||
}
|
||||
|
||||
// SameTurn says whether two turns are the same value, so the gate writes only when they differ.
|
||||
func SameTurn(a, b Turn) bool {
|
||||
if len(a.Waiting) != len(b.Waiting) || a.Repository != b.Repository || a.V != b.V {
|
||||
return false
|
||||
}
|
||||
for i := range a.Waiting {
|
||||
x, y := a.Waiting[i], b.Waiting[i]
|
||||
if !x.Approved.Equal(y.Approved) || !x.Since.Equal(y.Since) || !x.WaitsSince.Equal(y.WaitsSince) {
|
||||
return false
|
||||
}
|
||||
x.Approved, y.Approved, x.Since, y.Since, x.WaitsSince, y.WaitsSince = time.Time{}, time.Time{}, time.Time{}, time.Time{}, time.Time{}, time.Time{}
|
||||
if !reflect.DeepEqual(x, y) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
@@ -0,0 +1,149 @@
|
||||
package delivery
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/seal"
|
||||
)
|
||||
|
||||
// **The grants part's user list, sealed to the bus node** (novox/hq ADR 0547 clauses 7 and 8, issue 1945): the contract
|
||||
// between the grants part (mesh-delivery modules/mesh-grants), which writes it, and the node-engine of the node holding
|
||||
// mesh-broker (mesh-host), which follows it once the operator's take-over says so.
|
||||
//
|
||||
// - The grants part composes the bus's user list (the accounts block, buscompose.ComposeAccounts, with the vault's
|
||||
// hashes), seals it to the bus node's sealing key (seal.Seal, the key the node says in its have of the bus module,
|
||||
// UserListWritten.Key), and puts the box as one object of its object bucket GrantsUserListObjects, named by the
|
||||
// list's digest (UserListObjectName).
|
||||
// - It writes the key GrantsUserListKey of its state `user-list` (bucket GrantsUserListBucket): the GrantsUserList
|
||||
// below, which names the object, the list's digest, its revision, its user count and the users left out, and which
|
||||
// source the bus node follows (Follow: FollowGrants from take-over, FollowController in shadow and after hand-back).
|
||||
// - The node-engine of the bus node watches that key alone. While it says FollowGrants it opens the object, checks
|
||||
// the digest, writes the bus's user-list file (mode 0600) only when the file's digest differs, and says what it
|
||||
// loaded in its have of the bus module (UserListWritten, Source GrantsUserListSource); otherwise it follows the
|
||||
// controller's compositions as before (clause 8). Never both.
|
||||
|
||||
// The names of the grants part's user list on the bus.
|
||||
const (
|
||||
// GrantsUserListBucket is the grants part's state `user-list`.
|
||||
GrantsUserListBucket = "mesh-grants_user-list"
|
||||
// GrantsUserListKey is the one key of it the bus node reads.
|
||||
GrantsUserListKey = "sealed"
|
||||
// GrantsUserListObjects is the grants part's object bucket, where each sealed list is one object.
|
||||
GrantsUserListObjects = "mesh-grants-user-list"
|
||||
// GrantsUserListObjectStream is the bucket's stream.
|
||||
GrantsUserListObjectStream = "OBJ_" + GrantsUserListObjects
|
||||
// GrantsUserListSource is the source a have names for a list taken from the grants part.
|
||||
GrantsUserListSource = GrantsUserListBucket + "." + GrantsUserListKey
|
||||
)
|
||||
|
||||
// Which list the bus node follows.
|
||||
const (
|
||||
FollowController = "controller"
|
||||
FollowGrants = "grants"
|
||||
)
|
||||
|
||||
// GrantsUserList is the value of the grants part's key `user-list.sealed`.
|
||||
type GrantsUserList struct {
|
||||
V int `json:"v"`
|
||||
// Follow is which list the bus node follows: FollowGrants from take-over, FollowController otherwise.
|
||||
Follow string `json:"follow"`
|
||||
// Digest is the list's digest, `sha256:<hex>` of the file's bytes as the bus loads them.
|
||||
Digest string `json:"digest"`
|
||||
// Object is the object's name in GrantsUserListObjects (UserListObjectName of Digest).
|
||||
Object string `json:"object"`
|
||||
// SealedTo is the fingerprint of the key the object is sealed to (SealingKeyID), and Node the node it holds.
|
||||
SealedTo string `json:"sealed_to"`
|
||||
Node string `json:"node"`
|
||||
// Revision counts the lists the grants part wrote: one more for each new digest.
|
||||
Revision uint64 `json:"revision"`
|
||||
Users int `json:"users"`
|
||||
// LeftOut are the users the list leaves out for want of a hash in the vault.
|
||||
LeftOut []string `json:"left_out"`
|
||||
At time.Time `json:"at"`
|
||||
}
|
||||
|
||||
// EncodeGrantsUserList is g as JSON at Version; it refuses a value the bus node could not follow.
|
||||
func EncodeGrantsUserList(g GrantsUserList) ([]byte, error) {
|
||||
g.V = Version
|
||||
if g.LeftOut == nil {
|
||||
g.LeftOut = []string{}
|
||||
}
|
||||
if err := g.check(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return json.Marshal(g)
|
||||
}
|
||||
|
||||
// DecodeGrantsUserList reads the key's value; it refuses another version and what Encode refuses.
|
||||
func DecodeGrantsUserList(data []byte) (GrantsUserList, error) {
|
||||
var g GrantsUserList
|
||||
if err := decode(data, &g); err != nil {
|
||||
return GrantsUserList{}, err
|
||||
}
|
||||
if err := g.check(); err != nil {
|
||||
return GrantsUserList{}, err
|
||||
}
|
||||
return g, nil
|
||||
}
|
||||
|
||||
func (g GrantsUserList) check() error {
|
||||
if g.Follow != FollowGrants && g.Follow != FollowController {
|
||||
return fmt.Errorf("delivery: the user list's follow %q is not %s or %s", g.Follow, FollowGrants, FollowController)
|
||||
}
|
||||
if g.Digest == "" {
|
||||
return nil // a switch with no list written yet: the bus node keeps what it follows
|
||||
}
|
||||
if !strings.HasPrefix(g.Digest, "sha256:") || g.Object != UserListObjectName(g.Digest) {
|
||||
return fmt.Errorf("delivery: the user list names object %q for digest %q; the object is the digest's hex", g.Object, g.Digest)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// UserListDigest is a list's digest as the have and the key say it, `sha256:<hex>` of its bytes.
|
||||
func UserListDigest(content []byte) string {
|
||||
sum := sha256.Sum256(content)
|
||||
return "sha256:" + hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
// UserListObjectName is the object a list of that digest is put under: the digest's hex.
|
||||
func UserListObjectName(digest string) string { return strings.TrimPrefix(digest, "sha256:") }
|
||||
|
||||
// SealingKeyID is a short fingerprint of a public key in PEM, `sha256:<16 hex>`, to say which key a box is sealed to
|
||||
// without carrying the key.
|
||||
func SealingKeyID(publicPEM string) string {
|
||||
sum := sha256.Sum256([]byte(strings.TrimSpace(publicPEM)))
|
||||
return "sha256:" + hex.EncodeToString(sum[:8])
|
||||
}
|
||||
|
||||
// SealUserList is the object a list is put as: the sealed box (seal.Seal) of its bytes to the bus node's key, as JSON.
|
||||
func SealUserList(list []byte, nodePEM string) ([]byte, error) {
|
||||
if !strings.Contains(nodePEM, "PUBLIC KEY") {
|
||||
return nil, fmt.Errorf("delivery: the bus node's key is not a public key in PEM")
|
||||
}
|
||||
box, err := seal.Seal(string(list), nodePEM)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("delivery: the user list could not be sealed: %w", err)
|
||||
}
|
||||
return json.Marshal(box)
|
||||
}
|
||||
|
||||
// OpenUserList opens an object with open (the node's own key) and checks it is the list of digest.
|
||||
func OpenUserList(object []byte, digest string, open func(seal.Box) ([]byte, error)) ([]byte, error) {
|
||||
var box seal.Box
|
||||
if err := json.Unmarshal(object, &box); err != nil {
|
||||
return nil, fmt.Errorf("delivery: the object is no sealed box: %w", err)
|
||||
}
|
||||
list, err := open(box)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if got := UserListDigest(list); got != digest {
|
||||
return nil, fmt.Errorf("delivery: the opened list's digest is %s, and the key names %s", got, digest)
|
||||
}
|
||||
return list, nil
|
||||
}
|
||||
@@ -101,7 +101,7 @@ func (k *KeyKeeper) ensure(ctx context.Context) error {
|
||||
}
|
||||
r := k.Run(ctx, nil, "ssh-keygen", "-q", "-t", "ed25519", "-N", "", "-C", k.Account+"@"+k.Node+" (mesh)", "-f", k.path())
|
||||
if r.Status != 0 || r.Err != "" {
|
||||
return fmt.Errorf("ssh-keygen: %s", firstLine(r.Err+" "+r.Stderr))
|
||||
return fmt.Errorf("ssh-keygen: %s", Said(r.Err+"\n"+r.Stderr))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
+58
-6
@@ -9,6 +9,7 @@ import (
|
||||
"os/exec"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
// Ran is what a command did: its output, its exit status, and why it never ran to an answer.
|
||||
@@ -43,7 +44,8 @@ func ExecRunnerFor(bound time.Duration) Runner {
|
||||
|
||||
// ExecRunnerClean is ExecRunnerFor whose commands get exactly the env argument as their environment, and nothing of
|
||||
// this process's (issue 1497; novox/hq ADR 0466 §4: no registry password in a fetch's environment). The caller
|
||||
// gives PATH and HOME itself. A directory is the command's own business (`git -C`, `go -C`).
|
||||
// gives PATH and HOME itself. A directory is the command's own business (`git -C`, `go -C`): it is started in the HOME
|
||||
// it is given, or the machine's temporary directory, never in this process's (stableDir).
|
||||
func ExecRunnerClean(bound time.Duration) Runner {
|
||||
return func(ctx context.Context, env []string, name string, args ...string) Ran {
|
||||
return execRun(ctx, bound, true, env, name, args...)
|
||||
@@ -62,6 +64,7 @@ func execRun(ctx context.Context, bound time.Duration, clean bool, env []string,
|
||||
} else {
|
||||
cmd.Env = append(os.Environ(), env...)
|
||||
}
|
||||
cmd.Dir = stableDir(clean, env)
|
||||
var out, errb bytes.Buffer
|
||||
cmd.Stdout, cmd.Stderr = &out, &errb
|
||||
err := cmd.Run()
|
||||
@@ -83,11 +86,60 @@ func execRun(ctx context.Context, bound time.Duration, clean bool, env []string,
|
||||
return r
|
||||
}
|
||||
|
||||
func firstLine(s string) string {
|
||||
for _, l := range strings.Split(s, "\n") {
|
||||
if l = strings.TrimSpace(l); l != "" {
|
||||
return l
|
||||
// stableDir is the directory a command starts in when it names none (novox/hq issue 734). **Never one that may be
|
||||
// deleted under it**: a module runs in the directory its build was unpacked in, which a new build replaces, and a
|
||||
// command started there inherits it — `go version` then fails "getwd: no such file or directory" before it reads
|
||||
// anything. A clean command starts in the HOME it is given, or the temporary directory; any other keeps this
|
||||
// process's directory while it exists, and starts in the temporary directory once it is gone. "" is this process's.
|
||||
func stableDir(clean bool, env []string) string {
|
||||
if !clean {
|
||||
if _, err := os.Getwd(); err == nil {
|
||||
return ""
|
||||
}
|
||||
return os.TempDir()
|
||||
}
|
||||
for i := len(env) - 1; i >= 0; i-- {
|
||||
if home, ok := strings.CutPrefix(env[i], "HOME="); ok {
|
||||
if info, err := os.Stat(home); err == nil && info.IsDir() {
|
||||
return home
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
return ""
|
||||
return os.TempDir()
|
||||
}
|
||||
|
||||
// SaidBound bounds what Said keeps of a command's words, so a reason stays a line of a step's record and of a log.
|
||||
const SaidBound = 600
|
||||
|
||||
// Said is what a command said of its failure in its own words, for a reason: every line that says something, trimmed
|
||||
// and joined with "; ", bounded at SaidBound bytes. A line that is a bare prefix and nothing else (git writes "error:"
|
||||
// alone before and after the forge's own message, "remote:" or "fatal:" likewise) is left out, so the cause is never
|
||||
// cut off behind it: the first line alone of `git fetch` from a repository the account cannot read was "error:".
|
||||
func Said(s string) string {
|
||||
var keep []string
|
||||
for _, l := range strings.Split(s, "\n") {
|
||||
l = strings.TrimSpace(l)
|
||||
if l == "" || barePrefix(l) {
|
||||
continue
|
||||
}
|
||||
keep = append(keep, l)
|
||||
}
|
||||
out := strings.Join(keep, "; ")
|
||||
if len(out) > SaidBound {
|
||||
cut := SaidBound
|
||||
for cut > 0 && !utf8.RuneStart(out[cut]) {
|
||||
cut--
|
||||
}
|
||||
out = out[:cut] + " …"
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// barePrefix says whether a line is one word and a colon, such as "error:" or "remote:", with nothing after it.
|
||||
func barePrefix(l string) bool {
|
||||
if !strings.HasSuffix(l, ":") {
|
||||
return false
|
||||
}
|
||||
return !strings.ContainsAny(strings.TrimSuffix(l, ":"), " \t:")
|
||||
}
|
||||
|
||||
@@ -48,13 +48,29 @@ func TestExecRunnerForZeroIsCutByItsContext(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// 1497/AC4: ExecRunnerClean(0) runs env with only the given words: PATH and HOME, not the process's secret.
|
||||
// 1497/AC4: ExecRunnerClean(0) runs env with only the given words: PATH and HOME, not the process's secret. And issue
|
||||
// 734: run from a process whose own directory was deleted under it (a module whose build a new one replaced), a clean
|
||||
// command starts in its HOME, and any other in the temporary directory, rather than failing to find where it is.
|
||||
func TestExecRunnerCleanGivesTheCommandOnlyTheEnvArgument(t *testing.T) {
|
||||
t.Setenv("MESH_TEST_SECRET", "x")
|
||||
r := ExecRunnerClean(0)(context.Background(), []string{"PATH=" + os.Getenv("PATH"), "HOME=/h"}, "env")
|
||||
if r.Status != 0 || !strings.Contains(r.Stdout, "HOME=/h") || !strings.Contains(r.Stdout, "PATH=") || strings.Contains(r.Stdout, "MESH_TEST_SECRET") {
|
||||
t.Errorf("ran = %+v", r)
|
||||
}
|
||||
|
||||
home, gone := t.TempDir(), t.TempDir()
|
||||
t.Chdir(gone)
|
||||
if err := os.Remove(gone); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r = ExecRunnerClean(0)(context.Background(), []string{"PATH=" + os.Getenv("PATH"), "HOME=" + home}, "pwd")
|
||||
if r.Status != 0 || strings.TrimSpace(r.Stdout) != home {
|
||||
t.Errorf("a clean command run from a deleted directory = %+v, want it started in its HOME %s", r, home)
|
||||
}
|
||||
r = ExecRunnerFor(0)(context.Background(), nil, "pwd")
|
||||
if r.Status != 0 || strings.TrimSpace(r.Stdout) != os.TempDir() {
|
||||
t.Errorf("a command run from a deleted directory = %+v, want it started in %s", r, os.TempDir())
|
||||
}
|
||||
}
|
||||
|
||||
// 1497/AC5: ExecRunnerFor(0) is unchanged: the env argument is added to the process's environment.
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
package forgekey
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// git's own words survive as the reason: a bare "error:" or "fatal:" line is never the whole reason (the mirror of
|
||||
// jschoubben/car-hunter "is not used: git -C: error:", 2026-10-26), every line that says something is kept, and the
|
||||
// words are bounded.
|
||||
func TestSaidKeepsTheCauseBehindABarePrefix(t *testing.T) {
|
||||
forge := "error: \nerror: User: 22:nox-builder with Key: 30:mesh:ace.mesh-build-go is not authorized to read jschoubben/car-hunter.\nerror: \nfatal: Could not read from remote repository.\n\nPlease make sure you have the correct access rights\nand the repository exists.\n"
|
||||
for _, c := range []struct{ name, in, want string }{
|
||||
{"forge refusal", forge, "error: User: 22:nox-builder with Key: 30:mesh:ace.mesh-build-go is not authorized to read jschoubben/car-hunter.; fatal: Could not read from remote repository.; Please make sure you have the correct access rights; and the repository exists."},
|
||||
{"one line", "fatal: couldn't find remote ref x\n", "fatal: couldn't find remote ref x"},
|
||||
{"bare prefixes only", "error:\nremote:\n", ""},
|
||||
{"a colon inside stays", " ENOENT \nfatal: bad: thing\n", "ENOENT; fatal: bad: thing"},
|
||||
{"empty", "", ""},
|
||||
} {
|
||||
if got := Said(c.in); got != c.want {
|
||||
t.Errorf("%s: Said = %q, want %q", c.name, got, c.want)
|
||||
}
|
||||
}
|
||||
long := Said(strings.Repeat("é", SaidBound))
|
||||
if !strings.HasSuffix(long, " …") || len(long) > SaidBound+len(" …") || !strings.HasPrefix(long, "é") {
|
||||
t.Errorf("long: %d bytes, ends %q", len(long), long[len(long)-6:])
|
||||
}
|
||||
}
|
||||
@@ -1,11 +1,19 @@
|
||||
package process
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
)
|
||||
|
||||
// 1171/AC2: given an account file broker-<process id> in the controller's form {url, fingerprint, node,
|
||||
@@ -46,6 +54,97 @@ func TestReadAccountToleratesNoClaimsAtAll(t *testing.T) {
|
||||
// 1171/AC2: a file missing url, user or password is refused, naming the field; the password never appears
|
||||
// in the error even when it is the field present and another is missing.
|
||||
func TestReadAccountRefusesAFileMissingAField(t *testing.T) {
|
||||
t.Run("1945: a refused bucket stops a start only when the part cannot work without it", func(t *testing.T) {
|
||||
refusal := errors.New("nats: permissions violation for publish to $JS.API.STREAM.INFO.KV_mesh-delivery-gate_source")
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
err error
|
||||
}{{"refused", refusal}, {"not answered", context.DeadlineExceeded}} {
|
||||
var said []string
|
||||
logf := func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) }
|
||||
if kv := OptionalBucket(context.Background(), refusingBus{c.err}, "b", "b is not heard", logf); kv != nil {
|
||||
t.Errorf("%s: an optional bucket answered %v", c.name, kv)
|
||||
}
|
||||
NotWatched(logf, "b", c.err, "b is not heard")
|
||||
if len(said) != 2 || !strings.Contains(said[0], c.err.Error()) || !strings.Contains(said[1], "cannot be watched") {
|
||||
t.Errorf("%s: said %q", c.name, said)
|
||||
}
|
||||
if _, err := EssentialBucket(context.Background(), refusingBus{c.err}, "builder", "b", "build"); err == nil ||
|
||||
!strings.Contains(err.Error(), "grant of b") || !strings.Contains(err.Error(), "the builder cannot build") {
|
||||
t.Errorf("%s: an essential bucket: %v", c.name, err)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("1790: a bucket missing at start is bound when the bus says it was created, and what was written meanwhile is kept", func(t *testing.T) {
|
||||
bus := &laterBus{}
|
||||
hear := &heldSubscriber{}
|
||||
var said []string
|
||||
logf := func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) }
|
||||
b := AwaitBucket(context.Background(), bus, hear, "m_hearing", "the health tool cannot say it", logf)
|
||||
if hear.subject != "$JS.EVENT.ADVISORY.STREAM.CREATED.KV_m_hearing" {
|
||||
t.Fatalf("heard %q", hear.subject)
|
||||
}
|
||||
ready := make(chan jetstream.KeyValue, 1)
|
||||
b.Ready(func(kv jetstream.KeyValue) { ready <- kv })
|
||||
for _, v := range []string{"one", "two"} {
|
||||
if _, err := b.Put(context.Background(), "dead-letters", []byte(v)); !errors.Is(err, ErrNotYet) {
|
||||
t.Fatalf("a write while missing: %v", err)
|
||||
}
|
||||
}
|
||||
if b.KV() != nil || len(ready) > 0 {
|
||||
t.Fatal("bound before the bucket exists")
|
||||
}
|
||||
bus.create()
|
||||
hear.deliver()
|
||||
select {
|
||||
case <-ready:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("not bound within 5s of the bus saying the bucket was created")
|
||||
}
|
||||
if got := bus.kv.puts; strings.Join(got, ",") != "dead-letters=two" {
|
||||
t.Errorf("written when it appeared: %v, want the latest value only", got)
|
||||
}
|
||||
if _, err := b.Put(context.Background(), "dead-letters", []byte("three")); err != nil {
|
||||
t.Errorf("a write once bound: %v", err)
|
||||
}
|
||||
if len(said) != 2 || !strings.Contains(said[0], "cannot be read yet") || !strings.Contains(said[1], "appeared") {
|
||||
t.Errorf("said %q", said)
|
||||
}
|
||||
// The bus refuses the subscription to the creation: said once, loudly; a reconnect hears it again and binds.
|
||||
refusedBus, refusing := &laterBus{}, &heldSubscriber{refused: true}
|
||||
var loud []string
|
||||
r := AwaitBucket(context.Background(), refusedBus, refusing, "m_reasks", "x",
|
||||
func(f string, a ...any) { loud = append(loud, fmt.Sprintf(f, a...)) })
|
||||
bound := make(chan struct{})
|
||||
r.Ready(func(jetstream.KeyValue) { close(bound) })
|
||||
if len(loud) != 2 || !strings.Contains(loud[0], "cannot hear the creation of m_reasks") ||
|
||||
!strings.Contains(loud[0], "permissions violation") {
|
||||
t.Fatalf("a refused subscription was said as %q", loud)
|
||||
}
|
||||
refusing.mu.Lock()
|
||||
refusing.refused = false
|
||||
refusing.mu.Unlock()
|
||||
refusedBus.create()
|
||||
refusing.status <- nats.CONNECTED
|
||||
select {
|
||||
case <-bound:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("not bound after the connection connected again")
|
||||
}
|
||||
if refusing.subscribed != 2 {
|
||||
t.Errorf("subscribed %d times, want again at the reconnect", refusing.subscribed)
|
||||
}
|
||||
|
||||
// Without the advisory heard, the next use binds it.
|
||||
quiet := &laterBus{}
|
||||
c := AwaitBucket(context.Background(), quiet, nil, "m_hearing", "x", nil)
|
||||
quiet.create()
|
||||
if c.KV() == nil {
|
||||
t.Error("a use after the bucket appeared did not bind it")
|
||||
}
|
||||
})
|
||||
|
||||
for _, tc := range []struct {
|
||||
field string
|
||||
values map[string]any
|
||||
@@ -82,3 +181,79 @@ func writeAccount(t *testing.T, fields map[string]any) string {
|
||||
}
|
||||
return path
|
||||
}
|
||||
|
||||
// refusingBus refuses every bucket with err.
|
||||
type refusingBus struct{ err error }
|
||||
|
||||
func (r refusingBus) KeyValue(context.Context, string) (jetstream.KeyValue, error) { return nil, r.err }
|
||||
|
||||
// laterBus has no bucket until create, and then one.
|
||||
type laterBus struct {
|
||||
mu sync.Mutex
|
||||
kv *keptKV
|
||||
}
|
||||
|
||||
func (l *laterBus) create() { l.mu.Lock(); l.kv = &keptKV{}; l.mu.Unlock() }
|
||||
|
||||
func (l *laterBus) KeyValue(context.Context, string) (jetstream.KeyValue, error) {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
if l.kv == nil {
|
||||
return nil, jetstream.ErrBucketNotFound
|
||||
}
|
||||
return l.kv, nil
|
||||
}
|
||||
|
||||
// keptKV records each put as key=value.
|
||||
type keptKV struct {
|
||||
jetstream.KeyValue
|
||||
puts []string
|
||||
}
|
||||
|
||||
func (k *keptKV) Put(_ context.Context, key string, value []byte) (uint64, error) {
|
||||
k.puts = append(k.puts, key+"="+string(value))
|
||||
return uint64(len(k.puts)), nil
|
||||
}
|
||||
|
||||
// heldSubscriber keeps the last subscription made, for the test to deliver to; refused, the bus refuses it as a
|
||||
// *nats.Conn says it (the connection's last error after a round trip). status stands for the connection's CONNECTED.
|
||||
type heldSubscriber struct {
|
||||
mu sync.Mutex
|
||||
subject string
|
||||
handle nats.MsgHandler
|
||||
subscribed int
|
||||
refused bool
|
||||
last error
|
||||
status chan nats.Status
|
||||
}
|
||||
|
||||
func (h *heldSubscriber) Subscribe(subject string, handle nats.MsgHandler) (*nats.Subscription, error) {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
h.subject, h.handle = subject, handle
|
||||
h.subscribed++
|
||||
if h.refused {
|
||||
h.last = fmt.Errorf("%w: Permissions Violation for Subscription to %q", nats.ErrPermissionViolation, subject)
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (h *heldSubscriber) FlushTimeout(time.Duration) error { return nil }
|
||||
|
||||
func (h *heldSubscriber) LastError() error { h.mu.Lock(); defer h.mu.Unlock(); return h.last }
|
||||
|
||||
func (h *heldSubscriber) StatusChanged(...nats.Status) chan nats.Status {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
h.status = make(chan nats.Status, 1)
|
||||
return h.status
|
||||
}
|
||||
|
||||
func (h *heldSubscriber) RemoveStatusListener(chan nats.Status) {}
|
||||
|
||||
func (h *heldSubscriber) deliver() {
|
||||
h.mu.Lock()
|
||||
handle, subject := h.handle, h.subject
|
||||
h.mu.Unlock()
|
||||
handle(&nats.Msg{Subject: subject})
|
||||
}
|
||||
|
||||
@@ -0,0 +1,322 @@
|
||||
package process
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
)
|
||||
|
||||
// **A part does not stop for a bucket it can work without** (novox/hq issue 1945, met twice on 2026-10-25: the delivery
|
||||
// builder exited on a refused read of the gate's source and crash-looped). A grant reaches the bus with a delivery, so a
|
||||
// bucket, a watch or a consumer the bus refuses, or times out on, at start is often only not granted yet: exiting for it
|
||||
// stops the part until a person's hand. A part opens what it cannot do its job at all without with EssentialBucket, which
|
||||
// stops its start with an error naming the bucket and its grant, and everything else with OptionalBucket, which says what
|
||||
// is not heard and goes on; a watch the bus refuses on an optional bucket is said with NotWatched. Here, beside State,
|
||||
// because this is where a process opens what it keeps and reads on the bus; the parts that open buckets with their own
|
||||
// JetStream (the delivery parts) take the same three.
|
||||
|
||||
// BucketOpener is the part of the bus a start opens buckets with: a jetstream.JetStream, or a test's fake.
|
||||
type BucketOpener interface {
|
||||
KeyValue(ctx context.Context, bucket string) (jetstream.KeyValue, error)
|
||||
}
|
||||
|
||||
// EssentialBucket opens a bucket part cannot work without (it cannot `does` without it); its error is EssentialErr's.
|
||||
func EssentialBucket(ctx context.Context, js BucketOpener, part, name, does string) (jetstream.KeyValue, error) {
|
||||
kv, err := js.KeyValue(ctx, name)
|
||||
if err != nil {
|
||||
return nil, EssentialErr(part, name, does, err)
|
||||
}
|
||||
return kv, nil
|
||||
}
|
||||
|
||||
// EssentialErr is the error of a bucket, consumer or stream named name that part cannot work without: it says what part
|
||||
// cannot do, that it stops, and the grant to look at.
|
||||
func EssentialErr(part, name, does string, err error) error {
|
||||
return fmt.Errorf("%s cannot be read (%v): the %s cannot %s without it, so it stops; where the bus refused it or did not "+
|
||||
"answer, this module's grant of %s (its manifest's reads, state or consumes, issued with its credential) has not "+
|
||||
"reached the bus yet", name, err, part, does, name)
|
||||
}
|
||||
|
||||
// OptionalBucket opens a bucket the part works without: any error is said with what is not heard (without), and nil
|
||||
// answered.
|
||||
func OptionalBucket(ctx context.Context, js BucketOpener, name, without string, logf func(string, ...any)) jetstream.KeyValue {
|
||||
kv, err := js.KeyValue(ctx, name)
|
||||
if err != nil {
|
||||
logf("the bucket %s cannot be read (%v): %s", name, err, without)
|
||||
return nil
|
||||
}
|
||||
return kv
|
||||
}
|
||||
|
||||
// NotWatched says that an optional bucket opened but its watch was refused or not answered: what is not heard.
|
||||
func NotWatched(logf func(string, ...any), name string, err error, without string) {
|
||||
logf("the bucket %s cannot be watched (%v): %s", name, err, without)
|
||||
}
|
||||
|
||||
// ErrNotYet is a write to an awaited bucket that is not on the bus yet: the value is kept, the latest per key, and
|
||||
// written once the bucket appears.
|
||||
var ErrNotYet = errors.New("the bucket is not on the bus yet: the value is kept and written when it appears")
|
||||
|
||||
// Subscriber is the part of a connection that hears what the bus says: a *nats.Conn, or a test's fake.
|
||||
type Subscriber interface {
|
||||
Subscribe(subject string, handle nats.MsgHandler) (*nats.Subscription, error)
|
||||
}
|
||||
|
||||
// createdAdvisory is the subject the bus says a bucket's stream was created on: the server's own advisory, which a
|
||||
// principal is granted for each bucket it may bind (mesh-sdk buscompose StreamCreatedAdvisory, stateCreatedHeard).
|
||||
func createdAdvisory(bucket string) string { return "$JS.EVENT.ADVISORY.STREAM.CREATED.KV_" + bucket }
|
||||
|
||||
// bindWithin bounds one bind of an awaited bucket: one STREAM.INFO.
|
||||
const bindWithin = 10 * time.Second
|
||||
|
||||
// AwaitedBucket is a bucket a part keeps or reads that may not be on the bus at its start, bound once it is (novox/hq
|
||||
// issue 1790, met 2026-10-26: since the grants part creates a manifest's buckets after a delivery, ADR 0547, a module
|
||||
// restarted by that delivery opened its new state `hearing` before it existed, got "bucket not found", and never looked
|
||||
// again). It binds at AwaitBucket when it can; otherwise when the bus says the bucket's stream was created (heard on
|
||||
// nc), and at each use before then (one STREAM.INFO, the cheap check), never on a timer. It says once that it waits and
|
||||
// once that the bucket appeared.
|
||||
type AwaitedBucket struct {
|
||||
ctx context.Context
|
||||
js BucketOpener
|
||||
name string
|
||||
without string
|
||||
logf func(string, ...any)
|
||||
|
||||
nc Subscriber
|
||||
|
||||
mu sync.Mutex
|
||||
kv jetstream.KeyValue
|
||||
refusedSaid bool
|
||||
stopStatus func()
|
||||
pending map[string][]byte
|
||||
order []string
|
||||
ready []func(jetstream.KeyValue)
|
||||
sub *nats.Subscription
|
||||
}
|
||||
|
||||
// AwaitBucket opens the bucket name now if the bus holds it, and otherwise waits for it as AwaitedBucket says; without
|
||||
// is what is not done while it is missing. nc may be nil: then only a use binds it. ctx is the part's life: a bind
|
||||
// after it ends is not tried.
|
||||
func AwaitBucket(ctx context.Context, js BucketOpener, nc Subscriber, name, without string, logf func(string, ...any)) *AwaitedBucket {
|
||||
if logf == nil {
|
||||
logf = func(string, ...any) {}
|
||||
}
|
||||
b := &AwaitedBucket{ctx: ctx, js: js, name: name, without: without, logf: logf, pending: map[string][]byte{}}
|
||||
// Heard before the first try, so a creation between the try and the subscription is not missed.
|
||||
if nc != nil {
|
||||
b.nc = nc
|
||||
b.hear()
|
||||
b.followReconnects()
|
||||
}
|
||||
ctxB, cancel := context.WithTimeout(ctx, bindWithin)
|
||||
kv, err := js.KeyValue(ctxB, name)
|
||||
cancel()
|
||||
if err == nil {
|
||||
b.bound(kv, false)
|
||||
return b
|
||||
}
|
||||
logf("the bucket %s cannot be read yet (%v): %s until it appears; it is bound when the bus says it was created, or at "+
|
||||
"its next use", name, err, without)
|
||||
return b
|
||||
}
|
||||
|
||||
// KV is the bound bucket, or nil while it is not on the bus; while nil, the call tries to bind it once.
|
||||
func (b *AwaitedBucket) KV() jetstream.KeyValue {
|
||||
b.mu.Lock()
|
||||
kv := b.kv
|
||||
b.mu.Unlock()
|
||||
if kv != nil {
|
||||
return kv
|
||||
}
|
||||
return b.bind()
|
||||
}
|
||||
|
||||
// Ready runs f with the bucket: now when it is bound, otherwise once it appears (on the goroutine that bound it).
|
||||
func (b *AwaitedBucket) Ready(f func(jetstream.KeyValue)) {
|
||||
b.mu.Lock()
|
||||
kv := b.kv
|
||||
if kv == nil {
|
||||
b.ready = append(b.ready, f)
|
||||
}
|
||||
b.mu.Unlock()
|
||||
if kv != nil {
|
||||
f(kv)
|
||||
}
|
||||
}
|
||||
|
||||
// Put writes key, as jetstream.KeyValue's Put. While the bucket is missing it keeps value as key's latest and answers
|
||||
// ErrNotYet; the kept values are written, in the order their keys were first kept, when it appears.
|
||||
func (b *AwaitedBucket) Put(ctx context.Context, key string, value []byte) (uint64, error) {
|
||||
kv := b.KV()
|
||||
if kv == nil {
|
||||
b.mu.Lock()
|
||||
if b.kv == nil {
|
||||
if _, ok := b.pending[key]; !ok {
|
||||
b.order = append(b.order, key)
|
||||
}
|
||||
b.pending[key] = append([]byte(nil), value...)
|
||||
b.mu.Unlock()
|
||||
return 0, fmt.Errorf("%s: %w", b.name, ErrNotYet)
|
||||
}
|
||||
kv = b.kv
|
||||
b.mu.Unlock()
|
||||
}
|
||||
return kv.Put(ctx, key, value)
|
||||
}
|
||||
|
||||
// bind tries one STREAM.INFO of the bucket and, when it is there, binds it; it answers the bucket or nil.
|
||||
func (b *AwaitedBucket) bind() jetstream.KeyValue {
|
||||
if b.ctx.Err() != nil {
|
||||
return nil
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(b.ctx, bindWithin)
|
||||
defer cancel()
|
||||
kv, err := b.js.KeyValue(ctx, b.name)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
return b.bound(kv, true)
|
||||
}
|
||||
|
||||
// bound keeps kv as the bucket (unless another bind did first), writes what was kept and runs what waited on it.
|
||||
func (b *AwaitedBucket) bound(kv jetstream.KeyValue, appeared bool) jetstream.KeyValue {
|
||||
b.mu.Lock()
|
||||
if b.kv != nil {
|
||||
kv = b.kv
|
||||
b.mu.Unlock()
|
||||
return kv
|
||||
}
|
||||
// The kept values are written before the bucket is handed to anyone, so no later write is overtaken by an earlier.
|
||||
written, failed := 0, 0
|
||||
for _, key := range b.order {
|
||||
ctx, cancel := context.WithTimeout(b.ctx, bindWithin)
|
||||
_, err := kv.Put(ctx, key, b.pending[key])
|
||||
cancel()
|
||||
if err != nil {
|
||||
failed++
|
||||
b.logf("the bucket %s appeared but the value kept for %s could not be written: %v", b.name, key, err)
|
||||
continue
|
||||
}
|
||||
written++
|
||||
}
|
||||
b.kv, b.pending, b.order = kv, nil, nil
|
||||
ready, sub, stop := b.ready, b.sub, b.stopStatus
|
||||
b.ready, b.sub, b.stopStatus = nil, nil, nil
|
||||
b.mu.Unlock()
|
||||
if sub != nil {
|
||||
_ = sub.Unsubscribe()
|
||||
}
|
||||
if stop != nil {
|
||||
stop()
|
||||
}
|
||||
if appeared {
|
||||
b.logf("the bucket %s appeared and is bound: %d kept value(s) written, %d not", b.name, written, failed)
|
||||
}
|
||||
for _, f := range ready {
|
||||
f(kv)
|
||||
}
|
||||
return kv
|
||||
}
|
||||
|
||||
// refusalReader is the part of a *nats.Conn that says whether the bus refused a subscription: the server's refusal comes
|
||||
// back after a round trip as the connection's last error, naming the subject.
|
||||
type refusalReader interface {
|
||||
FlushTimeout(time.Duration) error
|
||||
LastError() error
|
||||
}
|
||||
|
||||
// statusFollower is the part of a *nats.Conn that says when it connected again, without taking its reconnect handler.
|
||||
type statusFollower interface {
|
||||
StatusChanged(statuses ...nats.Status) chan nats.Status
|
||||
RemoveStatusListener(ch chan nats.Status)
|
||||
}
|
||||
|
||||
// hear subscribes to the bucket's creation (again), and says once, loudly, when the bus refuses it: then only a use or
|
||||
// a reconnect binds the bucket.
|
||||
func (b *AwaitedBucket) hear() {
|
||||
subject := createdAdvisory(b.name)
|
||||
b.mu.Lock()
|
||||
old, bound := b.sub, b.kv != nil
|
||||
b.sub = nil
|
||||
b.mu.Unlock()
|
||||
if old != nil {
|
||||
_ = old.Unsubscribe()
|
||||
}
|
||||
if bound {
|
||||
return
|
||||
}
|
||||
checks, _ := b.nc.(refusalReader)
|
||||
var before error
|
||||
if checks != nil {
|
||||
before = checks.LastError()
|
||||
}
|
||||
sub, err := b.nc.Subscribe(subject, func(*nats.Msg) { go b.bind() })
|
||||
if err == nil && checks != nil && checks.FlushTimeout(bindWithin) == nil {
|
||||
if after := checks.LastError(); after != nil && after != before && errors.Is(after, nats.ErrPermissionViolation) &&
|
||||
strings.Contains(after.Error(), subject) {
|
||||
err = after
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
if sub != nil {
|
||||
_ = sub.Unsubscribe()
|
||||
}
|
||||
b.mu.Lock()
|
||||
said := b.refusedSaid
|
||||
b.refusedSaid = true
|
||||
b.mu.Unlock()
|
||||
if !said {
|
||||
b.logf("cannot hear the creation of %s: %v; it binds at its next use, or when the connection reconnects", b.name, err)
|
||||
}
|
||||
return
|
||||
}
|
||||
b.mu.Lock()
|
||||
if b.kv != nil {
|
||||
b.mu.Unlock()
|
||||
if sub != nil {
|
||||
_ = sub.Unsubscribe()
|
||||
}
|
||||
return
|
||||
}
|
||||
b.sub, b.refusedSaid = sub, false
|
||||
b.mu.Unlock()
|
||||
}
|
||||
|
||||
// followReconnects hears the bucket's creation again and tries a bind each time the connection connects again (a grant
|
||||
// reaches the bus with a new user list; a lost subscription is made again), until the bucket is bound or the part ends.
|
||||
func (b *AwaitedBucket) followReconnects() {
|
||||
w, ok := b.nc.(statusFollower)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
ch := w.StatusChanged(nats.CONNECTED)
|
||||
b.mu.Lock()
|
||||
if b.kv != nil {
|
||||
b.mu.Unlock()
|
||||
w.RemoveStatusListener(ch)
|
||||
return
|
||||
}
|
||||
b.stopStatus = func() { w.RemoveStatusListener(ch) }
|
||||
b.mu.Unlock()
|
||||
go func() {
|
||||
for {
|
||||
select {
|
||||
case <-b.ctx.Done():
|
||||
w.RemoveStatusListener(ch)
|
||||
return
|
||||
case _, open := <-ch:
|
||||
if !open {
|
||||
return
|
||||
}
|
||||
b.hear()
|
||||
b.KV()
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
@@ -57,8 +57,10 @@ func grantedSendProcess(t *testing.T) (url string, account Account) {
|
||||
Users: []*server.User{
|
||||
{Username: adminUser, Password: adminPass},
|
||||
{Username: sendUser, Password: sendPassword, Permissions: &server.Permissions{
|
||||
Publish: &server.SubjectPermission{Allow: sendGrants},
|
||||
Subscribe: &server.SubjectPermission{Allow: []string{"_INBOX." + sendUser + ".>"}},
|
||||
Publish: &server.SubjectPermission{Allow: sendGrants},
|
||||
// And the creation of its bucket, as mesh-sdk's composer grants it since issue 1790 (stateCreatedHeard).
|
||||
Subscribe: &server.SubjectPermission{Allow: []string{"_INBOX." + sendUser + ".>",
|
||||
"$JS.EVENT.ADVISORY.STREAM.CREATED.KV_mesh-mail_sends"}},
|
||||
}},
|
||||
},
|
||||
}
|
||||
|
||||
@@ -18,12 +18,40 @@ import (
|
||||
// key, reads it, and updates it with the revision it read, then each succeeds and returns the new
|
||||
// revision; an update with a stale revision fails as stdio.ErrChanged; a create of an existing key fails
|
||||
// as stdio.ErrChanged.
|
||||
//
|
||||
// 1790: and before that, the bucket is not on the bus when the process opens it (a delivery restarted it before the
|
||||
// grants part created its new state): it is bound when the bus says the bucket was created, as the process's own
|
||||
// account hears it, and the value written while it was missing is there.
|
||||
func TestStateCreateGetUpdateOnAGrantedBucket(t *testing.T) {
|
||||
url, account := grantedSendProcess(t)
|
||||
_, adminJetStream := adminJS(t, url)
|
||||
|
||||
nc, err := nats.Connect(url, nats.UserInfo(sendUser, sendPassword), nats.CustomInboxPrefix("_INBOX."+sendUser))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer nc.Close()
|
||||
js, err := jetstream.New(nc)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
awaited := AwaitBucket(ctxFor(t), js, nc, "mesh-mail_sends", "nothing is kept", t.Logf)
|
||||
appeared := make(chan struct{})
|
||||
awaited.Ready(func(jetstream.KeyValue) { close(appeared) })
|
||||
if _, err := awaited.Put(ctxFor(t), "hearing", []byte(`{"heard":false}`)); !errors.Is(err, ErrNotYet) {
|
||||
t.Fatalf("a write before the bucket exists: %v, want ErrNotYet", err)
|
||||
}
|
||||
if _, err := awaited.Put(ctxFor(t), "hearing", []byte(`{"heard":true}`)); !errors.Is(err, ErrNotYet) {
|
||||
t.Fatalf("a second write before the bucket exists: %v, want ErrNotYet", err)
|
||||
}
|
||||
if _, err := adminJetStream.CreateKeyValue(ctxFor(t), jetstream.KeyValueConfig{Bucket: "mesh-mail_sends"}); err != nil {
|
||||
t.Fatalf("the admin could not make the bucket: %v", err)
|
||||
}
|
||||
select {
|
||||
case <-appeared:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("the bucket was not bound within 5s of its creation, with no use since")
|
||||
}
|
||||
|
||||
conn, err := Connect(account)
|
||||
if err != nil {
|
||||
@@ -31,6 +59,9 @@ func TestStateCreateGetUpdateOnAGrantedBucket(t *testing.T) {
|
||||
}
|
||||
defer conn.Close()
|
||||
s := conn.State("sends")
|
||||
if kept, err := s.Get("hearing"); err != nil || kept == nil || string(kept.Value) != `{"heard":true}` {
|
||||
t.Fatalf("the value written while the bucket was missing: %+v, %v; want the latest", kept, err)
|
||||
}
|
||||
|
||||
rev, err := s.Create("key", map[string]any{"v": 1})
|
||||
if err != nil {
|
||||
|
||||
Reference in New Issue
Block a user