Compare commits
177
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e0d1de0475 | ||
|
|
db26f7e8b6 | ||
|
|
d4ae4b6790 | ||
|
|
2f6741b7f8 | ||
|
|
20a6dc4920 | ||
|
|
c4f37c279a | ||
|
|
7f89cb51de | ||
|
|
518dae5ec9 | ||
|
|
e8f8ef63fa | ||
|
|
4f4d5ca9c9 | ||
|
|
a8fbabbfc3 | ||
|
|
b9911dc1c1 | ||
|
|
e6b2df6506 | ||
|
|
f9d11acf23 | ||
|
|
10b6e20232 | ||
|
|
da7f045575 | ||
|
|
748c564cdf | ||
|
|
312878befb | ||
|
|
d7a9b9688b | ||
|
|
7f9cc0e55c | ||
|
|
e45bdc0891 | ||
|
|
a523a8cbb0 | ||
|
|
5da87a0dda | ||
|
|
e64486b0e6 | ||
|
|
64a502d16a | ||
|
|
de5b33843f | ||
|
|
d8da8f1d36 | ||
|
|
d506f2196a | ||
|
|
d6cef5829b | ||
|
|
826fd73cf2 | ||
|
|
aee971c198 | ||
|
|
bfc3883af9 | ||
|
|
19c1d35939 | ||
|
|
bc0c25e510 | ||
|
|
8c6e18e822 | ||
|
|
d9fa5536a8 | ||
|
|
dd0d156b9e | ||
|
|
51bc6a65b6 | ||
|
|
66bf26c0d7 | ||
|
|
34f210e5eb | ||
|
|
4e0036cc95 | ||
|
|
4784762a70 | ||
|
|
668fc2a722 | ||
|
|
20d18eb4f5 | ||
|
|
e1f393bbb1 | ||
|
|
35e86b00d3 | ||
|
|
53a1de2aaa | ||
|
|
403a5d72ab | ||
|
|
1ff6d33978 | ||
|
|
a2c813a6b4 | ||
|
|
a9560890ee | ||
|
|
bec9585d50 | ||
|
|
fc4f57c4b3 | ||
|
|
88fa74ce5e | ||
|
|
9205222a98 | ||
|
|
285566eed1 | ||
|
|
51756ee9a4 | ||
|
|
d7375c9811 | ||
|
|
232ded26ea | ||
|
|
691c97ec31 | ||
|
|
ba092d543f | ||
|
|
c3915ffb11 | ||
|
|
34338df950 | ||
|
|
07462c4caa | ||
|
|
6003da0a58 | ||
|
|
3e5d4c30ff | ||
|
|
1672f40233 | ||
|
|
e1dca0a39f | ||
|
|
4b503f8d0d | ||
|
|
8cd22892a3 | ||
|
|
1f6fc027c0 | ||
|
|
f625d98713 | ||
|
|
c2967432e6 | ||
|
|
69041c5302 | ||
|
|
792fb2ccfd | ||
|
|
89aa6fc17c | ||
|
|
a1d73bdcf0 | ||
|
|
d9384bcb23 | ||
|
|
b9b9c5120c | ||
|
|
37eff6fe5c | ||
|
|
687d715e99 | ||
|
|
ec2bd3b990 | ||
|
|
2248b985bf | ||
|
|
780be2fae5 | ||
|
|
5ab7ccf47e | ||
|
|
6147a20454 | ||
|
|
7aacbc1620 | ||
|
|
22b70c82b0 | ||
|
|
6aa8e869a3 | ||
|
|
104533e8c0 | ||
|
|
259bfd0c2b | ||
|
|
875c680d98 | ||
|
|
cdfc63cf10 | ||
|
|
d1af87a83a | ||
|
|
e76a5a31a0 | ||
|
|
8892a11768 | ||
|
|
4fac7810a1 | ||
|
|
b07e741c41 | ||
|
|
72a477eb1c | ||
|
|
8c3bfad86b | ||
|
|
9cb4d59093 | ||
|
|
fd2ae83431 | ||
|
|
4dd967e62f | ||
|
|
ab2ca6e3a7 | ||
|
|
a90f523edc | ||
|
|
fc7c4e97e6 | ||
|
|
f7cd91e49d | ||
|
|
3b3cabd2b0 | ||
|
|
f8b8f3a666 | ||
|
|
1f4ff20488 | ||
|
|
39a2144cd6 | ||
|
|
2aab86c5cb | ||
|
|
33387ac3e5 | ||
|
|
57d2f31f05 | ||
|
|
bb8b3bb43b | ||
|
|
d13bc6ecc6 | ||
|
|
ad1f6e6ac2 | ||
|
|
97895851f8 | ||
|
|
d14a6d301c | ||
|
|
7cf73ed76b | ||
|
|
d32743499c | ||
|
|
8d65cb3016 | ||
|
|
0c8d0f73fa | ||
|
|
dfc2c858d9 | ||
|
|
06fe1ee8b3 | ||
|
|
bd7731f358 | ||
|
|
c154bdedd3 | ||
|
|
5ea6c215a4 | ||
|
|
2b8ada5eb9 | ||
|
|
5652312f09 | ||
|
|
076e1f7947 | ||
|
|
26517ac00b | ||
|
|
637c33b7f8 | ||
|
|
c61212eba7 | ||
|
|
c6e669f71e | ||
|
|
580da61a91 | ||
|
|
78364e318b | ||
|
|
afbe371003 | ||
|
|
9aaa5c80ac | ||
|
|
07a65927c3 | ||
|
|
92fdc8c86a | ||
|
|
0730ef6092 | ||
|
|
4495a16608 | ||
|
|
b944d07158 | ||
|
|
6d0c2076a3 | ||
|
|
4375ed8b2c | ||
|
|
c0bbff3b4b | ||
|
|
725590e06e | ||
|
|
5792c3b959 | ||
|
|
d4deecdb38 | ||
|
|
216d2b4411 | ||
|
|
86f9a1a5c5 | ||
|
|
36d9eba51d | ||
|
|
90b20f5a3b | ||
|
|
85b5a2687e | ||
|
|
b5d8eb03ce | ||
|
|
863a981e01 | ||
|
|
b68fc75589 | ||
|
|
6c857e1c02 | ||
|
|
f8a9420952 | ||
|
|
f0f66d5b8d | ||
|
|
66e8fdaa62 | ||
|
|
88375c5786 | ||
|
|
757509366e | ||
|
|
c8e35097e8 | ||
|
|
ddfa119720 | ||
|
|
a86d607e48 | ||
|
|
86ac7596c9 | ||
|
|
cb54756c38 | ||
|
|
89dd57bed0 | ||
|
|
465beb4ea5 | ||
|
|
c9f978d330 | ||
|
|
efd3d224e1 | ||
|
|
081dc616ff | ||
|
|
b7b81e040f | ||
|
|
66b052aeee | ||
|
|
ea1b2ab395 |
@@ -1,2 +1,3 @@
|
||||
node_modules/
|
||||
dist/
|
||||
vendor/
|
||||
|
||||
@@ -0,0 +1,126 @@
|
||||
// Package announce is how a module says on the bus which states it keeps and which of them others may read (novox/hq
|
||||
// ADR 0495 clauses 2 and 3; issue 1790): the subject it says it on, the value it says, and where the controller keeps
|
||||
// the last one said by each module. What a module announced is the truth of what it keeps; a reader is granted a read
|
||||
// from it, never from a list or from the owner's manifest.
|
||||
//
|
||||
// A module announces from its code (process.Conn.Keeps, stdio.Keeps, the TypeScript SDK's keeps), never from its
|
||||
// manifest, which only the core reads (clause 1). A module that keeps no state on the bus announces nothing.
|
||||
package announce
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"sort"
|
||||
)
|
||||
|
||||
// Version is the version of the value this package writes.
|
||||
const Version = 1
|
||||
|
||||
// Event is the event type of an announcement, under the module's own namespace.
|
||||
const Event = "states"
|
||||
|
||||
// Subject is where module announces what it keeps: its own event subject, mesh.mod.<module>.event.states, which its
|
||||
// account may already publish to and the events stream keeps.
|
||||
func Subject(module string) string { return "mesh.mod." + module + ".event." + Event }
|
||||
|
||||
// SubjectAll is every module's announcement, what the controller consumes.
|
||||
const SubjectAll = "mesh.mod.*.event." + Event
|
||||
|
||||
// KeptBucket is the controller's bucket of the last announcement of each module, by Key.
|
||||
const KeptBucket = "mesh-controller_states-announced"
|
||||
|
||||
var name = regexp.MustCompile(`^[a-z0-9][a-z0-9-]*$`)
|
||||
|
||||
// Key is module's key in KeptBucket: the module itself, which must be one plain name, since a module's name is part of
|
||||
// its buckets' names (catalogue.StateProblems).
|
||||
func Key(module string) (string, error) {
|
||||
if !name.MatchString(module) {
|
||||
return "", fmt.Errorf("announce: %q is not a module that keeps state: lower-case letters, digits and hyphens", module)
|
||||
}
|
||||
return module, nil
|
||||
}
|
||||
|
||||
// State is one state a module keeps, by its local name, and whether others may read it. Others never write it: a
|
||||
// state another module writes is the owner's own code's business.
|
||||
type State struct {
|
||||
Name string `json:"name"`
|
||||
Readable bool `json:"readable,omitempty"`
|
||||
}
|
||||
|
||||
// States is an announcement: the module, the machine whose instance said it, and every state the module keeps, sorted
|
||||
// by name. An empty list says the module keeps nothing.
|
||||
type States struct {
|
||||
V int `json:"v"`
|
||||
Module string `json:"module"`
|
||||
Node string `json:"node,omitempty"`
|
||||
States []State `json:"states"`
|
||||
}
|
||||
|
||||
// Keeps says whether the module announced keeping the state.
|
||||
func (s States) Keeps(state string) bool {
|
||||
for _, k := range s.States {
|
||||
if k.Name == state {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Readable says whether the module announced the state readable by others.
|
||||
func (s States) Readable(state string) bool {
|
||||
for _, k := range s.States {
|
||||
if k.Name == state {
|
||||
return k.Readable
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (s *States) check() error {
|
||||
if s.Module == "" {
|
||||
return fmt.Errorf("announce: an announcement names no module")
|
||||
}
|
||||
if _, err := Key(s.Module); err != nil {
|
||||
return err
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
for _, k := range s.States {
|
||||
if !name.MatchString(k.Name) {
|
||||
return fmt.Errorf("announce: %s announces the state %q, and a state is named locally: lower-case letters, digits and hyphens", s.Module, k.Name)
|
||||
}
|
||||
if seen[k.Name] {
|
||||
return fmt.Errorf("announce: %s announces the state %q twice", s.Module, k.Name)
|
||||
}
|
||||
seen[k.Name] = true
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Encode is s as JSON at Version, its states sorted; it refuses a module or a state no bucket could be named by and a
|
||||
// state announced twice, naming it.
|
||||
func Encode(s States) ([]byte, error) {
|
||||
if err := s.check(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
s.V = Version
|
||||
s.States = append([]State{}, s.States...)
|
||||
sort.Slice(s.States, func(i, j int) bool { return s.States[i].Name < s.States[j].Name })
|
||||
return json.Marshal(s)
|
||||
}
|
||||
|
||||
// Decode reads an announcement; it refuses another version and what Encode refuses.
|
||||
func Decode(data []byte) (States, error) {
|
||||
var s States
|
||||
if err := json.Unmarshal(data, &s); err != nil {
|
||||
return States{}, fmt.Errorf("announce: the announcement is not JSON: %w", err)
|
||||
}
|
||||
if s.V != Version {
|
||||
return States{}, fmt.Errorf("announce: the announcement is version %d, and this reads version %d", s.V, Version)
|
||||
}
|
||||
if err := s.check(); err != nil {
|
||||
return States{}, err
|
||||
}
|
||||
sort.Slice(s.States, func(i, j int) bool { return s.States[i].Name < s.States[j].Name })
|
||||
return s, nil
|
||||
}
|
||||
@@ -0,0 +1,71 @@
|
||||
package announce
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// A module says on the bus which states it keeps and which of them others may read (novox/hq ADR 0495 clause 2), on its
|
||||
// own event subject, in a versioned value the controller keeps by module.
|
||||
func TestAnAnnouncementRoundTripsSortedAndSaysWhatIsReadable(t *testing.T) {
|
||||
data, err := Encode(States{Module: "mesh-controller", Node: "novox", States: []State{
|
||||
{Name: "plans-building", Readable: true}, {Name: "assignments", Readable: true}, {Name: "own"}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
back, err := Decode(data)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if back.V != Version || back.Module != "mesh-controller" || back.Node != "novox" {
|
||||
t.Fatalf("decoded %+v", back)
|
||||
}
|
||||
var names []string
|
||||
for _, s := range back.States {
|
||||
names = append(names, s.Name)
|
||||
}
|
||||
if strings.Join(names, ",") != "assignments,own,plans-building" {
|
||||
t.Fatalf("the states are not sorted: %v", names)
|
||||
}
|
||||
if !back.Readable("plans-building") || back.Readable("own") || back.Readable("absent") {
|
||||
t.Fatal("Readable does not say what was announced readable")
|
||||
}
|
||||
if !back.Keeps("own") || back.Keeps("absent") {
|
||||
t.Fatal("Keeps does not say what was announced")
|
||||
}
|
||||
if Subject("mesh-controller") != "mesh.mod.mesh-controller.event.states" || SubjectAll != "mesh.mod.*.event.states" {
|
||||
t.Fatalf("the subjects are %q and %q", Subject("mesh-controller"), SubjectAll)
|
||||
}
|
||||
if KeptBucket != "mesh-controller_states-announced" {
|
||||
t.Fatalf("the kept bucket is %q", KeptBucket)
|
||||
}
|
||||
if k, err := Key("mesh-delivery-builder"); err != nil || k != "mesh-delivery-builder" {
|
||||
t.Fatalf("Key = %q, %v", k, err)
|
||||
}
|
||||
if nothing, err := Encode(States{Module: "quiet"}); err != nil || !strings.Contains(string(nothing), `"states":[]`) {
|
||||
t.Fatalf("a module keeping nothing: %s, %v", nothing, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnAnnouncementRefusesWhatNoBucketCouldBe(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
s States
|
||||
says string
|
||||
}{
|
||||
{States{Module: ""}, "no module"},
|
||||
{States{Module: "a.b"}, `"a.b"`},
|
||||
{States{Module: "a", States: []State{{Name: "x_y"}}}, `"x_y"`},
|
||||
{States{Module: "a", States: []State{{Name: ""}}}, `""`},
|
||||
{States{Module: "a", States: []State{{Name: "x"}, {Name: "x", Readable: true}}}, "twice"},
|
||||
} {
|
||||
if _, err := Encode(c.s); err == nil || !strings.Contains(err.Error(), c.says) {
|
||||
t.Errorf("%+v: %v, wanted a refusal saying %s", c.s, err, c.says)
|
||||
}
|
||||
}
|
||||
if _, err := Decode([]byte(`{"v":2,"module":"a","states":[]}`)); err == nil {
|
||||
t.Error("an unknown version was read")
|
||||
}
|
||||
if _, err := Key("a.b"); err == nil {
|
||||
t.Error("a module with a dot became a key")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,91 @@
|
||||
// Package announcetest is the test each module runs with its own checks to hold that it announces on the bus every
|
||||
// state it keeps (novox/hq ADR 0495 clause 5; issue 1790): it starts the module against a bus of the test's own and
|
||||
// compares what the module announced with what its code declares. With the bind guard of process.Conn.Keeps,
|
||||
// stdio.Keeps and the TypeScript keeps, which refuses a state the code did not announce, announcement and code agree.
|
||||
//
|
||||
// It replaces the merge gate's check of one module's reads against another's manifest (clause 5); a test using it
|
||||
// carries `//go:build integration`, since it starts a bus.
|
||||
package announcetest
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"sort"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/announce"
|
||||
"git.novox.be/novox/mesh-sdk/go/testbus"
|
||||
)
|
||||
|
||||
// Wait is how long Check waits for the module's announcement once it started it.
|
||||
var Wait = 10 * time.Second
|
||||
|
||||
// Check starts a bus of the test's own, with a stream taking module's events, calls start with its address (start
|
||||
// runs the module, or its part that connects, against it, and answers how to stop it), waits for the module's
|
||||
// announcement and fails the test unless it is module's and names exactly want (in any order), readable as want says.
|
||||
// It answers the announcement.
|
||||
func Check(t testing.TB, module string, want []announce.State, start func(url string) (stop func())) announce.States {
|
||||
t.Helper()
|
||||
url := testbus.URL(t)
|
||||
nc, err := nats.Connect(url)
|
||||
if err != nil {
|
||||
t.Fatalf("announcetest: connecting to the test bus: %v", err)
|
||||
}
|
||||
defer nc.Close()
|
||||
js, err := jetstream.New(nc)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), Wait)
|
||||
defer cancel()
|
||||
stream, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "ANNOUNCETEST", Subjects: []string{"mesh.mod." + module + ".event.>"}})
|
||||
if err != nil {
|
||||
t.Fatalf("announcetest: making the events stream: %v", err)
|
||||
}
|
||||
stop := start(url)
|
||||
if stop != nil {
|
||||
defer stop()
|
||||
}
|
||||
got, err := waitFor(ctx, stream, module)
|
||||
if err != nil {
|
||||
t.Fatalf("announcetest: %v", err)
|
||||
}
|
||||
if problem := differs(got, module, want); problem != "" {
|
||||
t.Fatalf("announcetest: %s", problem)
|
||||
}
|
||||
return got
|
||||
}
|
||||
|
||||
// waitFor reads the stream until module's announcement is in it.
|
||||
func waitFor(ctx context.Context, stream jetstream.Stream, module string) (announce.States, error) {
|
||||
for {
|
||||
msg, err := stream.GetLastMsgForSubject(ctx, announce.Subject(module))
|
||||
if err == nil {
|
||||
return announce.Decode(msg.Data)
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return announce.States{}, fmt.Errorf("%s announced nothing on %s within %s", module, announce.Subject(module), Wait)
|
||||
case <-time.After(50 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// differs says how got is not module's announcement of want, or "".
|
||||
func differs(got announce.States, module string, want []announce.State) string {
|
||||
if got.Module != module {
|
||||
return fmt.Sprintf("the announcement names the module %q, not %q", got.Module, module)
|
||||
}
|
||||
w := append([]announce.State{}, want...)
|
||||
sort.Slice(w, func(i, j int) bool { return w[i].Name < w[j].Name })
|
||||
g := append([]announce.State{}, got.States...)
|
||||
if !reflect.DeepEqual(g, w) && !(len(g) == 0 && len(w) == 0) {
|
||||
return fmt.Sprintf("%s announced %v, and its code declares %v", module, g, w)
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
//go:build integration
|
||||
|
||||
package announcetest
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/announce"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
)
|
||||
|
||||
// 1790/S11: a module started against the test bus announces through process.Conn.Keeps, and Check reads that
|
||||
// announcement back against what the code declares.
|
||||
func TestCheckReadsTheModulesAnnouncementOffARealBus(t *testing.T) {
|
||||
declared := []announce.State{{Name: "accounts", Readable: true}, {Name: "drafts"}}
|
||||
got := Check(t, "mesh-mail", declared, func(url string) func() {
|
||||
conn, err := process.Connect(process.Account{URL: url, Module: "mesh-mail", Node: "novox", User: "u", Password: "p"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := conn.Keeps(declared...); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return conn.Close
|
||||
})
|
||||
if got.Node != "novox" {
|
||||
t.Fatalf("announced from %q", got.Node)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
package announcetest
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/announce"
|
||||
)
|
||||
|
||||
// The comparison is exact up to order: a state missing, one extra, a readability that differs or another module's
|
||||
// announcement each fail, naming what differs.
|
||||
func TestAnAnnouncementDiffersFromTheCodeByNameAndReadability(t *testing.T) {
|
||||
got := announce.States{Module: "m", States: []announce.State{{Name: "a", Readable: true}, {Name: "b"}}}
|
||||
if p := differs(got, "m", []announce.State{{Name: "b"}, {Name: "a", Readable: true}}); p != "" {
|
||||
t.Fatalf("an equal announcement differs: %s", p)
|
||||
}
|
||||
for _, want := range [][]announce.State{
|
||||
{{Name: "a", Readable: true}},
|
||||
{{Name: "a", Readable: true}, {Name: "b"}, {Name: "c"}},
|
||||
{{Name: "a"}, {Name: "b"}},
|
||||
} {
|
||||
if p := differs(got, "m", want); p == "" || !strings.Contains(p, "declares") {
|
||||
t.Errorf("want %v: %q", want, p)
|
||||
}
|
||||
}
|
||||
if p := differs(got, "other", nil); !strings.Contains(p, `"other"`) {
|
||||
t.Errorf("another module's announcement: %q", p)
|
||||
}
|
||||
if p := differs(announce.States{Module: "m"}, "m", nil); p != "" {
|
||||
t.Errorf("a module keeping nothing: %q", p)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,401 @@
|
||||
package buildstep
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"path"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// **One ask, one whole job** (novox/hq ADR 0525 §3). A build module runs a pulled ask with its own step tools' bodies,
|
||||
// the same ones its request and reply tools run (ADR 0525 §8), in the order the job names: a build ask fetches, then
|
||||
// builds or copies each artifact; a check ask fetches, digests the level, takes the kept verdicts and tests the rest,
|
||||
// the packages whose tests the change touches first. The answers are the tools' own, whole, so the asker judges them as
|
||||
// it judged the tools' answers.
|
||||
|
||||
// StepFunc is one step tool's body: its arguments in, its answer out.
|
||||
type StepFunc func(ctx context.Context, args map[string]any) (any, error)
|
||||
|
||||
// AskSteps are the step tools a build module runs an ask with. Copy is the image module's alone; Digest the Go module's.
|
||||
type AskSteps struct {
|
||||
Fetch, Build, Copy, Digest, Test StepFunc
|
||||
// Toolchain is the toolchain the module builds with now; a build ask naming another is refused. Nil checks nothing
|
||||
// (an image or an archive names none).
|
||||
Toolchain func(ctx context.Context) (delivery.ToolchainUsed, error)
|
||||
// Kept looks up a kept verdict by level and digest (the checker's kept bucket, delivery.KeptKey): whether one is
|
||||
// kept and how many tests it counted. Nil takes none.
|
||||
Kept func(ctx context.Context, level, digest string) (tests int, found bool, err error)
|
||||
// Release removes a workspace once the ask's steps on it ended (the module's release tool's body); nil leaves it to
|
||||
// the module's sweep.
|
||||
Release func(workspace string) error
|
||||
Logf func(format string, args ...any)
|
||||
}
|
||||
|
||||
// RunAsk runs ask's job with steps and answers a delivery.BuildAnswer or a delivery.CheckAnswer.
|
||||
func RunAsk(ctx context.Context, steps AskSteps, ask delivery.StepAsk) (any, error) {
|
||||
if ask.Build != nil {
|
||||
return steps.build(ctx, *ask.Build)
|
||||
}
|
||||
if ask.Check != nil {
|
||||
return steps.check(ctx, *ask.Check)
|
||||
}
|
||||
return nil, Refuse("the ask %s holds no job", ask.ID)
|
||||
}
|
||||
|
||||
func raw(v any) (json.RawMessage, error) {
|
||||
if r, ok := v.(json.RawMessage); ok {
|
||||
return r, nil
|
||||
}
|
||||
return json.Marshal(v)
|
||||
}
|
||||
|
||||
// fetched is what an ask reads of a fetch's answer.
|
||||
type fetched struct {
|
||||
Workspace string `json:"workspace"`
|
||||
Merge string `json:"merge"`
|
||||
Conflicts []string `json:"conflicts"`
|
||||
Changed []string `json:"changed"`
|
||||
Modules []string `json:"modules"`
|
||||
}
|
||||
|
||||
func (s AskSteps) fetch(ctx context.Context, args map[string]any) (json.RawMessage, fetched, error) {
|
||||
if s.Fetch == nil {
|
||||
return nil, fetched{}, Refuse("this build module fetches nothing")
|
||||
}
|
||||
out, err := s.Fetch(ctx, args)
|
||||
if err != nil {
|
||||
return nil, fetched{}, err
|
||||
}
|
||||
r, err := raw(out)
|
||||
if err != nil {
|
||||
return nil, fetched{}, err
|
||||
}
|
||||
var f fetched
|
||||
if err := json.Unmarshal(r, &f); err != nil || f.Workspace == "" {
|
||||
return r, f, fmt.Errorf("the fetch answered no workspace: %s", r)
|
||||
}
|
||||
return r, f, nil
|
||||
}
|
||||
|
||||
func (s AskSteps) build(ctx context.Context, job delivery.BuildJob) (any, error) {
|
||||
answer := delivery.BuildAnswer{}
|
||||
if job.Toolchain != nil && s.Toolchain != nil {
|
||||
tc, err := s.Toolchain(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if tc.Version != job.Toolchain.Version || tc.Recipe != job.Toolchain.Recipe {
|
||||
return nil, Refuse("the ask is for %s (recipe %s) and this node builds with %s (recipe %s)",
|
||||
job.Toolchain.Version, job.Toolchain.Recipe, tc.Version, tc.Recipe)
|
||||
}
|
||||
answer.Toolchain = &tc
|
||||
}
|
||||
var f fetched
|
||||
defer func() { s.release(f.Workspace) }()
|
||||
for _, a := range job.Artifacts {
|
||||
if a.From == "" && f.Workspace == "" {
|
||||
r, got, err := s.fetch(ctx, job.Fetch)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if _, candidate := job.Fetch["base"]; candidate {
|
||||
// A candidate is built from its head merged onto the trunk; a head that does not merge leaves the tree at the
|
||||
// trunk, and building that would store the trunk's build under the candidate's fingerprint.
|
||||
if len(got.Conflicts) > 0 {
|
||||
return nil, fmt.Errorf("the head does not merge into the trunk: files in conflict: %s", strings.Join(got.Conflicts, ", "))
|
||||
}
|
||||
if got.Merge == "" {
|
||||
return nil, fmt.Errorf("the fetch with a base answered no merge: %s", r)
|
||||
}
|
||||
}
|
||||
f = got
|
||||
}
|
||||
var (
|
||||
out any
|
||||
err error
|
||||
)
|
||||
if a.From != "" {
|
||||
if s.Copy == nil {
|
||||
return nil, Refuse("this build module copies no artifact (%s)", a.Artifact)
|
||||
}
|
||||
out, err = s.Copy(ctx, map[string]any{"module": job.Module, "artifact": a.Artifact, "from": a.From})
|
||||
} else {
|
||||
if s.Build == nil {
|
||||
return nil, Refuse("this build module builds no artifact (%s)", a.Artifact)
|
||||
}
|
||||
args := map[string]any{"workspace": f.Workspace, "module": job.Path, "artifact": a.Artifact}
|
||||
if job.Publish {
|
||||
args["publish"] = true
|
||||
}
|
||||
out, err = s.Build(ctx, args)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s: %w", a.Artifact, err)
|
||||
}
|
||||
r, err := raw(out)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var built struct {
|
||||
Digest string `json:"digest"`
|
||||
Reference string `json:"reference"`
|
||||
}
|
||||
if err := json.Unmarshal(r, &built); err != nil || built.Digest == "" {
|
||||
return nil, fmt.Errorf("the build of %s answered no digest: %s", a.Artifact, r)
|
||||
}
|
||||
answer.Artifacts = append(answer.Artifacts, delivery.AskBuilt{Artifact: a.Artifact, Digest: built.Digest, Reference: built.Reference})
|
||||
}
|
||||
return answer, nil
|
||||
}
|
||||
|
||||
func (s AskSteps) check(ctx context.Context, job delivery.CheckJob) (any, error) {
|
||||
r, f, err := s.fetch(ctx, job.Fetch)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer s.release(f.Workspace)
|
||||
answer := delivery.CheckAnswer{Fetch: r}
|
||||
if len(f.Conflicts) > 0 {
|
||||
return answer, nil // the asker reads the conflict from the fetch's answer: a verdict, not an error
|
||||
}
|
||||
if s.Test == nil {
|
||||
return nil, Refuse("this build module tests nothing")
|
||||
}
|
||||
runner := ""
|
||||
if job.Runner != "" && (job.RunnerWhen == "" || changesUnder(f.Changed, job.RunnerWhen)) {
|
||||
runner = job.Runner
|
||||
}
|
||||
for _, level := range job.Levels {
|
||||
la := delivery.LevelAnswer{Level: level}
|
||||
passed, err := s.level(ctx, job, level, f, runner, &la)
|
||||
answer.Levels = append(answer.Levels, la)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !passed {
|
||||
return answer, nil // the first level that did not pass ends the check (ADR 0484 clause 13)
|
||||
}
|
||||
}
|
||||
return answer, nil
|
||||
}
|
||||
|
||||
// level runs one level of a check ask and says whether it passed.
|
||||
func (s AskSteps) level(ctx context.Context, job delivery.CheckJob, level string, f fetched, runner string, la *delivery.LevelAnswer) (bool, error) {
|
||||
withRunner := func(args map[string]any) map[string]any {
|
||||
if runner != "" {
|
||||
args["runner"] = runner
|
||||
}
|
||||
return args
|
||||
}
|
||||
test := func(args map[string]any) (bool, error) {
|
||||
out, err := s.Test(ctx, args)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
r, err := raw(out)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
la.Tests = append(la.Tests, r)
|
||||
var t struct {
|
||||
Passed bool `json:"passed"`
|
||||
}
|
||||
_ = json.Unmarshal(r, &t)
|
||||
return t.Passed, nil
|
||||
}
|
||||
if s.Digest == nil {
|
||||
// A module with no digest (Node) tests each module directory of the tree, those whose tests changed first.
|
||||
for _, priority := range []bool{true, false} {
|
||||
for _, dir := range f.Modules {
|
||||
if changesTestIn(f.Changed, dir) != priority {
|
||||
continue
|
||||
}
|
||||
passed, err := test(withRunner(map[string]any{"workspace": f.Workspace, "module": dir, "level": level}))
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("the %s level of %s: %w", level, dir, err)
|
||||
}
|
||||
if !passed {
|
||||
return false, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
todo, all, keepRunner := s.plan(ctx, job, level, f, runner, withRunner, la)
|
||||
if !keepRunner {
|
||||
runner = "" // an installed module that ignores the runner judges the level as it judges any other (ADR 0535 §2)
|
||||
}
|
||||
for _, priority := range []bool{true, false} {
|
||||
args := withRunner(map[string]any{"workspace": f.Workspace, "level": level})
|
||||
if job.Run != "" {
|
||||
args["run"] = job.Run
|
||||
}
|
||||
if all {
|
||||
if priority {
|
||||
continue // nothing to put first: the level runs whole after
|
||||
}
|
||||
} else {
|
||||
var run []string
|
||||
for _, p := range todo {
|
||||
if changesTestOf(f.Changed, p) == priority {
|
||||
run = append(run, p)
|
||||
}
|
||||
}
|
||||
if len(run) == 0 {
|
||||
continue
|
||||
}
|
||||
args["packages"] = run
|
||||
}
|
||||
passed, err := test(args)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("the %s level: %w", level, err)
|
||||
}
|
||||
if !passed {
|
||||
return false, nil
|
||||
}
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// release removes an ask's workspace, saying a failure: the module's sweep removes it later.
|
||||
func (s AskSteps) release(workspace string) {
|
||||
if workspace == "" || s.Release == nil {
|
||||
return
|
||||
}
|
||||
if err := s.Release(workspace); err != nil && s.Logf != nil {
|
||||
s.Logf("the workspace %s is left to the sweep: %v", workspace, err)
|
||||
}
|
||||
}
|
||||
|
||||
// plan digests the level and takes the kept verdicts: the packages left to test, or all when the level runs whole, and
|
||||
// whether the runner stays. A try of some tests runs whole, digesting nothing (a partial run proves no package). Under the
|
||||
// candidate's runner the digest is asked as the candidate: when the candidate answers it, every package runs and nothing
|
||||
// is digested for keeping (ADR 0535 §4); when the installed module answers it (it does not take the runner), the level is
|
||||
// planned as any other and the runner is dropped. The level runs whole and takes nothing when the digest names no package
|
||||
// or does not cover the siblings the fetch placed (ADR 0444, L5).
|
||||
func (s AskSteps) plan(ctx context.Context, job delivery.CheckJob, level string, f fetched, runner string, withRunner func(map[string]any) map[string]any, la *delivery.LevelAnswer) ([]string, bool, bool) {
|
||||
if job.Run != "" {
|
||||
return nil, true, true
|
||||
}
|
||||
out, err := s.Digest(ctx, withRunner(map[string]any{"workspace": f.Workspace, "level": level}))
|
||||
if err != nil {
|
||||
if s.Logf != nil {
|
||||
s.Logf("no digest of the %s level, so it runs whole: %v", level, err)
|
||||
}
|
||||
return nil, true, true
|
||||
}
|
||||
r, err := raw(out)
|
||||
if err != nil {
|
||||
return nil, true, true
|
||||
}
|
||||
var d delivery.LevelDigest // packages kept as a sorted list, never keyed by a package path (issue 1930)
|
||||
if err := json.Unmarshal(r, &d); err != nil {
|
||||
return nil, true, true
|
||||
}
|
||||
if runner != "" {
|
||||
if strings.HasPrefix(d.JudgedBy, "candidate ") {
|
||||
la.Judged = d.JudgedBy
|
||||
return nil, true, true // the candidate decides which verdicts would be taken: all run, nothing kept
|
||||
}
|
||||
runner = ""
|
||||
}
|
||||
if len(d.Packages) == 0 || !slices.Equal(d.Beside, besideOf(job.Fetch)) {
|
||||
return nil, true, runner != "" // a digest that names nothing or not the siblings keys no verdict: the level runs whole
|
||||
}
|
||||
la.Digest = &d
|
||||
var todo []string
|
||||
for _, pd := range d.Packages {
|
||||
p := pd.Package
|
||||
if job.Kept && s.Kept != nil {
|
||||
tests, found, err := s.Kept(ctx, level, pd.Digest)
|
||||
if err != nil && s.Logf != nil {
|
||||
s.Logf("the kept verdict of %s at %s cannot be read, so it runs: %v", p, level, err)
|
||||
}
|
||||
if err == nil && found {
|
||||
la.Kept = append(la.Kept, delivery.KeptTaken{Package: p, Digest: pd.Digest, Tests: tests})
|
||||
continue
|
||||
}
|
||||
}
|
||||
todo = append(todo, p)
|
||||
}
|
||||
return todo, false, false
|
||||
}
|
||||
|
||||
// besideOf are the siblings a fetch's arguments place, as a digest echoes them: `<owner/repo> <commit>`, sorted.
|
||||
func besideOf(fetch map[string]any) []string {
|
||||
raw, err := json.Marshal(fetch["beside"])
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
var refs []struct {
|
||||
Repository string `json:"repository"`
|
||||
Commit string `json:"commit"`
|
||||
}
|
||||
_ = json.Unmarshal(raw, &refs)
|
||||
var out []string
|
||||
for _, r := range refs {
|
||||
out = append(out, r.Repository+" "+r.Commit)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// changesUnder says whether a path the merge changes is under prefix.
|
||||
func changesUnder(changed []string, prefix string) bool {
|
||||
for _, p := range changed {
|
||||
if strings.HasPrefix(p, prefix) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// isTestFile says whether a repository-relative path is a test file: Go's _test.go, and a TypeScript module's .test.
|
||||
// and .spec. files and test directories.
|
||||
func isTestFile(p string) bool {
|
||||
switch {
|
||||
case strings.HasSuffix(p, "_test.go"):
|
||||
return true
|
||||
case strings.Contains(p, ".test.") || strings.Contains(p, ".spec."):
|
||||
return true
|
||||
}
|
||||
for _, seg := range strings.Split(path.Dir(p), "/") {
|
||||
if seg == "__tests__" || seg == "test" || seg == "tests" {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// changesTestOf says whether the merge changes a test file of the Go package with the import path pkg: the file's
|
||||
// directory and the import path agree on their last segments.
|
||||
func changesTestOf(changed []string, pkg string) bool {
|
||||
for _, f := range changed {
|
||||
if !strings.HasSuffix(f, "_test.go") {
|
||||
continue
|
||||
}
|
||||
dir := path.Dir(f)
|
||||
if dir == pkg || strings.HasSuffix(pkg, "/"+dir) || strings.HasSuffix(dir, "/"+pkg) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// changesTestIn says whether the merge changes a test file inside the directory of a Node module.
|
||||
func changesTestIn(changed []string, dir string) bool {
|
||||
for _, f := range changed {
|
||||
if !isTestFile(f) || strings.HasSuffix(f, "_test.go") {
|
||||
continue
|
||||
}
|
||||
if dir == "." || dir == "" || strings.HasPrefix(f, strings.TrimSuffix(dir, "/")+"/") {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -0,0 +1,205 @@
|
||||
package buildstep
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
const runCommit = "0123456789abcdef0123456789abcdef01234567"
|
||||
|
||||
// recorder is a build module's step tools that record each call.
|
||||
type recorder struct {
|
||||
calls []string
|
||||
fetch map[string]any
|
||||
tests map[string]bool // package or module directory -> passed
|
||||
digests map[string]string
|
||||
}
|
||||
|
||||
func (r *recorder) steps() AskSteps {
|
||||
call := func(name string, answer func(args map[string]any) any) StepFunc {
|
||||
return func(ctx context.Context, args map[string]any) (any, error) {
|
||||
b, _ := json.Marshal(args)
|
||||
r.calls = append(r.calls, name+" "+string(b))
|
||||
return answer(args), nil
|
||||
}
|
||||
}
|
||||
return AskSteps{
|
||||
Fetch: call("fetch", func(map[string]any) any { return r.fetch }),
|
||||
Build: call("build", func(args map[string]any) any {
|
||||
return map[string]any{"digest": "sha256:" + args["artifact"].(string), "reference": "artifact-store://" + args["artifact"].(string)}
|
||||
}),
|
||||
Copy: call("copy", func(args map[string]any) any { return map[string]any{"digest": "sha256:copied"} }),
|
||||
Digest: call("digest", func(map[string]any) any { return map[string]any{"level": "unit", "packages": r.digests} }),
|
||||
Test: call("test", func(args map[string]any) any {
|
||||
passed := true
|
||||
if pkgs, ok := args["packages"].([]string); ok {
|
||||
for _, p := range pkgs {
|
||||
passed = passed && r.tests[p]
|
||||
}
|
||||
}
|
||||
if dir, ok := args["module"].(string); ok {
|
||||
passed = r.tests[dir]
|
||||
}
|
||||
return map[string]any{"passed": passed, "tests": 1}
|
||||
}),
|
||||
Kept: func(ctx context.Context, level, digest string) (int, bool, error) { return 7, digest == "d-kept", nil },
|
||||
}
|
||||
}
|
||||
|
||||
func ask(job any) delivery.StepAsk {
|
||||
a := delivery.StepAsk{ID: NewID(), Kind: delivery.AskCheck, Language: delivery.AskGo, Asker: "t", At: time.Now()}
|
||||
switch j := job.(type) {
|
||||
case delivery.BuildJob:
|
||||
a.Build = &j
|
||||
case delivery.CheckJob:
|
||||
a.Check = &j
|
||||
}
|
||||
return a
|
||||
}
|
||||
|
||||
// 1887/A3: a build ask fetches once, builds each artifact from the workspace with publish, copies a pinned one without a
|
||||
// fetch, and answers each artifact's digest and reference.
|
||||
func TestRunAskBuilds(t *testing.T) {
|
||||
r := &recorder{fetch: map[string]any{"workspace": "/w/1"}}
|
||||
steps := r.steps()
|
||||
var released []string
|
||||
steps.Release = func(ws string) error { released = append(released, ws); return nil }
|
||||
out, err := RunAsk(context.Background(), steps, ask(delivery.BuildJob{Module: "m", Path: "modules/m", Publish: true,
|
||||
Fetch: map[string]any{"repository": "novox/x", "commit": runCommit},
|
||||
Artifacts: []delivery.AskArtifact{{Artifact: "base", From: "registry/b@sha256:00"}, {Artifact: "bundle"}, {Artifact: "lib"}}}))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := strings.Join(r.calls, "\n")
|
||||
want := `copy {"artifact":"base","from":"registry/b@sha256:00","module":"m"}
|
||||
fetch {"commit":"` + runCommit + `","repository":"novox/x"}
|
||||
build {"artifact":"bundle","module":"modules/m","publish":true,"workspace":"/w/1"}
|
||||
build {"artifact":"lib","module":"modules/m","publish":true,"workspace":"/w/1"}`
|
||||
if got != want {
|
||||
t.Fatalf("calls:\n%s\nwant:\n%s", got, want)
|
||||
}
|
||||
if len(released) != 1 || released[0] != "/w/1" {
|
||||
t.Errorf("released %v, want the one workspace", released)
|
||||
}
|
||||
a := out.(delivery.BuildAnswer)
|
||||
if len(a.Artifacts) != 3 || a.Artifacts[1].Reference != "artifact-store://bundle" {
|
||||
t.Fatalf("answer %+v", a)
|
||||
}
|
||||
}
|
||||
|
||||
// A build ask for another toolchain is refused, so its step ends refused and the asker reads why.
|
||||
func TestRunAskRefusesAnotherToolchain(t *testing.T) {
|
||||
r := &recorder{fetch: map[string]any{"workspace": "/w"}}
|
||||
s := r.steps()
|
||||
s.Toolchain = func(context.Context) (delivery.ToolchainUsed, error) {
|
||||
return delivery.ToolchainUsed{Version: "go1.27", Recipe: "3"}, nil
|
||||
}
|
||||
_, err := RunAsk(context.Background(), s, ask(delivery.BuildJob{Module: "m", Path: ".", Fetch: map[string]any{"repository": "novox/x", "commit": runCommit},
|
||||
Artifacts: []delivery.AskArtifact{{Artifact: "bundle"}}, Toolchain: &delivery.ToolchainUsed{Version: "go1.26", Recipe: "3"}}))
|
||||
var ref *Refusal
|
||||
if !errors.As(err, &ref) || len(r.calls) != 0 {
|
||||
t.Fatalf("err %v, calls %v", err, r.calls)
|
||||
}
|
||||
}
|
||||
|
||||
// 1887/A3, A5a, issue 1887 design limit 1: a check ask runs the levels of one check after one fetch: it digests the unit
|
||||
// level, takes the kept verdicts, tests the packages whose tests changed first, and stops at the first failure, running
|
||||
// no later level.
|
||||
func TestRunAskChecksTheLevelsOfOneCheck(t *testing.T) {
|
||||
r := &recorder{
|
||||
fetch: map[string]any{"workspace": "/w", "changed": []string{"pkg/b/b_test.go"}},
|
||||
digests: map[string]string{"x/pkg/a": "d-a", "x/pkg/b": "d-b", "x/pkg/k": "d-kept"},
|
||||
tests: map[string]bool{"x/pkg/a": true, "x/pkg/b": false},
|
||||
}
|
||||
out, err := RunAsk(context.Background(), r.steps(), ask(delivery.CheckJob{Levels: []string{delivery.LevelUnit, delivery.LevelIntegration}, Kept: true,
|
||||
Runner: "workspace", RunnerWhen: "modules/mesh-build-go/", Fetch: map[string]any{"repository": "novox/x", "commit": runCommit, "base": runCommit}}))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := strings.Join(r.calls, "\n")
|
||||
want := `fetch {"base":"` + runCommit + `","commit":"` + runCommit + `","repository":"novox/x"}
|
||||
digest {"level":"unit","workspace":"/w"}
|
||||
test {"level":"unit","packages":["x/pkg/b"],"workspace":"/w"}`
|
||||
if got != want {
|
||||
t.Fatalf("calls:\n%s\nwant:\n%s", got, want)
|
||||
}
|
||||
a := out.(delivery.CheckAnswer)
|
||||
if len(a.Levels) != 1 {
|
||||
t.Fatalf("levels %+v, want the unit level alone", a.Levels)
|
||||
}
|
||||
u := a.Levels[0]
|
||||
if u.Level != delivery.LevelUnit || len(u.Kept) != 1 || u.Kept[0].Package != "x/pkg/k" || u.Kept[0].Tests != 7 || len(u.Tests) != 1 || u.Digest == nil {
|
||||
t.Fatalf("the unit level %+v", u)
|
||||
}
|
||||
}
|
||||
|
||||
// ADR 0535 §2, §4: a merge that changes the build module's own code is judged by the candidate's runner: its digest says
|
||||
// the candidate ran, and every package runs with nothing kept; an installed module that ignores the runner is planned as
|
||||
// any other, the runner dropped. A digest that does not cover the fetch's siblings runs the level whole too.
|
||||
func TestRunAskRunsWholeUnderTheRunnerOrWithoutTheSiblings(t *testing.T) {
|
||||
for _, tc := range []struct{ judged, want string }{
|
||||
{"candidate 999", `digest {"level":"unit","runner":"workspace","workspace":"/w"}
|
||||
test {"level":"unit","runner":"workspace","workspace":"/w"}`},
|
||||
{"installed", `digest {"level":"unit","runner":"workspace","workspace":"/w"}`},
|
||||
} {
|
||||
r := &recorder{fetch: map[string]any{"workspace": "/w", "changed": []string{"modules/mesh-build-go/x.go"}},
|
||||
digests: map[string]string{"x/pkg/a": "d-kept"}, tests: map[string]bool{}}
|
||||
steps := r.steps()
|
||||
digest := steps.Digest
|
||||
steps.Digest = func(ctx context.Context, args map[string]any) (any, error) {
|
||||
_, _ = digest(ctx, args)
|
||||
return map[string]any{"level": "unit", "packages": r.digests, "judged_by": tc.judged}, nil
|
||||
}
|
||||
out, err := RunAsk(context.Background(), steps, ask(delivery.CheckJob{Levels: []string{delivery.LevelUnit}, Kept: true,
|
||||
Runner: "workspace", RunnerWhen: "modules/mesh-build-go/", Fetch: map[string]any{"repository": "novox/x", "commit": runCommit}}))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := strings.Join(r.calls[1:], "\n"); got != tc.want {
|
||||
t.Errorf("judged by %s: calls\n%s\nwant\n%s", tc.judged, got, tc.want)
|
||||
}
|
||||
l := out.(delivery.CheckAnswer).Levels[0]
|
||||
if tc.judged == "installed" && (len(l.Kept) != 1 || l.Judged != "") {
|
||||
t.Errorf("an installed module's level %+v, want the kept verdict taken", l)
|
||||
}
|
||||
if tc.judged != "installed" && (len(l.Kept) != 0 || l.Digest != nil || l.Judged != tc.judged) {
|
||||
t.Errorf("the candidate's level %+v, want nothing kept and the program named", l)
|
||||
}
|
||||
}
|
||||
r := &recorder{fetch: map[string]any{"workspace": "/w"}, digests: map[string]string{"x/pkg/a": "d-kept"}, tests: map[string]bool{}}
|
||||
out, err := RunAsk(context.Background(), r.steps(), ask(delivery.CheckJob{Levels: []string{delivery.LevelUnit}, Kept: true,
|
||||
Fetch: map[string]any{"repository": "novox/x", "commit": runCommit, "beside": []any{map[string]any{"repository": "novox/mesh-sdk", "commit": runCommit}}}}))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if l := out.(delivery.CheckAnswer).Levels[0]; len(l.Kept) != 0 || l.Digest != nil {
|
||||
t.Fatalf("a digest without the siblings kept %+v", l)
|
||||
}
|
||||
}
|
||||
|
||||
// A Node check tests each module directory, the one whose tests changed first; a conflict ends the ask with the fetch
|
||||
// alone, a verdict for the asker.
|
||||
func TestRunAskChecksNodeModulesAndConflicts(t *testing.T) {
|
||||
r := &recorder{fetch: map[string]any{"workspace": "/w", "modules": []string{"a", "b"}, "changed": []string{"b/test/x.test.ts"}},
|
||||
tests: map[string]bool{"a": true, "b": true}}
|
||||
s := r.steps()
|
||||
s.Digest = nil
|
||||
if _, err := RunAsk(context.Background(), s, ask(delivery.CheckJob{Levels: []string{delivery.LevelUnit}, Fetch: map[string]any{"repository": "novox/x", "commit": runCommit}})); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if fmt.Sprint(r.calls[1:]) != `[test {"level":"unit","module":"b","workspace":"/w"} test {"level":"unit","module":"a","workspace":"/w"}]` {
|
||||
t.Fatalf("calls %v", r.calls)
|
||||
}
|
||||
r = &recorder{fetch: map[string]any{"workspace": "/w", "conflicts": []string{"go.mod"}}}
|
||||
out, err := RunAsk(context.Background(), r.steps(), ask(delivery.CheckJob{Levels: []string{delivery.LevelUnit}, Fetch: map[string]any{"repository": "novox/x", "commit": runCommit}}))
|
||||
if err != nil || len(r.calls) != 1 || len(out.(delivery.CheckAnswer).Levels) != 0 {
|
||||
t.Fatalf("a conflict: %v %v", err, r.calls)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,170 @@
|
||||
package buildstep
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
)
|
||||
|
||||
// RegistryWait is how long a push waits for an artifact store that refuses connections, longer than its nightly
|
||||
// collection holds it still (about a minute and a half, issue 457); RegistryFirstPause is the first pause, doubling
|
||||
// to RegistryMostPause. Variables so a test shortens them.
|
||||
var (
|
||||
RegistryWait = 5 * time.Minute
|
||||
RegistryFirstPause = time.Second
|
||||
RegistryMostPause = 10 * time.Second
|
||||
)
|
||||
|
||||
// BlobReference is where a pushed bundle or archive is, as a done key names it and the controller delivers from
|
||||
// (novox/hq ADR 0498 clause 1): artifact-store://<module>/<artifact>/blobs/<digest>, the store's address left out
|
||||
// (issue 102).
|
||||
func BlobReference(module, artifact, digest string) string {
|
||||
return "artifact-store://" + module + "/" + artifact + "/blobs/" + digest
|
||||
}
|
||||
|
||||
// PushBlob puts body into the artifact store at registry (host:port) as a blob of repository (<module>/<artifact>) by
|
||||
// its digest (`sha256:` and hex), from the build module's own process, after the build step ended (novox/hq ADR 0498
|
||||
// clauses 1 and 2): a blob the store already holds is done without an upload; otherwise it is uploaded (POST
|
||||
// blobs/uploads/, then PUT with the digest). A store refusing connections is waited out up to RegistryWait; any other
|
||||
// refusal fails, naming the store's status and its own words, for the caller's `not stored:` error. Bytes that are not
|
||||
// their digest are refused before anything is sent.
|
||||
func PushBlob(ctx context.Context, client *http.Client, registry, repository, digest string, body []byte) error {
|
||||
sum := sha256.Sum256(body)
|
||||
if got := "sha256:" + hex.EncodeToString(sum[:]); got != digest {
|
||||
return fmt.Errorf("the blob is %s, not the digest %s it was to be pushed as", got, digest)
|
||||
}
|
||||
if client == nil {
|
||||
client = http.DefaultClient
|
||||
}
|
||||
base := "http://" + registry + "/v2/" + repository + "/blobs/"
|
||||
var present bool
|
||||
if err := waitForStore(ctx, registry, func() error {
|
||||
var err error
|
||||
present, err = blobPresent(ctx, client, base+digest)
|
||||
return err
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
if present {
|
||||
return nil
|
||||
}
|
||||
var location string
|
||||
if err := waitForStore(ctx, registry, func() error {
|
||||
var err error
|
||||
location, err = startUpload(ctx, client, base+"uploads/")
|
||||
return err
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
target, err := url.Parse(base)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
loc, err := target.Parse(location)
|
||||
if err != nil {
|
||||
return fmt.Errorf("the store answered the upload's location %q, which is not a URL: %w", location, err)
|
||||
}
|
||||
q := loc.Query()
|
||||
q.Set("digest", digest)
|
||||
loc.RawQuery = q.Encode()
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPut, loc.String(), bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/octet-stream")
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return fmt.Errorf("putting %s of %s: %w", digest, repository, err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusCreated {
|
||||
return storeSaid(resp, "putting "+digest+" of "+repository)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func blobPresent(ctx context.Context, client *http.Client, at string) (bool, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodHead, at, nil)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
resp.Body.Close()
|
||||
switch resp.StatusCode {
|
||||
case http.StatusOK:
|
||||
return true, nil
|
||||
case http.StatusNotFound:
|
||||
return false, nil
|
||||
}
|
||||
return false, fmt.Errorf("asking the store for %s: it answered %s", at, resp.Status)
|
||||
}
|
||||
|
||||
func startUpload(ctx context.Context, client *http.Client, at string) (string, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, at, nil)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusAccepted {
|
||||
return "", storeSaid(resp, "starting an upload at "+at)
|
||||
}
|
||||
location := resp.Header.Get("Location")
|
||||
if location == "" {
|
||||
return "", fmt.Errorf("starting an upload at %s: the store named no location", at)
|
||||
}
|
||||
return location, nil
|
||||
}
|
||||
|
||||
// storeSaid is the store's refusal: its status and the first of its own words.
|
||||
func storeSaid(resp *http.Response, what string) error {
|
||||
words, _ := io.ReadAll(io.LimitReader(resp.Body, 2048))
|
||||
return fmt.Errorf("%s: the store answered %s: %s", what, resp.Status, strings.TrimSpace(string(words)))
|
||||
}
|
||||
|
||||
// refused is whether an error is a connection refused.
|
||||
func refused(err error) bool {
|
||||
return err != nil && (errors.Is(err, syscall.ECONNREFUSED) || strings.Contains(err.Error(), "connection refused"))
|
||||
}
|
||||
|
||||
// waitForStore runs try, and while it fails because the store refuses connections, tries again with a growing pause
|
||||
// until RegistryWait has passed. A refusal is the one error waited for: nothing was sent, so trying again does nothing
|
||||
// twice.
|
||||
func waitForStore(ctx context.Context, registry string, try func() error) error {
|
||||
err := try()
|
||||
if !refused(err) {
|
||||
return err
|
||||
}
|
||||
started, pause := time.Now(), RegistryFirstPause
|
||||
for {
|
||||
left := RegistryWait - time.Since(started)
|
||||
if left <= 0 {
|
||||
return fmt.Errorf("the artifact store at %s refused every connection for %s, longer than its nightly collection holds it still, so it is down, not paused: %w",
|
||||
registry, time.Since(started).Round(time.Millisecond), err)
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return fmt.Errorf("stopped while waiting for the artifact store at %s: %w (last: %v)", registry, ctx.Err(), err)
|
||||
case <-time.After(min(pause, left)):
|
||||
}
|
||||
pause = min(pause*2, RegistryMostPause)
|
||||
if err = try(); !refused(err) {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,151 @@
|
||||
package buildstep
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
func digestOfBytes(b []byte) string {
|
||||
h := sha256.Sum256(b)
|
||||
return "sha256:" + hex.EncodeToString(h[:])
|
||||
}
|
||||
|
||||
// registry is an artifact store as far as a blob push needs one: HEAD a blob, start an upload, PUT it with its digest.
|
||||
type registry struct {
|
||||
mu sync.Mutex
|
||||
blobs map[string][]byte
|
||||
calls []string
|
||||
put int // the status a PUT answers, 201 when 0
|
||||
}
|
||||
|
||||
func (r *registry) ServeHTTP(w http.ResponseWriter, req *http.Request) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
r.calls = append(r.calls, req.Method+" "+req.URL.Path)
|
||||
switch {
|
||||
case req.Method == http.MethodHead && strings.Contains(req.URL.Path, "/blobs/sha256:"):
|
||||
if _, ok := r.blobs[req.URL.Path[strings.LastIndex(req.URL.Path, "/")+1:]]; ok {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
case req.Method == http.MethodPost && strings.HasSuffix(req.URL.Path, "/blobs/uploads/"):
|
||||
w.Header().Set("Location", strings.TrimSuffix(req.URL.Path, "/")+"/u1?state=x")
|
||||
w.WriteHeader(http.StatusAccepted)
|
||||
case req.Method == http.MethodPut && strings.Contains(req.URL.Path, "/blobs/uploads/u1"):
|
||||
if r.put != 0 {
|
||||
w.WriteHeader(r.put)
|
||||
_, _ = io.WriteString(w, `{"errors":[{"code":"DIGEST_INVALID","message":"provided digest did not match uploaded content"}]}`)
|
||||
return
|
||||
}
|
||||
body, _ := io.ReadAll(req.Body)
|
||||
d := req.URL.Query().Get("digest")
|
||||
if req.URL.Query().Get("state") != "x" || d != digestOfBytes(body) {
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
r.blobs[d] = body
|
||||
w.WriteHeader(http.StatusCreated)
|
||||
default:
|
||||
w.WriteHeader(http.StatusMethodNotAllowed)
|
||||
}
|
||||
}
|
||||
|
||||
// A done build's bundle or archive is pushed to the artifact store by its build module (novox/hq ADR 0498 clauses 1
|
||||
// and 2): a blob the store already holds is done with no upload; one it lacks is uploaded by digest; its reference is
|
||||
// the artifact-store form the builder's key and the controller check.
|
||||
func TestABlobIsPushedOnceByItsDigest(t *testing.T) {
|
||||
reg := ®istry{blobs: map[string][]byte{}}
|
||||
srv := httptest.NewServer(reg)
|
||||
defer srv.Close()
|
||||
addr := strings.TrimPrefix(srv.URL, "http://")
|
||||
body := []byte("the packed bundle")
|
||||
d := digestOfBytes(body)
|
||||
if err := PushBlob(context.Background(), srv.Client(), addr, "mail/code", d, body); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if string(reg.blobs[d]) != string(body) {
|
||||
t.Fatalf("the store holds %q", reg.blobs[d])
|
||||
}
|
||||
reg.calls = nil
|
||||
if err := PushBlob(context.Background(), srv.Client(), addr, "mail/code", d, body); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(reg.calls) != 1 || !strings.HasPrefix(reg.calls[0], "HEAD ") {
|
||||
t.Fatalf("a blob the store holds was uploaded again: %v", reg.calls)
|
||||
}
|
||||
ref := BlobReference("mail", "code", d)
|
||||
if ref != "artifact-store://mail/code/blobs/"+d {
|
||||
t.Fatalf("BlobReference = %q", ref)
|
||||
}
|
||||
b := delivery.BuilderBuild{State: delivery.BuildDone, Manifest: strings.Repeat("4", 64),
|
||||
Artifacts: []delivery.BuiltArtifact{{Name: "code", Kind: "bundle", Digest: d, Reference: ref}}}
|
||||
if _, err := delivery.EncodeBuilderBuild(b); err != nil {
|
||||
t.Fatalf("the key refuses the reference BlobReference gives: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// A store that refuses the upload fails the push naming the store's own words; bytes that are not the digest are
|
||||
// refused before anything is sent.
|
||||
func TestAPushTheStoreRefusesSaysTheStoresWords(t *testing.T) {
|
||||
reg := ®istry{blobs: map[string][]byte{}, put: http.StatusBadRequest}
|
||||
srv := httptest.NewServer(reg)
|
||||
defer srv.Close()
|
||||
addr := strings.TrimPrefix(srv.URL, "http://")
|
||||
body := []byte("x")
|
||||
err := PushBlob(context.Background(), srv.Client(), addr, "mail/code", digestOfBytes(body), body)
|
||||
if err == nil || !strings.Contains(err.Error(), "provided digest did not match") || !strings.Contains(err.Error(), "400") {
|
||||
t.Fatalf("a refused upload said %v", err)
|
||||
}
|
||||
reg.calls = nil
|
||||
err = PushBlob(context.Background(), srv.Client(), addr, "mail/code", digestOfBytes([]byte("other")), body)
|
||||
if err == nil || len(reg.calls) != 0 {
|
||||
t.Fatalf("bytes that are not their digest: %v, calls %v", err, reg.calls)
|
||||
}
|
||||
}
|
||||
|
||||
// A store refusing connections (its nightly collection holds it still) is waited out, up to RegistryWait (issue 457).
|
||||
func TestAPushWaitsOutAStoreRefusingConnections(t *testing.T) {
|
||||
l, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
addr := l.Addr().String()
|
||||
l.Close() // refusing now
|
||||
reg := ®istry{blobs: map[string][]byte{}}
|
||||
go func() {
|
||||
time.Sleep(300 * time.Millisecond)
|
||||
l2, err := net.Listen("tcp", addr)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
_ = http.Serve(l2, reg)
|
||||
}()
|
||||
beforeWait, beforePause := RegistryWait, RegistryFirstPause
|
||||
RegistryWait, RegistryFirstPause = 3*time.Second, 100*time.Millisecond
|
||||
defer func() { RegistryWait, RegistryFirstPause = beforeWait, beforePause }()
|
||||
body := []byte("late")
|
||||
if err := PushBlob(context.Background(), http.DefaultClient, addr, "mail/code", digestOfBytes(body), body); err != nil {
|
||||
t.Fatalf("a store that came back was not waited for: %v", err)
|
||||
}
|
||||
|
||||
l3, _ := net.Listen("tcp", "127.0.0.1:0")
|
||||
gone := l3.Addr().String()
|
||||
l3.Close()
|
||||
RegistryWait = 300 * time.Millisecond
|
||||
err = PushBlob(context.Background(), http.DefaultClient, gone, "mail/code", digestOfBytes(body), body)
|
||||
if err == nil || !strings.Contains(err.Error(), "refused every connection") {
|
||||
t.Fatalf("a store that stayed down said %v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
package buildstep
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||
)
|
||||
|
||||
// CapacitySetting is the build modules' setting of each node's build capacity (novox/hq ADR 0525 §5): `<node>=<asks at
|
||||
// once>` pairs separated by commas, as the builder's was (ADR 0485 §1). Each build module declares it with
|
||||
// DefaultCapacities as its default, and the controller lays it into the module's settings file.
|
||||
const CapacitySetting = "capacity"
|
||||
|
||||
// DefaultCapacities is the setting's default, the builder's before R4g: the control-node (novox) and the laptop (g14)
|
||||
// build nothing.
|
||||
const DefaultCapacities = "shanks=3,ace=2,g14=0,novox=0"
|
||||
|
||||
// NodeCapacity is node's build capacity, read from the module's settings file at path at every call, so a change takes
|
||||
// effect at the next pull; with no path it reads fallback. A node the setting does not name runs one ask at a time (as
|
||||
// the builder's did). A file or a setting that cannot be read gives 0 and says why: pulling on a setting nobody can read
|
||||
// could build on a node that must not.
|
||||
func NodeCapacity(path, fallback, node string, logf func(string, ...any)) func() int {
|
||||
say := func(format string, args ...any) {
|
||||
if logf != nil {
|
||||
logf(format, args...)
|
||||
}
|
||||
}
|
||||
return func() int {
|
||||
setting := fallback
|
||||
if path != "" {
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
say("the build capacity cannot be read, so %s pulls nothing: %v", node, err)
|
||||
return 0
|
||||
}
|
||||
var s map[string]any
|
||||
if err := json.Unmarshal(raw, &s); err != nil {
|
||||
say("the settings file %s is not JSON, so %s pulls nothing: %v", path, node, err)
|
||||
return 0
|
||||
}
|
||||
setting, _ = s[CapacitySetting].(string)
|
||||
}
|
||||
n, err := capacityOf(setting, node)
|
||||
if err != nil {
|
||||
say("%v, so %s pulls nothing", err, node)
|
||||
return 0
|
||||
}
|
||||
return n
|
||||
}
|
||||
}
|
||||
|
||||
// capacityOf is node's capacity in a setting's `<node>=<asks>` pairs: DefaultCapacity for a node it does not name.
|
||||
func capacityOf(setting, node string) (int, error) {
|
||||
caps, err := buildrun.ParseCapacity(setting)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("the build capacity: %w", err)
|
||||
}
|
||||
return (&buildrun.Places{Capacity: caps}).CapacityOf(node), nil
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
package buildstep
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// 1887/A7: the node's build capacity is the build module's setting, read at every pull: the control-node at 0 pulls
|
||||
// nothing, a change takes effect at the next read, and a setting that cannot be read pulls nothing.
|
||||
func TestNodeCapacity(t *testing.T) {
|
||||
if got := NodeCapacity("", DefaultCapacities, "novox", nil)(); got != 0 {
|
||||
t.Errorf("novox = %d by default, want 0", got)
|
||||
}
|
||||
if got := NodeCapacity("", DefaultCapacities, "shanks", nil)(); got != 3 {
|
||||
t.Errorf("shanks = %d by default, want 3", got)
|
||||
}
|
||||
file := filepath.Join(t.TempDir(), "settings.json")
|
||||
os.WriteFile(file, []byte(`{"capacity":"shanks=1,ace=2"}`), 0o600)
|
||||
capacity := NodeCapacity(file, DefaultCapacities, "shanks", nil)
|
||||
if got := capacity(); got != 1 {
|
||||
t.Errorf("shanks = %d from the file, want 1", got)
|
||||
}
|
||||
os.WriteFile(file, []byte(`{"capacity":"shanks=4"}`), 0o600)
|
||||
if got := capacity(); got != 4 {
|
||||
t.Errorf("shanks = %d after the change, want 4", got)
|
||||
}
|
||||
if got := NodeCapacity(file, "", "pi", nil)(); got != 1 {
|
||||
t.Errorf("a node not named = %d, want 1", got)
|
||||
}
|
||||
os.WriteFile(file, []byte(`{"capacity":"shanks=lots"}`), 0o600)
|
||||
if got := capacity(); got != 0 {
|
||||
t.Errorf("an unreadable setting = %d, want 0", got)
|
||||
}
|
||||
if got := NodeCapacity(filepath.Join(t.TempDir(), "none"), DefaultCapacities, "shanks", nil)(); got != 0 {
|
||||
t.Errorf("a missing file = %d, want 0", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,234 @@
|
||||
package buildstep
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// credentialField is the node runtime's rule for a state value (mesh-tools node-tools/internal/bus/state.go, novox/hq
|
||||
// ADR 0201), copied whole: the first field anywhere in the value whose name, lower-cased and without - _ and spaces, ends
|
||||
// in a credential word, or "". A step's end is refused when it answers a field.
|
||||
func credentialField(value json.RawMessage) string {
|
||||
endings := []string{"password", "passwd", "secret", "token", "credential", "credentials",
|
||||
"authorization", "apikey", "privatekey", "accesskey", "cookie"}
|
||||
var v any
|
||||
if json.Unmarshal(value, &v) != nil {
|
||||
return ""
|
||||
}
|
||||
var walk func(any) string
|
||||
walk = func(v any) string {
|
||||
switch t := v.(type) {
|
||||
case map[string]any:
|
||||
keys := make([]string, 0, len(t))
|
||||
for k := range t {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
for _, k := range keys {
|
||||
norm := strings.NewReplacer("-", "", "_", "", " ", "").Replace(strings.ToLower(k))
|
||||
for _, end := range endings {
|
||||
if strings.HasSuffix(norm, end) {
|
||||
return k
|
||||
}
|
||||
}
|
||||
if found := walk(t[k]); found != "" {
|
||||
return found
|
||||
}
|
||||
}
|
||||
case []any:
|
||||
for _, x := range t {
|
||||
if found := walk(x); found != "" {
|
||||
return found
|
||||
}
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
return walk(v)
|
||||
}
|
||||
|
||||
// runtimeState is a steps state that refuses a value as the node runtime does, and keeps every value it took.
|
||||
type runtimeState struct {
|
||||
mu sync.Mutex
|
||||
took map[string][]delivery.Step
|
||||
refused int
|
||||
}
|
||||
|
||||
func newRuntimeState() *runtimeState { return &runtimeState{took: map[string][]delivery.Step{}} }
|
||||
|
||||
func (s *runtimeState) put(key string, value any) (uint64, error) {
|
||||
raw, _ := value.(json.RawMessage)
|
||||
if f := credentialField(raw); f != "" {
|
||||
s.mu.Lock()
|
||||
s.refused++
|
||||
s.mu.Unlock()
|
||||
return 0, fmt.Errorf("steps.%s carries a field %q, which names a credential", key, f)
|
||||
}
|
||||
st, err := delivery.DecodeStep(raw)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.took[key] = append(s.took[key], st)
|
||||
return uint64(len(s.took[key])), nil
|
||||
}
|
||||
|
||||
func (s *runtimeState) last(key string) (delivery.Step, bool) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
t := s.took[key]
|
||||
if len(t) == 0 {
|
||||
return delivery.Step{}, false
|
||||
}
|
||||
return t[len(t)-1], true
|
||||
}
|
||||
|
||||
func fastEnds(t *testing.T) {
|
||||
attempts, retry := EndAttempts, EndRetry
|
||||
EndAttempts, EndRetry = 2, time.Millisecond
|
||||
t.Cleanup(func() { EndAttempts, EndRetry = attempts, retry })
|
||||
}
|
||||
|
||||
// 1930/AC1: given a check of a repository with the package .../internal/token, when its ask's answer is stored as the
|
||||
// step's end, then no field is keyed by a package path: the runtime's credential rule takes the value, and the package
|
||||
// and its digest read back from the stored end.
|
||||
func TestADigestOfAPackageNamedTokenIsStoredAsAList(t *testing.T) {
|
||||
fastEnds(t)
|
||||
r := &recorder{
|
||||
fetch: map[string]any{"workspace": "/w"},
|
||||
digests: map[string]string{"x/internal/token": "d-token", "x/pkg/a": "d-a"},
|
||||
tests: map[string]bool{"x/internal/token": true, "x/pkg/a": true},
|
||||
}
|
||||
out, err := RunAsk(context.Background(), r.steps(), ask(delivery.CheckJob{Levels: []string{delivery.LevelUnit},
|
||||
Fetch: map[string]any{"repository": "novox/x", "commit": runCommit}}))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
old, _ := json.Marshal(map[string]any{"levels": []any{map[string]any{"level": "unit", "digest": map[string]any{"packages": r.digests}}}})
|
||||
if credentialField(old) != "x/internal/token" {
|
||||
t.Fatalf("the rule does not refuse the form before issue 1930 (%s): the test proves nothing", old)
|
||||
}
|
||||
st := newRuntimeState()
|
||||
if err := finish(st.put, "shanks.a1", delivery.Step{ID: "a1", Tool: "check ask", Node: "shanks", State: delivery.StepRunning, Started: time.Now()}, out, nil, t.Logf); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
s, ok := st.last("shanks.a1")
|
||||
if !ok || s.State != delivery.StepPassed || st.refused != 0 {
|
||||
t.Fatalf("the end %+v (written %v, refused %d), want it passed and taken at once", s, ok, st.refused)
|
||||
}
|
||||
var a delivery.CheckAnswer
|
||||
if err := json.Unmarshal(s.Result, &a); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(a.Levels) != 1 || a.Levels[0].Digest == nil {
|
||||
t.Fatalf("levels %+v, want the unit level with its digest", a.Levels)
|
||||
}
|
||||
want := []delivery.PackageDigest{{Package: "x/internal/token", Digest: "d-token"}, {Package: "x/pkg/a", Digest: "d-a"}}
|
||||
if got := a.Levels[0].Digest.Packages; fmt.Sprint(got) != fmt.Sprint(want) {
|
||||
t.Errorf("packages %v, want %v", got, want)
|
||||
}
|
||||
if !strings.Contains(string(s.Result), `"package":"x/internal/token"`) {
|
||||
t.Errorf("the stored end does not name the package as a value: %s", s.Result)
|
||||
}
|
||||
}
|
||||
|
||||
// 1930/AC2: given an end the state refuses every time, when finish writes it, then it logs the refusal and writes a small
|
||||
// failed end without the result that names the write's error; and when no end can be written at all it answers an error.
|
||||
func TestARefusedEndIsSaidAndEndsTheStepFailed(t *testing.T) {
|
||||
fastEnds(t)
|
||||
big := errors.New("the value carries a field \"x/internal/token\", which names a credential")
|
||||
var refused int
|
||||
var stored []delivery.Step
|
||||
put := func(key string, value any) (uint64, error) {
|
||||
st, _ := delivery.DecodeStep(value.(json.RawMessage))
|
||||
if len(st.Result) > 0 {
|
||||
refused++
|
||||
return 0, big
|
||||
}
|
||||
stored = append(stored, st)
|
||||
return 1, nil
|
||||
}
|
||||
var logged []string
|
||||
logf := func(format string, args ...any) { logged = append(logged, fmt.Sprintf(format, args...)) }
|
||||
err := finish(put, "shanks.a2", delivery.Step{ID: "a2", Tool: "check ask", Node: "shanks", State: delivery.StepRunning, Started: time.Now()}, map[string]any{"levels": 1}, nil, logf)
|
||||
if err != nil {
|
||||
t.Fatalf("finish answered %v, want the small end written", err)
|
||||
}
|
||||
if refused != EndAttempts || len(stored) != 1 {
|
||||
t.Fatalf("refused %d, stored %d, want %d refusals and one small end", refused, len(stored), EndAttempts)
|
||||
}
|
||||
s := stored[0]
|
||||
if s.State != delivery.StepFailed || len(s.Result) != 0 || !strings.Contains(s.Reason, "could not be written") ||
|
||||
!strings.Contains(s.Reason, big.Error()) || !strings.Contains(s.Reason, "passed") {
|
||||
t.Errorf("the small end %+v, want failed, no result, naming the write's error and that it passed", s)
|
||||
}
|
||||
if len(logged) == 0 || !strings.Contains(logged[0], big.Error()) {
|
||||
t.Errorf("logged %q, want the refusal said", logged)
|
||||
}
|
||||
none := func(string, any) (uint64, error) { return 0, errors.New("the bus is gone") }
|
||||
if err := finish(none, "shanks.a3", delivery.Step{ID: "a3", Tool: "check ask", Node: "shanks", Started: time.Now()}, nil, nil, logf); err == nil || !strings.Contains(err.Error(), "the bus is gone") {
|
||||
t.Errorf("finish with no write possible answered %v, want the error", err)
|
||||
}
|
||||
}
|
||||
|
||||
// 1930/AC2: given an ask whose end the state refuses, when it runs, then the ask is acknowledged only once its small
|
||||
// failed end is written; when no end can be written at all, it is given back, never acknowledged as done.
|
||||
func TestAnAskWhoseEndCannotBeWrittenIsNotAcked(t *testing.T) {
|
||||
fastEnds(t)
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
smallOK bool
|
||||
wantAcked bool
|
||||
}{{"the small end written", true, true}, {"no end written", false, false}} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
src := NewFakeAsks()
|
||||
a := testAsk(delivery.AskCheck)
|
||||
_ = src.PublishAsk(a)
|
||||
var mu sync.Mutex
|
||||
var ends []delivery.Step
|
||||
put := func(key string, value any) (uint64, error) {
|
||||
st, err := delivery.DecodeStep(value.(json.RawMessage))
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if st.State != delivery.StepRunning && (len(st.Result) > 0 || !tc.smallOK) {
|
||||
return 0, errors.New("refused")
|
||||
}
|
||||
mu.Lock()
|
||||
ends = append(ends, st)
|
||||
mu.Unlock()
|
||||
return 1, nil
|
||||
}
|
||||
p := &Puller{Source: src, Language: delivery.AskGo, Node: "shanks", Put: put, Capacity: func() int { return 1 },
|
||||
PullWait: 10 * time.Millisecond, Logf: t.Logf,
|
||||
Run: func(ctx context.Context, a delivery.StepAsk) (any, error) { return map[string]any{"ran": true}, nil }}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
go p.Pull(ctx)
|
||||
waitFor(t, "the ask's fate", func() bool { ac, tm, nk := src.Fates(); return len(ac)+len(tm)+len(nk) > 0 })
|
||||
cancel()
|
||||
acked, termed, naked := src.Fates()
|
||||
if tc.wantAcked && (len(acked) != 1 || len(naked) != 0) || !tc.wantAcked && (len(acked) != 0 || len(naked) == 0) || len(termed) != 0 {
|
||||
t.Fatalf("acked %v naked %v termed %v", acked, naked, termed)
|
||||
}
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if tc.smallOK {
|
||||
last := ends[len(ends)-1]
|
||||
if last.State != delivery.StepFailed || !strings.Contains(last.Reason, "refused") {
|
||||
t.Errorf("the end %+v, want the small failed end", last)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
+88
-1
@@ -4,6 +4,8 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
@@ -13,12 +15,37 @@ import (
|
||||
// Option changes FetchCommit.
|
||||
type Option func(*fetchOptions)
|
||||
|
||||
type fetchOptions struct{ allowFile bool }
|
||||
type fetchOptions struct {
|
||||
allowFile bool
|
||||
mirror string
|
||||
}
|
||||
|
||||
// AllowFileRemote lets FetchCommit take a file:// remote. It is for a test's local repository; production
|
||||
// code does not pass it, and FetchCommit then refuses anything but ssh://.
|
||||
func AllowFileRemote() Option { return func(o *fetchOptions) { o.allowFile = true } }
|
||||
|
||||
// WithMirror makes FetchCommit (and MergeInto) borrow objects from mirror, a bare repository that UpdateMirror keeps
|
||||
// in the build module's own cache (novox/hq ADR 0536 §2; issue 890): the workspace's .git/objects/info/alternates
|
||||
// names the mirror's objects directory, so git transfers only the objects the mirror lacks. It is a plain file, never
|
||||
// a symlink. mirror must be an absolute path of an existing directory; the workspace never writes to it.
|
||||
func WithMirror(mirror string) Option { return func(o *fetchOptions) { o.mirror = mirror } }
|
||||
|
||||
// useMirror writes the alternates file of the repository in dir, when a mirror is set.
|
||||
func useMirror(o fetchOptions, dir string) error {
|
||||
if o.mirror == "" {
|
||||
return nil
|
||||
}
|
||||
info := filepath.Join(dir, ".git", "objects", "info")
|
||||
if err := os.MkdirAll(info, 0o755); err != nil {
|
||||
return fmt.Errorf("buildstep: the mirror cannot be used: %w", err)
|
||||
}
|
||||
line := filepath.Join(o.mirror, "objects") + "\n"
|
||||
if err := os.WriteFile(filepath.Join(info, "alternates"), []byte(line), 0o644); err != nil {
|
||||
return fmt.Errorf("buildstep: the mirror cannot be used: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
var (
|
||||
commitOK = regexp.MustCompile(`^[0-9a-f]{40}$`)
|
||||
refOK = regexp.MustCompile(`^refs/[A-Za-z0-9._/-]+$`)
|
||||
@@ -51,6 +78,9 @@ func FetchCommit(ctx context.Context, run forgekey.Runner, dir, remote, commit,
|
||||
if _, err := git("init", "-q"); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := useMirror(o, dir); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := git("remote", "add", "origin", remote); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -78,10 +108,67 @@ func FetchCommit(ctx context.Context, run forgekey.Runner, dir, remote, commit,
|
||||
return nil
|
||||
}
|
||||
|
||||
// UpdateMirror brings the bare repository mirror to commit: it makes the mirror when it is absent (git init --bare,
|
||||
// origin set to remote), then fetches the commit, or the ref when only the ref holds it, with the same rules as
|
||||
// FetchCommit (no prompt, no credential helper, ssh:// or an allowed file:// remote, a 40-hex commit). The mirror
|
||||
// keeps every commit it was brought to, so later fetches through WithMirror transfer only what is new. Callers
|
||||
// serialise updates of one mirror.
|
||||
func UpdateMirror(ctx context.Context, run forgekey.Runner, mirror, remote, commit, ref string, opts ...Option) error {
|
||||
var o fetchOptions
|
||||
for _, opt := range opts {
|
||||
opt(&o)
|
||||
}
|
||||
if !filepath.IsAbs(mirror) {
|
||||
return fmt.Errorf("buildstep: the mirror %q is not an absolute path", mirror)
|
||||
}
|
||||
if err := checkFetch(o, mirror, remote, commit, ref); err != nil {
|
||||
return err
|
||||
}
|
||||
env := []string{"GIT_TERMINAL_PROMPT=0"}
|
||||
git := func(args ...string) error {
|
||||
r := run(ctx, env, "git", append([]string{"-c", "credential.helper="}, args...)...)
|
||||
if r.Status != 0 || r.Err != "" {
|
||||
return fmt.Errorf("git %s: %s", args[0], firstLine(r.Err+" "+r.Stderr))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
in := func(args ...string) error { return git(append([]string{"-C", mirror}, args...)...) }
|
||||
if _, err := os.Stat(filepath.Join(mirror, "HEAD")); err != nil {
|
||||
if err := git("init", "-q", "--bare", mirror); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := in("remote", "add", "origin", remote); err != nil {
|
||||
return err
|
||||
}
|
||||
} else if err := in("remote", "set-url", "origin", remote); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := in("fetch", "--no-tags", "origin", commit); err != nil {
|
||||
if ref == "" {
|
||||
return err
|
||||
}
|
||||
if rerr := in("fetch", "--no-tags", "origin", ref); rerr != nil {
|
||||
return fmt.Errorf("%v; then the ref %s: %w", err, ref, rerr)
|
||||
}
|
||||
}
|
||||
if err := in("cat-file", "-e", commit+"^{commit}"); err != nil {
|
||||
return fmt.Errorf("the commit %s is not in the mirror: %w", commit, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func checkFetch(o fetchOptions, dir, remote, commit, ref string) error {
|
||||
if dir == "" {
|
||||
return fmt.Errorf("buildstep: no workspace directory")
|
||||
}
|
||||
if o.mirror != "" {
|
||||
if !filepath.IsAbs(o.mirror) {
|
||||
return fmt.Errorf("buildstep: the mirror %q is not an absolute path", o.mirror)
|
||||
}
|
||||
if fi, err := os.Stat(o.mirror); err != nil || !fi.IsDir() {
|
||||
return fmt.Errorf("buildstep: the mirror %q is not an existing directory", o.mirror)
|
||||
}
|
||||
}
|
||||
if !commitOK.MatchString(commit) {
|
||||
return fmt.Errorf("buildstep: %q is not a full commit (40 hex digits)", commit)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,175 @@
|
||||
package buildstep
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/forgekey"
|
||||
)
|
||||
|
||||
// Bubblewrap is the program a run step runs in (novox/hq ADR 0448): a build module declares its package, which needs
|
||||
// unprivileged user namespaces (issue 1308).
|
||||
const Bubblewrap = "bwrap"
|
||||
|
||||
// Isolation is how a run step's command is shut off (novox/hq ADR 0448; issue 1309, the lead's decision L2 of R4e: one
|
||||
// runner in the SDK for every build module). The fetch step holds the forge key and the network and runs none of the
|
||||
// change's code; the run step runs the change's build and tests in bubblewrap with:
|
||||
//
|
||||
// - no network: its own network namespace, with only a loopback, so a test may start a bus or a store of its own but
|
||||
// resolves no name and reaches no socket of the machine;
|
||||
// - none of the account: not its home, not the forge key, not a bus login, not the runtime directory (/run), since
|
||||
// none of them is bound; HOME is a directory in the step's own /tmp unless Env names one;
|
||||
// - an environment of exactly Env and the runner's env: nothing of the build module's own passes;
|
||||
// - the system's programs and libraries read-only, and of /etc only what a program needs to run (users, libraries,
|
||||
// certificates, time zone), with a hosts file naming only the loopback;
|
||||
// - the Workspace and Writable directories read-write, the ReadOnly ones (the toolchain, the modules the fetch step
|
||||
// downloaded) read-only;
|
||||
// - the shared warm Caches read-only under a throwaway writable layer (S18), or read-write for a trunk build alone
|
||||
// (WriteCaches), so a candidate's code never changes what a later build reads.
|
||||
type Isolation struct {
|
||||
Workspace string
|
||||
Writable []string
|
||||
ReadOnly []string
|
||||
Caches []string
|
||||
WriteCaches bool
|
||||
Env []string
|
||||
// Hosts is the file bound as /etc/hosts; Isolated writes one naming only the loopback.
|
||||
Hosts string
|
||||
}
|
||||
|
||||
// etcNeeded is what of /etc a program in the step may read: who the users are, where the libraries are, the
|
||||
// certificates, the time zone. Not resolv.conf, not hosts: the step resolves no name.
|
||||
var etcNeeded = []string{"/etc/passwd", "/etc/group", "/etc/nsswitch.conf", "/etc/ld.so.cache", "/etc/ld.so.conf",
|
||||
"/etc/ld.so.conf.d", "/etc/ssl", "/etc/ca-certificates", "/etc/pki", "/etc/localtime", "/etc/os-release", "/etc/alternatives"}
|
||||
|
||||
// systemArgs binds the system's programs and libraries read-only: /usr, and /bin, /sbin, /lib, /lib64 as the symlinks
|
||||
// they are on a merged-usr system, or bound when they are directories. A variable so a unit test fixes it.
|
||||
var systemArgs = func() []string {
|
||||
args := []string{"--ro-bind", "/usr", "/usr"}
|
||||
for _, d := range []string{"/bin", "/sbin", "/lib", "/lib64"} {
|
||||
info, err := os.Lstat(d)
|
||||
switch {
|
||||
case err != nil:
|
||||
case info.Mode()&os.ModeSymlink != 0:
|
||||
if target, err := os.Readlink(d); err == nil {
|
||||
args = append(args, "--symlink", target, d)
|
||||
}
|
||||
case info.IsDir():
|
||||
args = append(args, "--ro-bind", d, d)
|
||||
}
|
||||
}
|
||||
for _, e := range etcNeeded {
|
||||
args = append(args, "--ro-bind-try", e, e)
|
||||
}
|
||||
return args
|
||||
}
|
||||
|
||||
// stepHome is the HOME of a step that is given none: inside its own /tmp, gone with it.
|
||||
const stepHome = "/tmp/home"
|
||||
|
||||
func checkDir(what, dir string) error {
|
||||
if dir == "" || !strings.HasPrefix(dir, "/") || path.Clean(dir) != dir || dir == "/" {
|
||||
return fmt.Errorf("buildstep: the %s %q is not an absolute, clean directory other than the root", what, dir)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Command is the bubblewrap command line that runs name with args shut off as iso says, env added to iso.Env. It
|
||||
// refuses a directory that is not absolute and clean, or is the root, and an environment word without "=".
|
||||
func (iso Isolation) Command(name string, args ...string) ([]string, error) {
|
||||
return iso.command(nil, name, args...)
|
||||
}
|
||||
|
||||
func (iso Isolation) command(env []string, name string, args ...string) ([]string, error) {
|
||||
if err := checkDir("workspace", iso.Workspace); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, group := range []struct {
|
||||
what string
|
||||
dirs []string
|
||||
}{{"writable directory", iso.Writable}, {"read-only directory", iso.ReadOnly}, {"cache", iso.Caches}} {
|
||||
for _, d := range group.dirs {
|
||||
if err := checkDir(group.what, d); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
}
|
||||
words := append(append([]string{}, iso.Env...), env...)
|
||||
home := false
|
||||
for _, w := range words {
|
||||
k, _, ok := strings.Cut(w, "=")
|
||||
if !ok || k == "" {
|
||||
return nil, fmt.Errorf("buildstep: the environment word %q is not NAME=value", w)
|
||||
}
|
||||
home = home || k == "HOME"
|
||||
}
|
||||
argv := []string{"--unshare-all", "--die-with-parent", "--new-session", "--clearenv",
|
||||
"--proc", "/proc", "--dev", "/dev", "--tmpfs", "/tmp", "--dir", stepHome}
|
||||
argv = append(argv, systemArgs()...)
|
||||
for _, d := range iso.ReadOnly {
|
||||
argv = append(argv, "--ro-bind", d, d)
|
||||
}
|
||||
argv = append(argv, cacheArgs(iso.Caches, iso.WriteCaches)...)
|
||||
argv = append(argv, "--bind", iso.Workspace, iso.Workspace)
|
||||
for _, d := range iso.Writable {
|
||||
argv = append(argv, "--bind", d, d)
|
||||
}
|
||||
if iso.Hosts != "" {
|
||||
argv = append(argv, "--ro-bind", iso.Hosts, "/etc/hosts")
|
||||
}
|
||||
for _, w := range words {
|
||||
k, v, _ := strings.Cut(w, "=")
|
||||
argv = append(argv, "--setenv", k, v)
|
||||
}
|
||||
if !home {
|
||||
argv = append(argv, "--setenv", "HOME", stepHome)
|
||||
}
|
||||
argv = append(argv, "--chdir", iso.Workspace, "--", name)
|
||||
return append(argv, args...), nil
|
||||
}
|
||||
|
||||
// cacheArgs binds the shared warm caches (S18): for a trunk build read-write, so the next build is warm; for any other
|
||||
// read-only beneath an overlay whose writes go to a tmpfs of the step's own, so the step writes as a build does and
|
||||
// nothing it wrote outlives it (bubblewrap's --tmp-overlay, an overlay mount in the step's user namespace).
|
||||
func cacheArgs(caches []string, write bool) []string {
|
||||
var argv []string
|
||||
for _, c := range caches {
|
||||
if write {
|
||||
argv = append(argv, "--bind", c, c)
|
||||
continue
|
||||
}
|
||||
argv = append(argv, "--overlay-src", c, "--tmp-overlay", c)
|
||||
}
|
||||
return argv
|
||||
}
|
||||
|
||||
// loopbackHosts is the hosts file of a step: the loopback and nothing else.
|
||||
const loopbackHosts = "127.0.0.1 localhost\n::1 localhost\n"
|
||||
|
||||
// Isolated is run with every command shut off as iso says (Command), run's env going inside the step: the build modules'
|
||||
// run steps call their test, build and digest commands through it, never run itself. Each command gets a hosts file of
|
||||
// its own, naming only the loopback, removed when it ends.
|
||||
func Isolated(run forgekey.Runner, iso Isolation) forgekey.Runner {
|
||||
return func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran {
|
||||
dir, err := os.MkdirTemp("", "step-hosts-")
|
||||
if err != nil {
|
||||
return forgekey.Ran{Status: 1, Err: "the step's hosts file: " + err.Error()}
|
||||
}
|
||||
defer os.RemoveAll(dir)
|
||||
hosts := filepath.Join(dir, "hosts")
|
||||
if err := os.WriteFile(hosts, []byte(loopbackHosts), 0o644); err != nil {
|
||||
return forgekey.Ran{Status: 1, Err: "the step's hosts file: " + err.Error()}
|
||||
}
|
||||
iso := iso
|
||||
iso.Hosts = hosts
|
||||
argv, err := iso.command(env, name, args...)
|
||||
if err != nil {
|
||||
return forgekey.Ran{Status: 1, Err: err.Error()}
|
||||
}
|
||||
return run(ctx, nil, Bubblewrap, argv...)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
package buildstep
|
||||
|
||||
import "testing"
|
||||
|
||||
// The shared warm caches are read under a throwaway writable layer (novox/hq ADR 0448; issue 1309): a candidate's step
|
||||
// writes into them as a build does, and nothing it writes outlives it or reaches a later build; only a trunk build
|
||||
// writes them.
|
||||
func TestACandidatesStepWritesTheCachesOnlyIntoAThrowawayLayer(t *testing.T) {
|
||||
fixedSystem(t)
|
||||
iso := Isolation{Workspace: "/w", Caches: []string{"/var/cache/go-build", "/var/cache/npm"}}
|
||||
argv, err := iso.Command("go", "test")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, c := range iso.Caches {
|
||||
if !holds(argv, "--overlay-src", c, "--tmp-overlay", c) {
|
||||
t.Errorf("the cache %s is not under a throwaway layer: %v", c, argv)
|
||||
}
|
||||
if holds(argv, "--bind", c, c) {
|
||||
t.Errorf("a candidate's step writes the cache %s: %v", c, argv)
|
||||
}
|
||||
}
|
||||
iso.WriteCaches = true
|
||||
argv, _ = iso.Command("go", "build")
|
||||
for _, c := range iso.Caches {
|
||||
if !holds(argv, "--bind", c, c) || holds(argv, "--tmp-overlay", c) {
|
||||
t.Errorf("a trunk build does not write the cache %s: %v", c, argv)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
//go:build integration && isolation
|
||||
|
||||
package buildstep
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/forgekey"
|
||||
)
|
||||
|
||||
// 1309/S19, the issue's proof: a run step, in bubblewrap as Isolated runs it, reads no key and no login, resolves no
|
||||
// name, reaches no TCP or unix socket of the machine, finds no home of the account, and leaves the shared cache it
|
||||
// wrote unchanged; the workspace it writes keeps what it wrote. It needs bubblewrap and unprivileged user namespaces
|
||||
// (issue 1308) on the machine that runs it, and fails naming them when they are not there.
|
||||
func TestARunStepReachesNothingAndLeavesTheCachesUnchanged(t *testing.T) {
|
||||
if _, err := exec.LookPath(Bubblewrap); err != nil {
|
||||
t.Fatalf("bubblewrap is not installed here, so the isolation of a run step cannot be proved (issue 1309): %v", err)
|
||||
}
|
||||
root := t.TempDir()
|
||||
account := filepath.Join(root, "account")
|
||||
key, login := filepath.Join(account, ".ssh", "id_ed25519"), filepath.Join(account, "bus-login.json")
|
||||
writeFile(t, filepath.Dir(key), filepath.Base(key), "PRIVATE KEY")
|
||||
writeFile(t, account, "bus-login.json", `{"password":"secret"}`)
|
||||
cacheDir := filepath.Join(root, "cache")
|
||||
writeFile(t, cacheDir, "warm", "warm\n")
|
||||
workspace := filepath.Join(root, "work", "tree")
|
||||
if err := os.MkdirAll(workspace, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
probe := filepath.Join(workspace, "probe")
|
||||
if out, err := exec.Command("go", "build", "-o", probe, "./testdata/isolated-probe").CombinedOutput(); err != nil {
|
||||
t.Fatalf("building the probe: %v\n%s", err, out)
|
||||
}
|
||||
tcp, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer tcp.Close()
|
||||
socket := filepath.Join(account, "bus.sock")
|
||||
unix, err := net.Listen("unix", socket)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer unix.Close()
|
||||
|
||||
run := Isolated(forgekey.ExecRunnerFor(0), Isolation{Workspace: workspace, Caches: []string{cacheDir}})
|
||||
r := run(context.Background(), []string{"PATH=/usr/bin:/bin"}, probe, key, login, tcp.Addr().String(), socket, "localhost.localdomain.invalid", filepath.Join(cacheDir, "warm"))
|
||||
if r.Status != 0 || r.Err != "" || strings.TrimSpace(r.Stdout) != "shut off" {
|
||||
t.Fatalf("the run step was not shut off (status %d, %s):\n%s%s", r.Status, r.Err, r.Stdout, r.Stderr)
|
||||
}
|
||||
if b, _ := os.ReadFile(filepath.Join(cacheDir, "warm")); string(b) != "warm\n" {
|
||||
t.Fatalf("the step changed the shared cache: %q", b)
|
||||
}
|
||||
|
||||
// The workspace is the step's own: what it writes there stays.
|
||||
r = run(context.Background(), nil, "/bin/sh", "-c", "echo built > out")
|
||||
if b, err := os.ReadFile(filepath.Join(workspace, "out")); r.Status != 0 || err != nil || string(b) != "built\n" {
|
||||
t.Fatalf("the step's write to its workspace did not stay (status %d, %s%s): %q %v", r.Status, r.Stderr, r.Err, b, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,115 @@
|
||||
package buildstep
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/forgekey"
|
||||
)
|
||||
|
||||
// pairs says whether argv holds the words of want one after another.
|
||||
func holds(argv []string, want ...string) bool {
|
||||
for i := 0; i+len(want) <= len(argv); i++ {
|
||||
match := true
|
||||
for j, w := range want {
|
||||
if argv[i+j] != w {
|
||||
match = false
|
||||
break
|
||||
}
|
||||
}
|
||||
if match {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func fixedSystem(t *testing.T) {
|
||||
t.Helper()
|
||||
before := systemArgs
|
||||
systemArgs = func() []string { return []string{"--ro-bind", "/usr", "/usr", "--symlink", "usr/bin", "/bin"} }
|
||||
t.Cleanup(func() { systemArgs = before })
|
||||
}
|
||||
|
||||
// A run step's command runs in bubblewrap (novox/hq ADR 0448; issue 1309): no network, no namespace shared, an
|
||||
// environment of exactly what the step is given, the workspace the one tree it writes, the toolchain read-only, and
|
||||
// nothing else of the machine's: not the account's home, not the forge key or a bus login, not the runtime directory,
|
||||
// and no name service.
|
||||
func TestARunStepRunsShutOffInBubblewrap(t *testing.T) {
|
||||
fixedSystem(t)
|
||||
iso := Isolation{Workspace: "/work/ab12/tree", ReadOnly: []string{"/usr/lib/go"}, Writable: []string{"/work/ab12/out"},
|
||||
Env: []string{"GOFLAGS=-mod=mod", "PATH=/usr/bin"}, Hosts: "/work/ab12/hosts"}
|
||||
argv, err := iso.Command("go", "test", "./...")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, want := range [][]string{
|
||||
{"--unshare-all"}, {"--die-with-parent"}, {"--new-session"}, {"--clearenv"},
|
||||
{"--proc", "/proc"}, {"--dev", "/dev"}, {"--tmpfs", "/tmp"},
|
||||
{"--ro-bind", "/usr", "/usr"}, {"--symlink", "usr/bin", "/bin"},
|
||||
{"--ro-bind", "/usr/lib/go", "/usr/lib/go"},
|
||||
{"--bind", "/work/ab12/tree", "/work/ab12/tree"}, {"--bind", "/work/ab12/out", "/work/ab12/out"},
|
||||
{"--ro-bind", "/work/ab12/hosts", "/etc/hosts"},
|
||||
{"--setenv", "GOFLAGS", "-mod=mod"}, {"--setenv", "PATH", "/usr/bin"}, {"--setenv", "HOME", "/tmp/home"},
|
||||
{"--chdir", "/work/ab12/tree"},
|
||||
{"--", "go", "test", "./..."},
|
||||
} {
|
||||
if !holds(argv, want...) {
|
||||
t.Errorf("the command has no %v: %v", want, argv)
|
||||
}
|
||||
}
|
||||
if holds(argv, "--share-net") || strings.Contains(strings.Join(argv, " "), "resolv.conf") || strings.Contains(strings.Join(argv, " "), "/run") {
|
||||
t.Errorf("the command reaches the network, the name service or the runtime directory: %v", argv)
|
||||
}
|
||||
// The workspace is mounted after /tmp's tmpfs, so a workspace under /tmp is not hidden by it.
|
||||
if strings.Index(strings.Join(argv, "\x00"), "--tmpfs\x00/tmp") > strings.Index(strings.Join(argv, "\x00"), "--bind\x00/work/ab12/tree") {
|
||||
t.Errorf("the tmpfs of /tmp comes after the workspace: %v", argv)
|
||||
}
|
||||
// A HOME the step is given is its own.
|
||||
iso.Env = append(iso.Env, "HOME=/work/ab12/home")
|
||||
argv, _ = iso.Command("true")
|
||||
if holds(argv, "--setenv", "HOME", "/tmp/home") || !holds(argv, "--setenv", "HOME", "/work/ab12/home") {
|
||||
t.Errorf("the step's own HOME was replaced: %v", argv)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnIsolationNamesNoPathItCannotPin(t *testing.T) {
|
||||
for _, iso := range []Isolation{
|
||||
{Workspace: ""},
|
||||
{Workspace: "tree"},
|
||||
{Workspace: "/"},
|
||||
{Workspace: "/w", ReadOnly: []string{"relative"}},
|
||||
{Workspace: "/w", Writable: []string{"/"}},
|
||||
{Workspace: "/w", Caches: []string{"/a/../b"}},
|
||||
{Workspace: "/w", Env: []string{"NOEQUALS"}},
|
||||
} {
|
||||
if _, err := iso.Command("true"); err == nil {
|
||||
t.Errorf("%+v was accepted", iso)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Isolated runs every command through bwrap, with the step's environment inside and a hosts file that names only the
|
||||
// loopback, removed after.
|
||||
func TestIsolatedRunsEachCommandThroughBubblewrap(t *testing.T) {
|
||||
fixedSystem(t)
|
||||
var ran []string
|
||||
var hosts string
|
||||
inner := func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran {
|
||||
ran = append([]string{name}, args...)
|
||||
for i, a := range args {
|
||||
if a == "/etc/hosts" && i > 0 {
|
||||
hosts = args[i-1]
|
||||
}
|
||||
}
|
||||
return forgekey.Ran{Stdout: "ok"}
|
||||
}
|
||||
r := Isolated(inner, Isolation{Workspace: "/w"})(context.Background(), []string{"A=1"}, "go", "build")
|
||||
if r.Stdout != "ok" || ran[0] != Bubblewrap || !holds(ran, "--setenv", "A", "1") || !holds(ran, "--", "go", "build") {
|
||||
t.Fatalf("ran %v: %+v", ran, r)
|
||||
}
|
||||
if hosts == "" {
|
||||
t.Fatalf("no hosts file was bound: %v", ran)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
package buildstep
|
||||
|
||||
import "strings"
|
||||
|
||||
// LauncherSuffix is what a TypeScript entrypoint's launcher is called beside it: index.js is started as
|
||||
// index.serve.mjs (novox/hq ADR 0193), an ES module by its own extension whatever the bundle's package.json says.
|
||||
const LauncherSuffix = ".serve.mjs"
|
||||
|
||||
// Launchers are the launchers a bundle's build writes, by entrypoint, relative to the bundle's root: for TypeScript
|
||||
// every entrypoint `x.js` is started by `x.serve.mjs` beside it; an entrypoint that is not JavaScript and every other
|
||||
// language, whose build is already executable, get none. The one rule mesh-build-node, build-agent and the controller
|
||||
// share (R4e), so the name a build writes is the name a reader expects.
|
||||
func Launchers(language string, entrypoints []string) map[string]string {
|
||||
out := map[string]string{}
|
||||
if language != "typescript" {
|
||||
return out
|
||||
}
|
||||
for _, entry := range entrypoints {
|
||||
if strings.HasSuffix(entry, ".js") {
|
||||
out[entry] = strings.TrimSuffix(entry, ".js") + LauncherSuffix
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
package buildstep
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// A TypeScript bundle's entrypoint x.js is started as x.serve.mjs beside it (novox/hq ADR 0193), the rule mesh-build-node
|
||||
// and build-agent both apply; an entrypoint that is not .js gets none, and another language gets none.
|
||||
func TestATypeScriptEntrypointIsStartedByTheLauncherBesideIt(t *testing.T) {
|
||||
got := Launchers("typescript", []string{"index.js", "tools/x.js", "data.json"})
|
||||
want := map[string]string{"index.js": "index.serve.mjs", "tools/x.js": "tools/x.serve.mjs"}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("Launchers(typescript) = %v, want %v", got, want)
|
||||
}
|
||||
if got := Launchers("go", []string{"index.js"}); len(got) != 0 {
|
||||
t.Fatalf("a Go bundle got launchers: %v", got)
|
||||
}
|
||||
if got := Launchers("typescript", nil); len(got) != 0 {
|
||||
t.Fatalf("a bundle with no entrypoint got launchers: %v", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,115 @@
|
||||
package buildstep
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/forgekey"
|
||||
)
|
||||
|
||||
// MergeResult is what MergeInto made of a head and a trunk commit: the trunk commit (Base) and the head, whole, and the
|
||||
// merge commit and the paths the merge changes against the trunk commit; or, when the head does not merge cleanly, the
|
||||
// files in conflict, with Merge and Changed empty.
|
||||
type MergeResult struct {
|
||||
Base, Head, Merge string
|
||||
Conflicts []string
|
||||
Changed []string
|
||||
}
|
||||
|
||||
// MergeInto makes, in the workspace dir that FetchCommit left at commit, the merge of commit into the trunk commit
|
||||
// base, as the forge's default merge would (a merge commit, never a fast-forward), and leaves the workspace at the merge
|
||||
// (novox/hq ADR 0484 clause 3, ADR 0323). It is the one merge the build modules' fetch steps make, with the identity
|
||||
// and flags of the controller's check merge (mesh-controller internal/builder/check_merge.go): a name nobody owns, no
|
||||
// signature, no hook, nothing of the machine's git configuration that changes a merge. Nothing is pushed.
|
||||
//
|
||||
// A merge needs both histories, so it makes the fetch whole (the shallow clone FetchCommit made is deepened) and
|
||||
// fetches base by its sha. A head that does not merge cleanly is an answer, not an error: the files in conflict, the
|
||||
// merge undone and the workspace left at base. A head already in base answers a Merge equal to Base and no changed
|
||||
// paths. An error is that the merge could not be tried: a base or commit that is not a full sha, no base, a workspace
|
||||
// not at commit, or a fetch or git command that failed. remote and the options are those of FetchCommit; the runner's
|
||||
// bound must allow a whole fetch.
|
||||
func MergeInto(ctx context.Context, run forgekey.Runner, dir, remote, base, commit string, opts ...Option) (MergeResult, error) {
|
||||
var o fetchOptions
|
||||
for _, opt := range opts {
|
||||
opt(&o)
|
||||
}
|
||||
res := MergeResult{Base: base, Head: commit}
|
||||
if base == "" {
|
||||
return res, fmt.Errorf("buildstep: no base: a merge needs the trunk commit it is made onto")
|
||||
}
|
||||
if err := checkFetch(o, dir, remote, base, ""); err != nil {
|
||||
return res, err
|
||||
}
|
||||
if err := checkFetch(o, dir, remote, commit, ""); err != nil {
|
||||
return res, err
|
||||
}
|
||||
if err := useMirror(o, dir); err != nil {
|
||||
return res, err
|
||||
}
|
||||
env := []string{"GIT_TERMINAL_PROMPT=0"}
|
||||
git := func(args ...string) (string, error) {
|
||||
full := append([]string{"-c", "credential.helper=", "-C", dir}, args...)
|
||||
r := run(ctx, env, "git", full...)
|
||||
if r.Status != 0 || r.Err != "" {
|
||||
return "", fmt.Errorf("git %s: %s", args[0], firstLine(r.Err+" "+r.Stderr))
|
||||
}
|
||||
return r.Stdout, nil
|
||||
}
|
||||
at, err := git("rev-parse", "HEAD")
|
||||
if err != nil {
|
||||
return res, err
|
||||
}
|
||||
if head := strings.TrimSpace(at); head != commit {
|
||||
return res, fmt.Errorf("buildstep: the workspace is at %s, not at the commit %s", head, commit)
|
||||
}
|
||||
shallow, err := git("rev-parse", "--is-shallow-repository")
|
||||
if err != nil {
|
||||
return res, err
|
||||
}
|
||||
fetch := []string{"fetch", "--no-tags", "origin", base}
|
||||
if strings.TrimSpace(shallow) == "true" {
|
||||
fetch = []string{"fetch", "--no-tags", "--unshallow", "origin", base}
|
||||
}
|
||||
if _, err := git(fetch...); err != nil {
|
||||
return res, fmt.Errorf("the trunk commit %s: %w", base, err)
|
||||
}
|
||||
if _, err := git("checkout", "-q", "--detach", base); err != nil {
|
||||
return res, fmt.Errorf("the trunk commit %s cannot be checked out: %w", base, err)
|
||||
}
|
||||
message := fmt.Sprintf("Check the merge of %s into %s (made for a check, never pushed)", short(commit), short(base))
|
||||
args := append(append([]string{}, mergeIdentity...), "merge", "--no-ff", "--no-edit", "--no-verify", "--quiet", "-m", message, commit)
|
||||
if _, err := git(args...); err != nil {
|
||||
out, _ := git("diff", "--name-only", "--diff-filter=U")
|
||||
res.Conflicts = strings.Fields(out)
|
||||
_, _ = git("merge", "--abort")
|
||||
if len(res.Conflicts) == 0 {
|
||||
return res, fmt.Errorf("the head %s could not be merged into the trunk commit %s: %w", short(commit), short(base), err)
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
merge, err := git("rev-parse", "HEAD")
|
||||
if err != nil {
|
||||
return res, fmt.Errorf("the merge cannot be read: %w", err)
|
||||
}
|
||||
res.Merge = strings.TrimSpace(merge)
|
||||
changed, err := git("diff", "--name-only", "--no-renames", base, res.Merge)
|
||||
if err != nil {
|
||||
return res, fmt.Errorf("the paths the merge changes cannot be read: %w", err)
|
||||
}
|
||||
res.Changed = strings.Fields(changed)
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// mergeIdentity is who a check's merge is made as: it is never pushed, so it names nobody's account. No hook of the
|
||||
// clone's or the build machine's git configuration runs, and no setting there changes the merge (the controller's
|
||||
// check merge, internal/builder/check_merge.go).
|
||||
var mergeIdentity = []string{"-c", "user.name=mesh merge check", "-c", "user.email=merge-check@mesh.invalid",
|
||||
"-c", "commit.gpgsign=false", "-c", "core.hooksPath=/dev/null", "-c", "merge.verifySignatures=false"}
|
||||
|
||||
func short(commit string) string {
|
||||
if len(commit) > 12 {
|
||||
return commit[:12]
|
||||
}
|
||||
return commit
|
||||
}
|
||||
@@ -0,0 +1,172 @@
|
||||
//go:build integration
|
||||
|
||||
package buildstep
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/forgekey"
|
||||
)
|
||||
|
||||
// The merge on real git repositories starts git processes, so it is an integration test (the unit level keeps its bound,
|
||||
// issue 895); the unit tests of merge_test.go prove the commands and the refusals with a fake runner.
|
||||
|
||||
func realGit(t *testing.T, dir string, args ...string) string {
|
||||
t.Helper()
|
||||
cmd := exec.Command("git", append([]string{"-C", dir, "-c", "user.name=t", "-c", "user.email=t@example.invalid", "-c", "commit.gpgsign=false"}, args...)...)
|
||||
out, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
t.Fatalf("git %v: %v\n%s", args, err, out)
|
||||
}
|
||||
return strings.TrimSpace(string(out))
|
||||
}
|
||||
|
||||
func writeFile(t *testing.T, dir, name, content string) {
|
||||
t.Helper()
|
||||
p := filepath.Join(dir, name)
|
||||
if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(p, []byte(content), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// mergeRig is a local bare repository standing in for the forge, a work repository that pushes to it, and an empty
|
||||
// workspace: the fetch step's shape (FetchCommit of the head into the workspace, then MergeInto).
|
||||
type mergeRig struct {
|
||||
t *testing.T
|
||||
bare, work, ws string
|
||||
fork, trunk string // the commit both histories share, and the trunk's head
|
||||
remote string
|
||||
}
|
||||
|
||||
func newMergeRig(t *testing.T) *mergeRig {
|
||||
root := t.TempDir()
|
||||
r := &mergeRig{t: t, bare: filepath.Join(root, "r.git"), work: filepath.Join(root, "work"), ws: filepath.Join(root, "ws")}
|
||||
for _, d := range []string{r.bare, r.work, r.ws} {
|
||||
if err := os.MkdirAll(d, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
r.remote = "file://" + r.bare
|
||||
realGit(t, r.bare, "init", "-q", "--bare")
|
||||
realGit(t, r.work, "init", "-q", "-b", "main")
|
||||
writeFile(t, r.work, "a.txt", "one\ntwo\nthree\n")
|
||||
writeFile(t, r.work, "modules/m/module.json", "{}\n")
|
||||
realGit(t, r.work, "add", ".")
|
||||
realGit(t, r.work, "commit", "-q", "-m", "fork point")
|
||||
r.fork = realGit(t, r.work, "rev-parse", "HEAD")
|
||||
return r
|
||||
}
|
||||
|
||||
// head makes a commit on a branch off the fork point, pushes it as the pull request's head ref and answers its sha.
|
||||
func (r *mergeRig) head(files map[string]string) string {
|
||||
realGit(r.t, r.work, "checkout", "-q", "-B", "pr", r.fork)
|
||||
for n, c := range files {
|
||||
writeFile(r.t, r.work, n, c)
|
||||
}
|
||||
realGit(r.t, r.work, "add", ".")
|
||||
realGit(r.t, r.work, "commit", "-q", "-m", "head")
|
||||
h := realGit(r.t, r.work, "rev-parse", "HEAD")
|
||||
realGit(r.t, r.work, "push", "-q", r.bare, "HEAD:refs/pull/1/head")
|
||||
return h
|
||||
}
|
||||
|
||||
// advance moves the trunk past the fork point with a commit changing files, pushes it and answers its sha.
|
||||
func (r *mergeRig) advance(files map[string]string) string {
|
||||
realGit(r.t, r.work, "checkout", "-q", "-B", "main", r.fork)
|
||||
for n, c := range files {
|
||||
writeFile(r.t, r.work, n, c)
|
||||
}
|
||||
realGit(r.t, r.work, "add", ".")
|
||||
realGit(r.t, r.work, "commit", "-q", "-m", "trunk moves")
|
||||
r.trunk = realGit(r.t, r.work, "rev-parse", "HEAD")
|
||||
realGit(r.t, r.work, "push", "-q", "-f", r.bare, "HEAD:refs/heads/main")
|
||||
return r.trunk
|
||||
}
|
||||
|
||||
func (r *mergeRig) fetch(head string) {
|
||||
if err := FetchCommit(context.Background(), forgekey.ExecRunner, r.ws, r.remote, head, "refs/pull/1/head", AllowFileRemote()); err != nil {
|
||||
r.t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// 1674/AC1: a clean merge answers the merge commit and the changed files, and leaves the workspace at the merge, which
|
||||
// holds both sides' changes.
|
||||
func TestMergeIntoACleanMergeAnswersTheMergeCommitAndTheChangedFiles(t *testing.T) {
|
||||
r := newMergeRig(t)
|
||||
head := r.head(map[string]string{"b.txt": "head\n"})
|
||||
base := r.advance(map[string]string{"a.txt": "one\ntwo\nthree\nfour\n"})
|
||||
r.fetch(head)
|
||||
res, err := MergeInto(context.Background(), forgekey.ExecRunner, r.ws, r.remote, base, head, AllowFileRemote())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if res.Base != base || res.Head != head || len(res.Merge) != 40 || res.Merge == base || res.Merge == head || len(res.Conflicts) != 0 {
|
||||
t.Fatalf("result = %+v", res)
|
||||
}
|
||||
if !reflect.DeepEqual(res.Changed, []string{"b.txt"}) {
|
||||
t.Errorf("changed = %v, want only what the head adds to the trunk: b.txt", res.Changed)
|
||||
}
|
||||
if at := realGit(t, r.ws, "rev-parse", "HEAD"); at != res.Merge {
|
||||
t.Errorf("the workspace is at %s, not at the merge %s", at, res.Merge)
|
||||
}
|
||||
for f, want := range map[string]string{"b.txt": "head\n", "a.txt": "one\ntwo\nthree\nfour\n"} {
|
||||
if got, _ := os.ReadFile(filepath.Join(r.ws, f)); string(got) != want {
|
||||
t.Errorf("%s = %q, want %q", f, got, want)
|
||||
}
|
||||
}
|
||||
if parents := strings.Fields(realGit(t, r.ws, "rev-list", "--parents", "-n", "1", res.Merge)); len(parents) != 3 || parents[1] != base || parents[2] != head {
|
||||
t.Errorf("the merge commit's parents = %v, want the trunk commit and the head", parents)
|
||||
}
|
||||
if author := realGit(t, r.ws, "log", "-1", "--format=%an <%ae>", res.Merge); author != "mesh merge check <merge-check@mesh.invalid>" {
|
||||
t.Errorf("the merge is made as %q", author)
|
||||
}
|
||||
}
|
||||
|
||||
// 1674/AC2: a conflict is an answer: the files in conflict, no merge commit, and the workspace left at the trunk commit.
|
||||
func TestMergeIntoAConflictAnswersTheFilesAndLeavesTheTreeAtTheTrunk(t *testing.T) {
|
||||
r := newMergeRig(t)
|
||||
head := r.head(map[string]string{"a.txt": "one\nHEAD\nthree\n", "c.txt": "c\n"})
|
||||
base := r.advance(map[string]string{"a.txt": "one\nTRUNK\nthree\n"})
|
||||
r.fetch(head)
|
||||
res, err := MergeInto(context.Background(), forgekey.ExecRunner, r.ws, r.remote, base, head, AllowFileRemote())
|
||||
if err != nil {
|
||||
t.Fatalf("a conflict is an answer, not an error: %v", err)
|
||||
}
|
||||
if !reflect.DeepEqual(res.Conflicts, []string{"a.txt"}) || res.Merge != "" || len(res.Changed) != 0 || res.Base != base || res.Head != head {
|
||||
t.Fatalf("result = %+v", res)
|
||||
}
|
||||
if at := realGit(t, r.ws, "rev-parse", "HEAD"); at != base {
|
||||
t.Errorf("the workspace is at %s, not at the trunk commit %s", at, base)
|
||||
}
|
||||
if got, _ := os.ReadFile(filepath.Join(r.ws, "a.txt")); string(got) != "one\nTRUNK\nthree\n" {
|
||||
t.Errorf("a.txt = %q, want the trunk's", got)
|
||||
}
|
||||
if st := realGit(t, r.ws, "status", "--porcelain"); st != "" {
|
||||
t.Errorf("the tree is not clean after the aborted merge: %q", st)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(r.ws, "c.txt")); err == nil {
|
||||
t.Error("a file of the head is in a tree left at the trunk")
|
||||
}
|
||||
}
|
||||
|
||||
// A head already in the trunk merges into nothing: the merge is the trunk commit and no path changes.
|
||||
func TestMergeIntoAHeadAlreadyInTheTrunkChangesNothing(t *testing.T) {
|
||||
r := newMergeRig(t)
|
||||
head := r.head(map[string]string{"b.txt": "head\n"})
|
||||
r.fork = head // the trunk continues from the head
|
||||
base := r.advance(map[string]string{"a.txt": "one\ntwo\nthree\nfour\n"})
|
||||
r.fetch(head)
|
||||
res, err := MergeInto(context.Background(), forgekey.ExecRunner, r.ws, r.remote, base, head, AllowFileRemote())
|
||||
if err != nil || res.Merge != base || len(res.Changed) != 0 || len(res.Conflicts) != 0 {
|
||||
t.Errorf("result = %+v, %v", res, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
package buildstep
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/forgekey"
|
||||
)
|
||||
|
||||
// 1674/AC3: no base is refused before anything runs; so is a base or a commit that is not a full sha, a workspace that
|
||||
// is not at the commit, and a remote that is not ssh://.
|
||||
func TestMergeIntoRefusesWhatIsNotAMerge(t *testing.T) {
|
||||
f := &fakeGit{}
|
||||
for name, c := range map[string]struct{ remote, base, commit string }{
|
||||
"no base": {"ssh://mesh-forge/o/r.git", "", sha},
|
||||
"a short base": {"ssh://mesh-forge/o/r.git", "abc123", sha},
|
||||
"a branch as base": {"ssh://mesh-forge/o/r.git", "main", sha},
|
||||
"a short commit": {"ssh://mesh-forge/o/r.git", sha, "abc"},
|
||||
"an https remote": {"https://forge.example/o/r.git", sha, sha},
|
||||
} {
|
||||
_, err := MergeInto(context.Background(), f.run, "/w", c.remote, c.base, c.commit)
|
||||
if err == nil {
|
||||
t.Errorf("%s accepted", name)
|
||||
}
|
||||
}
|
||||
if len(f.calls) != 0 {
|
||||
t.Errorf("git ran for a refused merge: %v", f.calls)
|
||||
}
|
||||
g := &fakeGit{rule: func(l string) forgekey.Ran {
|
||||
if strings.Contains(l, "rev-parse HEAD") {
|
||||
return forgekey.Ran{Stdout: strings.Repeat("f", 40) + "\n"}
|
||||
}
|
||||
return forgekey.Ran{}
|
||||
}}
|
||||
_, err := MergeInto(context.Background(), g.run, "/w", "ssh://mesh-forge/o/r.git", strings.Repeat("a", 40), sha)
|
||||
if err == nil || !strings.Contains(err.Error(), "not at the commit") || len(g.calls) != 1 {
|
||||
t.Errorf("a workspace at another commit: %v after %d commands", err, len(g.calls))
|
||||
}
|
||||
}
|
||||
|
||||
// 1674/AC4: the merge is made with the check merge's identity and flags, hooks off, and fetches with no credential.
|
||||
func TestMergeIntoRunsTheCheckMergesCommands(t *testing.T) {
|
||||
base := strings.Repeat("b", 40)
|
||||
f := &fakeGit{rule: func(l string) forgekey.Ran {
|
||||
switch {
|
||||
case strings.HasSuffix(l, "rev-parse HEAD"):
|
||||
return forgekey.Ran{Stdout: sha + "\n"}
|
||||
case strings.Contains(l, "--is-shallow-repository"):
|
||||
return forgekey.Ran{Stdout: "true\n"}
|
||||
}
|
||||
return forgekey.Ran{}
|
||||
}}
|
||||
if _, err := MergeInto(context.Background(), f.run, "/w", "ssh://mesh-forge/o/r.git", base, sha); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
all := strings.Join(f.calls, "\n")
|
||||
for _, want := range []string{
|
||||
pre + "fetch --no-tags --unshallow origin " + base,
|
||||
pre + "checkout -q --detach " + base,
|
||||
"user.name=mesh merge check", "user.email=merge-check@mesh.invalid", "commit.gpgsign=false", "core.hooksPath=/dev/null",
|
||||
"merge --no-ff --no-edit --no-verify --quiet -m",
|
||||
pre + "diff --name-only --no-renames " + base,
|
||||
} {
|
||||
if !strings.Contains(all, want) {
|
||||
t.Errorf("the commands lack %q:\n%s", want, all)
|
||||
}
|
||||
}
|
||||
for _, c := range f.calls {
|
||||
if strings.Contains(c, "https://") || !strings.Contains(c, "credential.helper= ") {
|
||||
t.Errorf("command %q carries a credential or https", c)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
//go:build integration
|
||||
|
||||
package buildstep
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/forgekey"
|
||||
)
|
||||
|
||||
// Issue 890 C1 (AC3): a mirror brought to two commits in turn; a fetch through it checks out each commit, the
|
||||
// alternates name the mirror, and no symlink is in the workspace.
|
||||
func TestMirrorServesTwoCommitsWithoutSymlinks(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
src := filepath.Join(root, "src.git")
|
||||
work := filepath.Join(root, "work")
|
||||
for _, d := range []string{src, work} {
|
||||
if err := os.MkdirAll(d, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
gitIn(t, src, "init", "-q", "--bare")
|
||||
gitIn(t, work, "init", "-q")
|
||||
var commits []string
|
||||
for _, c := range []string{"one", "two"} {
|
||||
if err := os.WriteFile(filepath.Join(work, "f"), []byte(c), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
gitIn(t, work, "add", "f")
|
||||
gitIn(t, work, "commit", "-q", "-m", c)
|
||||
commits = append(commits, gitIn(t, work, "rev-parse", "HEAD"))
|
||||
gitIn(t, work, "push", "-q", src, "HEAD:refs/heads/main")
|
||||
}
|
||||
remote := "file://" + src
|
||||
mirror := filepath.Join(root, "mirror.git")
|
||||
for i, commit := range commits {
|
||||
if err := UpdateMirror(context.Background(), forgekey.ExecRunner, mirror, remote, commit, "", AllowFileRemote()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ws := filepath.Join(root, "ws"+string(rune('0'+i)))
|
||||
if err := os.MkdirAll(ws, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := FetchCommit(context.Background(), forgekey.ExecRunner, ws, remote, commit, "", AllowFileRemote(), WithMirror(mirror)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if head := gitIn(t, ws, "rev-parse", "HEAD"); head != commit {
|
||||
t.Errorf("HEAD = %s, want %s", head, commit)
|
||||
}
|
||||
b, _ := os.ReadFile(filepath.Join(ws, ".git", "objects", "info", "alternates"))
|
||||
if !strings.Contains(string(b), filepath.Join(mirror, "objects")) {
|
||||
t.Errorf("alternates %q", b)
|
||||
}
|
||||
_ = filepath.Walk(ws, func(p string, fi os.FileInfo, err error) error {
|
||||
if err == nil && fi.Mode()&os.ModeSymlink != 0 {
|
||||
t.Errorf("symlink %s", p)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,106 @@
|
||||
package buildstep
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/forgekey"
|
||||
)
|
||||
|
||||
// Issue 890 C1 (ADR 0536 §2): a fetch through a bare mirror names the mirror in .git/objects/info/alternates, a
|
||||
// plain file, and never links it.
|
||||
func TestFetchCommitWithMirrorWritesAlternates(t *testing.T) {
|
||||
mirror := t.TempDir()
|
||||
ws := t.TempDir()
|
||||
f := &fakeGit{}
|
||||
// the fake's init does not create .git; make the objects directory as git init would.
|
||||
if err := os.MkdirAll(filepath.Join(ws, ".git", "objects", "info"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
err := FetchCommit(context.Background(), f.run, ws, "ssh://mesh-forge/novox/mesh-sdk.git", sha, "", WithMirror(mirror))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
p := filepath.Join(ws, ".git", "objects", "info", "alternates")
|
||||
b, err := os.ReadFile(p)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got, want := strings.TrimSpace(string(b)), filepath.Join(mirror, "objects"); got != want {
|
||||
t.Errorf("alternates %q, want %q", got, want)
|
||||
}
|
||||
if fi, err := os.Lstat(p); err != nil || !fi.Mode().IsRegular() {
|
||||
t.Errorf("alternates is not a plain file: %v %v", fi, err)
|
||||
}
|
||||
// the alternates exist before the fetch runs
|
||||
if len(f.calls) < 3 || !strings.Contains(f.calls[2], "fetch") {
|
||||
t.Errorf("calls %v", f.calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWithMirrorRefusesARelativeOrMissingMirror(t *testing.T) {
|
||||
for _, m := range []string{"rel/mirror", filepath.Join(t.TempDir(), "absent")} {
|
||||
f := &fakeGit{}
|
||||
err := FetchCommit(context.Background(), f.run, t.TempDir(), "ssh://mesh-forge/novox/x.git", sha, "", WithMirror(m))
|
||||
if err == nil || len(f.calls) != 0 {
|
||||
t.Errorf("mirror %q: err %v, calls %v", m, err, f.calls)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpdateMirrorInitsBareThenFetches(t *testing.T) {
|
||||
mirror := filepath.Join(t.TempDir(), "mirror.git")
|
||||
f := &fakeGit{}
|
||||
if err := UpdateMirror(context.Background(), f.run, mirror, "ssh://mesh-forge/novox/x.git", sha, ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pm := "git -c credential.helper= -C " + mirror + " "
|
||||
want := []string{
|
||||
"git -c credential.helper= init -q --bare " + mirror,
|
||||
pm + "remote add origin ssh://mesh-forge/novox/x.git",
|
||||
pm + "fetch --no-tags origin " + sha,
|
||||
pm + "cat-file -e " + sha + "^{commit}",
|
||||
}
|
||||
if strings.Join(f.calls, "\n") != strings.Join(want, "\n") {
|
||||
t.Errorf("commands:\n%s\nwant:\n%s", strings.Join(f.calls, "\n"), strings.Join(want, "\n"))
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpdateMirrorKeepsAnExistingMirrorAndFallsBackToTheRef(t *testing.T) {
|
||||
mirror := t.TempDir() // exists: no init
|
||||
if err := os.WriteFile(filepath.Join(mirror, "HEAD"), []byte("ref: refs/heads/main\n"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f := &fakeGit{}
|
||||
f.rule = func(args string) (r forgekey.Ran) {
|
||||
if strings.HasSuffix(args, "origin "+sha) {
|
||||
r.Status = 1
|
||||
r.Stderr = "not our ref"
|
||||
}
|
||||
return
|
||||
}
|
||||
if err := UpdateMirror(context.Background(), f.run, mirror, "ssh://mesh-forge/novox/x.git", sha, "refs/pull/3/head"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
all := strings.Join(f.calls, "\n")
|
||||
if strings.Contains(all, "init") || strings.Contains(all, "remote add") || !strings.Contains(all, "fetch --no-tags origin refs/pull/3/head") {
|
||||
t.Errorf("commands:\n%s", all)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpdateMirrorRefusesBeforeRunning(t *testing.T) {
|
||||
f := &fakeGit{}
|
||||
for _, c := range []struct{ remote, commit string }{
|
||||
{"https://x/y.git", sha}, {"ssh://u@h/o/r.git", sha}, {"ssh://h/o/r.git", "abc"},
|
||||
} {
|
||||
if err := UpdateMirror(context.Background(), f.run, t.TempDir(), c.remote, c.commit, ""); err == nil {
|
||||
t.Errorf("%v accepted", c)
|
||||
}
|
||||
}
|
||||
if len(f.calls) != 0 {
|
||||
t.Errorf("ran %v", f.calls)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,71 @@
|
||||
package buildstep
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// narrowKept are the files of a module's directory a narrowed build keeps beside its named paths: its manifest and
|
||||
// its lock files, each an input of the module's key (ModuleTree's Manifest and Locks). package.json and go.mod are
|
||||
// not kept: neither is in the named tree, so a build that needs one names it among its artifact's paths, which puts
|
||||
// it in the key (novox/hq ADR 0499 clause 4).
|
||||
var narrowKept = append([]string{"module.json"}, lockFiles...)
|
||||
|
||||
// Narrow leaves in the fetched tree only what a build of an artifact that names its paths may see (novox/hq ADR 0499
|
||||
// clause 4): r.Dir's named paths, its manifest and lock files, and .git; everything else is removed, so a file outside
|
||||
// them cannot change what is built without changing its key. A named path absent at the tree is an error naming it,
|
||||
// and nothing is removed then. With no paths in r it changes nothing. Both build modules' fetch steps call it.
|
||||
func Narrow(tree string, r FetchRequest) error {
|
||||
if len(r.Paths) == 0 {
|
||||
return nil
|
||||
}
|
||||
keep := map[string]bool{".git": true}
|
||||
for _, p := range r.Paths {
|
||||
rel := path.Join(r.Dir, p)
|
||||
if strings.HasPrefix(rel, "../") || rel == ".." || path.IsAbs(p) {
|
||||
return fmt.Errorf("the named path %q leaves the module's directory", p)
|
||||
}
|
||||
if _, err := os.Lstat(filepath.Join(tree, filepath.FromSlash(rel))); err != nil {
|
||||
return fmt.Errorf("the named path %q is not in the fetched tree: %w", p, err)
|
||||
}
|
||||
keep[rel] = true
|
||||
}
|
||||
for _, f := range narrowKept {
|
||||
keep[path.Join(r.Dir, f)] = true
|
||||
}
|
||||
return prune(tree, "", keep)
|
||||
}
|
||||
|
||||
// prune removes from dir (rel within the tree) every entry that is neither kept nor on the way to one.
|
||||
func prune(root, rel string, keep map[string]bool) error {
|
||||
entries, err := os.ReadDir(filepath.Join(root, filepath.FromSlash(rel)))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, e := range entries {
|
||||
p := path.Join(rel, e.Name())
|
||||
if keep[p] {
|
||||
continue
|
||||
}
|
||||
ancestor := false
|
||||
for k := range keep {
|
||||
if strings.HasPrefix(k, p+"/") {
|
||||
ancestor = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if ancestor && e.IsDir() {
|
||||
if err := prune(root, p, keep); err != nil {
|
||||
return err
|
||||
}
|
||||
continue
|
||||
}
|
||||
if err := os.RemoveAll(filepath.Join(root, filepath.FromSlash(p))); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,82 @@
|
||||
package buildstep
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func filesUnder(t *testing.T, root string) []string {
|
||||
t.Helper()
|
||||
var out []string
|
||||
_ = filepath.Walk(root, func(p string, info os.FileInfo, err error) error {
|
||||
if err == nil && !info.IsDir() {
|
||||
rel, _ := filepath.Rel(root, p)
|
||||
out = append(out, filepath.ToSlash(rel))
|
||||
}
|
||||
return nil
|
||||
})
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// A build of an artifact that names its paths sees only those paths, its manifest and its lock files (novox/hq ADR 0499
|
||||
// clause 4): Narrow removes everything else of the fetched tree but .git, package.json and go.mod among it, since
|
||||
// neither is in the named tree nor among the lock files; with no paths it changes nothing.
|
||||
func TestNarrowLeavesOnlyTheNamedPathsTheManifestAndTheLocks(t *testing.T) {
|
||||
tree := t.TempDir()
|
||||
for _, f := range []string{".git/HEAD", "module.json", "package.json", "package-lock.json", "go.sum", "src/index.ts",
|
||||
"src/deep/a.ts", "test/a.test.ts", "tsconfig.json", "go/delivery/x.go", "README.md", "other/module.json"} {
|
||||
writeTreeFile(t, tree, f, "x")
|
||||
}
|
||||
if err := Narrow(tree, FetchRequest{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := len(filesUnder(t, tree)); got != 12 {
|
||||
t.Fatalf("a request without paths changed the tree: %d files", got)
|
||||
}
|
||||
if err := Narrow(tree, FetchRequest{Dir: ".", Paths: []string{"src", "test", "tsconfig.json"}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := ".git/HEAD,go.sum,module.json,package-lock.json,src/deep/a.ts,src/index.ts,test/a.test.ts,tsconfig.json"
|
||||
if got := strings.Join(filesUnder(t, tree), ","); got != want {
|
||||
t.Fatalf("the narrowed tree holds %s\nwant %s", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNarrowKeepsAModuleDirectorysPathsAndFailsOnAnAbsentOne(t *testing.T) {
|
||||
tree := t.TempDir()
|
||||
for _, f := range []string{".git/config", "modules/a/module.json", "modules/a/go.mod", "modules/a/src/x.ts",
|
||||
"modules/a/notes.md", "modules/b/module.json", "root.txt"} {
|
||||
writeTreeFile(t, tree, f, "x")
|
||||
}
|
||||
err := Narrow(tree, FetchRequest{Dir: "modules/a", Paths: []string{"src", "gone"}})
|
||||
if err == nil || !strings.Contains(err.Error(), `"gone"`) {
|
||||
t.Fatalf("an absent named path gave %v", err)
|
||||
}
|
||||
if len(filesUnder(t, tree)) != 7 {
|
||||
t.Fatal("a refused narrowing removed files")
|
||||
}
|
||||
if err := Narrow(tree, FetchRequest{Dir: "modules/a", Paths: []string{"src"}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := strings.Join(filesUnder(t, tree), ","); got != ".git/config,modules/a/module.json,modules/a/src/x.ts" {
|
||||
t.Fatalf("the narrowed tree holds %s", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A build that needs package.json or go.mod names it among its paths, and so keys on it (ADR 0499 clause 4).
|
||||
func TestNarrowKeepsAManifestOfTheLanguageOnlyWhenNamed(t *testing.T) {
|
||||
tree := t.TempDir()
|
||||
for _, f := range []string{"module.json", "package.json", "go.mod", "src/a.ts"} {
|
||||
writeTreeFile(t, tree, f, "x")
|
||||
}
|
||||
if err := Narrow(tree, FetchRequest{Dir: ".", Paths: []string{"src", "package.json"}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := strings.Join(filesUnder(t, tree), ","); got != "module.json,package.json,src/a.ts" {
|
||||
t.Fatalf("the narrowed tree holds %s", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,445 @@
|
||||
package buildstep
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// **The pull loop the build modules share** (novox/hq ADR 0525 §4-§6, ADR 0536 §4; issue 1887). A build module on a node
|
||||
// whose build capacity is above zero pulls the asks of its language's seat: from the delivery worker first, then check,
|
||||
// then recheck, holding at most its capacity at once. Each ask runs as a step under the ask's own id, so its end is the
|
||||
// key <node>.<ask id> in the module's steps state, which the asker watches; the ask is acknowledged when that end is
|
||||
// written, kept alive while it runs by an in-progress signal, and redelivered by the stream to another node if this one
|
||||
// stops. No part keeps a timer for an ask: the pull's own wait and the stream's acknowledgement wait are the bus's.
|
||||
|
||||
// Pulled is one ask a source handed over, with what decides its fate on the bus.
|
||||
type Pulled struct {
|
||||
// Kind is the worker it came from, filled by the puller.
|
||||
Kind string
|
||||
Data []byte
|
||||
// Delivered is how many times the bus has handed it over, this time included.
|
||||
Delivered uint64
|
||||
// Ack ends it: it is never delivered again. InProgress tells the bus it is still being worked, restarting its
|
||||
// acknowledgement wait. Nak hands it back to be delivered again at once (to this node or another). Term drops it.
|
||||
Ack, InProgress, Nak, Term func() error
|
||||
}
|
||||
|
||||
// AskSource is where a puller takes asks from: the seat's workers on the bus (JetStreamAsks) or a test double
|
||||
// (FakeAsks).
|
||||
type AskSource interface {
|
||||
// Next takes one ask of kind: with wait 0 only one already waiting, else waiting for one at most wait. It answers
|
||||
// nil and no error when none came.
|
||||
Next(ctx context.Context, kind string, wait time.Duration) (*Pulled, error)
|
||||
}
|
||||
|
||||
// Defaults of a Puller.
|
||||
var (
|
||||
// DefaultPullWait is how long one pull of the delivery worker waits for an ask while every worker is empty: the
|
||||
// pull's own wait (as process.Consume's), after which the check and recheck workers are looked at again.
|
||||
DefaultPullWait = 5 * time.Second
|
||||
// DefaultHeartbeat is how often a running ask tells the bus it is still being worked; under the seat worker's
|
||||
// acknowledgement wait of 60 s (mesh-controller internal/broker HolderConsumerFor).
|
||||
DefaultHeartbeat = 20 * time.Second
|
||||
)
|
||||
|
||||
// Puller pulls a build seat's asks and runs them.
|
||||
type Puller struct {
|
||||
Source AskSource
|
||||
Language string // the seat's language: delivery.AskGo, AskNode or AskImage
|
||||
Node string // the node this holder runs on, the first part of its step keys
|
||||
Put Put // writes the module's steps state
|
||||
// Capacity is how many asks the node runs at once; read before every pull, so a change takes effect at the next
|
||||
// one. 0 pulls nothing.
|
||||
Capacity func() int
|
||||
// Run runs one ask's job; a *Refusal ends its step refused. Its result is the step's result.
|
||||
Run func(ctx context.Context, ask delivery.StepAsk) (any, error)
|
||||
// PullWait and Heartbeat default to DefaultPullWait and DefaultHeartbeat.
|
||||
PullWait time.Duration
|
||||
Heartbeat time.Duration
|
||||
Logf func(format string, args ...any)
|
||||
// Load reads one key of the state the node's pause is kept in (nil and no error when it is absent). When set, Pull
|
||||
// reads the node's pause key (delivery.PauseKey) before it pulls, and Pause and Resume write it with PausePut, so a
|
||||
// pause outlives a restart.
|
||||
Load func(key string) ([]byte, error)
|
||||
// PausePut writes the node's pause key: the module's pauses state (delivery.PausesState), which has no age, so a
|
||||
// pause never ends by itself while the steps state drops its old keys. Put when nil.
|
||||
PausePut Put
|
||||
// WatchConfig, when set, follows the module's config value `capacity` (CapacitySetting) as mesh-config resolves it
|
||||
// for this node, and wakes the puller on each change (watchCapacity): WatchConfigServed in a module, a test's own
|
||||
// double. Until it hands a value the puller judges by Capacity.
|
||||
WatchConfig func(key string, handle func(stdio.ProvisionChange) error) error
|
||||
|
||||
mu sync.Mutex
|
||||
inHand int
|
||||
running map[string]RunningAsk
|
||||
paused bool
|
||||
wake chan struct{}
|
||||
once sync.Once
|
||||
// watched is the capacity the config watch handed last, used in place of Capacity once hasWatched.
|
||||
watched int
|
||||
hasWatched bool
|
||||
}
|
||||
|
||||
func (p *Puller) init() {
|
||||
p.once.Do(func() { p.wake = make(chan struct{}, 1) })
|
||||
}
|
||||
|
||||
func (p *Puller) logf(format string, args ...any) {
|
||||
if p.Logf != nil {
|
||||
p.Logf(format, args...)
|
||||
}
|
||||
}
|
||||
|
||||
// Wake makes a waiting puller look again: after a capacity change, a resume, or a step's end.
|
||||
func (p *Puller) Wake() {
|
||||
p.init()
|
||||
select {
|
||||
case p.wake <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
// Pause stops the puller taking asks; the asks it holds run to their end. by names who paused it. With Load set the
|
||||
// pause is written on the node's pause key first, and an error there leaves the puller as it was.
|
||||
func (p *Puller) Pause(by string) error { return p.setPaused(true, by) }
|
||||
|
||||
// Resume lets the puller take asks again, written as Pause's is.
|
||||
func (p *Puller) Resume(by string) error { return p.setPaused(false, by) }
|
||||
|
||||
func (p *Puller) setPaused(paused bool, by string) error {
|
||||
if p.Load != nil {
|
||||
key, err := delivery.PauseKey(p.Node)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
raw, err := delivery.EncodePause(delivery.Pause{Paused: paused, By: by, At: now()})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
put := p.PausePut
|
||||
if put == nil {
|
||||
put = p.Put
|
||||
}
|
||||
if _, err := put(key, json.RawMessage(raw)); err != nil {
|
||||
return fmt.Errorf("buildstep: writing the pause of %s: %w", p.Node, err)
|
||||
}
|
||||
}
|
||||
p.mu.Lock()
|
||||
p.paused = paused
|
||||
p.mu.Unlock()
|
||||
p.Wake()
|
||||
return nil
|
||||
}
|
||||
|
||||
// restore reads the node's pause key, when Load is set.
|
||||
func (p *Puller) restore() error {
|
||||
if p.Load == nil {
|
||||
return nil
|
||||
}
|
||||
key, err := delivery.PauseKey(p.Node)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
raw, err := p.Load(key)
|
||||
if err != nil {
|
||||
return fmt.Errorf("buildstep: reading the pause of %s: %w", p.Node, err)
|
||||
}
|
||||
if raw == nil {
|
||||
return nil
|
||||
}
|
||||
v, err := delivery.DecodePause(raw)
|
||||
if err != nil {
|
||||
return fmt.Errorf("buildstep: the pause of %s: %w", p.Node, err)
|
||||
}
|
||||
p.mu.Lock()
|
||||
p.paused = v.Paused
|
||||
p.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
|
||||
// Paused says whether the puller takes no asks.
|
||||
func (p *Puller) Paused() bool {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
return p.paused
|
||||
}
|
||||
|
||||
// InHand is how many asks run now.
|
||||
func (p *Puller) InHand() int {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
return p.inHand
|
||||
}
|
||||
|
||||
// RunningAsk is an ask a puller runs now, as its seat's current verb says it.
|
||||
type RunningAsk struct {
|
||||
ID string `json:"id"`
|
||||
Kind string `json:"kind"`
|
||||
Module string `json:"module,omitempty"`
|
||||
Started time.Time `json:"started"`
|
||||
}
|
||||
|
||||
// Running are the asks the puller runs now, oldest first.
|
||||
func (p *Puller) Running() []RunningAsk {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
out := make([]RunningAsk, 0, len(p.running))
|
||||
for _, r := range p.running {
|
||||
out = append(out, r)
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool { return out[i].Started.Before(out[j].Started) })
|
||||
return out
|
||||
}
|
||||
|
||||
// room says whether the puller may take one more ask now.
|
||||
func (p *Puller) room() bool {
|
||||
ok, _, _ := p.roomNow()
|
||||
return ok
|
||||
}
|
||||
|
||||
// roomNow is room, with the capacity and the asks in hand it was judged on.
|
||||
func (p *Puller) roomNow() (ok bool, capacity, inHand int) {
|
||||
p.mu.Lock()
|
||||
watched, has := p.watched, p.hasWatched
|
||||
p.mu.Unlock()
|
||||
if has {
|
||||
capacity = watched
|
||||
} else if p.Capacity != nil {
|
||||
capacity = p.Capacity()
|
||||
}
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
return !p.paused && p.inHand < capacity, capacity, p.inHand
|
||||
}
|
||||
|
||||
// CapacityNow is the capacity the puller judges by: the config watch's last value, else Capacity.
|
||||
func (p *Puller) CapacityNow() int {
|
||||
_, capacity, _ := p.roomNow()
|
||||
return capacity
|
||||
}
|
||||
|
||||
// Pull takes and runs asks until ctx ends. An ask still running then is neither ended nor acknowledged: the stream
|
||||
// delivers it again, to another node or to this one restarted.
|
||||
func (p *Puller) Pull(ctx context.Context) error {
|
||||
p.init()
|
||||
if p.Source == nil || p.Put == nil || p.Run == nil {
|
||||
return errors.New("buildstep: a puller needs its source, its steps' put and its run")
|
||||
}
|
||||
if err := p.restore(); err != nil {
|
||||
// An unreadable pause degrades to not paused, said once: a puller that stops silently is the worse failure, and a
|
||||
// node kept from building is held by its capacity (0), not by a pause.
|
||||
p.logf("the pause of %s cannot be read: %v; pulling as not paused", p.Node, err)
|
||||
}
|
||||
go p.watchCapacity() // WatchConfigServed waits for Serve, which a module starts after its pull
|
||||
wait := p.PullWait
|
||||
if wait <= 0 {
|
||||
wait = DefaultPullWait
|
||||
}
|
||||
for {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if ok, capacity, inHand := p.roomNow(); !ok {
|
||||
// Parked until something wakes it: a capacity change (the config watch), a resume, or a step's end. Every
|
||||
// park logs once, so a puller that sits idle says why (issue 1887).
|
||||
p.logf("the %s puller on %s parks: %d of capacity %d in hand, paused %v", p.Language, p.Node, inHand, capacity, p.Paused())
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-p.wake:
|
||||
}
|
||||
_, capacity, inHand = p.roomNow()
|
||||
p.logf("the %s puller on %s wakes: %d of capacity %d in hand, paused %v", p.Language, p.Node, inHand, capacity, p.Paused())
|
||||
continue
|
||||
}
|
||||
msg, err := p.next(ctx, wait)
|
||||
if err != nil {
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
p.logf("the %s asks cannot be pulled: %v", p.Language, err)
|
||||
select { // the pull's own wait before the next try, as an empty pull's
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-p.wake:
|
||||
case <-time.After(wait):
|
||||
}
|
||||
continue
|
||||
}
|
||||
if msg == nil {
|
||||
continue
|
||||
}
|
||||
p.take(ctx, msg)
|
||||
}
|
||||
}
|
||||
|
||||
// next is one ask from the workers in their order: one already waiting on delivery, check or recheck, and when none
|
||||
// waits, a pull of the delivery worker that waits for one.
|
||||
func (p *Puller) next(ctx context.Context, wait time.Duration) (*Pulled, error) {
|
||||
for _, kind := range delivery.AskKinds {
|
||||
msg, err := p.Source.Next(ctx, kind, 0)
|
||||
if err != nil || msg != nil {
|
||||
return withKind(msg, kind), err
|
||||
}
|
||||
}
|
||||
msg, err := p.Source.Next(ctx, delivery.AskDelivery, wait)
|
||||
return withKind(msg, delivery.AskDelivery), err
|
||||
}
|
||||
|
||||
func withKind(msg *Pulled, kind string) *Pulled {
|
||||
if msg != nil {
|
||||
msg.Kind = kind
|
||||
}
|
||||
return msg
|
||||
}
|
||||
|
||||
// take starts one ask: it decodes it, refuses one that is not of this seat, and runs it as a step under its own id.
|
||||
func (p *Puller) take(ctx context.Context, msg *Pulled) {
|
||||
kind := msg.Kind
|
||||
ask, err := delivery.DecodeStepAsk(msg.Data)
|
||||
if err == nil && (ask.Language != p.Language || (kind != "" && ask.Kind != kind)) {
|
||||
err = fmt.Errorf("the ask %s is a %s ask of %s, pulled from the %s %s worker", ask.ID, ask.Kind, ask.Language, p.Language, kind)
|
||||
}
|
||||
if err != nil {
|
||||
// An ask that is not one runs nowhere: delivering it again would only fail again.
|
||||
p.logf("an ask dropped: %v", err)
|
||||
_ = msg.Term()
|
||||
return
|
||||
}
|
||||
step, key, err := begin(p.Put, p.Node, ask.Kind+" ask", ask.ID)
|
||||
if err != nil {
|
||||
p.logf("the ask %s cannot start on %s: %v", ask.ID, p.Node, err)
|
||||
_ = msg.Nak()
|
||||
return
|
||||
}
|
||||
if msg.Delivered > delivery.AskRuns {
|
||||
// Run AskRuns times and never ended: each run took its node down or died with it. It is run no more, and said
|
||||
// (issue 1907): the asker's key fails with why.
|
||||
why := fmt.Errorf("the ask was taken %d times and no run of it ended: each took its build node down or died with it, so it is run no more (poison)", msg.Delivered-1)
|
||||
p.logf("the ask %s is ended on %s: %v", ask.ID, p.Node, why)
|
||||
_ = finish(p.Put, key, step, nil, why, p.Logf) // run no more either way: the asker's bound says it then
|
||||
_ = msg.Term()
|
||||
return
|
||||
}
|
||||
module := ""
|
||||
if ask.Build != nil {
|
||||
module = ask.Build.Module
|
||||
}
|
||||
p.mu.Lock()
|
||||
p.inHand++
|
||||
if p.running == nil {
|
||||
p.running = map[string]RunningAsk{}
|
||||
}
|
||||
p.running[ask.ID] = RunningAsk{ID: ask.ID, Kind: ask.Kind, Module: module, Started: step.Started}
|
||||
p.mu.Unlock()
|
||||
go p.run(ctx, msg, ask, key, step)
|
||||
}
|
||||
|
||||
func (p *Puller) run(ctx context.Context, msg *Pulled, ask delivery.StepAsk, key string, step delivery.Step) {
|
||||
defer func() {
|
||||
p.mu.Lock()
|
||||
p.inHand--
|
||||
delete(p.running, ask.ID)
|
||||
p.mu.Unlock()
|
||||
p.Wake()
|
||||
}()
|
||||
beat := p.Heartbeat
|
||||
if beat <= 0 {
|
||||
beat = DefaultHeartbeat
|
||||
}
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
t := time.NewTicker(beat)
|
||||
defer t.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-done:
|
||||
return
|
||||
case <-t.C:
|
||||
if msg.InProgress != nil {
|
||||
_ = msg.InProgress()
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
result, err := protect(ctx, func(ctx context.Context) (any, error) { return p.Run(ctx, ask) })
|
||||
close(done)
|
||||
if ctx.Err() != nil {
|
||||
// The process stops: the ask is neither ended nor acknowledged here, so the stream gives it to a node that runs.
|
||||
return
|
||||
}
|
||||
if ferr := finish(p.Put, key, step, result, err, p.Logf); ferr != nil {
|
||||
// No end was written: the ask is not acknowledged as done, so the stream gives it again and a run that can
|
||||
// write its end (or the poison rule, issue 1907) ends it where its asker sees it (issue 1930).
|
||||
p.logf("the ask %s ran on %s and no end of it could be written, so it is given back: %v", ask.ID, p.Node, ferr)
|
||||
_ = msg.Nak()
|
||||
return
|
||||
}
|
||||
if aerr := msg.Ack(); aerr != nil {
|
||||
p.logf("the ask %s ended on %s and its acknowledgement failed: %v", ask.ID, p.Node, aerr)
|
||||
}
|
||||
}
|
||||
|
||||
// watchCapacity follows the module's config value `capacity` when WatchConfig is set (issue 1887, 2026-10-25: g14's
|
||||
// puller started at capacity 0 sat parked with capacity 2 set, because nothing woke it to read its settings file
|
||||
// again). The value comes as mesh-config resolved it for this node, the event of the change itself: the runtime hands
|
||||
// the value now and then each change its machine's provisioner client says moved, so nothing polls, and the puller does
|
||||
// not wait for the node-engine to fill its settings file again. Each value handed is kept, used in place of Capacity,
|
||||
// and wakes the puller. A watch that cannot start leaves the puller on Capacity, said once.
|
||||
func (p *Puller) watchCapacity() {
|
||||
if p.WatchConfig == nil {
|
||||
return
|
||||
}
|
||||
err := p.WatchConfig(CapacitySetting, func(c stdio.ProvisionChange) error {
|
||||
n, why := p.capacityFrom(c)
|
||||
p.mu.Lock()
|
||||
before, had := p.watched, p.hasWatched
|
||||
p.watched, p.hasWatched = n, true
|
||||
p.mu.Unlock()
|
||||
switch {
|
||||
case why != "":
|
||||
p.logf("the %s puller on %s pulls nothing: %s", p.Language, p.Node, why)
|
||||
case !had || before != n:
|
||||
p.logf("the %s build capacity of %s is %d, as mesh-config resolved it", p.Language, p.Node, n)
|
||||
}
|
||||
p.Wake()
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
p.logf("the %s build capacity of %s cannot be followed on the bus, so it is read from the settings file when the "+
|
||||
"puller wakes: %v", p.Language, p.Node, err)
|
||||
}
|
||||
}
|
||||
|
||||
// WatchConfigServed is stdio.WatchConfig once Serve has the runtime's channel (stdio.Served): a build module starts its
|
||||
// pull before it serves, and a watch asked before that would be refused.
|
||||
func WatchConfigServed(key string, handle func(stdio.ProvisionChange) error) error {
|
||||
<-stdio.Served()
|
||||
return stdio.WatchConfig(key, handle)
|
||||
}
|
||||
|
||||
// capacityFrom is the node's capacity in one value the watch handed, and why it is 0 when it cannot be read.
|
||||
func (p *Puller) capacityFrom(c stdio.ProvisionChange) (int, string) {
|
||||
if c.Refused != "" {
|
||||
return 0, "its capacity is no longer answered: " + c.Refused
|
||||
}
|
||||
setting := ""
|
||||
if len(c.Value) > 0 && string(c.Value) != "null" {
|
||||
if err := json.Unmarshal(c.Value, &setting); err != nil {
|
||||
return 0, fmt.Sprintf("its capacity %s is not a string", c.Value)
|
||||
}
|
||||
}
|
||||
n, err := capacityOf(setting, p.Node)
|
||||
if err != nil {
|
||||
return 0, err.Error()
|
||||
}
|
||||
return n, ""
|
||||
}
|
||||
@@ -0,0 +1,102 @@
|
||||
package buildstep
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// JetStreamAsks is a build seat's workers on the bus: the shared pull consumer of each kind on the seat's stream
|
||||
// (delivery.AskStream, AskWorker), which the broker makes when the seat is declared; a holder only binds them.
|
||||
type JetStreamAsks struct {
|
||||
JS jetstream.JetStream
|
||||
Language string
|
||||
|
||||
mu sync.Mutex
|
||||
cons map[string]jetstream.Consumer
|
||||
}
|
||||
|
||||
func (s *JetStreamAsks) consumer(ctx context.Context, kind string) (jetstream.Consumer, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if c, ok := s.cons[kind]; ok {
|
||||
return c, nil
|
||||
}
|
||||
stream, err := delivery.AskStream(s.Language)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
worker, err := delivery.AskWorker(s.Language, kind)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
c, err := s.JS.Consumer(ctx, stream, worker)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("binding the %s worker %s on %s: %w", kind, worker, stream, err)
|
||||
}
|
||||
if s.cons == nil {
|
||||
s.cons = map[string]jetstream.Consumer{}
|
||||
}
|
||||
s.cons[kind] = c
|
||||
return c, nil
|
||||
}
|
||||
|
||||
// Next is AskSource's: a fetch of one with no wait when wait is 0, else one that waits at most wait.
|
||||
func (s *JetStreamAsks) Next(ctx context.Context, kind string, wait time.Duration) (*Pulled, error) {
|
||||
c, err := s.consumer(ctx, kind)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var batch jetstream.MessageBatch
|
||||
if wait <= 0 {
|
||||
batch, err = c.FetchNoWait(1)
|
||||
} else {
|
||||
batch, err = c.Fetch(1, jetstream.FetchMaxWait(wait))
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
select {
|
||||
case msg, ok := <-batch.Messages():
|
||||
if !ok {
|
||||
return nil, batch.Error()
|
||||
}
|
||||
var delivered uint64 = 1
|
||||
if meta, err := msg.Metadata(); err == nil && meta != nil {
|
||||
delivered = meta.NumDelivered
|
||||
}
|
||||
return &Pulled{Data: append([]byte(nil), msg.Data()...), Delivered: delivered,
|
||||
Ack: msg.Ack, InProgress: msg.InProgress, Nak: msg.Nak, Term: msg.Term}, nil
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
// DeclareAskWorkers makes language's seat stream and its three workers on a bus with no broker: a test's (the build
|
||||
// modules' integration tests start one). On the mesh the broker declares them from the seat, as it declares every
|
||||
// seat's stream (mesh-controller internal/broker SeatStreams); a holder never does.
|
||||
func DeclareAskWorkers(ctx context.Context, js jetstream.JetStream, language string, ackWait time.Duration) error {
|
||||
stream, err := delivery.AskStream(language)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
seat, _ := delivery.BuildSeat(language)
|
||||
if _, err := js.CreateOrUpdateStream(ctx, jetstream.StreamConfig{Name: stream, Subjects: []string{"mesh.seat." + seat + ".accept.>"},
|
||||
Retention: jetstream.WorkQueuePolicy}); err != nil {
|
||||
return fmt.Errorf("the stream %s: %w", stream, err)
|
||||
}
|
||||
for _, kind := range delivery.AskKinds {
|
||||
worker, _ := delivery.AskWorker(language, kind)
|
||||
subject, _ := delivery.AskSubject(language, kind)
|
||||
if _, err := js.CreateOrUpdateConsumer(ctx, stream, jetstream.ConsumerConfig{Durable: worker, FilterSubject: subject,
|
||||
AckPolicy: jetstream.AckExplicitPolicy, AckWait: ackWait, MaxDeliver: 5}); err != nil {
|
||||
return fmt.Errorf("the worker %s: %w", worker, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
package buildstep
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
)
|
||||
|
||||
// configWatch is a double of stdio.WatchConfig: it hands the value now at the start, as the runtime does, and a test
|
||||
// hands each change after with set.
|
||||
type configWatch struct {
|
||||
mu sync.Mutex
|
||||
value string
|
||||
handle func(stdio.ProvisionChange) error
|
||||
asked string
|
||||
}
|
||||
|
||||
func (w *configWatch) watch(key string, handle func(stdio.ProvisionChange) error) error {
|
||||
w.mu.Lock()
|
||||
w.asked, w.handle = key, handle
|
||||
value := w.value
|
||||
w.mu.Unlock()
|
||||
return handle(stdio.ProvisionChange{Kind: "config", Key: key, Value: json.RawMessage(`"` + value + `"`)})
|
||||
}
|
||||
|
||||
func (w *configWatch) set(t *testing.T, value string) {
|
||||
t.Helper()
|
||||
w.mu.Lock()
|
||||
handle, key := w.handle, w.asked
|
||||
w.mu.Unlock()
|
||||
if handle == nil {
|
||||
t.Fatal("the puller watches no config value")
|
||||
}
|
||||
if err := handle(stdio.ProvisionChange{Kind: "config", Key: key, Value: json.RawMessage(`"` + value + `"`)}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,113 @@
|
||||
package buildstep
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// FakeAsks is a test double of a seat's workers for the build modules' unit tests: Publish puts an ask on a kind's
|
||||
// worker, Next hands them out in their order, and the fates the puller gives them are counted. Safe for concurrent use.
|
||||
type FakeAsks struct {
|
||||
mu sync.Mutex
|
||||
waiting map[string][][]byte
|
||||
added chan struct{}
|
||||
// Acked, Termed and Naked are the asks' ids or data by fate, InProgress how many heartbeats were given.
|
||||
Acked, Termed, Naked []string
|
||||
InProgress int
|
||||
// Delivered, when set, is the delivery count every ask handed over says (1 when 0): a test of a poisoned ask sets it.
|
||||
Delivered uint64
|
||||
}
|
||||
|
||||
// NewFakeAsks is an empty double.
|
||||
func NewFakeAsks() *FakeAsks {
|
||||
return &FakeAsks{waiting: map[string][][]byte{}, added: make(chan struct{}, 1)}
|
||||
}
|
||||
|
||||
// Publish puts an encoded ask on kind's worker.
|
||||
func (f *FakeAsks) Publish(kind string, data []byte) {
|
||||
f.mu.Lock()
|
||||
f.waiting[kind] = append(f.waiting[kind], data)
|
||||
f.mu.Unlock()
|
||||
select {
|
||||
case f.added <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
// PublishAsk encodes a and puts it on its kind's worker.
|
||||
func (f *FakeAsks) PublishAsk(a delivery.StepAsk) error {
|
||||
raw, err := delivery.EncodeStepAsk(a)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
f.Publish(a.Kind, raw)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Waiting is how many asks wait on kind's worker.
|
||||
func (f *FakeAsks) Waiting(kind string) int {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
return len(f.waiting[kind])
|
||||
}
|
||||
|
||||
// Fates is a copy of the acked, termed and naked lists.
|
||||
func (f *FakeAsks) Fates() (acked, termed, naked []string) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
return append([]string(nil), f.Acked...), append([]string(nil), f.Termed...), append([]string(nil), f.Naked...)
|
||||
}
|
||||
|
||||
// Next is AskSource's.
|
||||
func (f *FakeAsks) Next(ctx context.Context, kind string, wait time.Duration) (*Pulled, error) {
|
||||
var deadline <-chan time.Time
|
||||
if wait > 0 {
|
||||
t := time.NewTimer(wait)
|
||||
defer t.Stop()
|
||||
deadline = t.C
|
||||
}
|
||||
for {
|
||||
f.mu.Lock()
|
||||
if q := f.waiting[kind]; len(q) > 0 {
|
||||
data := q[0]
|
||||
f.waiting[kind] = q[1:]
|
||||
f.mu.Unlock()
|
||||
return f.pulled(data), nil
|
||||
}
|
||||
f.mu.Unlock()
|
||||
if wait <= 0 {
|
||||
return nil, nil
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
case <-deadline:
|
||||
return nil, nil
|
||||
case <-f.added:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (f *FakeAsks) pulled(data []byte) *Pulled {
|
||||
id := string(data)
|
||||
if a, err := delivery.DecodeStepAsk(data); err == nil {
|
||||
id = a.ID
|
||||
}
|
||||
fate := func(list *[]string) func() error {
|
||||
return func() error {
|
||||
f.mu.Lock()
|
||||
*list = append(*list, id)
|
||||
f.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
}
|
||||
delivered := f.Delivered
|
||||
if delivered == 0 {
|
||||
delivered = 1
|
||||
}
|
||||
return &Pulled{Data: data, Delivered: delivered, Ack: fate(&f.Acked), Nak: fate(&f.Naked), Term: fate(&f.Termed),
|
||||
InProgress: func() error { f.mu.Lock(); f.InProgress++; f.mu.Unlock(); return nil }}
|
||||
}
|
||||
@@ -0,0 +1,144 @@
|
||||
//go:build integration
|
||||
|
||||
package buildstep
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/testbus"
|
||||
)
|
||||
|
||||
func askBus(t *testing.T, ackWait time.Duration) jetstream.JetStream {
|
||||
t.Helper()
|
||||
nc, err := nats.Connect(testbus.URL(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(nc.Close)
|
||||
js, err := jetstream.New(nc)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := DeclareAskWorkers(context.Background(), js, delivery.AskGo, ackWait); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return js
|
||||
}
|
||||
|
||||
func publishAsk(t *testing.T, js jetstream.JetStream, a delivery.StepAsk) {
|
||||
t.Helper()
|
||||
raw, err := delivery.EncodeStepAsk(a)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
subject, _ := delivery.AskSubject(a.Language, a.Kind)
|
||||
if _, err := js.Publish(context.Background(), subject, raw); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// 1887/A2a, 1887/AC-order, 1887/AC-pull on the real server: a check ask published before a delivery ask runs after it;
|
||||
// with capacity 2 and three asks two run, the third waits in the stream and starts when one ends.
|
||||
func TestPullOrderAndCapacityOnTheBus(t *testing.T) {
|
||||
js := askBus(t, 30*time.Second)
|
||||
check, first, second := testAsk(delivery.AskCheck), testAsk(delivery.AskDelivery), testAsk(delivery.AskDelivery)
|
||||
publishAsk(t, js, check)
|
||||
publishAsk(t, js, first)
|
||||
publishAsk(t, js, second)
|
||||
release := make(chan struct{})
|
||||
var mu sync.Mutex
|
||||
var order []string
|
||||
st := newSteps()
|
||||
p := &Puller{Source: &JetStreamAsks{JS: js, Language: delivery.AskGo}, Language: delivery.AskGo, Node: "shanks", Put: st.put,
|
||||
Capacity: func() int { return 2 }, PullWait: 200 * time.Millisecond,
|
||||
Run: func(ctx context.Context, a delivery.StepAsk) (any, error) {
|
||||
mu.Lock()
|
||||
order = append(order, a.ID)
|
||||
mu.Unlock()
|
||||
<-release
|
||||
return nil, nil
|
||||
}}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
go p.Pull(ctx)
|
||||
waitFor(t, "two in hand", func() bool { return p.InHand() == 2 })
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
mu.Lock()
|
||||
if len(order) != 2 || order[0] != first.ID || order[1] != second.ID {
|
||||
t.Fatalf("ran %v, want the two delivery asks %s %s first", order, first.ID, second.ID)
|
||||
}
|
||||
mu.Unlock()
|
||||
release <- struct{}{}
|
||||
waitFor(t, "the check taken", func() bool { mu.Lock(); defer mu.Unlock(); return len(order) == 3 })
|
||||
close(release)
|
||||
waitFor(t, "every step ended", func() bool {
|
||||
for _, a := range []delivery.StepAsk{check, first, second} {
|
||||
key, _ := delivery.AskStepKey("shanks", a.ID)
|
||||
if s, ok := st.get(key); !ok || s.State != delivery.StepPassed {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
})
|
||||
info, err := js.Stream(ctx, "SEAT_NODE_BUILD_GO")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waitFor(t, "the stream empty (every ask acknowledged)", func() bool {
|
||||
si, err := info.Info(ctx)
|
||||
return err == nil && si.State.Msgs == 0
|
||||
})
|
||||
}
|
||||
|
||||
// 1887/A2c, ADR 0525 §6: a killed puller's ask is redelivered by the stream's acknowledgement wait to the other node,
|
||||
// whose step key names the reference.
|
||||
func TestPullRedeliversAKilledPullersAsk(t *testing.T) {
|
||||
js := askBus(t, 2*time.Second)
|
||||
a := testAsk(delivery.AskDelivery)
|
||||
publishAsk(t, js, a)
|
||||
st := newSteps()
|
||||
taken := make(chan struct{})
|
||||
dying := &Puller{Source: &JetStreamAsks{JS: js, Language: delivery.AskGo}, Language: delivery.AskGo, Node: "ace", Put: st.put,
|
||||
Capacity: func() int { return 1 }, PullWait: 200 * time.Millisecond, Heartbeat: time.Hour,
|
||||
Run: func(ctx context.Context, a delivery.StepAsk) (any, error) {
|
||||
close(taken)
|
||||
<-ctx.Done()
|
||||
return nil, ctx.Err()
|
||||
}}
|
||||
kill, killed := context.WithCancel(context.Background())
|
||||
go dying.Pull(kill)
|
||||
<-taken
|
||||
killed()
|
||||
other := &Puller{Source: &JetStreamAsks{JS: js, Language: delivery.AskGo}, Language: delivery.AskGo, Node: "shanks", Put: st.put,
|
||||
Capacity: func() int { return 1 }, PullWait: 200 * time.Millisecond,
|
||||
Run: func(ctx context.Context, a delivery.StepAsk) (any, error) {
|
||||
return delivery.BuildAnswer{Artifacts: []delivery.AskBuilt{{Artifact: "bundle", Digest: "sha256:00", Reference: "artifact-store://x"}}}, nil
|
||||
}}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
go other.Pull(ctx)
|
||||
key, _ := delivery.AskStepKey("shanks", a.ID)
|
||||
deadline := time.Now().Add(10 * time.Second)
|
||||
for {
|
||||
if s, ok := st.get(key); ok && s.State == delivery.StepPassed {
|
||||
if want := `artifact-store://x`; !contains(string(s.Result), want) {
|
||||
t.Fatalf("the step's result %s names no %s", s.Result, want)
|
||||
}
|
||||
return
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("the killed puller's ask was not taken by shanks within 10 s")
|
||||
}
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
|
||||
func contains(s, sub string) bool { return strings.Contains(s, sub) }
|
||||
@@ -0,0 +1,44 @@
|
||||
package buildstep
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// SeatAsks is a build seat's workers as a tools bundle reaches them: through its runtime's seat pull and settle
|
||||
// (stdio.SeatPull, SeatSettle; issue 1887 A11), one message per pull, only from the workers the module's membership
|
||||
// lists.
|
||||
type SeatAsks struct {
|
||||
Language string
|
||||
// Pull and Settle are stdio.SeatPull and stdio.SeatSettle when nil; a test gives its own.
|
||||
Pull func(worker string, wait time.Duration) (*stdio.Pulled, error)
|
||||
Settle func(id, fate string) error
|
||||
}
|
||||
|
||||
// Next is AskSource's.
|
||||
func (s SeatAsks) Next(ctx context.Context, kind string, wait time.Duration) (*Pulled, error) {
|
||||
pull, settle := s.Pull, s.Settle
|
||||
if pull == nil {
|
||||
pull = stdio.SeatPull
|
||||
}
|
||||
if settle == nil {
|
||||
settle = stdio.SeatSettle
|
||||
}
|
||||
worker, err := delivery.AskWorker(s.Language, kind)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
p, err := pull(worker, wait)
|
||||
if err != nil || p == nil {
|
||||
return nil, err
|
||||
}
|
||||
fate := func(f string) func() error { return func() error { return settle(p.ID, f) } }
|
||||
return &Pulled{Data: p.Body, Delivered: p.Delivered, Ack: fate(stdio.SettleAck), InProgress: fate(stdio.SettleProgress),
|
||||
Nak: fate(stdio.SettleNak), Term: fate(stdio.SettleTerm)}, nil
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
package buildstep
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// 1887/A11: a bundle's source pulls the kind's worker by its seat's name and settles each ask by the runtime's id.
|
||||
func TestSeatAsksPullsTheKindsWorkerAndSettlesByID(t *testing.T) {
|
||||
var pulled []string
|
||||
var settled []string
|
||||
src := SeatAsks{Language: delivery.AskGo,
|
||||
Pull: func(worker string, wait time.Duration) (*stdio.Pulled, error) {
|
||||
pulled = append(pulled, worker)
|
||||
if worker != "SEAT_NODE_BUILD_GO_CHECK_worker" {
|
||||
return nil, nil
|
||||
}
|
||||
return &stdio.Pulled{ID: "p9", Worker: worker, Body: []byte(`{"v":1}`), Delivered: 2}, nil
|
||||
},
|
||||
Settle: func(id, fate string) error { settled = append(settled, id+" "+fate); return nil }}
|
||||
if p, err := src.Next(context.Background(), delivery.AskDelivery, 0); err != nil || p != nil {
|
||||
t.Fatalf("delivery = %+v, %v", p, err)
|
||||
}
|
||||
p, err := src.Next(context.Background(), delivery.AskCheck, time.Second)
|
||||
if err != nil || p == nil || string(p.Data) != `{"v":1}` || p.Delivered != 2 {
|
||||
t.Fatalf("check = %+v, %v", p, err)
|
||||
}
|
||||
_ = p.InProgress()
|
||||
_ = p.Ack()
|
||||
if len(pulled) != 2 || pulled[0] != "SEAT_NODE_BUILD_GO_DELIVERY_worker" {
|
||||
t.Errorf("pulled %v", pulled)
|
||||
}
|
||||
if len(settled) != 2 || settled[0] != "p9 progress" || settled[1] != "p9 ack" {
|
||||
t.Errorf("settled %v", settled)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,364 @@
|
||||
package buildstep
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// steps records a steps state's puts: the last value of each key.
|
||||
type steps struct {
|
||||
mu sync.Mutex
|
||||
keys map[string]delivery.Step
|
||||
}
|
||||
|
||||
func newSteps() *steps { return &steps{keys: map[string]delivery.Step{}} }
|
||||
|
||||
func (s *steps) put(key string, value any) (uint64, error) {
|
||||
raw, _ := value.(json.RawMessage)
|
||||
st, err := delivery.DecodeStep(raw)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
s.mu.Lock()
|
||||
s.keys[key] = st
|
||||
s.mu.Unlock()
|
||||
return 1, nil
|
||||
}
|
||||
|
||||
func (s *steps) get(key string) (delivery.Step, bool) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
st, ok := s.keys[key]
|
||||
return st, ok
|
||||
}
|
||||
|
||||
func testAsk(kind string) delivery.StepAsk {
|
||||
id := NewID()
|
||||
a := delivery.StepAsk{ID: id, Kind: kind, Language: delivery.AskGo, Asker: "mesh-delivery-builder", At: time.Now()}
|
||||
fetch := map[string]any{"repository": "novox/x", "commit": "0123456789abcdef0123456789abcdef01234567"}
|
||||
if kind == delivery.AskDelivery {
|
||||
b := buildJobFixture
|
||||
b.Fetch = fetch
|
||||
a.Build = &b
|
||||
} else {
|
||||
a.Check = &delivery.CheckJob{Fetch: fetch, Levels: []string{delivery.LevelUnit}}
|
||||
}
|
||||
return a
|
||||
}
|
||||
|
||||
var buildJobFixture = delivery.BuildJob{Module: "m", Path: ".", Artifacts: []delivery.AskArtifact{{Artifact: "bundle"}}}
|
||||
|
||||
func waitFor(t *testing.T, what string, cond func() bool) {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(2 * time.Second)
|
||||
for !cond() {
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("waited 2 s for %s", what)
|
||||
}
|
||||
time.Sleep(time.Millisecond)
|
||||
}
|
||||
}
|
||||
|
||||
// 1887/A2a: given a check ask published before a delivery ask and one slot, the delivery ask runs first.
|
||||
func TestPullTakesDeliveryBeforeCheck(t *testing.T) {
|
||||
src, st := NewFakeAsks(), newSteps()
|
||||
check, recheck, deliver := testAsk(delivery.AskCheck), testAsk(delivery.AskRecheck), testAsk(delivery.AskDelivery)
|
||||
for _, a := range []delivery.StepAsk{recheck, check, deliver} {
|
||||
if err := src.PublishAsk(a); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
var mu sync.Mutex
|
||||
var order []string
|
||||
p := &Puller{Source: src, Language: delivery.AskGo, Node: "shanks", Put: st.put, Capacity: func() int { return 1 },
|
||||
PullWait: 10 * time.Millisecond, Run: func(ctx context.Context, a delivery.StepAsk) (any, error) {
|
||||
mu.Lock()
|
||||
order = append(order, a.Kind)
|
||||
mu.Unlock()
|
||||
return map[string]string{"ran": a.ID}, nil
|
||||
}}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
go p.Pull(ctx)
|
||||
waitFor(t, "three asks acked", func() bool { a, _, _ := src.Fates(); return len(a) == 3 })
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if want := []string{delivery.AskDelivery, delivery.AskCheck, delivery.AskRecheck}; len(order) != 3 || order[0] != want[0] || order[1] != want[1] || order[2] != want[2] {
|
||||
t.Fatalf("order = %v, want %v", order, want)
|
||||
}
|
||||
key, _ := delivery.AskStepKey("shanks", deliver.ID)
|
||||
s, ok := st.get(key)
|
||||
if !ok || s.State != delivery.StepPassed || string(s.Result) != `{"ran":"`+deliver.ID+`"}` {
|
||||
t.Fatalf("the step %s = %+v (%v)", key, s, ok)
|
||||
}
|
||||
}
|
||||
|
||||
// 1887/A2a, 1887/AC-pull: given capacity 2 and three asks waiting, two run and the third waits, and it starts when
|
||||
// one ends, with no timer.
|
||||
func TestPullHoldsAtMostItsCapacity(t *testing.T) {
|
||||
src, st := NewFakeAsks(), newSteps()
|
||||
for i := 0; i < 3; i++ {
|
||||
_ = src.PublishAsk(testAsk(delivery.AskDelivery))
|
||||
}
|
||||
release := make(chan struct{})
|
||||
var mu sync.Mutex
|
||||
started := 0
|
||||
p := &Puller{Source: src, Language: delivery.AskGo, Node: "shanks", Put: st.put, Capacity: func() int { return 2 },
|
||||
PullWait: 10 * time.Millisecond, Run: func(ctx context.Context, a delivery.StepAsk) (any, error) {
|
||||
mu.Lock()
|
||||
started++
|
||||
mu.Unlock()
|
||||
<-release
|
||||
return nil, nil
|
||||
}}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
go p.Pull(ctx)
|
||||
waitFor(t, "two in hand", func() bool { return p.InHand() == 2 })
|
||||
time.Sleep(30 * time.Millisecond) // three pull waits: the third must still wait
|
||||
if p.InHand() != 2 || src.Waiting(delivery.AskDelivery) != 1 {
|
||||
t.Fatalf("in hand %d, waiting %d; want 2 and 1", p.InHand(), src.Waiting(delivery.AskDelivery))
|
||||
}
|
||||
release <- struct{}{}
|
||||
waitFor(t, "the third taken", func() bool { return src.Waiting(delivery.AskDelivery) == 0 })
|
||||
// Taken from the source comes a moment before its run starts: wait for the start, not only the take.
|
||||
waitFor(t, "the third started", func() bool { mu.Lock(); defer mu.Unlock(); return started == 3 })
|
||||
close(release)
|
||||
waitFor(t, "all acked", func() bool { a, _, _ := src.Fates(); return len(a) == 3 })
|
||||
}
|
||||
|
||||
// An ask that is not one, or not this seat's, is dropped and runs nowhere.
|
||||
func TestPullDropsWhatIsNoAsk(t *testing.T) {
|
||||
src, st := NewFakeAsks(), newSteps()
|
||||
src.Publish(delivery.AskDelivery, []byte(`{"v":1}`))
|
||||
other := testAsk(delivery.AskDelivery)
|
||||
other.Language = delivery.AskNode
|
||||
_ = src.PublishAsk(other)
|
||||
ran := false
|
||||
p := &Puller{Source: src, Language: delivery.AskGo, Node: "shanks", Put: st.put, Capacity: func() int { return 1 },
|
||||
PullWait: 10 * time.Millisecond, Run: func(ctx context.Context, a delivery.StepAsk) (any, error) { ran = true; return nil, nil }}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
go p.Pull(ctx)
|
||||
waitFor(t, "two termed", func() bool { _, tm, _ := src.Fates(); return len(tm) == 2 })
|
||||
if ran {
|
||||
t.Fatal("a wrong ask ran")
|
||||
}
|
||||
}
|
||||
|
||||
// load reads the recorded steps state's raw values: the pause key is not a step, so it is kept apart.
|
||||
type kv struct {
|
||||
mu sync.Mutex
|
||||
vals map[string][]byte
|
||||
}
|
||||
|
||||
func (k *kv) put(key string, value any) (uint64, error) {
|
||||
raw, _ := value.(json.RawMessage)
|
||||
k.mu.Lock()
|
||||
defer k.mu.Unlock()
|
||||
if k.vals == nil {
|
||||
k.vals = map[string][]byte{}
|
||||
}
|
||||
k.vals[key] = append([]byte(nil), raw...)
|
||||
return 1, nil
|
||||
}
|
||||
|
||||
func (k *kv) load(key string) ([]byte, error) {
|
||||
k.mu.Lock()
|
||||
defer k.mu.Unlock()
|
||||
return k.vals[key], nil
|
||||
}
|
||||
|
||||
// 1887/A2b, 1887/AC-zero, 1887/AC-wake: at capacity 0 the puller takes nothing; a capacity raised takes effect at the
|
||||
// next pull, or, with the config watch (g14 idle at capacity 2, 2026-10-25), at once when the value moves, with no
|
||||
// pause, resume or step end to wake it; a refused value pulls nothing, and a watch that cannot start leaves the
|
||||
// settings file's capacity.
|
||||
func TestPullAtCapacityZeroTakesNothing(t *testing.T) {
|
||||
t.Run("settings file", func(t *testing.T) {
|
||||
src, st := NewFakeAsks(), &kv{}
|
||||
_ = src.PublishAsk(testAsk(delivery.AskDelivery))
|
||||
var mu sync.Mutex
|
||||
capacity := 0
|
||||
p := &Puller{Source: src, Language: delivery.AskGo, Node: "novox", Put: st.put, PullWait: 10 * time.Millisecond,
|
||||
Capacity: func() int { mu.Lock(); defer mu.Unlock(); return capacity },
|
||||
Run: func(ctx context.Context, a delivery.StepAsk) (any, error) { return nil, nil }}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
go p.Pull(ctx)
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
if src.Waiting(delivery.AskDelivery) != 1 {
|
||||
t.Fatal("a node at capacity 0 took an ask")
|
||||
}
|
||||
mu.Lock()
|
||||
capacity = 1
|
||||
mu.Unlock()
|
||||
p.Wake()
|
||||
waitFor(t, "the ask taken at capacity 1", func() bool { a, _, _ := src.Fates(); return len(a) == 1 })
|
||||
})
|
||||
t.Run("config watch wakes", func(t *testing.T) {
|
||||
src, st := NewFakeAsks(), newSteps()
|
||||
_ = src.PublishAsk(testAsk(delivery.AskDelivery))
|
||||
w := &configWatch{value: "shanks=3,g14=0"}
|
||||
p := &Puller{Source: src, Language: delivery.AskGo, Node: "g14", Put: st.put, PullWait: 10 * time.Millisecond,
|
||||
Capacity: func() int { return 0 }, // the settings file, not refilled yet
|
||||
WatchConfig: w.watch,
|
||||
Run: func(ctx context.Context, a delivery.StepAsk) (any, error) { return nil, nil }}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
go p.Pull(ctx)
|
||||
waitFor(t, "the watch of the capacity", func() bool { w.mu.Lock(); defer w.mu.Unlock(); return w.handle != nil })
|
||||
time.Sleep(30 * time.Millisecond)
|
||||
if src.Waiting(delivery.AskDelivery) != 1 {
|
||||
t.Fatal("a node at capacity 0 took an ask")
|
||||
}
|
||||
if w.asked != CapacitySetting {
|
||||
t.Fatalf("the puller watches %q, want %q", w.asked, CapacitySetting)
|
||||
}
|
||||
w.set(t, "shanks=3,g14=2")
|
||||
waitFor(t, "the ask taken once the capacity moved to 2", func() bool { a, _, _ := src.Fates(); return len(a) == 1 })
|
||||
})
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
watch func(string, func(stdio.ProvisionChange) error) error
|
||||
room bool
|
||||
}{
|
||||
{"refused value leaves no room", func(key string, handle func(stdio.ProvisionChange) error) error {
|
||||
return handle(stdio.ProvisionChange{Kind: "config", Key: key, Refused: "the module is no longer on g14"})
|
||||
}, false},
|
||||
{"watch that cannot start keeps the settings file", func(string, func(stdio.ProvisionChange) error) error {
|
||||
return errors.New("no runtime")
|
||||
}, true},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
p := &Puller{Source: NewFakeAsks(), Language: delivery.AskGo, Node: "g14", Put: newSteps().put,
|
||||
Capacity: func() int { return 1 }, WatchConfig: c.watch,
|
||||
Run: func(ctx context.Context, a delivery.StepAsk) (any, error) { return nil, nil }}
|
||||
p.watchCapacity()
|
||||
if p.room() != c.room {
|
||||
t.Fatalf("room = %v, want %v", p.room(), c.room)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// 1887/A2b: a paused puller takes nothing, its pause is kept on the pause key, and a puller started again on that state
|
||||
// stays paused until resumed.
|
||||
func TestPullPauseOutlivesARestart(t *testing.T) {
|
||||
src, st, steps := NewFakeAsks(), &kv{}, &kv{}
|
||||
run := func(ctx context.Context, a delivery.StepAsk) (any, error) { return nil, nil }
|
||||
p := &Puller{Source: src, Language: delivery.AskGo, Node: "shanks", Put: steps.put, PausePut: st.put, Load: st.load,
|
||||
PullWait: 10 * time.Millisecond, Capacity: func() int { return 2 }, Run: run}
|
||||
if err := p.Pause("the operator"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
key, _ := delivery.PauseKey("shanks")
|
||||
raw, _ := st.load(key)
|
||||
if v, err := delivery.DecodePause(raw); err != nil || !v.Paused || v.By != "the operator" {
|
||||
t.Fatalf("the pause key holds %s (%v)", raw, err)
|
||||
}
|
||||
if raw, _ := steps.load(key); raw != nil {
|
||||
t.Fatalf("the pause was written in the steps state, whose keys age out: %s", raw)
|
||||
}
|
||||
// The restart: a new puller on the same state.
|
||||
again := &Puller{Source: src, Language: delivery.AskGo, Node: "shanks", Put: steps.put, PausePut: st.put, Load: st.load,
|
||||
PullWait: 10 * time.Millisecond, Capacity: func() int { return 2 }, Run: run}
|
||||
_ = src.PublishAsk(testAsk(delivery.AskDelivery))
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
go again.Pull(ctx)
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
if !again.Paused() || src.Waiting(delivery.AskDelivery) != 1 {
|
||||
t.Fatalf("restarted: paused %v, waiting %d; want paused and 1", again.Paused(), src.Waiting(delivery.AskDelivery))
|
||||
}
|
||||
if err := again.Resume("the operator"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waitFor(t, "the ask taken after the resume", func() bool { a, _, _ := src.Fates(); return len(a) == 1 })
|
||||
|
||||
t.Run("a pause that cannot be read is said and pulled as not paused", func(t *testing.T) {
|
||||
src := NewFakeAsks()
|
||||
var said atomic.Value
|
||||
p := &Puller{Source: src, Language: delivery.AskGo, Node: "shanks", Put: steps.put, PausePut: st.put,
|
||||
Load: func(string) ([]byte, error) { return nil, errors.New("no runtime yet") },
|
||||
PullWait: 10 * time.Millisecond, Capacity: func() int { return 2 }, Run: run,
|
||||
Logf: func(f string, a ...any) {
|
||||
if m := fmt.Sprintf(f, a...); strings.Contains(m, "cannot be read") {
|
||||
said.Store(m)
|
||||
}
|
||||
}}
|
||||
_ = src.PublishAsk(testAsk(delivery.AskDelivery))
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
go p.Pull(ctx)
|
||||
waitFor(t, "the ask taken", func() bool { a, _, _ := src.Fates(); return len(a) == 1 })
|
||||
if m, _ := said.Load().(string); !strings.Contains(m, "pulling as not paused") {
|
||||
t.Fatalf("the unreadable pause was not said: %q", m)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// 1887/A2c: a long step tells the bus it is still being worked; a holder that stops mid-step neither ends nor
|
||||
// acknowledges its ask, so the stream gives it to another node.
|
||||
func TestPullHeartbeatAndStop(t *testing.T) {
|
||||
src, st := NewFakeAsks(), newSteps()
|
||||
a := testAsk(delivery.AskDelivery)
|
||||
_ = src.PublishAsk(a)
|
||||
running := make(chan struct{})
|
||||
p := &Puller{Source: src, Language: delivery.AskGo, Node: "shanks", Put: st.put, Capacity: func() int { return 1 },
|
||||
PullWait: 10 * time.Millisecond, Heartbeat: 5 * time.Millisecond,
|
||||
Run: func(ctx context.Context, a delivery.StepAsk) (any, error) {
|
||||
close(running)
|
||||
<-ctx.Done()
|
||||
return nil, ctx.Err()
|
||||
}}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
stopped := make(chan struct{})
|
||||
go func() { p.Pull(ctx); close(stopped) }()
|
||||
<-running
|
||||
waitFor(t, "two heartbeats", func() bool { src.mu.Lock(); defer src.mu.Unlock(); return src.InProgress >= 2 })
|
||||
cancel()
|
||||
<-stopped
|
||||
waitFor(t, "the step's goroutine gone", func() bool { return p.InHand() == 0 })
|
||||
acked, termed, naked := src.Fates()
|
||||
if len(acked)+len(termed)+len(naked) != 0 {
|
||||
t.Fatalf("a stopped holder gave its ask a fate: acked %v termed %v naked %v", acked, termed, naked)
|
||||
}
|
||||
key, _ := delivery.AskStepKey("shanks", a.ID)
|
||||
if s, ok := st.get(key); !ok || s.State != delivery.StepRunning {
|
||||
t.Fatalf("the step %s = %+v, want left running", key, s)
|
||||
}
|
||||
}
|
||||
|
||||
// 1907: an ask taken past AskRuns (each run took its node down) is run no more: its step ends failed saying why, so the
|
||||
// asker's key fails, and it is ended on the stream, not offered to another node.
|
||||
func TestAPoisonedAskIsSaidAndRunNoMore(t *testing.T) {
|
||||
src, st := NewFakeAsks(), newSteps()
|
||||
src.Delivered = delivery.AskRuns + 1
|
||||
a := testAsk(delivery.AskDelivery)
|
||||
_ = src.PublishAsk(a)
|
||||
ran := false
|
||||
p := &Puller{Source: src, Language: delivery.AskGo, Node: "shanks", Put: st.put, Capacity: func() int { return 1 },
|
||||
PullWait: 10 * time.Millisecond, Run: func(ctx context.Context, a delivery.StepAsk) (any, error) { ran = true; return nil, nil }}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
go p.Pull(ctx)
|
||||
waitFor(t, "the ask termed", func() bool { _, tm, _ := src.Fates(); return len(tm) == 1 })
|
||||
key, _ := delivery.AskStepKey("shanks", a.ID)
|
||||
s, ok := st.get(key)
|
||||
if ran || !ok || s.State != delivery.StepFailed || !strings.Contains(s.Reason, "taken 3 times") || !strings.Contains(s.Reason, "poison") {
|
||||
t.Fatalf("ran %v, step %+v", ran, s)
|
||||
}
|
||||
if acked, _, naked := src.Fates(); len(acked)+len(naked) != 0 {
|
||||
t.Errorf("acked %v naked %v, want it termed alone", acked, naked)
|
||||
}
|
||||
}
|
||||
@@ -1,5 +1,7 @@
|
||||
package buildstep
|
||||
|
||||
import "fmt"
|
||||
|
||||
// Refusal is a step that is not run on purpose, as against one that failed: the toolchain installed is not the
|
||||
// recipe's, the integration level's postgres major is not the one it is built for (novox/hq ADR 0469 §1, ADR 0470
|
||||
// §4; issue 1473). A step's run returns one and the runner ends the step refused with its text.
|
||||
@@ -11,3 +13,6 @@ type Refusal struct {
|
||||
}
|
||||
|
||||
func (r *Refusal) Error() string { return r.Text }
|
||||
|
||||
// Refuse is a Refusal with its text formatted.
|
||||
func Refuse(format string, args ...any) *Refusal { return &Refusal{Text: fmt.Sprintf(format, args...)} }
|
||||
|
||||
@@ -0,0 +1,115 @@
|
||||
package buildstep
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"path"
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
var (
|
||||
requestCommitOK = regexp.MustCompile(`^[0-9a-f]{40}$`)
|
||||
requestRepositoryOK = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]*/[A-Za-z0-9][A-Za-z0-9._-]*$`)
|
||||
)
|
||||
|
||||
// FetchRequest is what a build module's fetch tool is asked: the repository and the full commit, the pull request's
|
||||
// ref when only its head carries the commit, the trunk commit to merge onto for a merge preview, and, for a build of a
|
||||
// module whose every artifact names its paths, that module's directory and its named paths (novox/hq ADR 0499 clause 4).
|
||||
//
|
||||
// **The paths come from the builder, not from a manifest** (ADR 0495 clause 1; the lead's decision L8 of R4e): the
|
||||
// builder fills Dir and Paths from the indexer's Module value (Path, and KeptInputs2.Paths' keys), and the build module
|
||||
// gives the build and test steps a workspace holding those paths, the manifest and its lock files. A request without
|
||||
// them builds from the module's directory, as before.
|
||||
type FetchRequest struct {
|
||||
Repository string `json:"repository"`
|
||||
Commit string `json:"commit"`
|
||||
Ref string `json:"ref,omitempty"`
|
||||
Base string `json:"base,omitempty"`
|
||||
Dir string `json:"dir,omitempty"`
|
||||
Paths []string `json:"paths,omitempty"`
|
||||
}
|
||||
|
||||
// FetchRequestProperties are the JSON schema properties of FetchRequest's dir and paths, for a fetch tool's Input.
|
||||
func FetchRequestProperties() map[string]any {
|
||||
return map[string]any{
|
||||
"dir": map[string]any{"type": "string", "description": "the module's directory in the repository (\".\" for its root), with paths"},
|
||||
"paths": map[string]any{"type": "array", "items": map[string]any{"type": "string"},
|
||||
"description": "the paths the module names, relative to dir, from the indexer's kept inputs: the build sees only these, the manifest and its lock files"},
|
||||
}
|
||||
}
|
||||
|
||||
// Args is r as a tool call's arguments.
|
||||
func (r FetchRequest) Args() map[string]any {
|
||||
args := map[string]any{"repository": r.Repository, "commit": r.Commit}
|
||||
for k, v := range map[string]string{"ref": r.Ref, "base": r.Base, "dir": r.Dir} {
|
||||
if v != "" {
|
||||
args[k] = v
|
||||
}
|
||||
}
|
||||
if len(r.Paths) > 0 {
|
||||
args["paths"] = append([]string(nil), r.Paths...)
|
||||
}
|
||||
return args
|
||||
}
|
||||
|
||||
// ParseFetchRequest reads a fetch tool's arguments; arguments it does not know (a build module's own) are left to the
|
||||
// caller. It refuses a repository that is not <owner>/<name>, a commit or a base that is not 40 hex digits, paths with no
|
||||
// module directory or a directory with no path, a directory leaving the repository and a path leaving the directory,
|
||||
// naming each.
|
||||
func ParseFetchRequest(args map[string]any) (FetchRequest, error) {
|
||||
var r FetchRequest
|
||||
r.Repository, _ = args["repository"].(string)
|
||||
r.Commit, _ = args["commit"].(string)
|
||||
r.Ref, _ = args["ref"].(string)
|
||||
r.Base, _ = args["base"].(string)
|
||||
r.Dir, _ = args["dir"].(string)
|
||||
switch v := args["paths"].(type) {
|
||||
case nil:
|
||||
case []string:
|
||||
r.Paths = append([]string(nil), v...)
|
||||
case []any:
|
||||
for _, p := range v {
|
||||
s, ok := p.(string)
|
||||
if !ok {
|
||||
return FetchRequest{}, fmt.Errorf("paths is a list of strings")
|
||||
}
|
||||
r.Paths = append(r.Paths, s)
|
||||
}
|
||||
default:
|
||||
return FetchRequest{}, fmt.Errorf("paths is a list of strings")
|
||||
}
|
||||
return r, r.Check()
|
||||
}
|
||||
|
||||
// Check is what ParseFetchRequest refuses.
|
||||
func (r FetchRequest) Check() error {
|
||||
if !requestRepositoryOK.MatchString(r.Repository) || strings.Contains(r.Repository, "..") {
|
||||
return fmt.Errorf("%q is not a repository, <owner>/<name>", r.Repository)
|
||||
}
|
||||
if !requestCommitOK.MatchString(r.Commit) {
|
||||
return fmt.Errorf("%q is not a full commit (40 hex digits)", r.Commit)
|
||||
}
|
||||
if r.Base != "" && !requestCommitOK.MatchString(r.Base) {
|
||||
return fmt.Errorf("%q is not a trunk commit (40 hex digits)", r.Base)
|
||||
}
|
||||
switch {
|
||||
case len(r.Paths) > 0 && r.Dir == "":
|
||||
return fmt.Errorf("the request names paths and names no module directory (dir) they are relative to")
|
||||
case len(r.Paths) == 0 && r.Dir != "":
|
||||
return fmt.Errorf("the request names a module directory and no path")
|
||||
}
|
||||
if r.Dir != "" && (outside(r.Dir) || strings.HasPrefix(r.Dir, "/")) {
|
||||
return fmt.Errorf("the module directory %q leaves the repository", r.Dir)
|
||||
}
|
||||
for _, p := range r.Paths {
|
||||
if p == "" || strings.HasPrefix(p, "/") || outside(p) || path.Clean(p) == "." {
|
||||
return fmt.Errorf("the path %q leaves the module's directory or is not a path in it", p)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func outside(p string) bool {
|
||||
c := path.Clean(p)
|
||||
return c == ".." || strings.HasPrefix(c, "../")
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
package buildstep
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// A build's fetch is handed the paths of a module that names them (novox/hq ADR 0499 clause 4): the builder fills them
|
||||
// from the indexer's kept inputs and the build module reads them from the call, never from the manifest (ADR 0495
|
||||
// clause 1). The request survives the trip through a tool call's arguments.
|
||||
func TestAFetchRequestCarriesTheNamedPathsThroughTheToolCall(t *testing.T) {
|
||||
req := FetchRequest{Repository: "novox/mesh-sdk", Commit: strings.Repeat("a", 40), Ref: "refs/pull/7/head",
|
||||
Dir: ".", Paths: []string{"package.json", "src"}}
|
||||
args := req.Args()
|
||||
// A tool call's arguments arrive as JSON: lists of any.
|
||||
args["paths"] = []any{"package.json", "src"}
|
||||
got, err := ParseFetchRequest(args)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(got, req) {
|
||||
t.Fatalf("got %+v, want %+v", got, req)
|
||||
}
|
||||
plain, err := ParseFetchRequest(map[string]any{"repository": "novox/x", "commit": strings.Repeat("b", 40),
|
||||
"base": strings.Repeat("c", 40), "beside": []any{}})
|
||||
if err != nil || plain.Paths != nil || plain.Base != strings.Repeat("c", 40) {
|
||||
t.Fatalf("a request without paths: %+v, %v", plain, err)
|
||||
}
|
||||
|
||||
ok := map[string]any{"repository": "novox/x", "commit": strings.Repeat("b", 40)}
|
||||
for _, c := range []struct {
|
||||
set map[string]any
|
||||
says string
|
||||
}{
|
||||
{map[string]any{"repository": "novox"}, "is not a repository"},
|
||||
{map[string]any{"repository": "novox/../x"}, "is not a repository"},
|
||||
{map[string]any{"commit": "abc"}, "is not a full commit"},
|
||||
{map[string]any{"base": "abc"}, "is not a trunk commit"},
|
||||
{map[string]any{"paths": []any{"src"}}, "names no module directory"},
|
||||
{map[string]any{"dir": "modules/a", "paths": []any{"../b"}}, "leaves the module's directory"},
|
||||
{map[string]any{"dir": "../a", "paths": []any{"src"}}, "leaves the repository"},
|
||||
{map[string]any{"dir": "modules/a", "paths": []any{3}}, "paths is a list of strings"},
|
||||
{map[string]any{"dir": "modules/a"}, "names a module directory and no path"},
|
||||
} {
|
||||
args := map[string]any{}
|
||||
for k, v := range ok {
|
||||
args[k] = v
|
||||
}
|
||||
for k, v := range c.set {
|
||||
args[k] = v
|
||||
}
|
||||
if _, err := ParseFetchRequest(args); err == nil || !strings.Contains(err.Error(), c.says) {
|
||||
t.Errorf("%v: %v, wanted a refusal saying %q", c.set, err, c.says)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
package buildstep
|
||||
|
||||
import (
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
)
|
||||
|
||||
// SeatTools are a build seat's verbs on one node, as its holder serves them (novox/hq ADR 0525 §10; mesh-sdk catalogue
|
||||
// buildSeatVerbs): `<seat>.current` (what the node runs, its capacity, whether it is paused), `<seat>.pause` and
|
||||
// `<seat>.resume`, the pause kept on the node's pause key by the puller. what names the asks in the descriptions ("Go").
|
||||
func SeatTools(seat, what string, p *Puller) []stdio.Tool {
|
||||
verb := func(name, description string, run func() (any, error)) stdio.Tool {
|
||||
return stdio.Tool{Name: seat + "." + name, Description: description,
|
||||
Input: map[string]any{"type": "object", "properties": map[string]any{}},
|
||||
Run: func(map[string]any) (any, error) { return run() }}
|
||||
}
|
||||
current := func() (any, error) {
|
||||
return map[string]any{"node": p.Node, "capacity": p.CapacityNow(), "running": p.Running(), "paused": p.Paused()}, nil
|
||||
}
|
||||
return []stdio.Tool{
|
||||
verb("current", "The "+what+" asks this node runs now (each one's id, kind and when it started), its build capacity, "+
|
||||
"and whether it is paused.", current),
|
||||
verb("pause", "Take no new "+what+" ask on this node until resumed; an ask running here ends in its place. Kept "+
|
||||
"across a restart (the node's pause key).", func() (any, error) {
|
||||
if err := p.Pause("the seat's pause verb"); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return current()
|
||||
}),
|
||||
verb("resume", "Take "+what+" asks again on this node.", func() (any, error) {
|
||||
if err := p.Resume("the seat's resume verb"); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return current()
|
||||
}),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
package buildstep
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// 1887/A9a, ADR 0525 §10: a build seat's verbs are <seat>.current, .pause and .resume; current names the ask running,
|
||||
// pause stops the puller taking asks, resume lets it take them again.
|
||||
func TestSeatToolsCurrentPauseResume(t *testing.T) {
|
||||
src, st := NewFakeAsks(), newSteps()
|
||||
release := make(chan struct{})
|
||||
p := &Puller{Source: src, Language: delivery.AskGo, Node: "shanks", Put: st.put, Capacity: func() int { return 1 },
|
||||
PullWait: 10 * time.Millisecond, Run: func(ctx context.Context, a delivery.StepAsk) (any, error) { <-release; return nil, nil }}
|
||||
tools := map[string]func(map[string]any) (any, error){}
|
||||
for _, tool := range SeatTools("node-build-go", "Go", p) {
|
||||
tools[tool.Name] = tool.Run
|
||||
}
|
||||
for _, name := range []string{"node-build-go.current", "node-build-go.pause", "node-build-go.resume"} {
|
||||
if tools[name] == nil {
|
||||
t.Fatalf("no %s", name)
|
||||
}
|
||||
}
|
||||
a := testAsk(delivery.AskDelivery)
|
||||
_ = src.PublishAsk(a)
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
go p.Pull(ctx)
|
||||
waitFor(t, "the ask running", func() bool { return p.InHand() == 1 })
|
||||
out, _ := tools["node-build-go.current"](nil)
|
||||
cur := out.(map[string]any)
|
||||
running := cur["running"].([]RunningAsk)
|
||||
if len(running) != 1 || running[0].ID != a.ID || running[0].Module != "m" || cur["capacity"] != 1 || cur["paused"] != false {
|
||||
t.Fatalf("current = %+v", cur)
|
||||
}
|
||||
if out, err := tools["node-build-go.pause"](nil); err != nil || out.(map[string]any)["paused"] != true {
|
||||
t.Fatalf("pause = %v, %v", out, err)
|
||||
}
|
||||
close(release)
|
||||
_ = src.PublishAsk(testAsk(delivery.AskDelivery))
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
if src.Waiting(delivery.AskDelivery) != 1 {
|
||||
t.Fatal("a paused node took an ask")
|
||||
}
|
||||
if _, err := tools["node-build-go.resume"](nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waitFor(t, "the ask taken after the resume", func() bool { return src.Waiting(delivery.AskDelivery) == 0 })
|
||||
}
|
||||
+78
-31
@@ -10,6 +10,7 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
@@ -42,43 +43,89 @@ func NewID() string {
|
||||
// the error's text, or refused when the error is a *Refusal; a panic in run ends the step failed, naming it, and
|
||||
// the process goes on. If the running write fails, nothing was started and Start answers the error.
|
||||
func Start(ctx context.Context, put Put, node, tool string, run func(ctx context.Context) (result any, err error)) (string, error) {
|
||||
step := delivery.Step{ID: NewID(), Tool: tool, Node: node, State: delivery.StepRunning, Started: now().UTC()}
|
||||
key, err := delivery.StepKey(node, step.ID)
|
||||
step, key, err := begin(put, node, tool, NewID())
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := write(put, key, step); err != nil {
|
||||
return "", fmt.Errorf("buildstep: writing the running step %s: %w", key, err)
|
||||
}
|
||||
go func() {
|
||||
result, err := protect(ctx, run)
|
||||
step.Ended = now().UTC()
|
||||
switch ref := (*Refusal)(nil); {
|
||||
case err == nil:
|
||||
step.State = delivery.StepPassed
|
||||
if result != nil {
|
||||
raw, merr := json.Marshal(result)
|
||||
if merr != nil {
|
||||
step.State, step.Reason = delivery.StepFailed, "the result is not JSON: "+merr.Error()
|
||||
} else {
|
||||
step.Result = raw
|
||||
}
|
||||
}
|
||||
case errors.As(err, &ref):
|
||||
step.State, step.Reason = delivery.StepRefused, err.Error()
|
||||
default:
|
||||
step.State, step.Reason = delivery.StepFailed, err.Error()
|
||||
}
|
||||
for attempt := 1; ; attempt++ {
|
||||
if werr := write(put, key, step); werr == nil || attempt >= EndAttempts {
|
||||
return
|
||||
}
|
||||
time.Sleep(EndRetry)
|
||||
}
|
||||
}()
|
||||
go end(ctx, put, key, step, run)
|
||||
return step.ID, nil
|
||||
}
|
||||
|
||||
// begin writes the step running under id.
|
||||
func begin(put Put, node, tool, id string) (delivery.Step, string, error) {
|
||||
step := delivery.Step{ID: id, Tool: tool, Node: node, State: delivery.StepRunning, Started: now().UTC()}
|
||||
key, err := delivery.StepKey(node, step.ID)
|
||||
if err != nil {
|
||||
return delivery.Step{}, "", err
|
||||
}
|
||||
if err := write(put, key, step); err != nil {
|
||||
return delivery.Step{}, "", fmt.Errorf("buildstep: writing the running step %s: %w", key, err)
|
||||
}
|
||||
return step, key, nil
|
||||
}
|
||||
|
||||
// end runs run and writes the step's end.
|
||||
func end(ctx context.Context, put Put, key string, step delivery.Step, run func(ctx context.Context) (any, error)) {
|
||||
result, err := protect(ctx, run)
|
||||
_ = finish(put, key, step, result, err, log.Printf) // finish says a refused end itself
|
||||
}
|
||||
|
||||
// finish writes the step's end from run's result or error: passed, failed or refused (a *Refusal). A write that fails is
|
||||
// tried EndAttempts times. When the whole end is refused every time (a result the state will not take: issue 1930), the
|
||||
// refusal is logged and a small failed end without the result is written instead, naming the write's error, so the
|
||||
// step's reader learns it is over and why. finish answers an error only when no end could be written at all, and then
|
||||
// the step's ask is not acknowledged as done.
|
||||
func finish(put Put, key string, step delivery.Step, result any, err error, logf func(format string, args ...any)) error {
|
||||
step.Ended = now().UTC()
|
||||
switch ref := (*Refusal)(nil); {
|
||||
case err == nil:
|
||||
step.State = delivery.StepPassed
|
||||
if result != nil {
|
||||
raw, merr := json.Marshal(result)
|
||||
if merr != nil {
|
||||
step.State, step.Reason = delivery.StepFailed, "the result is not JSON: "+merr.Error()
|
||||
} else {
|
||||
step.Result = raw
|
||||
}
|
||||
}
|
||||
case errors.As(err, &ref):
|
||||
step.State, step.Reason = delivery.StepRefused, err.Error()
|
||||
default:
|
||||
step.State, step.Reason = delivery.StepFailed, err.Error()
|
||||
}
|
||||
werr := writeEnd(put, key, step)
|
||||
if werr == nil {
|
||||
return nil
|
||||
}
|
||||
if logf != nil {
|
||||
logf("the end of the step %s (%s) could not be written in %d tries: %v", key, step.State, EndAttempts, werr)
|
||||
}
|
||||
ended := step.State
|
||||
why := fmt.Sprintf("the step ended %s, and its end could not be written: %v", ended, werr)
|
||||
if step.Reason != "" {
|
||||
why = fmt.Sprintf("the step ended %s (%s), and its end could not be written: %v", ended, step.Reason, werr)
|
||||
}
|
||||
step.State, step.Reason, step.Result = delivery.StepFailed, why, nil
|
||||
if ferr := writeEnd(put, key, step); ferr != nil {
|
||||
if logf != nil {
|
||||
logf("the failed end of the step %s could not be written either: %v", key, ferr)
|
||||
}
|
||||
return fmt.Errorf("buildstep: no end of the step %s could be written: %w", key, ferr)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// writeEnd writes the step's end, trying EndAttempts times.
|
||||
func writeEnd(put Put, key string, step delivery.Step) error {
|
||||
for attempt := 1; ; attempt++ {
|
||||
werr := write(put, key, step)
|
||||
if werr == nil || attempt >= EndAttempts {
|
||||
return werr
|
||||
}
|
||||
time.Sleep(EndRetry)
|
||||
}
|
||||
}
|
||||
|
||||
// protect is run with a panic turned into the error that ends the step.
|
||||
func protect(ctx context.Context, run func(ctx context.Context) (any, error)) (result any, err error) {
|
||||
defer func() {
|
||||
|
||||
+49
@@ -0,0 +1,49 @@
|
||||
// The probe a run step's isolation is proved with (novox/hq ADR 0448; issue 1309, S19): run inside the step, it says
|
||||
// on stdout every way it reached what a step must not, and "shut off" when it reached none. Its arguments: the files
|
||||
// it must not read (the forge key, a bus login), a TCP address and a unix socket of the machine it must not reach, a
|
||||
// name it must not resolve, and a cache file it writes, which must be unchanged after the step.
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net"
|
||||
"os"
|
||||
"time"
|
||||
)
|
||||
|
||||
func main() {
|
||||
if len(os.Args) != 7 {
|
||||
fmt.Println("usage: probe <key> <login> <tcp address> <unix socket> <name> <cache file>")
|
||||
os.Exit(2)
|
||||
}
|
||||
key, login, tcp, socket, name, cache := os.Args[1], os.Args[2], os.Args[3], os.Args[4], os.Args[5], os.Args[6]
|
||||
reached := 0
|
||||
say := func(f string, a ...any) { reached++; fmt.Printf(f+"\n", a...) }
|
||||
for _, f := range []string{key, login} {
|
||||
if _, err := os.ReadFile(f); err == nil {
|
||||
say("read %s", f)
|
||||
}
|
||||
}
|
||||
if c, err := net.DialTimeout("tcp", tcp, 2*time.Second); err == nil {
|
||||
c.Close()
|
||||
say("reached %s", tcp)
|
||||
}
|
||||
if c, err := net.DialTimeout("unix", socket, 2*time.Second); err == nil {
|
||||
c.Close()
|
||||
say("reached %s", socket)
|
||||
}
|
||||
if addrs, err := net.LookupHost(name); err == nil {
|
||||
say("resolved %s to %v", name, addrs)
|
||||
}
|
||||
if home, err := os.UserHomeDir(); err == nil {
|
||||
if entries, err := os.ReadDir(home); err == nil && len(entries) > 0 {
|
||||
say("the home %s holds %d entries", home, len(entries))
|
||||
}
|
||||
}
|
||||
if err := os.WriteFile(cache, []byte("written by the step\n"), 0o644); err != nil {
|
||||
say("could not write the cache under its throwaway layer: %v", err)
|
||||
}
|
||||
if reached == 0 {
|
||||
fmt.Println("shut off")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,123 @@
|
||||
package buildstep
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/forgekey"
|
||||
)
|
||||
|
||||
// ModuleTree is what a merge holds of one module directory: its tree id, the sha256 of its module.json and the sha256
|
||||
// of each lock file it has (go.sum, package-lock.json), from which the checker and the indexer take the module's inputs
|
||||
// (novox/hq ADR 0484 clause 3).
|
||||
type ModuleTree struct {
|
||||
Tree string `json:"tree"`
|
||||
Manifest string `json:"manifest"`
|
||||
Locks map[string]string `json:"locks,omitempty"`
|
||||
// Paths are, of a module whose every artifact names its paths, each named path's git object id at the commit; Tree
|
||||
// is then their delivery.NamedTree with Manifest (novox/hq ADR 0499 clause 2). Absent otherwise.
|
||||
Paths map[string]string `json:"paths,omitempty"`
|
||||
}
|
||||
|
||||
var lockFiles = []string{"go.sum", "package-lock.json"}
|
||||
|
||||
// ModuleTrees are the trees, in commit, of every directory of the workspace dir that holds a module.json, by directory
|
||||
// (the repository's root is "."); for a module whose every artifact names its paths, the named tree of those paths and
|
||||
// their objects (novox/hq ADR 0499 clause 2). The workspace is at commit, so the manifests and the lock files are read from it. env is
|
||||
// the environment git runs with (nil for none). Both build modules' fetch steps answer it for a merge.
|
||||
func ModuleTrees(ctx context.Context, run forgekey.Runner, env []string, dir, commit string) (map[string]ModuleTree, error) {
|
||||
r := run(ctx, env, "git", "-C", dir, "ls-files", "--", "module.json", "*/module.json")
|
||||
if r.Status != 0 || r.Err != "" {
|
||||
return nil, fmt.Errorf("git ls-files: %s", firstLine(r.Err+" "+r.Stderr))
|
||||
}
|
||||
trees := map[string]ModuleTree{}
|
||||
for _, l := range strings.Split(r.Stdout, "\n") {
|
||||
if l = strings.TrimSpace(l); l == "" {
|
||||
continue
|
||||
}
|
||||
mdir := filepath.ToSlash(filepath.Dir(l))
|
||||
spec := commit + ":" + mdir
|
||||
if mdir == "." {
|
||||
spec = commit + "^{tree}"
|
||||
}
|
||||
t := run(ctx, env, "git", "-C", dir, "rev-parse", spec)
|
||||
if t.Status != 0 || t.Err != "" {
|
||||
return nil, fmt.Errorf("git rev-parse %s: %s", spec, firstLine(t.Err+" "+t.Stderr))
|
||||
}
|
||||
mt := ModuleTree{Tree: strings.TrimSpace(t.Stdout)}
|
||||
raw, err := os.ReadFile(filepath.Join(dir, filepath.FromSlash(l)))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
mt.Manifest = sha256Hex(raw)
|
||||
if err := narrow(ctx, run, env, dir, commit, mdir, raw, &mt); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, lock := range lockFiles {
|
||||
if raw, err := os.ReadFile(filepath.Join(dir, filepath.FromSlash(mdir), lock)); err == nil {
|
||||
if mt.Locks == nil {
|
||||
mt.Locks = map[string]string{}
|
||||
}
|
||||
mt.Locks[lock] = sha256Hex(raw)
|
||||
}
|
||||
}
|
||||
trees[mdir] = mt
|
||||
}
|
||||
return trees, nil
|
||||
}
|
||||
|
||||
// narrow sets mt's Paths and its named Tree when the module's manifest names paths on every artifact (novox/hq ADR 0499
|
||||
// clause 2): one `git ls-tree -z` gives each named path's object at commit. The merge preview reads the manifest here, as
|
||||
// the indexer does, with catalogue.ParseManifestForInputs (issue 1751), and no other build step reads paths from one (ADR 0495 clause 1). A manifest that does not parse
|
||||
// is not narrowed: the indexer gives it none. A named path absent at commit fails, naming the module and the path.
|
||||
func narrow(ctx context.Context, run forgekey.Runner, env []string, dir, commit, mdir string, raw []byte, mt *ModuleTree) error {
|
||||
m, err := catalogue.ParseManifestForInputs(raw)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
paths, every := m.NamedPaths()
|
||||
if !every {
|
||||
return nil
|
||||
}
|
||||
full := make([]string, len(paths))
|
||||
for i, p := range paths {
|
||||
full[i] = path.Join(mdir, p)
|
||||
}
|
||||
r := run(ctx, env, "git", append([]string{"-C", dir, "ls-tree", "-z", commit, "--"}, full...)...)
|
||||
if r.Status != 0 || r.Err != "" {
|
||||
return fmt.Errorf("git ls-tree %s of the module %s: %s", commit, mdir, firstLine(r.Err+" "+r.Stderr))
|
||||
}
|
||||
objects := map[string]string{}
|
||||
for _, l := range strings.Split(r.Stdout, "\x00") {
|
||||
meta, name, ok := strings.Cut(l, "\t")
|
||||
fields := strings.Fields(meta)
|
||||
if !ok || len(fields) != 3 {
|
||||
continue
|
||||
}
|
||||
objects[name] = fields[2]
|
||||
}
|
||||
mt.Paths = map[string]string{}
|
||||
for i, p := range paths {
|
||||
object := objects[full[i]]
|
||||
if object == "" {
|
||||
return fmt.Errorf("the module %s names the path %q, which is not in %s", mdir, p, commit)
|
||||
}
|
||||
mt.Paths[p] = object
|
||||
}
|
||||
named, err := delivery.NamedTree(mt.Manifest, mt.Paths)
|
||||
if err != nil {
|
||||
return fmt.Errorf("the module %s: %w", mdir, err)
|
||||
}
|
||||
mt.Tree = named
|
||||
return nil
|
||||
}
|
||||
|
||||
func sha256Hex(b []byte) string { h := sha256.Sum256(b); return hex.EncodeToString(h[:]) }
|
||||
@@ -0,0 +1,63 @@
|
||||
//go:build integration
|
||||
|
||||
package buildstep
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/forgekey"
|
||||
)
|
||||
|
||||
// On a real repository shaped like mesh-sdk (novox/hq ADR 0499 clauses 2 and 6), a commit touching only the Go package
|
||||
// leaves the TypeScript package's tree value equal, and a commit touching its source changes it; a module that names no
|
||||
// path keeps its directory's tree id.
|
||||
func TestAGoOnlyCommitLeavesTheNamedTreeEqualOnRealGit(t *testing.T) {
|
||||
repo := t.TempDir()
|
||||
realGit(t, repo, "init", "-q", "-b", "main")
|
||||
writeFile(t, repo, "module.json", `{"module":"mesh-sdk","version":"1","build":{"artifacts":[`+
|
||||
`{"name":"lib","kind":"package","language":"typescript","paths":["src","test","package.json","package-lock.json","tsconfig.json"]}]}}`)
|
||||
writeFile(t, repo, "src/index.ts", "export const a = 1\n")
|
||||
writeFile(t, repo, "test/a.test.ts", "// a\n")
|
||||
writeFile(t, repo, "package.json", "{}\n")
|
||||
writeFile(t, repo, "package-lock.json", "{}\n")
|
||||
writeFile(t, repo, "tsconfig.json", "{}\n")
|
||||
writeFile(t, repo, "go/delivery/x.go", "package delivery\n")
|
||||
writeFile(t, repo, "other/module.json", `{"module":"other","version":"1"}`)
|
||||
writeFile(t, repo, "other/a.txt", "a\n")
|
||||
commit := func(msg string) string {
|
||||
realGit(t, repo, "add", "-A")
|
||||
realGit(t, repo, "commit", "-q", "-m", msg)
|
||||
return realGit(t, repo, "rev-parse", "HEAD")
|
||||
}
|
||||
trees := func(c string) map[string]ModuleTree {
|
||||
got, err := ModuleTrees(context.Background(), forgekey.ExecRunner, nil, repo, c)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return got
|
||||
}
|
||||
first := trees(commit("first"))
|
||||
if !strings.HasPrefix(first["."].Tree, "named:") || len(first["."].Paths) != 5 {
|
||||
t.Fatalf("the root module was not narrowed: %+v", first["."])
|
||||
}
|
||||
if want := realGit(t, repo, "rev-parse", "HEAD:other"); first["other"].Tree != want || first["other"].Paths != nil {
|
||||
t.Fatalf("a module naming no path: %+v, want its directory's tree %s", first["other"], want)
|
||||
}
|
||||
|
||||
writeFile(t, repo, "go/delivery/x.go", "package delivery\n\nconst B = 2\n")
|
||||
goOnly := trees(commit("go only"))
|
||||
if goOnly["."].Tree != first["."].Tree {
|
||||
t.Fatalf("a Go-only commit changed the TypeScript package's tree: %s, was %s", goOnly["."].Tree, first["."].Tree)
|
||||
}
|
||||
|
||||
writeFile(t, repo, "src/index.ts", "export const a = 2\n")
|
||||
src := trees(commit("src"))
|
||||
if src["."].Tree == first["."].Tree {
|
||||
t.Fatal("a commit to src/ left the TypeScript package's tree equal")
|
||||
}
|
||||
if src["."].Paths["src"] != realGit(t, repo, "rev-parse", "HEAD:src") {
|
||||
t.Fatalf("the object of src is %s, git says %s", src["."].Paths["src"], realGit(t, repo, "rev-parse", "HEAD:src"))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,113 @@
|
||||
package buildstep
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/forgekey"
|
||||
)
|
||||
|
||||
// A module whose every artifact names its paths takes its tree from them (novox/hq ADR 0499 clause 2): ModuleTrees asks
|
||||
// git for each named path's object at the commit, keeps them in Paths and answers their delivery.NamedTree as Tree; a
|
||||
// module naming none keeps its directory's tree, as before.
|
||||
func TestModuleTreesNarrowsAModuleWhoseArtifactsNameTheirPaths(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
sdk := `{"module":"mesh-sdk","version":"1","build":{"artifacts":[{"name":"lib","kind":"package","language":"typescript","paths":["src","package.json"]}]}}`
|
||||
writeTreeFile(t, root, "module.json", sdk)
|
||||
writeTreeFile(t, root, "modules/b/module.json", `{"module":"b","version":"1"}`)
|
||||
merge := strings.Repeat("c", 40)
|
||||
var lsTree []string
|
||||
run := func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran {
|
||||
switch args[2] {
|
||||
case "ls-files":
|
||||
return forgekey.Ran{Stdout: "module.json\nmodules/b/module.json\n"}
|
||||
case "rev-parse":
|
||||
return forgekey.Ran{Stdout: "tree-of-" + args[3] + "\n"}
|
||||
case "ls-tree":
|
||||
lsTree = append(lsTree, strings.Join(args[4:], " "))
|
||||
return forgekey.Ran{Stdout: "100644 blob " + strings.Repeat("b", 40) + "\tpackage.json\x00" +
|
||||
"040000 tree " + strings.Repeat("a", 40) + "\tsrc\x00"}
|
||||
}
|
||||
return forgekey.Ran{Status: 1, Stderr: "unexpected " + args[2]}
|
||||
}
|
||||
got, err := ModuleTrees(context.Background(), run, nil, root, merge)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
paths := map[string]string{"src": strings.Repeat("a", 40), "package.json": strings.Repeat("b", 40)}
|
||||
want, _ := delivery.NamedTree(hexOf(sdk), paths)
|
||||
if g := got["."]; g.Tree != want || len(g.Paths) != 2 || g.Paths["src"] != paths["src"] || g.Paths["package.json"] != paths["package.json"] {
|
||||
t.Fatalf("the root module = %+v, want tree %s and paths %v", g, want, paths)
|
||||
}
|
||||
if len(lsTree) != 1 || lsTree[0] != merge+" -- package.json src" {
|
||||
t.Fatalf("git ls-tree ran as %v", lsTree)
|
||||
}
|
||||
if g := got["modules/b"]; g.Tree != "tree-of-"+merge+":modules/b" || g.Paths != nil {
|
||||
t.Fatalf("a module naming no path was narrowed: %+v", g)
|
||||
}
|
||||
}
|
||||
|
||||
// A named path absent at the commit fails the answer, naming the module and the path: a tree computed without it would
|
||||
// pin a build source that does not exist.
|
||||
func TestModuleTreesFailsNamingANamedPathAbsentAtTheCommit(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
writeTreeFile(t, root, "modules/a/module.json", `{"module":"a","version":"1","build":{"artifacts":[{"name":"lib","kind":"package","language":"typescript","paths":["src","gone"]}]}}`)
|
||||
run := func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran {
|
||||
switch args[2] {
|
||||
case "ls-files":
|
||||
return forgekey.Ran{Stdout: "modules/a/module.json\n"}
|
||||
case "rev-parse":
|
||||
return forgekey.Ran{Stdout: "t\n"}
|
||||
case "ls-tree":
|
||||
if strings.Join(args[4:], " ") != strings.Repeat("c", 40)+" -- modules/a/gone modules/a/src" {
|
||||
return forgekey.Ran{Status: 1, Stderr: "wrong arguments " + strings.Join(args[3:], " ")}
|
||||
}
|
||||
return forgekey.Ran{Stdout: "040000 tree " + strings.Repeat("a", 40) + "\tmodules/a/src\x00"}
|
||||
}
|
||||
return forgekey.Ran{Status: 1}
|
||||
}
|
||||
_, err := ModuleTrees(context.Background(), run, nil, root, strings.Repeat("c", 40))
|
||||
if err == nil || !strings.Contains(err.Error(), `"gone"`) || !strings.Contains(err.Error(), "modules/a") {
|
||||
t.Fatalf("an absent named path gave %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// A manifest whose config value falls back to a predefined key is narrowed here as the indexer narrows it: both read it
|
||||
// with catalogue.ParseManifestForInputs (issue 1751), so a build module holding no published list computes the same
|
||||
// tree (ADR 0499 clause 2).
|
||||
func TestModuleTreesNarrowsAManifestFallingBackToAPredefinedKey(t *testing.T) {
|
||||
was := catalogue.ReplacePublished(nil) // the build modules' case: no published list
|
||||
defer catalogue.ReplacePublished(was)
|
||||
root := t.TempDir()
|
||||
raw := `{"module":"xterm","version":"1","config":{"size":{"type":"number","description":"The text size.","predefined":"font-size","default":12,"runtime":true}},` +
|
||||
`"build":{"artifacts":[{"name":"lib","kind":"package","language":"typescript","paths":["src"]}]}}`
|
||||
if _, err := catalogue.ParseManifest([]byte(raw)); err == nil {
|
||||
t.Fatal("ParseManifest reads the manifest with no list; the test proves nothing")
|
||||
}
|
||||
if _, err := catalogue.ParseManifestForInputs([]byte(raw)); err != nil {
|
||||
t.Fatalf("the indexer's read refuses the manifest: %v", err)
|
||||
}
|
||||
writeTreeFile(t, root, "module.json", raw)
|
||||
run := func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran {
|
||||
switch args[2] {
|
||||
case "ls-files":
|
||||
return forgekey.Ran{Stdout: "module.json\n"}
|
||||
case "rev-parse":
|
||||
return forgekey.Ran{Stdout: "whole\n"}
|
||||
case "ls-tree":
|
||||
return forgekey.Ran{Stdout: "040000 tree " + strings.Repeat("a", 40) + "\tsrc\x00"}
|
||||
}
|
||||
return forgekey.Ran{Status: 1}
|
||||
}
|
||||
got, err := ModuleTrees(context.Background(), run, nil, root, strings.Repeat("c", 40))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want, _ := delivery.NamedTree(hexOf(raw), map[string]string{"src": strings.Repeat("a", 40)})
|
||||
if g := got["."]; g.Tree != want || g.Paths["src"] == "" {
|
||||
t.Fatalf("the module = %+v, want the named tree %s", g, want)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,93 @@
|
||||
package buildstep
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/forgekey"
|
||||
)
|
||||
|
||||
func hexOf(s string) string { h := sha256.Sum256([]byte(s)); return hex.EncodeToString(h[:]) }
|
||||
|
||||
func writeTreeFile(t *testing.T, root, name, content string) {
|
||||
t.Helper()
|
||||
p := filepath.Join(root, filepath.FromSlash(name))
|
||||
if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(p, []byte(content), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// ModuleTrees answers, for each directory holding a module.json (the root as "."), the tree id git gives, the sha256 of
|
||||
// module.json and of each lock file present, and reads them from the workspace.
|
||||
func TestModuleTreesAnswersTreeManifestAndLocksPerModuleDirectory(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
writeTreeFile(t, root, "module.json", `{"root":1}`)
|
||||
writeTreeFile(t, root, "go.sum", "rootsum")
|
||||
writeTreeFile(t, root, "modules/a/module.json", `{"a":1}`)
|
||||
writeTreeFile(t, root, "modules/a/go.sum", "asum")
|
||||
writeTreeFile(t, root, "modules/a/package-lock.json", "alock")
|
||||
writeTreeFile(t, root, "modules/b/module.json", `{"b":1}`)
|
||||
merge := strings.Repeat("c", 40)
|
||||
var calls []string
|
||||
var envs [][]string
|
||||
run := func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran {
|
||||
calls = append(calls, name+" "+strings.Join(args[2:], " "))
|
||||
envs = append(envs, env)
|
||||
switch args[2] {
|
||||
case "ls-files":
|
||||
return forgekey.Ran{Stdout: "module.json\nmodules/a/module.json\nmodules/b/module.json\n"}
|
||||
case "rev-parse":
|
||||
return forgekey.Ran{Stdout: "tree-of-" + args[3] + "\n"}
|
||||
}
|
||||
return forgekey.Ran{Status: 1, Stderr: "unexpected"}
|
||||
}
|
||||
got, err := ModuleTrees(context.Background(), run, []string{"X=1"}, root, merge)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := map[string]ModuleTree{
|
||||
".": {Tree: "tree-of-" + merge + "^{tree}", Manifest: hexOf(`{"root":1}`), Locks: map[string]string{"go.sum": hexOf("rootsum")}},
|
||||
"modules/a": {Tree: "tree-of-" + merge + ":modules/a", Manifest: hexOf(`{"a":1}`), Locks: map[string]string{"go.sum": hexOf("asum"), "package-lock.json": hexOf("alock")}},
|
||||
"modules/b": {Tree: "tree-of-" + merge + ":modules/b", Manifest: hexOf(`{"b":1}`)},
|
||||
}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Errorf("trees = %+v\nwant %+v", got, want)
|
||||
}
|
||||
if got["modules/b"].Locks != nil {
|
||||
t.Error("a module with no lock file carries a lock map")
|
||||
}
|
||||
for _, e := range envs {
|
||||
if len(e) != 1 || e[0] != "X=1" {
|
||||
t.Errorf("git ran with env %v, want the one given", e)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestModuleTreesFailsNamingTheGitCommandOrTheUnreadableManifest(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
fail := func(on string) forgekey.Runner {
|
||||
return func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran {
|
||||
if args[2] == on {
|
||||
return forgekey.Ran{Status: 128, Stderr: "fatal: no\n"}
|
||||
}
|
||||
return forgekey.Ran{Stdout: "modules/a/module.json\n"}
|
||||
}
|
||||
}
|
||||
for _, on := range []string{"ls-files", "rev-parse"} {
|
||||
if _, err := ModuleTrees(context.Background(), fail(on), nil, root, sha); err == nil || !strings.Contains(err.Error(), on) || !strings.Contains(err.Error(), "fatal: no") {
|
||||
t.Errorf("%s failing: %v", on, err)
|
||||
}
|
||||
}
|
||||
if _, err := ModuleTrees(context.Background(), fail("none"), nil, root, sha); err == nil {
|
||||
t.Error("a module.json that is not in the workspace was read")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,160 @@
|
||||
package buildstep
|
||||
|
||||
// A build module's room to work and its workspaces (novox/hq issue 1839, ADR 0509): a step is refused while the
|
||||
// filesystem it writes to has less free than the floor, and a workspace is removed when its caller releases it, or,
|
||||
// unheld, once its last use is past a bound (a lost caller's), never while a running step holds it.
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"sort"
|
||||
"sync"
|
||||
"syscall"
|
||||
"time"
|
||||
)
|
||||
|
||||
// FreeSpaceFloor is the free space a build module needs on its work and cache filesystems to take a step: the floor
|
||||
// build-agent keeps (mesh-controller internal/builder/floor.go, issue 1834).
|
||||
const FreeSpaceFloor uint64 = 20 << 30
|
||||
|
||||
// UnheldBound is how long a workspace no step holds is kept after its last use, when its caller never released it
|
||||
// (ADR 0509: a lost caller's workspace goes after two hours).
|
||||
const UnheldBound = 2 * time.Hour
|
||||
|
||||
// FreeBytes reads the bytes an unprivileged writer can still use on the filesystem holding path.
|
||||
func FreeBytes(path string) (uint64, error) {
|
||||
var s syscall.Statfs_t
|
||||
if err := syscall.Statfs(path, &s); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return s.Bavail * uint64(s.Bsize), nil
|
||||
}
|
||||
|
||||
// Floor refuses a step while a filesystem it writes to has less free than Bytes. A zero Bytes checks nothing.
|
||||
type Floor struct {
|
||||
// Node is named in the refusal.
|
||||
Node string
|
||||
Bytes uint64
|
||||
// Free reads a path's free bytes; FreeBytes when nil. A test answers for a disk it does not have.
|
||||
Free func(path string) (uint64, error)
|
||||
}
|
||||
|
||||
// Check answers a *Refusal, "refused: <node> has N GiB free under the floor of F GiB (on <path>)", when any path's
|
||||
// filesystem has less free than the floor, or its free space cannot be read; nil otherwise. A path not made yet is
|
||||
// read at its nearest folder that is; an empty path is skipped.
|
||||
func (f Floor) Check(paths ...string) error {
|
||||
if f.Bytes == 0 {
|
||||
return nil
|
||||
}
|
||||
free := f.Free
|
||||
if free == nil {
|
||||
free = FreeBytes
|
||||
}
|
||||
for _, p := range paths {
|
||||
if p == "" {
|
||||
continue
|
||||
}
|
||||
at := p
|
||||
for {
|
||||
if _, err := os.Stat(at); err == nil || filepath.Dir(at) == at {
|
||||
break
|
||||
}
|
||||
at = filepath.Dir(at)
|
||||
}
|
||||
n, err := free(at)
|
||||
if err != nil {
|
||||
return &Refusal{Text: fmt.Sprintf("refused: the free space of %s on %s cannot be read: %v", p, f.Node, err)}
|
||||
}
|
||||
if n < f.Bytes {
|
||||
return &Refusal{Text: fmt.Sprintf("refused: %s has %d GiB free under the floor of %d GiB (on %s)", f.Node, n>>30, f.Bytes>>30, p),
|
||||
Want: fmt.Sprintf("%d GiB", f.Bytes>>30), Have: fmt.Sprintf("%d GiB", n>>30)}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
var workspaceID = regexp.MustCompile(`^[a-z0-9]+$`)
|
||||
|
||||
// Workspaces are a build module's workspaces, each <Dir>/<id>, and which of them running steps hold.
|
||||
type Workspaces struct {
|
||||
Dir string
|
||||
|
||||
mu sync.Mutex
|
||||
held map[string]int
|
||||
}
|
||||
|
||||
// Hold marks the workspace held by a running step until the answered func is called; both mark it used now, so the
|
||||
// sweep counts its bound from the step's end. An id that is not a workspace's holds nothing.
|
||||
func (w *Workspaces) Hold(id string) func() {
|
||||
if !workspaceID.MatchString(id) {
|
||||
return func() {}
|
||||
}
|
||||
w.mu.Lock()
|
||||
if w.held == nil {
|
||||
w.held = map[string]int{}
|
||||
}
|
||||
w.held[id]++
|
||||
w.mu.Unlock()
|
||||
w.touch(id)
|
||||
var once sync.Once
|
||||
return func() {
|
||||
once.Do(func() {
|
||||
w.touch(id)
|
||||
w.mu.Lock()
|
||||
if w.held[id]--; w.held[id] <= 0 {
|
||||
delete(w.held, id)
|
||||
}
|
||||
w.mu.Unlock()
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func (w *Workspaces) touch(id string) {
|
||||
now := time.Now()
|
||||
_ = os.Chtimes(filepath.Join(w.Dir, id), now, now) // a workspace not made yet has nothing to mark
|
||||
}
|
||||
|
||||
// Release removes the workspace at once: its caller's chain of steps has ended. One already gone is no error; one a
|
||||
// running step holds is refused and kept; an id that is not a workspace's is refused.
|
||||
func (w *Workspaces) Release(id string) error {
|
||||
if !workspaceID.MatchString(id) {
|
||||
return fmt.Errorf("%q is not a workspace id", id)
|
||||
}
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
if w.held[id] > 0 {
|
||||
return fmt.Errorf("the workspace %s is held by a running step", id)
|
||||
}
|
||||
if err := os.RemoveAll(filepath.Join(w.Dir, id)); err != nil {
|
||||
return fmt.Errorf("the workspace %s could not be removed: %w", id, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Sweep removes every workspace no step holds whose last use is older than bound, and answers the ids it removed,
|
||||
// sorted. It is the safety net for a caller that never released its workspace.
|
||||
func (w *Workspaces) Sweep(now time.Time, bound time.Duration) []string {
|
||||
entries, err := os.ReadDir(w.Dir)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
removed := []string{}
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
for _, e := range entries {
|
||||
if !e.IsDir() || !workspaceID.MatchString(e.Name()) || w.held[e.Name()] > 0 {
|
||||
continue
|
||||
}
|
||||
info, err := e.Info()
|
||||
if err != nil || now.Sub(info.ModTime()) < bound {
|
||||
continue
|
||||
}
|
||||
if os.RemoveAll(filepath.Join(w.Dir, e.Name())) == nil {
|
||||
removed = append(removed, e.Name())
|
||||
}
|
||||
}
|
||||
sort.Strings(removed)
|
||||
return removed
|
||||
}
|
||||
@@ -0,0 +1,113 @@
|
||||
package buildstep
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// 1839/AC1: a filesystem under the floor refuses the step, naming the node, what is free and the floor.
|
||||
func TestAFilesystemUnderTheFloorRefusesNamingNodeFreeAndFloor(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
f := Floor{Node: "ace", Bytes: 20 << 30, Free: func(string) (uint64, error) { return 4<<30 + 5, nil }}
|
||||
err := f.Check(dir)
|
||||
var ref *Refusal
|
||||
if !errors.As(err, &ref) {
|
||||
t.Fatalf("Check = %v, want a *Refusal", err)
|
||||
}
|
||||
if want := "refused: ace has 4 GiB free under the floor of 20 GiB (on " + dir + ")"; err.Error() != want {
|
||||
t.Errorf("Check = %q, want %q", err, want)
|
||||
}
|
||||
}
|
||||
|
||||
// 1839/AC1: at or above the floor, and with no floor set, nothing is refused.
|
||||
func TestAtTheFloorOrWithNoFloorNothingIsRefused(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
if err := (Floor{Node: "ace", Bytes: 20 << 30, Free: func(string) (uint64, error) { return 20 << 30, nil }}).Check(dir); err != nil {
|
||||
t.Errorf("at the floor: %v", err)
|
||||
}
|
||||
if err := (Floor{Node: "ace", Free: func(string) (uint64, error) { return 0, nil }}).Check(dir); err != nil {
|
||||
t.Errorf("no floor: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// 1839/AC1: a path not made yet is read at its nearest folder that is; a free space that cannot be read is a refusal.
|
||||
func TestAPathNotMadeIsReadAtItsParentAndAnUnreadableOneRefuses(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
var read string
|
||||
f := Floor{Node: "ace", Bytes: 1, Free: func(p string) (uint64, error) { read = p; return 2, nil }}
|
||||
if err := f.Check(filepath.Join(dir, "not", "made")); err != nil || read != dir {
|
||||
t.Errorf("Check = %v, read %q, want nil and %q", err, read, dir)
|
||||
}
|
||||
f.Free = func(string) (uint64, error) { return 0, errors.New("statfs: boom") }
|
||||
var ref *Refusal
|
||||
if err := f.Check(dir); !errors.As(err, &ref) {
|
||||
t.Errorf("Check = %v, want a refusal", err)
|
||||
}
|
||||
}
|
||||
|
||||
func makeWorkspace(t *testing.T, w *Workspaces, id string, age time.Duration) string {
|
||||
t.Helper()
|
||||
p := filepath.Join(w.Dir, id)
|
||||
if err := os.MkdirAll(filepath.Join(p, "tree"), 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
at := time.Now().Add(-age)
|
||||
if err := os.Chtimes(p, at, at); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
func gone(p string) bool { _, err := os.Stat(p); return os.IsNotExist(err) }
|
||||
|
||||
// 1839/AC2: a released workspace is gone at once; one already gone is no error; one a running step holds is refused
|
||||
// and kept; an id that is not a workspace's is refused.
|
||||
func TestReleaseRemovesAWorkspaceAtOnce(t *testing.T) {
|
||||
w := &Workspaces{Dir: t.TempDir()}
|
||||
p := makeWorkspace(t, w, "abc1", 0)
|
||||
if err := w.Release("abc1"); err != nil || !gone(p) {
|
||||
t.Fatalf("Release = %v, gone %v", err, gone(p))
|
||||
}
|
||||
if err := w.Release("abc1"); err != nil {
|
||||
t.Errorf("a second release = %v, want nil", err)
|
||||
}
|
||||
held := makeWorkspace(t, w, "abc2", 0)
|
||||
done := w.Hold("abc2")
|
||||
if err := w.Release("abc2"); err == nil || gone(held) {
|
||||
t.Errorf("Release of a held workspace = %v, gone %v; want refused and kept", err, gone(held))
|
||||
}
|
||||
done()
|
||||
if err := w.Release("abc2"); err != nil || !gone(held) {
|
||||
t.Errorf("Release after the hold ended = %v, gone %v", err, gone(held))
|
||||
}
|
||||
if err := w.Release("../etc"); err == nil {
|
||||
t.Errorf("Release of ../etc answered nil")
|
||||
}
|
||||
}
|
||||
|
||||
// 1839/AC3: the sweep removes a workspace no step holds whose last use is past the bound, and keeps a held one and a
|
||||
// recent one; a hold ending counts as a use.
|
||||
func TestSweepRemovesUnheldWorkspacesPastTheBound(t *testing.T) {
|
||||
w := &Workspaces{Dir: t.TempDir()}
|
||||
old := makeWorkspace(t, w, "old1", 3*time.Hour)
|
||||
heldOld := makeWorkspace(t, w, "held1", 3*time.Hour)
|
||||
recent := makeWorkspace(t, w, "new1", time.Minute)
|
||||
if err := os.WriteFile(filepath.Join(w.Dir, "not-a-workspace.txt"), nil, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
done := w.Hold("held1")
|
||||
removed := w.Sweep(time.Now(), 2*time.Hour)
|
||||
if !gone(old) || gone(heldOld) || gone(recent) {
|
||||
t.Errorf("after the sweep: old gone %v, held kept %v, recent kept %v", gone(old), !gone(heldOld), !gone(recent))
|
||||
}
|
||||
if len(removed) != 1 || removed[0] != "old1" {
|
||||
t.Errorf("Sweep answered %v, want [old1]", removed)
|
||||
}
|
||||
done()
|
||||
if w.Sweep(time.Now(), 2*time.Hour); gone(heldOld) {
|
||||
t.Errorf("a workspace whose hold just ended was swept: its end is a use")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,124 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"sort"
|
||||
)
|
||||
|
||||
// BucketShare is one entry of `shares` on a module's requirement of `s3-bucket` (novox/hq ADR 0468 §8): who else may
|
||||
// put or get in the consumer's bucket, and with which methods. Not the NFS shares of shares.go (ADR 0263).
|
||||
//
|
||||
// "requires": [{"name": "s3-bucket", "shares": [{"to": "seat:node-files", "methods": ["put"]},
|
||||
// {"to": "print-and-scan", "methods": ["get"]}]}]
|
||||
//
|
||||
// `to` is a module name, or `seat:<seat name>`. `methods` is a non-empty subset of get and put, without a repeat.
|
||||
// Shares are allowed on `s3-bucket` only, and no two entries name the same `to`.
|
||||
type BucketShare struct {
|
||||
To string `json:"to"`
|
||||
Methods []string `json:"methods"`
|
||||
}
|
||||
|
||||
// BucketRequirement is the requirement `shares` may be written on.
|
||||
const BucketRequirement = "s3-bucket"
|
||||
|
||||
// RequireSharesKey is where a stored manifest keeps the shares of its requirements: beside `requires`, which stays a
|
||||
// plain list of names, for the reason OptionalRequiresKey is beside it. An older controller sets the unknown key
|
||||
// aside instead of failing the catalogue.
|
||||
const RequireSharesKey = "requires-shares"
|
||||
|
||||
// SharesValue is the key of the value a consumer gives the provider of s3-bucket in its grant: the list of
|
||||
// {to, methods} sorted by `to`. A consumer's own `contributes` may not name it.
|
||||
const SharesValue = "shares"
|
||||
|
||||
var shareTarget = regexp.MustCompile(`^(seat:)?[a-z0-9][a-z0-9-]*$`)
|
||||
|
||||
// checkBucketShares refuses a malformed `shares` of one requirement, naming the entry and ADR 0468 §8.
|
||||
func checkBucketShares(requirement string, shares []BucketShare) error {
|
||||
if len(shares) == 0 {
|
||||
return nil
|
||||
}
|
||||
if requirement != BucketRequirement {
|
||||
return fmt.Errorf("requires %s: `shares` is allowed on %s only (novox/hq ADR 0468 §8)", requirement, BucketRequirement)
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
for i, s := range shares {
|
||||
at := fmt.Sprintf("requires %s: shares[%d] (to %q)", requirement, i, s.To)
|
||||
if !shareTarget.MatchString(s.To) {
|
||||
return fmt.Errorf("%s: `to` is a module name or seat:<seat name> (novox/hq ADR 0468 §8)", at)
|
||||
}
|
||||
if seen[s.To] {
|
||||
return fmt.Errorf("%s: a second entry for the same `to` (novox/hq ADR 0468 §8)", at)
|
||||
}
|
||||
seen[s.To] = true
|
||||
if len(s.Methods) == 0 {
|
||||
return fmt.Errorf("%s: `methods` is empty; name get, put or both (novox/hq ADR 0468 §8)", at)
|
||||
}
|
||||
used := map[string]bool{}
|
||||
for _, method := range s.Methods {
|
||||
if method != "get" && method != "put" {
|
||||
return fmt.Errorf("%s: method %q is not get or put (novox/hq ADR 0468 §8)", at, method)
|
||||
}
|
||||
if used[method] {
|
||||
return fmt.Errorf("%s: method %q twice (novox/hq ADR 0468 §8)", at, method)
|
||||
}
|
||||
used[method] = true
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// readRequireShares reads the stored form's side key.
|
||||
func readRequireShares(raw json.RawMessage) (map[string][]BucketShare, error) {
|
||||
var out map[string][]BucketShare
|
||||
if err := json.Unmarshal(raw, &out); err != nil {
|
||||
return nil, fmt.Errorf("%s: an object of requirement to a list of {to, methods}: %w", RequireSharesKey, err)
|
||||
}
|
||||
for requirement, shares := range out {
|
||||
if err := checkBucketShares(requirement, shares); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// sortedShares is a requirement's shares as they are handed to the provider: sorted by `to` (a file whose lines move
|
||||
// looks changed), each entry's methods sorted, and a copy so the manifest is not reordered.
|
||||
func sortedShares(shares []BucketShare) []BucketShare {
|
||||
out := make([]BucketShare, 0, len(shares))
|
||||
for _, s := range shares {
|
||||
methods := append([]string(nil), s.Methods...)
|
||||
sort.Strings(methods)
|
||||
out = append(out, BucketShare{To: s.To, Methods: methods})
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool { return out[i].To < out[j].To })
|
||||
return out
|
||||
}
|
||||
|
||||
// withShares adds this module's shares of the requirement to the values it gives the provider. A module whose own
|
||||
// `contributes` already names `shares` is refused: the field is the only way to say it (ADR 0468 §8).
|
||||
func withShares(m Manifest, requirement string, values map[string]any) (map[string]any, error) {
|
||||
shares := m.RequireShares[requirement]
|
||||
if _, named := values[SharesValue]; named && requirement == BucketRequirement {
|
||||
return nil, fmt.Errorf("%s contributing to %s names `shares`; write it on its requirement instead (novox/hq ADR 0468 §8)",
|
||||
m.Module, requirement)
|
||||
}
|
||||
if len(shares) == 0 {
|
||||
return values, nil
|
||||
}
|
||||
out := make(map[string]any, len(values)+1)
|
||||
for k, v := range values {
|
||||
out[k] = v
|
||||
}
|
||||
listed := []any{}
|
||||
for _, s := range sortedShares(shares) {
|
||||
methods := []any{}
|
||||
for _, method := range s.Methods {
|
||||
methods = append(methods, method)
|
||||
}
|
||||
listed = append(listed, map[string]any{"to": s.To, "methods": methods})
|
||||
}
|
||||
out[SharesValue] = listed
|
||||
return out, nil
|
||||
}
|
||||
@@ -0,0 +1,185 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// Issue 1591: `shares` on a requirement of s3-bucket (novox/hq ADR 0468 §8).
|
||||
|
||||
const filesWithShares = `{"module":"files","version":"1","requires":["shell",
|
||||
{"name":"s3-bucket","shares":[{"to":"seat:node-files","methods":["put"]},{"to":"print-and-scan","methods":["get","put"]}]}]}`
|
||||
|
||||
func minioLike() Manifest {
|
||||
return Manifest{Module: "minio", Version: "1", Provides: Offers("s3-bucket"),
|
||||
Receives: map[string]string{"s3-bucket": "/grants/mesh.json"}}
|
||||
}
|
||||
|
||||
func givenToMinio(t *testing.T, out []map[string]any) []Contribution {
|
||||
t.Helper()
|
||||
for _, r := range out {
|
||||
if r["path"] != "/grants/mesh.json" {
|
||||
continue
|
||||
}
|
||||
var parsed struct {
|
||||
Given []Contribution `json:"given"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(r["content"].(string)), &parsed); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return parsed.Given
|
||||
}
|
||||
t.Fatalf("minio was given no file: %v", out)
|
||||
return nil
|
||||
}
|
||||
|
||||
func sharesOf(t *testing.T, values map[string]any) string {
|
||||
t.Helper()
|
||||
raw, err := json.Marshal(values["shares"])
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return string(raw)
|
||||
}
|
||||
|
||||
const sortedSharesJSON = `[{"methods":["get","put"],"to":"print-and-scan"},{"methods":["put"],"to":"seat:node-files"}]`
|
||||
|
||||
// 1591/AC1: parsed, the shares are on the manifest; stored, `requires` is plain names and the side key holds them.
|
||||
func TestSharesParseAndAreStoredBesideRequires(t *testing.T) {
|
||||
m := mustParse(t, filesWithShares)
|
||||
want := []BucketShare{{To: "seat:node-files", Methods: []string{"put"}}, {To: "print-and-scan", Methods: []string{"get", "put"}}}
|
||||
if !reflect.DeepEqual(m.RequireShares["s3-bucket"], want) {
|
||||
t.Fatalf("the shares read as %+v", m.RequireShares)
|
||||
}
|
||||
raw, err := json.Marshal(m)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var keys map[string]json.RawMessage
|
||||
if err := json.Unmarshal(raw, &keys); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if string(keys["requires"]) != `["shell","s3-bucket"]` {
|
||||
t.Errorf("the stored requires is not plain names: %s", keys["requires"])
|
||||
}
|
||||
if !strings.Contains(string(keys[RequireSharesKey]), `"seat:node-files"`) {
|
||||
t.Errorf("the side key does not hold the shares: %s", raw)
|
||||
}
|
||||
var again Manifest
|
||||
if err := json.Unmarshal(raw, &again); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(again, m) {
|
||||
t.Fatalf("not lossless:\n%+v\n%+v", again, m)
|
||||
}
|
||||
}
|
||||
|
||||
// 1591/AC2: a malformed share is refused, naming the entry and ADR 0468 §8.
|
||||
func TestAMalformedShareIsRefused(t *testing.T) {
|
||||
cases := map[string]string{
|
||||
"unknown method": `{"name":"s3-bucket","shares":[{"to":"a","methods":["delete"]}]}`,
|
||||
"empty to": `{"name":"s3-bucket","shares":[{"to":"","methods":["get"]}]}`,
|
||||
"malformed to": `{"name":"s3-bucket","shares":[{"to":"Seat:X y","methods":["get"]}]}`,
|
||||
"empty methods": `{"name":"s3-bucket","shares":[{"to":"a","methods":[]}]}`,
|
||||
"duplicate to": `{"name":"s3-bucket","shares":[{"to":"a","methods":["get"]},{"to":"a","methods":["put"]}]}`,
|
||||
"duplicate verb": `{"name":"s3-bucket","shares":[{"to":"a","methods":["get","get"]}]}`,
|
||||
"other require": `{"name":"route","shares":[{"to":"a","methods":["get"]}]}`,
|
||||
}
|
||||
for name, entry := range cases {
|
||||
_, err := ParseManifest([]byte(`{"module":"files","version":"1","requires":[` + entry + `]}`))
|
||||
if err == nil {
|
||||
t.Errorf("%s: accepted", name)
|
||||
continue
|
||||
}
|
||||
if !strings.Contains(err.Error(), "ADR 0468 §8") || !strings.Contains(err.Error(), "requires") {
|
||||
t.Errorf("%s: the error does not name the entry and the ADR: %v", name, err)
|
||||
}
|
||||
}
|
||||
// The stored form is held to the same rules.
|
||||
_, err := ParseManifest([]byte(`{"module":"files","version":"1","requires":["s3-bucket"],
|
||||
"requires-shares":{"s3-bucket":[{"to":"a","methods":[]}]}}`))
|
||||
if err == nil || !strings.Contains(err.Error(), "ADR 0468 §8") {
|
||||
t.Errorf("the stored form was accepted: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// 1591/AC3: the values a consumer gives carry the shares sorted by `to`, remote (grant) and co-located alike.
|
||||
func TestTheGrantCarriesTheSharesSorted(t *testing.T) {
|
||||
m := mustParse(t, filesWithShares)
|
||||
remote := Resolution{Node: "laptop", Modules: []Manifest{m}}
|
||||
values, asks, err := remote.ContributionsFrom("s3-bucket", "files", SettingsBy{})
|
||||
if err != nil || !asks {
|
||||
t.Fatalf("asks %v, err %v", asks, err)
|
||||
}
|
||||
if got := sharesOf(t, values); got != sortedSharesJSON {
|
||||
t.Fatalf("the grant values carry %s", got)
|
||||
}
|
||||
// As the provider's file shows it, through a grant.
|
||||
provider, err := Resolve(shelf(minioLike()), []string{"minio"}, workstation(), World{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out, err := provider.Declaration(Rendering{Grants: []Grant{{Provision: "s3-bucket", Consumer: "laptop", From: "files", Values: values}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
given := givenToMinio(t, out)
|
||||
if len(given) != 1 || sharesOf(t, given[0].Values) != sortedSharesJSON {
|
||||
t.Fatalf("the remote consumer's shares did not reach the file: %+v", given)
|
||||
}
|
||||
|
||||
// Co-located: the consumer contributes to s3-bucket too, and no grant is passed.
|
||||
co := mustParse(t, `{"module":"files","version":"1","contributes":{"s3-bucket":{"name":"files"}},
|
||||
"requires":[{"name":"s3-bucket","shares":[{"to":"seat:node-files","methods":["put"]},{"to":"print-and-scan","methods":["put","get"]}]}]}`)
|
||||
both, err := Resolve(shelf(minioLike(), co), []string{"files"}, workstation(), World{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out, err = both.Declaration(Rendering{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
given = givenToMinio(t, out)
|
||||
if len(given) != 1 || sharesOf(t, given[0].Values) != sortedSharesJSON || given[0].Values["name"] != "files" {
|
||||
t.Fatalf("the co-located consumer's shares did not reach the file: %+v", given)
|
||||
}
|
||||
}
|
||||
|
||||
// 1591/AC4: only the shares change, so the module is not recreated and held for no one; the provider's file changes.
|
||||
func TestChangingOnlyTheSharesDoesNotRecreateTheConsumer(t *testing.T) {
|
||||
was := mustParse(t, filesWithShares)
|
||||
now := mustParse(t, `{"module":"files","version":"1","requires":["shell",
|
||||
{"name":"s3-bucket","shares":[{"to":"print-and-scan","methods":["get"]}]}]}`)
|
||||
if said := Recreates(was, now).Say("files"); said != "" {
|
||||
t.Errorf("a change of shares recreates the module: %s", said)
|
||||
}
|
||||
if said := Holds(was, now); said != "" {
|
||||
t.Errorf("a change of shares holds the module: %s", said)
|
||||
}
|
||||
before := Resolution{Node: "laptop", Modules: []Manifest{was}}
|
||||
after := Resolution{Node: "laptop", Modules: []Manifest{now}}
|
||||
a, _, _ := before.ContributionsFrom("s3-bucket", "files", SettingsBy{})
|
||||
b, _, _ := after.ContributionsFrom("s3-bucket", "files", SettingsBy{})
|
||||
if sharesOf(t, a) == sharesOf(t, b) {
|
||||
t.Fatalf("the provider's values did not change: %s", sharesOf(t, a))
|
||||
}
|
||||
}
|
||||
|
||||
// 1591/AC5: a consumer whose contributes to s3-bucket names `shares` itself is refused.
|
||||
func TestAConsumerCannotContributeSharesItself(t *testing.T) {
|
||||
_, err := ParseManifest([]byte(`{"module":"files","version":"1",
|
||||
"contributes":{"s3-bucket":{"shares":"x"}},
|
||||
"requires":[{"name":"s3-bucket","shares":[{"to":"a","methods":["get"]}]}]}`))
|
||||
if err == nil || !strings.Contains(err.Error(), "ADR 0468 §8") {
|
||||
t.Errorf("both written, accepted: %v", err)
|
||||
}
|
||||
// Without the field the grant is still refused, so the provider is never told something the manifest did not say.
|
||||
m := Manifest{Module: "files", Requires: []string{"s3-bucket"},
|
||||
Contributes: map[string]map[string]any{"s3-bucket": {"shares": "x"}}}
|
||||
r := Resolution{Node: "laptop", Modules: []Manifest{m}}
|
||||
if _, _, err := r.ContributionsFrom("s3-bucket", "files", SettingsBy{}); err == nil {
|
||||
t.Error("a contributed `shares` reached the grant")
|
||||
}
|
||||
}
|
||||
@@ -394,6 +394,72 @@ func (b *Build) problems(module string) []string {
|
||||
"%s: %q mirrors %q, which names no tag or digest", module, a.Name, a.From))
|
||||
}
|
||||
problems = append(problems, upstreamFileProblems(module, a)...)
|
||||
problems = append(problems, pathsProblems(module, a)...)
|
||||
}
|
||||
return problems
|
||||
}
|
||||
|
||||
// NamedPaths are the paths a module's fingerprint takes its tree from (novox/hq ADR 0499 clause 2): the union of the
|
||||
// paths its artifacts name, cleaned and sorted, and every true only when the module builds something and every artifact
|
||||
// names paths; otherwise none, and the module keeps its directory's tree.
|
||||
//
|
||||
// **Read by the core alone** (ADR 0495 clause 1, the lead's decision L8 of R4e): the indexer and its merge preview
|
||||
// (buildstep.ModuleTrees) call it. A build module's fetch step is handed the paths by the builder, from the indexer's
|
||||
// kept inputs (delivery.KeptInputs2.Paths), and reads no manifest for them.
|
||||
func (m Manifest) NamedPaths() (paths []string, every bool) {
|
||||
if m.Build == nil || len(m.Build.Artifacts) == 0 {
|
||||
return nil, false
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
for _, a := range m.Build.Artifacts {
|
||||
if len(a.Paths) == 0 {
|
||||
return nil, false
|
||||
}
|
||||
for _, p := range a.Paths {
|
||||
seen[path.Clean(p)] = true
|
||||
}
|
||||
}
|
||||
for p := range seen {
|
||||
paths = append(paths, p)
|
||||
}
|
||||
sort.Strings(paths)
|
||||
return paths, true
|
||||
}
|
||||
|
||||
// pathsProblems is what is wrong with an artifact's `paths` (novox/hq ADR 0499 clause 1): named on a kind whose build
|
||||
// source is derived or named otherwise, or a path that leaves the module's directory, is its whole directory, or is a
|
||||
// pattern.
|
||||
func pathsProblems(module string, a Artifact) []string {
|
||||
if a.Paths == nil {
|
||||
return nil
|
||||
}
|
||||
switch {
|
||||
case a.Kind == ArtifactBundle && a.Language == "go":
|
||||
return []string{fmt.Sprintf("%s: %q is a Go program and names paths: its build source is derived from the "+
|
||||
"package it compiles (novox/hq ADR 0267 clause 1, ADR 0499 clause 1)", module, a.Name)}
|
||||
case a.Kind != ArtifactBundle && a.Kind != ArtifactPackage:
|
||||
return []string{fmt.Sprintf("%s: %q is a %q and names paths; only a package or a bundle in a language other "+
|
||||
"than Go names the paths it is built from (novox/hq ADR 0499 clause 1)", module, a.Name, a.Kind)}
|
||||
case len(a.Paths) == 0:
|
||||
return []string{fmt.Sprintf("%s: %q names no path in paths; leave the word out to build from the module's "+
|
||||
"directory", module, a.Name)}
|
||||
}
|
||||
var problems []string
|
||||
for _, p := range a.Paths {
|
||||
clean := path.Clean(p)
|
||||
switch {
|
||||
case strings.TrimSpace(p) == "":
|
||||
problems = append(problems, fmt.Sprintf("%s: %q names an empty path", module, a.Name))
|
||||
case strings.HasPrefix(p, "/") || clean == ".." || strings.HasPrefix(clean, "../"):
|
||||
problems = append(problems, fmt.Sprintf("%s: %q names the path %q, which leaves the module's directory",
|
||||
module, a.Name, p))
|
||||
case clean == ".":
|
||||
problems = append(problems, fmt.Sprintf("%s: %q names the module's whole directory (%q) as a path; leave "+
|
||||
"the word out for that", module, a.Name, p))
|
||||
case strings.ContainsAny(p, "*?[]{}\\"):
|
||||
problems = append(problems, fmt.Sprintf("%s: %q names %q, which is a pattern; paths names files and "+
|
||||
"directories", module, a.Name, p))
|
||||
}
|
||||
}
|
||||
return problems
|
||||
}
|
||||
|
||||
@@ -22,16 +22,12 @@ func catalogueRoot(t *testing.T) string {
|
||||
}
|
||||
|
||||
func TestEveryCatalogueManifestParses(t *testing.T) {
|
||||
root := catalogueRoot(t)
|
||||
found, err := filepath.Glob(filepath.Join(root, "modules", "*", "module.json"))
|
||||
if err != nil || len(found) == 0 {
|
||||
t.Fatalf("no manifests under %s: %v", root, err)
|
||||
}
|
||||
_, found := catalogueManifests(t)
|
||||
named, routed := 0, 0
|
||||
for _, p := range found {
|
||||
m, err := ReadTreeManifest(root, filepath.Dir(p))
|
||||
for _, r := range found {
|
||||
m, err := r.manifest, r.err
|
||||
if err != nil {
|
||||
t.Errorf("%s: %v", filepath.Base(filepath.Dir(p)), err)
|
||||
t.Errorf("%s: %v", filepath.Base(filepath.Dir(r.path)), err)
|
||||
continue
|
||||
}
|
||||
for _, l := range m.Listens {
|
||||
@@ -54,16 +50,12 @@ func TestEveryCatalogueManifestParses(t *testing.T) {
|
||||
// definition names a domain or a public address the mesh acts on, and every value that must for now
|
||||
// carries its reason (novox/hq ADR 0155, issue 134). The list it prints is the one that shrinks.
|
||||
func TestNoCatalogueManifestNamesAnInstallation(t *testing.T) {
|
||||
root := catalogueRoot(t)
|
||||
found, err := filepath.Glob(filepath.Join(root, "modules", "*", "module.json"))
|
||||
if err != nil || len(found) == 0 {
|
||||
t.Fatalf("no manifests under %s: %v", root, err)
|
||||
}
|
||||
_, found := catalogueManifests(t)
|
||||
var named []string
|
||||
for _, p := range found {
|
||||
m, err := ReadTreeManifest(root, filepath.Dir(p))
|
||||
for _, r := range found {
|
||||
m, err := r.manifest, r.err
|
||||
if err != nil {
|
||||
t.Errorf("%s: %v", p, err)
|
||||
t.Errorf("%s: %v", r.path, err)
|
||||
continue
|
||||
}
|
||||
named = append(named, InstallationProblems(m)...)
|
||||
@@ -78,15 +70,11 @@ func TestNoCatalogueManifestNamesAnInstallation(t *testing.T) {
|
||||
// backup line by hand, every directory a container writes is declared, and every irreplaceable item is
|
||||
// backed up — so a store added is a store backed up, without a list of stores to keep in step.
|
||||
func TestEveryCatalogueModuleDeclaresItsData(t *testing.T) {
|
||||
root := catalogueRoot(t)
|
||||
found, err := filepath.Glob(filepath.Join(root, "modules", "*", "module.json"))
|
||||
if err != nil || len(found) == 0 {
|
||||
t.Fatalf("no manifests under %s: %v", root, err)
|
||||
}
|
||||
_, found := catalogueManifests(t)
|
||||
shelf := Shelf{}
|
||||
granting := 0
|
||||
for _, p := range found {
|
||||
m, err := ReadTreeManifest(root, filepath.Dir(p))
|
||||
for _, r := range found {
|
||||
m, err := r.manifest, r.err
|
||||
if err != nil {
|
||||
continue // TestEveryCatalogueManifestParses says why
|
||||
}
|
||||
@@ -108,16 +96,12 @@ func TestEveryCatalogueModuleDeclaresItsData(t *testing.T) {
|
||||
// overflow fails here, in the pull request that introduces it, rather than on the provider's machine
|
||||
// the first time a real machine's name meets the module's (issue 263).
|
||||
func TestEveryCatalogueIdentityFitsWhatItRequires(t *testing.T) {
|
||||
root := catalogueRoot(t)
|
||||
found, err := filepath.Glob(filepath.Join(root, "modules", "*", "module.json"))
|
||||
if err != nil || len(found) == 0 {
|
||||
t.Fatalf("no manifests under %s: %v", root, err)
|
||||
}
|
||||
_, found := catalogueManifests(t)
|
||||
shelf := Shelf{}
|
||||
for _, p := range found {
|
||||
m, err := ReadTreeManifest(root, filepath.Dir(p))
|
||||
for _, r := range found {
|
||||
m, err := r.manifest, r.err
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", p, err)
|
||||
t.Fatalf("%s: %v", r.path, err)
|
||||
}
|
||||
shelf[m.Module] = m
|
||||
}
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// readManifest is one manifest of the catalogue as ReadTreeManifest read it.
|
||||
type readManifest struct {
|
||||
path string // the module.json
|
||||
manifest Manifest
|
||||
err error
|
||||
}
|
||||
|
||||
var (
|
||||
readOnce sync.Once
|
||||
readAll []readManifest
|
||||
readRoot string
|
||||
)
|
||||
|
||||
// catalogueManifests is every manifest of the catalogue (catalogueRoot), read through ReadTreeManifest once
|
||||
// per test binary and not once per test: the checks over the real catalogue each read and parsed every
|
||||
// manifest again, and together took 1.5 s of mesh-sdk's unit bound of 2 s (issue 1756). What it returns is
|
||||
// shared and read-only: a test that changes a manifest copies it first.
|
||||
func catalogueManifests(t *testing.T) (string, []readManifest) {
|
||||
t.Helper()
|
||||
root := catalogueRoot(t)
|
||||
readOnce.Do(func() {
|
||||
readRoot = root
|
||||
found, err := filepath.Glob(filepath.Join(root, "modules", "*", "module.json"))
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
for _, p := range found {
|
||||
m, err := ReadTreeManifest(root, filepath.Dir(p))
|
||||
readAll = append(readAll, readManifest{path: p, manifest: m, err: err})
|
||||
}
|
||||
})
|
||||
if len(readAll) == 0 {
|
||||
t.Fatalf("no manifests under %s", root)
|
||||
}
|
||||
return readRoot, readAll
|
||||
}
|
||||
@@ -482,3 +482,29 @@ func TestAnOperatorNamedValueAFileFallsBackOnMayHaveADefault(t *testing.T) {
|
||||
"default": "", "runtime": true}
|
||||
refused(t, runtime, `"url"`, `a key naming "url" is the operator's value`)
|
||||
}
|
||||
|
||||
// 1658/AC1: a value whose only placement is ${config:key} in an item's text is placed.
|
||||
func TestConfigCountsAnItemsTextAsAPlacement(t *testing.T) {
|
||||
m := map[string]any{
|
||||
"module": "m", "version": "1",
|
||||
"config": map[string]any{
|
||||
"k": map[string]any{"type": "string", "description": "a word the item says", "default": "x"},
|
||||
},
|
||||
"items": []any{map[string]any{"collection": "a", "content": "k = ${config:k}\n"}},
|
||||
}
|
||||
if _, err := parseRaw(t, m); err != nil {
|
||||
t.Fatalf("a value placed only in an item's text is refused: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// 1658/AC2: a declared value placed nowhere, with an item that does not say it, is still refused.
|
||||
func TestConfigStillRefusesAValueNoItemPlaces(t *testing.T) {
|
||||
m := map[string]any{
|
||||
"module": "m", "version": "1",
|
||||
"config": map[string]any{
|
||||
"k": map[string]any{"type": "string", "description": "a word nothing says", "default": "x"},
|
||||
},
|
||||
"items": []any{map[string]any{"collection": "a", "content": "k = fixed\n"}},
|
||||
}
|
||||
refused(t, m, `"k"`, "placed nowhere")
|
||||
}
|
||||
|
||||
@@ -1783,6 +1783,10 @@ func (r Resolution) contributions(settings SettingsBy, grants []Grant,
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// What the module shares of its bucket reaches the provider with the rest (novox/hq ADR 0468 §8).
|
||||
if values, err = withShares(m, to, values); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out[to] = append(out[to], Contribution{From: m.Module, Values: values})
|
||||
}
|
||||
// Several contributions to one requirement (ADR 0094's sibling for `contributes`): an
|
||||
@@ -2169,11 +2173,19 @@ func (r Resolution) ContributionsFrom(requirement, module string, settings Setti
|
||||
mine = append(mine, g.Values)
|
||||
}
|
||||
}
|
||||
var asker Manifest
|
||||
for _, m := range r.Modules {
|
||||
if m.Module == module {
|
||||
asker = m
|
||||
}
|
||||
}
|
||||
if len(mine) == 1 {
|
||||
// Its shares are already in, put there where its contribution was composed (contributions).
|
||||
return mine[0], true, nil
|
||||
}
|
||||
if len(mine) > 1 {
|
||||
return map[string]any{}, true, nil
|
||||
values, err := withShares(asker, requirement, map[string]any{})
|
||||
return values, err == nil, err
|
||||
}
|
||||
// It contributes no payload — but a require-only consumer of a parameterless provision (one whose
|
||||
// `serves` names no consumer-supplied key: `redis-cache`, `amqp`) still ASKS for it and must be
|
||||
@@ -2186,7 +2198,8 @@ func (r Resolution) ContributionsFrom(requirement, module string, settings Setti
|
||||
}
|
||||
for _, req := range m.Requires {
|
||||
if req == requirement {
|
||||
return map[string]any{}, true, nil
|
||||
values, err := withShares(m, requirement, map[string]any{})
|
||||
return values, err == nil, err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -9,7 +9,26 @@ package catalogue
|
||||
// DeliverySeat is the seat mesh-delivery holds.
|
||||
const DeliverySeat = "mesh-delivery"
|
||||
|
||||
// deliveryVerbs are the delivery seat's tools: six that read, and the acts of a person, a reviewer and healer H2.
|
||||
// RemovedDeliveryVerbs are the delivery seat's verbs release 6 removed (novox/hq ADR 0539 clause 17): with no plan the
|
||||
// gate keeps the merge's states and derives nothing after the merge. Each stays in deliveryVerbs as an optional row
|
||||
// with no input, so the holder of before and the holder of release 6 both hold the seat and a caller is answered by
|
||||
// name with its replacement, given here.
|
||||
var RemovedDeliveryVerbs = map[string]string{
|
||||
"times": "the delivery view (nox-delivery): each machine's want and have, and since when",
|
||||
"what-if": "the merge check's composition, which composes every machine a change touches",
|
||||
"release": "the assigner's asks (hold, restore, backup, record), answered on the phone or in the delivery view",
|
||||
"stop": "a bad fingerprint is held and its canary set back by the assigner (ADR 0429 §6, ADR 0539 clause 18)",
|
||||
"retire-history": "nothing to retire: the gate keeps no delivery after its merge",
|
||||
}
|
||||
|
||||
// deliveryRemovedInR6 is the row of a verb release 6 removed from the delivery seat.
|
||||
func deliveryRemovedInR6(name string) Verb {
|
||||
return Verb{Name: name, Description: "Removed in R6 (ADR 0539): refused by name, naming its replacement: " +
|
||||
RemovedDeliveryVerbs[name], Input: schema(nil, nil), Optional: true}
|
||||
}
|
||||
|
||||
// deliveryVerbs are the delivery seat's tools: what reads the merges and checks, and the acts of a person and healer H2
|
||||
// on a merge (ADR 0539 clause 17).
|
||||
func deliveryVerbs() []Verb {
|
||||
return []Verb{
|
||||
{Name: "deliveries", Description: "Every delivery not final, and those that ended in the last day, one line " +
|
||||
@@ -18,15 +37,7 @@ func deliveryVerbs() []Verb {
|
||||
Input: schema(map[string]string{"state": "one state, e.g. delivering or held",
|
||||
"repository": "owner/repository", "group": "a group's id (its branch name)",
|
||||
"all": "\"true\": the final ones of the last thirty days too"}, nil, "all")},
|
||||
// Delivery time against the delivery budgets (novox/hq ADR 0282): what probe D16 reads, optional until its
|
||||
// holder serves it.
|
||||
{Name: "times", Description: "Delivery time: from a merge to every machine of its walk running the build. " +
|
||||
"Each class's delivery budget (a leaf module three minutes, a core module ten), the last days' median and " +
|
||||
"worst, how many within and over, the newest delivery of each class, every delivery over its budget with its " +
|
||||
"longest phase, and the delivered ones whose time is not known. Given a delivery's id, its time phase by phase.",
|
||||
Input: schema(map[string]string{"days": "how many days back (default 7)", "id": "one delivery's id: its phases"}, nil),
|
||||
// Optional until mesh-delivery serves it: its holder lives in the catalogue (design 33 §7).
|
||||
Optional: true},
|
||||
deliveryRemovedInR6("times"),
|
||||
{Name: "show", Description: "One delivery or group whole: its delivery plan (what it builds, what each " +
|
||||
"machine receives, what is not an ordinary send), every transition with when and why, the machine " +
|
||||
"steps of its walk, its group and its order.",
|
||||
@@ -35,14 +46,10 @@ func deliveryVerbs() []Verb {
|
||||
"(declared, built by, version skew, engine before controller, by name), its composed check and its " +
|
||||
"state, derived from its members.",
|
||||
Input: schema(map[string]string{"all": "\"true\": the groups that ended too"}, nil, "all")},
|
||||
{Name: "what-if", Description: "The delivery plan a change would have, asked of the controller's planner " +
|
||||
"and kept nowhere: the modules it moves and their dependents in tiers, what each machine would receive, " +
|
||||
"the steps that are not an ordinary send.",
|
||||
Input: schema(map[string]string{"repository": "owner/repository",
|
||||
"paths": "the files it changes, comma-separated, from the repository's root",
|
||||
"base": "the branch it merges into (default main)"}, []string{"repository", "paths"})},
|
||||
deliveryRemovedInR6("what-if"),
|
||||
{Name: "checks", Description: "What the mesh's checks said of a pull request's head or of one commit: each " +
|
||||
"of the commit's mesh statuses (mesh/merge-gate, mesh/repo-check, mesh/delivery, …) with its state, " +
|
||||
"of the commit's mesh statuses (mesh/check, the one a main requires from R4e, hq's mesh/repo-check, " +
|
||||
"mesh/delivery, …) with its state, " +
|
||||
"description and when it was set; the merge check's full verdict as the controller said it — each " +
|
||||
"layer's summary, the machine that ran it, when, its build id and its report; and whether the branch's " +
|
||||
"protection would let it merge, every required status being success.",
|
||||
@@ -61,40 +68,13 @@ func deliveryVerbs() []Verb {
|
||||
"controller is asked for its check. With why.",
|
||||
Input: schema(map[string]string{"id": "the delivery's id", "why": "why, kept with the transition"},
|
||||
[]string{"id", "why"})},
|
||||
{Name: "release", Description: "A person's word that a held delivery goes on: it starts delivering. With why.",
|
||||
Input: schema(map[string]string{"id": "the delivery's id", "why": "why, kept with the transition"},
|
||||
[]string{"id", "why"})},
|
||||
{Name: "stop", Description: "Stop a delivery that is not final, with why; its walk is ended through the " +
|
||||
"controller, and in a group every member after it is stopped too, naming it.",
|
||||
Input: schema(map[string]string{"id": "the delivery's id", "why": "why, kept with the transition"},
|
||||
[]string{"id", "why"})},
|
||||
deliveryRemovedInR6("release"),
|
||||
deliveryRemovedInR6("stop"),
|
||||
{Name: "close", Description: "Healer H2's verb: take the one transition the table names for a delivery held " +
|
||||
"past its bound. Refused for any other.",
|
||||
Input: schema(map[string]string{"id": "the delivery's id", "why": "the condition it answers"},
|
||||
[]string{"id", "why"})},
|
||||
{Name: "retire-history", Description: "Retire held deliveries that are history, on a person's word, with " +
|
||||
"why: each made from the forge's word for a merge on the trunk heard long after it happened — replayed " +
|
||||
"when the holder first held the seat — with no walk of it open. Never released, delivered or put back: " +
|
||||
"retired, final. One by id, refused when the rule does not take it; or every held one, of one " +
|
||||
"repository when named, answering how many it retired and naming those it left held. dry \"true\" " +
|
||||
"answers what it would do and changes nothing.",
|
||||
Input: schema(map[string]string{"why": "why, kept with every transition",
|
||||
"id": "one delivery's id; every held one when empty",
|
||||
"repository": "owner/repository: only its held deliveries",
|
||||
"dry": "\"true\": what it would retire, changing nothing"}, []string{"why"}, "dry"),
|
||||
// Added after the seat's holder shipped (novox/hq issue 313): optional until mesh-delivery serves it.
|
||||
Optional: true},
|
||||
{Name: "mark-ready", Description: "A reviewer's word that a delivery's head is approved: the holder puts the " +
|
||||
"label mesh/ready on its pull request and then writes \"ready at <head>\", the order the forge counts, " +
|
||||
"as the mesh's own forge account, naming who reviewed it and why. Refused when the delivery's commit is " +
|
||||
"no longer its pull request's head. mesh-delivery then merges it once its checks pass (novox/hq ADR 0364).",
|
||||
Input: schema(map[string]string{"id": "the delivery's id (owner/repository@commit): the head reviewed",
|
||||
"reviewed-by": "who reviewed it, as who@node", "why": "the review's verdict in a line"},
|
||||
[]string{"id", "reviewed-by", "why"}),
|
||||
// Kept one more release, optional (novox/hq issue 1022): mesh-delivery no longer serves it, but the manifest
|
||||
// it still has on the mesh does until its catalogue member merges, and a seat refuses a holder that serves
|
||||
// a verb it does not promise. The next release removes this entry.
|
||||
Optional: true},
|
||||
deliveryRemovedInR6("retire-history"),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -58,107 +58,3 @@ func TestTheDeliverySeatPromisesChecks(t *testing.T) {
|
||||
t.Fatalf("a holder without %s: %v", before[0], err)
|
||||
}
|
||||
}
|
||||
|
||||
// The seat retires history on a person's word (novox/hq issue 313): `retire-history`, with why, one delivery by
|
||||
// id or every held one, dry to see first. Added after the holder shipped, it is optional, so the holder that
|
||||
// does not serve it yet still holds the seat, and one that does is not refused for a verb the seat lacks.
|
||||
func TestTheDeliverySeatPromisesRetireHistoryOptionally(t *testing.T) {
|
||||
seat, _ := SeatNamed(DeliverySeat)
|
||||
var retire *Verb
|
||||
for i := range seat.Serves {
|
||||
if seat.Serves[i].Name == "retire-history" {
|
||||
retire = &seat.Serves[i]
|
||||
}
|
||||
}
|
||||
if retire == nil || !retire.Optional {
|
||||
t.Fatalf("the delivery seat promises %v, retire-history optionally", VerbNames(seat.Serves))
|
||||
}
|
||||
props, _ := retire.Input["properties"].(map[string]any)
|
||||
for _, arg := range []string{"why", "id", "repository", "dry"} {
|
||||
if _, has := props[arg]; !has {
|
||||
t.Errorf("retire-history takes no %q", arg)
|
||||
}
|
||||
}
|
||||
if req, _ := retire.Input["required"].([]string); !reflect.DeepEqual(req, []string{"why"}) {
|
||||
t.Errorf("retire-history requires %v, wanted only why", req)
|
||||
}
|
||||
var without []string
|
||||
for _, v := range VerbNames(seat.Serves) {
|
||||
if v != "retire-history" {
|
||||
without = append(without, v)
|
||||
}
|
||||
}
|
||||
m := Manifest{Module: "mesh-delivery", Claims: []Claim{{Name: DeliverySeat, Scope: ScopeMesh, Serves: without}}}
|
||||
if err := CanHold(m, seat); err != nil {
|
||||
t.Fatalf("a holder without retire-history yet: %v", err)
|
||||
}
|
||||
m.Claims[0].Serves = VerbNames(seat.Serves)
|
||||
if err := CanHold(m, seat); err != nil {
|
||||
t.Fatalf("a holder serving retire-history: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// The seat says delivery times (novox/hq ADR 0282, issue 382): `times`, over some days or for one delivery. Added
|
||||
// after the holder shipped, it is optional, so the holder that does not serve it yet still holds the seat, and one
|
||||
// that does is not refused for a verb the seat lacks.
|
||||
func TestTheDeliverySeatPromisesTimesOptionally(t *testing.T) {
|
||||
seat, _ := SeatNamed(DeliverySeat)
|
||||
var times *Verb
|
||||
for i := range seat.Serves {
|
||||
if seat.Serves[i].Name == "times" {
|
||||
times = &seat.Serves[i]
|
||||
}
|
||||
}
|
||||
if times == nil || !times.Optional {
|
||||
t.Fatalf("the delivery seat promises %v, times optionally", VerbNames(seat.Serves))
|
||||
}
|
||||
props, _ := times.Input["properties"].(map[string]any)
|
||||
for _, arg := range []string{"days", "id"} {
|
||||
if _, has := props[arg]; !has {
|
||||
t.Errorf("times takes no %q", arg)
|
||||
}
|
||||
}
|
||||
var without []string
|
||||
for _, v := range VerbNames(seat.Serves) {
|
||||
if v != "times" {
|
||||
without = append(without, v)
|
||||
}
|
||||
}
|
||||
m := Manifest{Module: "mesh-delivery", Claims: []Claim{{Name: DeliverySeat, Scope: ScopeMesh, Serves: without}}}
|
||||
if err := CanHold(m, seat); err != nil {
|
||||
t.Fatalf("a holder without times yet: %v", err)
|
||||
}
|
||||
m.Claims[0].Serves = VerbNames(seat.Serves)
|
||||
if err := CanHold(m, seat); err != nil {
|
||||
t.Fatalf("a holder serving times: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// The seat still promises `mark-ready`, optionally, for one more release (issue 1022); it was added by novox/hq issue 913: `mark-ready`, with the delivery's id, who reviewed it
|
||||
// and why, so a reviewer marks a pull request through one audited verb, label and comment in the order the forge
|
||||
// counts, instead of two raw forge calls. Added after the holder shipped, it is optional.
|
||||
func TestTheDeliverySeatPromisesMarkReadyOptionally(t *testing.T) {
|
||||
seat, _ := SeatNamed(DeliverySeat)
|
||||
var mark *Verb
|
||||
for i := range seat.Serves {
|
||||
if seat.Serves[i].Name == "mark-ready" {
|
||||
mark = &seat.Serves[i]
|
||||
}
|
||||
}
|
||||
if mark == nil || !mark.Optional {
|
||||
t.Fatalf("the delivery seat promises %v, mark-ready optionally", VerbNames(seat.Serves))
|
||||
}
|
||||
if req, _ := mark.Input["required"].([]string); !reflect.DeepEqual(req, []string{"id", "reviewed-by", "why"}) {
|
||||
t.Errorf("mark-ready requires %v, wanted id, reviewed-by and why", req)
|
||||
}
|
||||
var without []string
|
||||
for _, v := range VerbNames(seat.Serves) {
|
||||
if v != "mark-ready" {
|
||||
without = append(without, v)
|
||||
}
|
||||
}
|
||||
m := Manifest{Module: "mesh-delivery", Claims: []Claim{{Name: DeliverySeat, Scope: ScopeMesh, Serves: without}}}
|
||||
if err := CanHold(m, seat); err != nil {
|
||||
t.Fatalf("a holder without mark-ready yet: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
package catalogue
|
||||
|
||||
import "testing"
|
||||
|
||||
// 1694/AC5: the capability a node-engine reports for files by digest (novox/hq ADR 0486 §5) is a known one.
|
||||
func TestFileReferencesIsAKnownCapability(t *testing.T) {
|
||||
if !oneOf(KnownCapabilities, "file-references") {
|
||||
t.Fatal("file-references is not a known capability")
|
||||
}
|
||||
}
|
||||
@@ -74,6 +74,22 @@ func filesVerbs() []Verb {
|
||||
map[string]string{"path": "an absolute path", "owner": "an account's name or number",
|
||||
"group": "a group's name or number (optional)", "recursive": "\"true\": also everything under a directory"},
|
||||
[]string{"path", "owner"}, []string{"group", "recursive"}, "chown", "chgrp"),
|
||||
Verb{Name: "upload", Description: "Send one file of this node into a bucket of the object store, over a short signed link " +
|
||||
"and never over the bus; answers the key (the file's SHA-256 when none is given), the size and the SHA-256. " +
|
||||
"At most 100 MiB. A place a secret is kept, a known credential file, or a link to one is refused by name, " +
|
||||
"before anything is sent. Read " + filesAsAccount + ".",
|
||||
Input: schema(map[string]string{"path": "an absolute path to a file", "bucket": "the bucket the file goes into",
|
||||
"key": "the object's key (optional; the file's SHA-256 when not given)"}, []string{"path", "bucket"})},
|
||||
// download is a plain Verb, not through write(): write() types its optional arguments as true/false
|
||||
// switches, which a hash and a reason are not (issue 1719).
|
||||
Verb{Name: "download", Description: "Write one object of a bucket to a file of this node, over a short signed link and " +
|
||||
"never over the bus; the old file goes to the node's trash. At most 100 MiB. The bytes must hash to sha256, or to " +
|
||||
"the key when the key is a SHA-256; otherwise no file is written. Refuses a path a module owns " +
|
||||
"unless override names a reason; done as the node's operator account, through sudo -n where its rights do not reach.",
|
||||
Input: schema(map[string]string{"bucket": "the bucket", "key": "the object's key", "path": "an absolute path",
|
||||
"sha256": "the SHA-256 the bytes must have (optional; the key when that is a SHA-256)",
|
||||
"reason": "why the file is changed, for the audit trail",
|
||||
"override": overrideWord}, []string{"bucket", "key", "path", "reason"})},
|
||||
Verb{Name: "trash", Description: "What the node's trash holds: each deleted or replaced item with its id, " +
|
||||
"original path and when it went.",
|
||||
Input: schema(map[string]string{}, nil), Replaces: []string{"ls ~/.local/share/Trash"}},
|
||||
|
||||
@@ -2,6 +2,7 @@ package catalogue
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
@@ -22,7 +23,8 @@ func TestNodeFilesIsANodeSeatWithEveryVerbOfADR0394(t *testing.T) {
|
||||
if v.Optional {
|
||||
t.Errorf("%s.%s is optional", FilesSeat, v.Name)
|
||||
}
|
||||
if len(v.Replaces) == 0 {
|
||||
// upload and download (ADR 0468 §9, issue 1718) move bytes through the object store: no command is theirs.
|
||||
if len(v.Replaces) == 0 && v.Name != "upload" && v.Name != "download" {
|
||||
t.Errorf("%s.%s says nothing it replaces", FilesSeat, v.Name)
|
||||
}
|
||||
for _, c := range v.Replaces {
|
||||
@@ -30,7 +32,7 @@ func TestNodeFilesIsANodeSeatWithEveryVerbOfADR0394(t *testing.T) {
|
||||
}
|
||||
}
|
||||
for _, name := range []string{"stat", "readlink", "list", "read", "search", "account", "write", "copy", "move",
|
||||
"delete", "mkdir", "chmod", "chown", "trash", "restore"} {
|
||||
"delete", "mkdir", "chmod", "chown", "upload", "download", "trash", "restore"} {
|
||||
if !have[name] {
|
||||
t.Errorf("%s serves no verb %s", FilesSeat, name)
|
||||
}
|
||||
@@ -109,3 +111,93 @@ func TestNodeFilesRestoreTakesIdAndReason(t *testing.T) {
|
||||
}
|
||||
t.Error("no restore verb")
|
||||
}
|
||||
|
||||
// 1718/AC1: node-files names upload and download, in mesh-files' order, before trash.
|
||||
func TestNodeFilesNamesUploadAndDownloadBeforeTrash(t *testing.T) {
|
||||
s, _ := SeatNamed(FilesSeat)
|
||||
var names []string
|
||||
for _, v := range s.Serves {
|
||||
names = append(names, v.Name)
|
||||
}
|
||||
joined := strings.Join(names, ",")
|
||||
if !strings.Contains(joined, "chown,upload,download,trash") {
|
||||
t.Errorf("1718/AC1: node-files' verbs are %s, want upload and download between chown and trash", joined)
|
||||
}
|
||||
}
|
||||
|
||||
// 1718/AC2: a holder serving every verb including upload and download is accepted; one naming a verb outside the
|
||||
// contract is still refused, and one leaving upload out is refused.
|
||||
func TestAHolderOfNodeFilesServesUploadAndDownloadAndNothingElse(t *testing.T) {
|
||||
s, _ := SeatNamed(FilesSeat)
|
||||
var verbs []string
|
||||
for _, v := range s.Serves {
|
||||
verbs = append(verbs, v.Name)
|
||||
}
|
||||
hold := func(serves []string) error {
|
||||
return CanHold(Manifest{Module: "mesh-files", Version: "1",
|
||||
Claims: []Claim{{Name: FilesSeat, Scope: ScopeNode, Serves: serves}}}, s)
|
||||
}
|
||||
if err := hold(verbs); err != nil {
|
||||
t.Errorf("1718/AC2: a holder serving every verb was refused: %v", err)
|
||||
}
|
||||
if err := hold(append(append([]string{}, verbs...), "sync")); err == nil || !strings.Contains(err.Error(), "does not promise") {
|
||||
t.Errorf("1718/AC2: a verb outside the contract was accepted: %v", err)
|
||||
}
|
||||
var without []string
|
||||
for _, v := range verbs {
|
||||
if v != "upload" {
|
||||
without = append(without, v)
|
||||
}
|
||||
}
|
||||
if err := hold(without); err == nil || !strings.Contains(err.Error(), "upload") {
|
||||
t.Errorf("1718/AC2: a holder leaving upload out was accepted: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// 1718/AC3: upload requires path and bucket; download requires bucket, key, path and reason; sha256, key (upload)
|
||||
// and override are plain strings, not true/false switches.
|
||||
func TestNodeFilesUploadAndDownloadInputs(t *testing.T) {
|
||||
s, _ := SeatNamed(FilesSeat)
|
||||
want := map[string]struct {
|
||||
required string
|
||||
strs []string
|
||||
}{
|
||||
"upload": {"path,bucket", []string{"path", "bucket", "key"}},
|
||||
"download": {"bucket,key,path,reason", []string{"bucket", "key", "path", "sha256", "reason", "override"}},
|
||||
}
|
||||
seen := 0
|
||||
for _, v := range s.Serves {
|
||||
w, ok := want[v.Name]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
seen++
|
||||
req, _ := v.Input["required"].([]string)
|
||||
got := append([]string{}, req...)
|
||||
sortStrings(got)
|
||||
exp := strings.Split(w.required, ",")
|
||||
sortStrings(exp)
|
||||
if strings.Join(got, ",") != strings.Join(exp, ",") {
|
||||
t.Errorf("1718/AC3: %s requires %v, want %s", v.Name, req, w.required)
|
||||
}
|
||||
props, _ := v.Input["properties"].(map[string]any)
|
||||
if len(props) != len(w.strs) {
|
||||
t.Errorf("1718/AC3: %s has %d inputs, want %v", v.Name, len(props), w.strs)
|
||||
}
|
||||
for _, name := range w.strs {
|
||||
p, has := props[name].(map[string]any)
|
||||
if !has {
|
||||
t.Errorf("1718/AC3: %s has no input %s", v.Name, name)
|
||||
continue
|
||||
}
|
||||
if _, enum := p["enum"]; enum || p["type"] != "string" {
|
||||
t.Errorf("1718/AC3: %s input %s is %v, want a plain string", v.Name, name, p)
|
||||
}
|
||||
}
|
||||
}
|
||||
if seen != 2 {
|
||||
t.Errorf("1718/AC3: found %d of upload and download", seen)
|
||||
}
|
||||
}
|
||||
|
||||
func sortStrings(s []string) { sort.Strings(s) }
|
||||
|
||||
@@ -0,0 +1,149 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// A container's supplementary groups may be an operator's value (novox/hq ADR 0480 clause 7, ADR 0348 §11,
|
||||
// issue 1636): `groups: ["${config:media-gid}"]`, each element a whole placeholder or a whole number, filled at
|
||||
// composition with the number the config value holds.
|
||||
|
||||
func aContainerWithGroups(groups ...any) map[string]any {
|
||||
return map[string]any{"id": "server", "type": "container", "name": "n8n", "image": "x@sha256:" + strings.Repeat("a", 64),
|
||||
"groups": groups}
|
||||
}
|
||||
|
||||
// 1636/AC3: a manifest container with groups: ["${setting:media-gid}"] resolves to the number the setting says.
|
||||
func TestAContainersGroupIsFilledFromTheSetting(t *testing.T) {
|
||||
r := aContainerWithGroups("${setting:media-gid}", float64(2001))
|
||||
layers := []Layer{{From: "ace", Values: map[string]any{"media-gid": float64(2000)}}}
|
||||
if err := settingInto(r, layers, "n8n"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if want := []any{2000, 2001}; !reflect.DeepEqual(r["groups"], want) {
|
||||
t.Fatalf("composed as %#v, want %#v", r["groups"], want)
|
||||
}
|
||||
// A value the operator wrote as text is read as the number it spells.
|
||||
r = aContainerWithGroups("${setting:media-gid}")
|
||||
if err := settingInto(r, []Layer{{From: "ace", Values: map[string]any{"media-gid": "2000"}}}, "n8n"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if want := []any{2000}; !reflect.DeepEqual(r["groups"], want) {
|
||||
t.Fatalf("composed as %#v, want %#v", r["groups"], want)
|
||||
}
|
||||
}
|
||||
|
||||
// 1636/AC3: an unset value refuses the module by name, and the key and the remedy are named; nothing is dropped.
|
||||
func TestAnUnsetGroupRefusesTheModuleByName(t *testing.T) {
|
||||
r := aContainerWithGroups("${setting:media-gid}")
|
||||
err := settingInto(r, nil, "n8n")
|
||||
if err == nil || !strings.Contains(err.Error(), "n8n") || !strings.Contains(err.Error(), "media-gid") {
|
||||
t.Fatalf("want a refusal naming n8n and media-gid, got %v", err)
|
||||
}
|
||||
if got, _ := r["groups"].([]any); len(got) != 1 || got[0] != "${setting:media-gid}" {
|
||||
t.Fatalf("the resource was changed: %#v", r["groups"])
|
||||
}
|
||||
}
|
||||
|
||||
// 1636/AC2: a value that is not a whole number above zero cannot be a group.
|
||||
func TestAGroupThatIsNotAWholeNumberAboveZeroIsRefused(t *testing.T) {
|
||||
for _, bad := range []any{"media", "-1", "0", float64(0), float64(2000.5), "2000:2000", "", true} {
|
||||
r := aContainerWithGroups("${setting:media-gid}")
|
||||
err := settingInto(r, []Layer{{From: "ace", Values: map[string]any{"media-gid": bad}}}, "n8n")
|
||||
if err == nil || !strings.Contains(err.Error(), "media-gid") {
|
||||
t.Errorf("value %#v: want a refusal naming the key, got %v", bad, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A manifest in the config form registers with the group a config value fills, and the value is placed (a container's
|
||||
// groups count as a place for it); a placeholder in any other shape is refused where the manifest is read.
|
||||
func TestAManifestWhoseContainerTakesItsGroupFromConfigRegisters(t *testing.T) {
|
||||
m := map[string]any{
|
||||
"module": "n8n", "version": "1",
|
||||
"config": map[string]any{"media-gid": map[string]any{"type": "number", "minimum": 1, "required": true,
|
||||
"description": "the group that owns the media library on this machine"}},
|
||||
"resources": []any{aContainerWithGroups("${config:media-gid}")},
|
||||
}
|
||||
if _, err := parseRaw(t, m); err != nil {
|
||||
t.Fatalf("refused: %v", err)
|
||||
}
|
||||
for _, shape := range []string{"x${config:media-gid}", "${config:media-gid}:1", "${config:other}"} {
|
||||
m["resources"] = []any{aContainerWithGroups(shape)}
|
||||
if _, err := parseRaw(t, m); err == nil {
|
||||
t.Errorf("groups %q registered", shape)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1636/AC3 composed for a machine, the way a push writes it: n8n's group is the number the machine's config value
|
||||
// says, written as a number, and the module is left out by name when the machine says none.
|
||||
func TestAComposedMachineGetsTheGroupTheConfigValueNames(t *testing.T) {
|
||||
manifest, err := parseRaw(t, map[string]any{
|
||||
"module": "n8n", "version": "1",
|
||||
"config": map[string]any{"media-gid": map[string]any{"type": "number", "minimum": 1, "required": true,
|
||||
"description": "the group that owns the media library on this machine"}},
|
||||
"resources": []any{aContainerWithGroups("${config:media-gid}")},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r := anAdoptedAnchor()
|
||||
r.Modules = append(r.Modules, manifest)
|
||||
with := anchorRendering(false)
|
||||
with.Settings["n8n"] = []Layer{{From: "anchor", Values: map[string]any{"media-gid": float64(2000)}}}
|
||||
composed, err := r.Compose(with)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if why, left := composed.LeftOut["n8n"]; left {
|
||||
t.Fatalf("left out: %s", why)
|
||||
}
|
||||
if got := byID(composed.Resources)["n8n.server"]["groups"]; !reflect.DeepEqual(got, []any{2000}) {
|
||||
t.Fatalf("composed groups %#v", got)
|
||||
}
|
||||
|
||||
without := anchorRendering(false)
|
||||
composed, err = r.Compose(without)
|
||||
if err == nil {
|
||||
why, left := composed.LeftOut["n8n"]
|
||||
if !left || !strings.Contains(why, "media-gid") {
|
||||
t.Fatalf("a machine that says no group kept n8n or left it out without the key: %q (%v)", why, left)
|
||||
}
|
||||
} else if !strings.Contains(err.Error(), "media-gid") {
|
||||
t.Fatalf("refused without the key: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// The node-engine reads a container's groups only when it reports the capability (novox/hq ADR 0480 clause 7, issue 1638):
|
||||
// a module with a container that names groups is hosted on a machine that reports container-groups, and left out of one
|
||||
// that does not, the capability named, so the field is never sent to an engine that would refuse it.
|
||||
func TestAModuleWhoseContainerNamesGroupsIsHeldOnAMachineThatDoesNotReadThem(t *testing.T) {
|
||||
n8n := Manifest{Module: "n8n", Resources: []map[string]any{aContainerWithGroups("${config:media-gid}")}}
|
||||
plain := Manifest{Module: "plain", Resources: []map[string]any{
|
||||
{"id": "server", "type": "container", "name": "p", "image": "x@sha256:" + strings.Repeat("b", 64)}}}
|
||||
for _, m := range []Manifest{n8n, plain} {
|
||||
if got := impliedCapabilities(m); (m.Module == "n8n") != reflect.DeepEqual(got, []string{ContainerGroups}) {
|
||||
t.Errorf("%s implies %v", m.Module, got)
|
||||
}
|
||||
}
|
||||
shelf := shelf(n8n, plain)
|
||||
reads := Node{Name: "ace", Capabilities: map[string]bool{"container-runtime": true, ContainerGroups: true}}
|
||||
got, err := Resolve(shelf, []string{"n8n", "plain"}, reads, World{})
|
||||
if err != nil || len(got.Unhostable) != 0 || len(got.Modules) != 2 {
|
||||
t.Fatalf("a machine that reads groups: %v, held %+v, hosted %v", err, got.Unhostable, names(got))
|
||||
}
|
||||
old := Node{Name: "g14", Capabilities: map[string]bool{"container-runtime": true}}
|
||||
got, err = Resolve(shelf, []string{"n8n", "plain"}, old, World{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(got.Unhostable) != 1 || got.Unhostable[0].Module != "n8n" || !reflect.DeepEqual(got.Unhostable[0].Missing, []string{ContainerGroups}) {
|
||||
t.Fatalf("n8n was not held, naming the capability: %+v", got.Unhostable)
|
||||
}
|
||||
if h := names(got); len(h) != 1 || h[0] != "plain" {
|
||||
t.Errorf("the module without groups is not the one hosted: %v", h)
|
||||
}
|
||||
}
|
||||
@@ -170,10 +170,10 @@ func TestAMembershipForTheNewBusIsComposedAsASealedFile(t *testing.T) {
|
||||
func TestAStoreRowWithoutAProtocolKeepsTheCompiledOne(t *testing.T) {
|
||||
was := Seats()
|
||||
t.Cleanup(func() { UseSeats(was) })
|
||||
UseSeats([]Seat{{Name: "mesh-build-machine", Scope: ScopeMesh, Decision: "row"}})
|
||||
got, ok := SeatNamed("mesh-build-machine")
|
||||
if !ok || len(got.Accepts) == 0 {
|
||||
t.Fatalf("the build machine's seat lost what it accepts when loaded from the store: %+v", got)
|
||||
UseSeats([]Seat{{Name: "node-build-go", Scope: ScopeNode, Decision: "row"}})
|
||||
got, ok := SeatNamed("node-build-go")
|
||||
if !ok || len(got.Accepts) == 0 || !got.PulledByKind {
|
||||
t.Fatalf("the Go build seat lost what it accepts when loaded from the store: %+v", got)
|
||||
}
|
||||
if got.Decision != "row" {
|
||||
t.Fatalf("the store's own columns were not kept: %+v", got)
|
||||
|
||||
@@ -494,6 +494,9 @@ type Manifest struct {
|
||||
// by UnmarshalJSON from the long form of a `requires` entry, {name, optional}, or from the stored
|
||||
// form's `optional-requires` list; never written by hand.
|
||||
OptionalRequires map[string]bool `json:"-"`
|
||||
// RequireShares is who else may use the bucket a module requires, by requirement (BucketShare, novox/hq ADR 0468
|
||||
// §8): filled from the long form's `shares`, or from the stored form's `requires-shares`; never written by hand.
|
||||
RequireShares map[string][]BucketShare `json:"-"`
|
||||
|
||||
// Emits are the events this module publishes, named **locally**: `order.placed`, not a subject
|
||||
// and not a routing key. The mesh derives where it lands (design 29 §1), so reorganising the
|
||||
@@ -1188,6 +1191,17 @@ type Artifact struct {
|
||||
// (novox/hq ADR 0193). Absent for nearly every bundle, which is then one file per entrypoint.
|
||||
External []string `json:"external,omitempty"`
|
||||
|
||||
// Paths are the files and directories, relative to the module's directory, a package or a bundle in a language
|
||||
// other than Go is built from (novox/hq ADR 0499 clause 1): with the module's manifest they are its build source,
|
||||
// and a module whose every artifact names them takes its fingerprint's tree from them alone (clause 2). Absent means
|
||||
// the module's directory, as before.
|
||||
//
|
||||
// **Named, because the build source of a package sharing a repository cannot be derived.** A Go program's is
|
||||
// (ADR 0267 clause 1), an archive's `from` names it and an image's is the context its manifest names, so each of those
|
||||
// refuses the word. A path stays inside the module's directory and is no pattern: a build source that could reach
|
||||
// out of it, or grow with a directory listing, would be one no fingerprint pins.
|
||||
Paths []string `json:"paths,omitempty"`
|
||||
|
||||
// Env is what a tools bundle is given on a machine (novox/hq ADR 0192): words and their values,
|
||||
// paths and constants composed with ${dir:…} and ${port:…} exactly as a container's environment
|
||||
// is, never a secret's content. The node's runtime hands it to this bundle and to no other.
|
||||
@@ -1538,8 +1552,9 @@ func (m *Manifest) UnmarshalJSON(raw []byte) error {
|
||||
delete(keys, "contributes")
|
||||
}
|
||||
var optionalRequires map[string]bool
|
||||
var requireShares map[string][]BucketShare
|
||||
if requires, ok := keys["requires"]; ok && string(requires) != "null" {
|
||||
names, optional, err := readRequires(requires)
|
||||
names, optional, shares, err := readRequires(requires)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -1549,6 +1564,23 @@ func (m *Manifest) UnmarshalJSON(raw []byte) error {
|
||||
}
|
||||
keys["requires"] = plainNames
|
||||
optionalRequires = optional
|
||||
requireShares = shares
|
||||
}
|
||||
// The stored form's shares (RequireSharesKey).
|
||||
if stored, ok := keys[RequireSharesKey]; ok {
|
||||
if string(stored) != "null" {
|
||||
stood, err := readRequireShares(stored)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for requirement, shares := range stood {
|
||||
if requireShares == nil {
|
||||
requireShares = map[string][]BucketShare{}
|
||||
}
|
||||
requireShares[requirement] = shares
|
||||
}
|
||||
}
|
||||
delete(keys, RequireSharesKey)
|
||||
}
|
||||
// The stored form: `requires` plain, and which are optional beside it (OptionalRequiresKey).
|
||||
if stored, ok := keys[OptionalRequiresKey]; ok {
|
||||
@@ -1599,6 +1631,7 @@ func (m *Manifest) UnmarshalJSON(raw []byte) error {
|
||||
m.unknown = unknown
|
||||
m.forms = forms
|
||||
m.OptionalRequires = optionalRequires
|
||||
m.RequireShares = requireShares
|
||||
if len(plain) > 0 {
|
||||
m.Secrets = plain
|
||||
}
|
||||
@@ -1611,6 +1644,21 @@ func (m *Manifest) UnmarshalJSON(raw []byte) error {
|
||||
if len(contributesMany) > 0 {
|
||||
m.ContributesMany = contributesMany
|
||||
}
|
||||
if err := m.checkSharesNotContributed(); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// checkSharesNotContributed refuses a module whose own `contributes` to s3-bucket names `shares` (ADR 0468 §8).
|
||||
func (m Manifest) checkSharesNotContributed() error {
|
||||
if len(m.RequireShares[BucketRequirement]) == 0 {
|
||||
return nil
|
||||
}
|
||||
if _, named := m.Contributes[BucketRequirement][SharesValue]; named {
|
||||
return fmt.Errorf("contributes.%s names `shares` and so does its requirement; keep one, the requirement's (novox/hq ADR 0468 §8)",
|
||||
BucketRequirement)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -1655,7 +1703,7 @@ func (m Manifest) marshalFields() ([]byte, error) {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(m.SecretsMany) == 0 && len(m.ContributesMany) == 0 && len(m.OptionalRequires) == 0 {
|
||||
if len(m.SecretsMany) == 0 && len(m.ContributesMany) == 0 && len(m.OptionalRequires) == 0 && len(m.RequireShares) == 0 {
|
||||
return raw, nil
|
||||
}
|
||||
var keys map[string]json.RawMessage
|
||||
@@ -1671,6 +1719,13 @@ func (m Manifest) marshalFields() ([]byte, error) {
|
||||
}
|
||||
keys[OptionalRequiresKey] = listed
|
||||
}
|
||||
if len(m.RequireShares) > 0 {
|
||||
listed, err := json.Marshal(m.RequireShares)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
keys[RequireSharesKey] = listed
|
||||
}
|
||||
if len(m.ContributesMany) > 0 {
|
||||
mergedContributes := map[string]any{}
|
||||
for to, values := range m.Contributes {
|
||||
@@ -2170,6 +2225,7 @@ func parseManifest(raw []byte, predefined PredefinedList) (Manifest, error) {
|
||||
problems = append(problems, replacesProblems(m)...)
|
||||
problems = append(problems, UnitSettingProblems(m)...)
|
||||
problems = append(problems, OwnerSettingProblems(m)...)
|
||||
problems = append(problems, GroupSettingProblems(m)...)
|
||||
problems = append(problems, endpointNameProblems(m)...)
|
||||
problems = append(problems, RouteProblems(m)...)
|
||||
for _, port := range m.Guards {
|
||||
|
||||
@@ -4,8 +4,6 @@ import (
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/internal/beside"
|
||||
)
|
||||
|
||||
// A bind mount the module never declared is refused where it is written (novox/hq 04-ISSUES/026,
|
||||
@@ -71,14 +69,10 @@ func TestTheRuntimeSocketIsGrantedByTheCapabilityAndNotOtherwise(t *testing.T) {
|
||||
// **Every manifest in the catalogue (internal/beside) passes**, so the rule is not one the catalogue
|
||||
// is already breaking.
|
||||
func TestEveryCatalogueManifestDeclaresWhatItMounts(t *testing.T) {
|
||||
files, _ := filepath.Glob(filepath.Join(beside.Catalogue(t), "*", "module.json"))
|
||||
if len(files) == 0 {
|
||||
t.Fatal("no manifests in the catalogue, so this proved nothing")
|
||||
}
|
||||
root := beside.Dir(t, "mesh-catalog") // once: it copies the captured tree
|
||||
for _, file := range files {
|
||||
if _, err := ReadTreeManifest(root, filepath.Dir(file)); err != nil {
|
||||
t.Errorf("%s: %v", filepath.Base(filepath.Dir(file)), err)
|
||||
_, files := catalogueManifests(t)
|
||||
for _, f := range files {
|
||||
if f.err != nil {
|
||||
t.Errorf("%s: %v", filepath.Base(filepath.Dir(f.path)), f.err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -32,17 +32,20 @@ const OnlyWhenBound = "only-when-bound"
|
||||
type requirementForm struct {
|
||||
Name string `json:"name"`
|
||||
Optional bool `json:"optional,omitempty"`
|
||||
// Shares is who else may use the consumer's bucket (BucketShare, novox/hq ADR 0468 §8), on s3-bucket only.
|
||||
Shares []BucketShare `json:"shares,omitempty"`
|
||||
}
|
||||
|
||||
// readRequires reads `requires` in both of its forms — a bare name, or {name, optional} — into the
|
||||
// names and which of them are optional.
|
||||
func readRequires(raw json.RawMessage) ([]string, map[string]bool, error) {
|
||||
func readRequires(raw json.RawMessage) ([]string, map[string]bool, map[string][]BucketShare, error) {
|
||||
var entries []json.RawMessage
|
||||
if err := json.Unmarshal(raw, &entries); err != nil {
|
||||
return nil, nil, fmt.Errorf("requires: a list of names, or of {name, optional}: %w", err)
|
||||
return nil, nil, nil, fmt.Errorf("requires: a list of names, or of {name, optional}: %w", err)
|
||||
}
|
||||
var names []string
|
||||
optional := map[string]bool{}
|
||||
var shares map[string][]BucketShare
|
||||
for i, e := range entries {
|
||||
var plain string
|
||||
if err := json.Unmarshal(e, &plain); err == nil {
|
||||
@@ -53,12 +56,21 @@ func readRequires(raw json.RawMessage) ([]string, map[string]bool, error) {
|
||||
dec := json.NewDecoder(bytes.NewReader(e))
|
||||
dec.DisallowUnknownFields()
|
||||
if err := dec.Decode(&full); err != nil {
|
||||
return nil, nil, fmt.Errorf("requires[%d]: a name, or {name, optional} (novox/hq ADR 0330): %w", i, err)
|
||||
return nil, nil, nil, fmt.Errorf("requires[%d]: a name, or {name, optional} (novox/hq ADR 0330): %w", i, err)
|
||||
}
|
||||
if strings.TrimSpace(full.Name) == "" {
|
||||
return nil, nil, fmt.Errorf("requires[%d] names no requirement: {name, optional}", i)
|
||||
return nil, nil, nil, fmt.Errorf("requires[%d] names no requirement: {name, optional}", i)
|
||||
}
|
||||
names = append(names, full.Name)
|
||||
if err := checkBucketShares(full.Name, full.Shares); err != nil {
|
||||
return nil, nil, nil, fmt.Errorf("requires[%d]: %w", i, err)
|
||||
}
|
||||
if len(full.Shares) > 0 {
|
||||
if shares == nil {
|
||||
shares = map[string][]BucketShare{}
|
||||
}
|
||||
shares[full.Name] = full.Shares
|
||||
}
|
||||
if full.Optional {
|
||||
optional[full.Name] = true
|
||||
}
|
||||
@@ -66,7 +78,7 @@ func readRequires(raw json.RawMessage) ([]string, map[string]bool, error) {
|
||||
if len(optional) == 0 {
|
||||
optional = nil
|
||||
}
|
||||
return names, optional, nil
|
||||
return names, optional, shares, nil
|
||||
}
|
||||
|
||||
// OptionalRequiresKey is where a stored manifest keeps which of its requirements are optional: beside
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"os"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// mesh-sdk's own manifest is judged by the controller that runs when it is checked, and that controller reads it
|
||||
// strictly: a manifest word ships one release ahead of its first user. R4e's group is judged by today's controller
|
||||
// (novox/hq ADR 0503 §4), which does not know `paths` (ADR 0499), so mesh-sdk's lib names no paths until a release
|
||||
// after R4e's delivery adds them back (ADR 0499 §6, issue 1758).
|
||||
func TestOwnManifestUsesNoWordTheRunningControllerRefuses(t *testing.T) {
|
||||
raw, err := os.ReadFile("../../module.json")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m, err := ParseManifest(raw)
|
||||
if err != nil {
|
||||
t.Fatalf("mesh-sdk's module.json does not parse: %v", err)
|
||||
}
|
||||
if m.Build == nil || len(m.Build.Artifacts) == 0 {
|
||||
t.Fatal("mesh-sdk's module.json declares no artifact")
|
||||
}
|
||||
for _, a := range m.Build.Artifacts {
|
||||
if a.Paths != nil {
|
||||
t.Errorf("artifact %q names paths %v: the running controller refuses the word until R4e is delivered", a.Name, a.Paths)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// An artifact names the paths it is built from with `paths` (novox/hq ADR 0499 clause 1): a package, or a bundle in a
|
||||
// language other than Go, may; a Go program's artifact, an archive, an image and an upstream artifact may not, and the
|
||||
// refusal names the artifact and its kind. Each path stays inside the module's directory: no `..`, no absolute path,
|
||||
// no pattern.
|
||||
func TestAnArtifactNamesThePathsItIsBuiltFrom(t *testing.T) {
|
||||
m, err := ParseManifest([]byte(`{"module":"mesh-sdk","version":"1","slug":"sdk","build":{"artifacts":[
|
||||
{"name":"lib","kind":"package","language":"typescript",
|
||||
"paths":["src","test","package.json","package-lock.json","tsconfig.json"]}]},"resources":[]}`))
|
||||
if err != nil {
|
||||
t.Fatalf("a package naming its paths was refused: %v", err)
|
||||
}
|
||||
if got := strings.Join(m.Build.Artifacts[0].Paths, ","); got != "src,test,package.json,package-lock.json,tsconfig.json" {
|
||||
t.Fatalf("the paths were not kept as written: %q", got)
|
||||
}
|
||||
if _, err := ParseManifest([]byte(`{"module":"a","version":"1","build":{"artifacts":[
|
||||
{"name":"tools","kind":"bundle","language":"typescript","entrypoints":["index.js"],"loads":["index.js"],"paths":["src","package.json"]}]}}`)); err != nil {
|
||||
t.Fatalf("a TypeScript bundle naming its paths was refused: %v", err)
|
||||
}
|
||||
|
||||
for _, c := range []struct{ artifact, says string }{
|
||||
{`{"name":"x","kind":"bundle","language":"go","system":"linux-amd64","paths":["cmd"]}`, `"x" is a Go program`},
|
||||
{`{"name":"x","kind":"archive","from":"files","paths":["files"]}`, `"x" is a "archive"`},
|
||||
{`{"name":"x","kind":"image","from":"Dockerfile","paths":["src"]}`, `"x" is a "image"`},
|
||||
{`{"name":"x","kind":"upstream","from":"nats:2.10","paths":["src"]}`, `"x" is a "upstream"`},
|
||||
{`{"name":"x","kind":"package","language":"typescript","paths":["../other"]}`, "leaves the module's directory"},
|
||||
{`{"name":"x","kind":"package","language":"typescript","paths":["src/../../x"]}`, "leaves the module's directory"},
|
||||
{`{"name":"x","kind":"package","language":"typescript","paths":["/etc"]}`, "leaves the module's directory"},
|
||||
{`{"name":"x","kind":"package","language":"typescript","paths":["src/*.ts"]}`, "is a pattern"},
|
||||
{`{"name":"x","kind":"package","language":"typescript","paths":["src/[ab]"]}`, "is a pattern"},
|
||||
{`{"name":"x","kind":"package","language":"typescript","paths":[""]}`, "names an empty path"},
|
||||
{`{"name":"x","kind":"package","language":"typescript","paths":["."]}`, "names the module's whole directory"},
|
||||
{`{"name":"x","kind":"package","language":"typescript","paths":[]}`, "names no path"},
|
||||
} {
|
||||
_, err := ParseManifest([]byte(`{"module":"a","version":"1","build":{"artifacts":[` + c.artifact + `]}}`))
|
||||
if err == nil || !strings.Contains(err.Error(), c.says) {
|
||||
t.Errorf("%s: %v, wanted a refusal saying %q", c.artifact, err, c.says)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The paths a module's tree is taken from are the union its artifacts name, cleaned and sorted, and only when every
|
||||
// artifact names some (novox/hq ADR 0499 clause 2); a module with an artifact that names none, or with none at all,
|
||||
// keeps its directory's tree.
|
||||
func TestAModulesNamedPathsAreTheUnionWhenEveryArtifactNamesThem(t *testing.T) {
|
||||
m := Manifest{Module: "a", Build: &Build{Artifacts: []Artifact{
|
||||
{Name: "lib", Kind: ArtifactPackage, Paths: []string{"src/", "package.json"}},
|
||||
{Name: "tools", Kind: ArtifactBundle, Paths: []string{"./src", "tools"}},
|
||||
}}}
|
||||
paths, every := m.NamedPaths()
|
||||
if !every || strings.Join(paths, ",") != "package.json,src,tools" {
|
||||
t.Fatalf("NamedPaths = %v, %v; want [package.json src tools], true", paths, every)
|
||||
}
|
||||
m.Build.Artifacts = append(m.Build.Artifacts, Artifact{Name: "img", Kind: ArtifactImage, From: "Dockerfile"})
|
||||
if paths, every := m.NamedPaths(); every || paths != nil {
|
||||
t.Fatalf("a module with an artifact naming no path gave %v, %v", paths, every)
|
||||
}
|
||||
if paths, every := (Manifest{Module: "b"}).NamedPaths(); every || paths != nil {
|
||||
t.Fatalf("a module that builds nothing gave %v, %v", paths, every)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
package catalogue
|
||||
|
||||
import "encoding/json"
|
||||
|
||||
// ParseManifestForInputs reads a manifest for the inputs of its build and its fingerprint: the indexer's and the
|
||||
// builder's read (novox/hq issue 1751). It is never a judge of config values. A value falling back to a predefined key
|
||||
// (ADR 0348 §7) is judged against the list the mesh published when this process holds one (PublishedPredefined);
|
||||
// where it holds none, as in the indexer and the builder, each key a value names is taken as the value declares it (its
|
||||
// type, trusted when it is), since the list is read where a manifest is judged: the merge gate from the tree's root,
|
||||
// the controller from what it published. No input of a fingerprint or a build comes from the list. Everything else
|
||||
// ParseManifest refuses is refused here too.
|
||||
func ParseManifestForInputs(raw []byte) (Manifest, error) {
|
||||
list := PublishedPredefined()
|
||||
if list == nil {
|
||||
list = fallbacksOf(raw)
|
||||
}
|
||||
return parseManifest(raw, list)
|
||||
}
|
||||
|
||||
// fallbacksOf is the list of the predefined keys a manifest's config values fall back to, each as the value declares
|
||||
// it. A manifest that is not JSON gives an empty list, and the parser says what is wrong with it.
|
||||
func fallbacksOf(raw []byte) PredefinedList {
|
||||
var m struct {
|
||||
Config map[string]struct {
|
||||
Type string `json:"type"`
|
||||
Description string `json:"description"`
|
||||
Predefined string `json:"predefined"`
|
||||
Trusted bool `json:"trusted"`
|
||||
} `json:"config"`
|
||||
}
|
||||
list := PredefinedList{}
|
||||
if json.Unmarshal(raw, &m) != nil {
|
||||
return list
|
||||
}
|
||||
for _, d := range m.Config {
|
||||
if d.Predefined != "" {
|
||||
list[d.Predefined] = PredefinedKey{Type: d.Type, Description: d.Description, Trusted: d.Trusted}
|
||||
}
|
||||
}
|
||||
return list
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// The indexer and the builder hold no published list of predefined keys; a manifest whose config values fall back to
|
||||
// one (claude-code, dunst, xterm at mesh-catalog 835bf22b) is still read for its build's and fingerprint's inputs
|
||||
// (issue 1751).
|
||||
func TestParseManifestForInputsReadsAValueFallingBackToAPredefinedKey(t *testing.T) { // issue 1751
|
||||
was := ReplacePublished(nil) // the indexer's and the builder's case: no published list
|
||||
defer ReplacePublished(was)
|
||||
for _, module := range []string{"claude-code", "dunst", "xterm"} {
|
||||
raw, err := os.ReadFile("testdata/catalogue-835bf22b-" + module + ".module.json")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := ParseManifest(raw); err == nil {
|
||||
t.Fatalf("%s: ParseManifest reads it with no list; the test proves nothing", module)
|
||||
}
|
||||
if m, err := ParseManifestForInputs(raw); err != nil || m.Module != module {
|
||||
t.Errorf("%s: %v", module, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseManifestForInputsStillRefusesWhatTheParserRefuses(t *testing.T) { // issue 1751
|
||||
was := ReplacePublished(nil)
|
||||
defer ReplacePublished(was)
|
||||
raw, _ := os.ReadFile("testdata/catalogue-835bf22b-xterm.module.json")
|
||||
bad := strings.Replace(string(raw), `"predefined": "font-size"`, `"predefined": "font-size", "trusted": "yes"`, 1)
|
||||
if bad == string(raw) {
|
||||
t.Fatal("the fixture no longer names font-size; the test proves nothing")
|
||||
}
|
||||
if _, err := ParseManifestForInputs([]byte(bad)); err == nil {
|
||||
t.Error("a manifest the parser refuses was read")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
These tests read files outside this directory (the repository's root files or other directories): mesh-build-go never keeps this package's verdict (issue 1935).
|
||||
+18
-1
@@ -904,7 +904,7 @@ func isModule(catalogue map[string]Manifest, want string) bool {
|
||||
// machine can host the module as far as its capabilities go.
|
||||
func missingCapabilities(m Manifest, node Node) []string {
|
||||
var missing []string
|
||||
for _, c := range m.Capabilities {
|
||||
for _, c := range append(append([]string(nil), m.Capabilities...), impliedCapabilities(m)...) {
|
||||
if !node.Capabilities[c] {
|
||||
missing = append(missing, c)
|
||||
}
|
||||
@@ -1457,3 +1457,20 @@ func packageNames(m map[string][]string) []string {
|
||||
sort.Strings(names)
|
||||
return names
|
||||
}
|
||||
|
||||
// ContainerGroups is the capability a node-engine reports when it reads a container's `groups` and passes each as a
|
||||
// supplementary group (novox/hq ADR 0480 clause 7, issue 1638). An older engine refuses the key (strict decoding).
|
||||
const ContainerGroups = "container-groups"
|
||||
|
||||
// impliedCapabilities are the capabilities a module needs for what its resources say, beyond those it declares: a
|
||||
// container that names groups needs an engine that reads them, so the module is left out of a machine whose report
|
||||
// lacks the capability, naming it, and the field is sent to no engine that would refuse it. None for a module whose
|
||||
// containers name no groups.
|
||||
func impliedCapabilities(m Manifest) []string {
|
||||
for _, r := range m.Resources {
|
||||
if fmt.Sprint(r["type"]) == "container" && len(groupsOf(r)) > 0 {
|
||||
return []string{ContainerGroups}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -66,7 +66,8 @@ var capabilityName = regexp.MustCompile(`^[a-z0-9][a-z0-9-]*$`)
|
||||
// names an `if-capability` may give (novox/hq ADR 0255). A name nothing detects would leave its
|
||||
// contribution out on every machine, silently, so it is refused. A detector added there is added here.
|
||||
var KnownCapabilities = []string{
|
||||
"battery", "container-runtime", "firewall", "graphical-session", "overlay", "package-manager",
|
||||
"battery", "container-groups", "container-runtime", "file-references", "firewall", "graphical-session", "overlay",
|
||||
"package-manager",
|
||||
"power-meter", "privileged", "seat", "service-manager", "uplink-dhcpcd", "uplink-networkmanager",
|
||||
"uplink-systemd-networkd", "user-namespaces", "virtualisation",
|
||||
}
|
||||
|
||||
+50
-41
@@ -45,6 +45,10 @@ type Seat struct {
|
||||
// today — which the bus refuses, because a namespace belongs to who it is named for.
|
||||
Accepts []string
|
||||
Emits []string
|
||||
// PulledByKind says each accepted kind is a worker of its own on the seat's stream, which its holders pull in the
|
||||
// order Accepts lists them (novox/hq ADR 0525 §2, ADR 0536 §4): the build seats'. Compiled, never stored, like
|
||||
// Receives.
|
||||
PulledByKind bool
|
||||
// Serves carries each verb in full — name, description, schema — because a role's tools are the
|
||||
// mesh's to define and an agent's to call (novox/hq ADR 0132, design 33 §2).
|
||||
Serves []Verb
|
||||
@@ -106,12 +110,18 @@ var defaultSeats = append([]Seat{
|
||||
"rolled-back",
|
||||
// A pull request's merge check, judged (novox/hq to-be 45 §9).
|
||||
"checked",
|
||||
// A walk kept in a new state, for the delivery it walks (novox/hq ADR 0239).
|
||||
// A walk kept in a new state, and a tier sent to a machine: kept until the catalogue's main consumes them no
|
||||
// more; nothing emits them (ADR 0539).
|
||||
"plan-moved",
|
||||
// A tier sent to a machine, once the send is kept (novox/hq ADR 0406 §5, issue 974).
|
||||
"sent",
|
||||
// A key of a record mirror written or removed (novox/hq ADR 0289, design 54 §3; design 57 F3).
|
||||
"assignment-changed", "declared-config-changed", "predefined-changed"},
|
||||
"assignment-changed", "declared-config-changed", "predefined-changed",
|
||||
// A builder's key not taken in because the artifact store does not hold an artifact it names (novox/hq
|
||||
// issue 1937): the builder builds it again.
|
||||
"artifacts-gone",
|
||||
// A bucket of a state this release deleted, deleted at the controller's first start (novox/hq ADR 0539
|
||||
// clause 13): the merge window's, the silent sends' and the building plans' hold, each said once.
|
||||
"bucket-deleted"},
|
||||
Serves: ControllerVerbs},
|
||||
// The store's first verbs (novox/hq ADR 0159): the smallest set that makes the store askable,
|
||||
// served by whichever module holds the seat with tools of these names.
|
||||
@@ -238,31 +248,19 @@ var defaultSeats = append([]Seat{
|
||||
// They keep their names until that migration is done deliberately, apart from the node-* pass.
|
||||
{Name: "npm-package-registry", Scope: ScopeMesh, Delivers: "npm-package-registry", Decision: "novox/hq ADR 0109"},
|
||||
{Name: "git", Scope: ScopeMesh, Delivers: "git", Decision: "novox/hq ADR 0111"},
|
||||
// A build is work submitted to this role and its outcome is the role's own event (ADR 0129).
|
||||
// One publish reaches whoever asked, the controller that records it, and the catalogue that
|
||||
// places it in the graph — what the old bus's shared exchange did for free.
|
||||
// A build says what it does as it does it (novox/hq ADR 0157): `started` when work is taken,
|
||||
// `log.<build id>` for every line, `built` for the outcome. The log's tail token is the build's
|
||||
// id, so a reader follows one build by subject alone.
|
||||
// **Node-scoped, and every holder takes from one queue** (novox/hq ADR 0190): a build is asked of
|
||||
// the role, and whichever machine holding the seat is idle pulls it. One holder per machine is
|
||||
// what the scope says; sharing the work is what a seat's queue has always done.
|
||||
//
|
||||
// **And its holder answers for the build it is running** (novox/hq ADR 0219): what it is
|
||||
// building, kill it, take nothing new, take again. The queue as a whole is the controller's to
|
||||
// show and change (`queue`, `cancel`, `clear`); what one machine does with an ask it already
|
||||
// took only that machine can do. `paused.<node>` is each holder saying whether it takes work, so
|
||||
// the controller can tell a plan waiting on a paused seat from one that is late.
|
||||
{Name: "node-build-agent", Scope: ScopeNode,
|
||||
Accepts: []string{"build"}, Emits: []string{"started", "built", "log.*", "paused.*"},
|
||||
Serves: buildAgentVerbs(),
|
||||
Decision: "novox/hq ADR 0190, ADR 0219"},
|
||||
// **Retired by ADR 0190, kept while a manifest still claims it.** The one build machine's seat.
|
||||
// A claim to a seat the mesh no longer defines is refused, and the module holding this one is
|
||||
// assigned on a live machine until build-agent replaces it — removing the row first would make
|
||||
// that machine unresolvable in the meantime. Deleted once no registered manifest claims it.
|
||||
{Name: "mesh-build-machine", Scope: ScopeMesh,
|
||||
Accepts: []string{"build"}, Emits: []string{"started", "built", "log.*"}, Decision: "novox/hq ADR 0190"},
|
||||
// **A build node's work, one seat per build language** (novox/hq ADR 0525 §2, §10; ADR 0536 §4; issue 1887). The
|
||||
// build module of each language claims its seat on every node it runs on; the builder and the checker put their
|
||||
// asks on the seat's accepted kinds, which the broker derives as one work queue with a shared worker per kind, and
|
||||
// each holder pulls delivery first, then check, then recheck, at most its node's build capacity at once. Its verbs
|
||||
// say what the node runs and pause or resume it there: every holder pulls from the same worker, so a node's pause is
|
||||
// its holder taking nothing, kept on the module's own pause key (issue 1887's design limit 2). They replace
|
||||
// node-build-agent, retired with build-agent (ADR 0524 §5).
|
||||
{Name: "node-build-go", Scope: ScopeNode, Accepts: buildAskKinds(), PulledByKind: true, Serves: buildSeatVerbs("Go"),
|
||||
Decision: "novox/hq ADR 0525, ADR 0536"},
|
||||
{Name: "node-build-node", Scope: ScopeNode, Accepts: buildAskKinds(), PulledByKind: true, Serves: buildSeatVerbs("Node"),
|
||||
Decision: "novox/hq ADR 0525, ADR 0536"},
|
||||
{Name: "node-build-image", Scope: ScopeNode, Accepts: buildAskKinds(), PulledByKind: true, Serves: buildSeatVerbs("image"),
|
||||
Decision: "novox/hq ADR 0525, ADR 0536"},
|
||||
// **The mesh's resolvers** (novox/hq ADR 0194, 0196, 0223): every node's internal domain, and
|
||||
// every node and container asks them and nothing else. Replicated since ADR 0223: held on more
|
||||
// than one machine, each answering the same names from the same roster, and every machine's
|
||||
@@ -498,6 +496,7 @@ func UseSeats(s []Seat) {
|
||||
row.Receives = d.Receives
|
||||
row.Paths = d.Paths
|
||||
row.Replicated = d.Replicated
|
||||
row.PulledByKind = d.PulledByKind
|
||||
row.Serves = optionalAsCompiled(row.Serves, d.Serves)
|
||||
}
|
||||
merged = append(merged, row)
|
||||
@@ -623,6 +622,10 @@ func claimProblems(m Manifest) []string {
|
||||
// after UseSeats has replaced the set with the store's.
|
||||
continue
|
||||
}
|
||||
if replaced, retired := RetiredSeats[c.Name]; retired {
|
||||
problems = append(problems, fmt.Sprintf("%s claims the seat %q, which is retired: its work is %s", m.Module, c.Name, replaced))
|
||||
continue
|
||||
}
|
||||
if isSystemSeatName(c.Name) {
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s claims %q, which is a seat in the mesh's own namespace (mesh-*/node-*) that it "+
|
||||
@@ -903,22 +906,28 @@ func loginShellVerbs() []Verb {
|
||||
}
|
||||
}
|
||||
|
||||
// buildAgentVerbs are what a holder of node-build-agent answers on its own machine (novox/hq ADR
|
||||
// 0219). One build at a time per holder (ADR 0190), so "the build running here" is one or none.
|
||||
func buildAgentVerbs() []Verb {
|
||||
// buildAskKinds are a build seat's accepted kinds, in the order a holder pulls them (ADR 0536 §4; mesh-sdk
|
||||
// delivery.AskKinds).
|
||||
func buildAskKinds() []string { return []string{"delivery", "check", "recheck"} }
|
||||
|
||||
// RetiredSeats are seats the mesh no longer defines, each with what replaced it: a claim or a use of one is refused
|
||||
// naming its replacement, not as a seat nobody declares.
|
||||
var RetiredSeats = map[string]string{
|
||||
"node-build-agent": "node-build-go, node-build-node and node-build-image, held by the build modules (novox/hq ADR 0525 §10)",
|
||||
"mesh-build-machine": "node-build-go, node-build-node and node-build-image, held by the build modules (novox/hq ADR 0190, ADR 0525 §10)",
|
||||
}
|
||||
|
||||
// buildSeatVerbs are what a build module answers on its own node for its seat (ADR 0525 §10).
|
||||
func buildSeatVerbs(language string) []Verb {
|
||||
return []Verb{
|
||||
{Name: "current", Description: "The build this machine is running — its id, repository, path, " +
|
||||
"the step it is at, when it started and for how long — or none; and whether this machine is " +
|
||||
"paused, taking no new build.",
|
||||
{Name: "current", Description: "The " + language + " asks this node runs now (each ask's id, kind, module and when it " +
|
||||
"started), its build capacity, and whether it is paused, taking no new ask.",
|
||||
Input: schema(map[string]string{}, nil)},
|
||||
{Name: "kill", Description: "End the build with this id, running here: its commands and the " +
|
||||
"containers it started are stopped, and its outcome is announced as failed, killed by hand — " +
|
||||
"settled, so it is not handed to another machine.",
|
||||
Input: schema(map[string]string{"id": "the build's id, as `queue` or `current` says it"}, []string{"id"})},
|
||||
{Name: "pause", Description: "Take no new build on this machine until resumed; a build running " +
|
||||
"here finishes. Kept across a restart of the holder.",
|
||||
{Name: "pause", Description: "Take no new " + language + " ask on this node until resumed; an ask running here " +
|
||||
"ends in its place. Kept across a restart of the build module (its pause key). A capacity of 0 is the " +
|
||||
"lasting way to keep a node from building.",
|
||||
Input: schema(map[string]string{}, nil)},
|
||||
{Name: "resume", Description: "Take builds again on this machine.",
|
||||
{Name: "resume", Description: "Take " + language + " asks again on this node.",
|
||||
Input: schema(map[string]string{}, nil)},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -364,6 +364,10 @@ func CatalogueProblems(shelf Shelf) []string {
|
||||
// A `uses` naming nothing is where ADR 0110's guarantee lands under a derived set: the
|
||||
// same refusal, at the same moment, from a set nobody maintains by hand.
|
||||
for _, u := range m.Uses {
|
||||
if replaced, retired := RetiredSeats[u]; retired && !exists(u) {
|
||||
problems = append(problems, fmt.Sprintf("%s uses the seat %q, which is retired: its work is %s", module, u, replaced))
|
||||
continue
|
||||
}
|
||||
if !exists(u) {
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s uses the seat %q, "+UndeclaredSeat,
|
||||
@@ -372,6 +376,10 @@ func CatalogueProblems(shelf Shelf) []string {
|
||||
}
|
||||
|
||||
for _, c := range m.Claims {
|
||||
if replaced, retired := RetiredSeats[c.Name]; retired && !exists(c.Name) {
|
||||
problems = append(problems, fmt.Sprintf("%s claims the seat %q, which is retired: its work is %s", module, c.Name, replaced))
|
||||
continue
|
||||
}
|
||||
if !exists(c.Name) {
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s claims the seat %q, "+UndeclaredSeat,
|
||||
@@ -423,13 +431,6 @@ func CatalogueProblems(shelf Shelf) []string {
|
||||
"operator's account, which every agent runs as (novox/hq ADR 0259)", module, why))
|
||||
}
|
||||
}
|
||||
// A read of a module's state that module does not keep (novox/hq ADR 0201) — said only where the
|
||||
// owner is on the shelf, as a consumer may be installed before its emitter.
|
||||
var manifests []Manifest
|
||||
for _, module := range shelfOrder(shelf) {
|
||||
manifests = append(manifests, shelf[module])
|
||||
}
|
||||
problems = append(problems, StateReadsNothingDeclares(manifests)...)
|
||||
sort.Strings(problems)
|
||||
return problems
|
||||
}
|
||||
|
||||
@@ -43,7 +43,8 @@ func TestTheSeatsAreAClosedSetAndEachNamesItsDecision(t *testing.T) {
|
||||
delivered[s.Delivers] = s.Name
|
||||
}
|
||||
}
|
||||
// Forty-five with node-files (novox/hq ADR 0394); forty-four again once node-reads goes.
|
||||
// Forty-six with the three build seats replacing node-build-agent and the retired mesh-build-machine (novox/hq
|
||||
// ADR 0525 §2, §10); forty-five with node-files (novox/hq ADR 0394); forty-four again once node-reads goes.
|
||||
// Forty-four with node-provisioner (novox/hq ADR 0349 clause 3); forty-three with mesh-provisioner (novox/hq ADR 0349 clause 3); forty-two with mesh-config (novox/hq ADR 0348 §9); forty-one with node-reads (novox/hq ADR 0310); forty with node-nfs-server and node-mounts (novox/hq ADR 0263); thirty-eight with node-resolver (novox/hq ADR 0247); thirty-seven with mesh-delivery (novox/hq ADR 0239); thirty-six since node-resolver-config retired
|
||||
// into node-uplink (novox/hq ADR 0223); thirty-seven
|
||||
// since the retired node-dns-resolver went (novox/hq ADR 0220); thirty-eight with
|
||||
@@ -55,8 +56,8 @@ func TestTheSeatsAreAClosedSetAndEachNamesItsDecision(t *testing.T) {
|
||||
// node-container-runtime (ADR 0207); nineteen with node-environment and node-login-shell (ADR 0203,
|
||||
// ADR 0204); seventeen with node-build-agent (ADR 0190). One fewer once the retired
|
||||
// mesh-build-machine row goes, when no registered manifest claims it.
|
||||
if len(Seats()) != 45 {
|
||||
t.Errorf("the mesh defines %d seats rather than 45; the set is closed, so a change here is "+
|
||||
if len(Seats()) != 46 {
|
||||
t.Errorf("the mesh defines %d seats rather than 46; the set is closed, so a change here is "+
|
||||
"a decision (novox/hq ADR 0110): %s", len(Seats()), seatNames())
|
||||
}
|
||||
}
|
||||
@@ -300,3 +301,35 @@ func TestAFormerNameResolvesAfterARename(t *testing.T) {
|
||||
t.Fatalf("the holder recorded under the former name was not matched: %+v found=%v", holder, found)
|
||||
}
|
||||
}
|
||||
|
||||
// 1887/A9a: the catalogue defines no seat node-build-agent, and a manifest claiming it is refused naming its
|
||||
// replacement; the build modules' seats accept the three kinds in their pull order and serve current, pause, resume.
|
||||
func TestNodeBuildAgentIsRetiredForTheBuildSeats(t *testing.T) {
|
||||
for _, gone := range []string{"node-build-agent", "mesh-build-machine"} {
|
||||
if _, ok := SeatNamed(gone); ok {
|
||||
t.Errorf("the mesh still defines %s", gone)
|
||||
}
|
||||
}
|
||||
_, err := ParseManifest([]byte(`{"module":"build-agent","version":"1","claims":[{"name":"node-build-agent","scope":"node"}]}`))
|
||||
if err == nil || !strings.Contains(err.Error(), `"node-build-agent", which is retired`) || !strings.Contains(err.Error(), "node-build-go") {
|
||||
t.Fatalf("the claim is not refused naming its replacement: %v", err)
|
||||
}
|
||||
// And over a catalogue whose store still holds no row for it, a use of it is refused the same way.
|
||||
user := Manifest{Module: "watcher", Uses: []string{"node-build-agent"}}
|
||||
if joined := strings.Join(CatalogueProblems(Shelf{user.Module: user}), "; "); !strings.Contains(joined, "which is retired: its work is node-build-go") {
|
||||
t.Fatalf("a use is not refused naming the replacement: %s", joined)
|
||||
}
|
||||
for _, name := range []string{"node-build-go", "node-build-node", "node-build-image"} {
|
||||
s, ok := SeatNamed(name)
|
||||
if !ok || s.Scope != ScopeNode || !s.PulledByKind || strings.Join(s.Accepts, ",") != "delivery,check,recheck" {
|
||||
t.Fatalf("%s = %+v", name, s)
|
||||
}
|
||||
var verbs []string
|
||||
for _, v := range s.Serves {
|
||||
verbs = append(verbs, v.Name)
|
||||
}
|
||||
if strings.Join(verbs, ",") != "current,pause,resume" {
|
||||
t.Errorf("%s serves %v", name, verbs)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -85,6 +85,9 @@ func settingInto(resource map[string]any, layers []Layer, module string) error {
|
||||
if fmt.Sprint(resource["type"]) == "directory" {
|
||||
return settingIntoOwner(resource, layers, module)
|
||||
}
|
||||
if fmt.Sprint(resource["type"]) == "container" {
|
||||
return settingIntoGroups(resource, layers, module)
|
||||
}
|
||||
if fmt.Sprint(resource["type"]) != "file" {
|
||||
return nil
|
||||
}
|
||||
@@ -324,7 +327,7 @@ func settingKeysUsedBy(m Manifest) map[string]bool {
|
||||
return used
|
||||
}
|
||||
|
||||
// settingKeysAskedOutsideOwners is every key a module's files, units, contributions and served facts
|
||||
// settingKeysAskedOutsideOwners is every key a module's files, units, items, contributions and served facts
|
||||
// ask for: what a directory's owner may follow (OwnerSettingProblems).
|
||||
func settingKeysAskedOutsideOwners(m Manifest) map[string]bool {
|
||||
used := map[string]bool{}
|
||||
@@ -343,8 +346,18 @@ func settingKeysAskedOutsideOwners(m Manifest) map[string]bool {
|
||||
if unit, ok := r["unit"].(string); ok {
|
||||
note(unit)
|
||||
}
|
||||
case "container":
|
||||
for _, g := range groupsOf(r) {
|
||||
if s, ok := g.(string); ok {
|
||||
note(s)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// An item's text, filled from its template (WithTemplatesFrom), places a key as a file's does (issue 1658).
|
||||
for _, it := range m.Items {
|
||||
note(it.Content)
|
||||
}
|
||||
inValues := func(values map[string]any) {
|
||||
for _, v := range values {
|
||||
if s, ok := v.(string); ok {
|
||||
@@ -365,3 +378,100 @@ func settingKeysAskedOutsideOwners(m Manifest) map[string]bool {
|
||||
}
|
||||
return used
|
||||
}
|
||||
|
||||
// A container's supplementary groups may be an operator's value (novox/hq ADR 0480 clause 7, ADR 0348 §11, issue 1636):
|
||||
// n8n joins the media library's group, whose number is the machine's, never the definition's (ADR 0112 §1). Each
|
||||
// element of `groups` is a whole placeholder or a whole number; a placeholder is filled with the number the
|
||||
// config value holds, and the list reaches the node-engine as numbers.
|
||||
|
||||
// groupSide is one element of a container's groups: a whole placeholder or a whole number.
|
||||
var groupSide = regexp.MustCompile(`^(?:\$\{setting:[a-z0-9][a-z0-9_.-]*\}|[0-9]+)$`)
|
||||
|
||||
// groupsOf is a container resource's groups as the manifest wrote them (text or numbers), none when it names none.
|
||||
func groupsOf(r map[string]any) []any {
|
||||
groups, _ := r["groups"].([]any)
|
||||
return groups
|
||||
}
|
||||
|
||||
// GroupSettingProblems refuses, where a manifest is read, a container whose groups hold anything but whole
|
||||
// placeholders and whole numbers, or a placeholder whose key the module neither declares nor asks for elsewhere.
|
||||
func GroupSettingProblems(m Manifest) []string {
|
||||
var problems []string
|
||||
for _, r := range m.Resources {
|
||||
if fmt.Sprint(r["type"]) != "container" {
|
||||
continue
|
||||
}
|
||||
for _, g := range groupsOf(r) {
|
||||
s, isText := g.(string)
|
||||
if !isText {
|
||||
continue // a number: the declaration judges its value
|
||||
}
|
||||
if !groupSide.MatchString(s) {
|
||||
problems = append(problems, fmt.Sprintf("%s: the container %v's group %q is not a whole "+
|
||||
"${config:key} or a whole number: a supplementary group is a number, and nothing else is "+
|
||||
"written into it (novox/hq ADR 0480 clause 7)", m.Module, r["id"], s))
|
||||
continue
|
||||
}
|
||||
if m.Templated() {
|
||||
continue
|
||||
}
|
||||
for _, key := range settingsUsed(s) {
|
||||
if _, declared := m.Settings[key]; !declared && !m.DeclaresConfig() {
|
||||
continue // the settings form judges its own keys
|
||||
}
|
||||
if _, declared := m.Settings[key]; !declared {
|
||||
problems = append(problems, fmt.Sprintf("%s: the container %v's group says ${config:%s}, and the module "+
|
||||
"declares no such value (novox/hq ADR 0348 §4)", m.Module, r["id"], key))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return problems
|
||||
}
|
||||
|
||||
// settingIntoGroups fills ${setting:…} in a container's groups, refusing — with the key and why, the resource left as
|
||||
// it was — a key nothing sets and a value that is not a whole number above zero. A number is kept; the list that
|
||||
// results is numbers alone.
|
||||
func settingIntoGroups(resource map[string]any, layers []Layer, module string) error {
|
||||
groups := groupsOf(resource)
|
||||
if len(groups) == 0 {
|
||||
return nil
|
||||
}
|
||||
filled := make([]any, 0, len(groups))
|
||||
for _, g := range groups {
|
||||
s, isText := g.(string)
|
||||
if !isText {
|
||||
if f, isNumber := g.(float64); isNumber && f == float64(int(f)) {
|
||||
g = int(f) // a number the manifest wrote, as the numbers a placeholder gives
|
||||
}
|
||||
filled = append(filled, g)
|
||||
continue
|
||||
}
|
||||
keys := settingsUsed(s)
|
||||
if len(keys) != 1 || settingRef.ReplaceAllString(s, "") != "" {
|
||||
filled = append(filled, g) // not a whole placeholder: the manifest check refuses it
|
||||
continue
|
||||
}
|
||||
key := keys[0]
|
||||
value, set := settingValue(layers, key)
|
||||
if !set {
|
||||
return &UnsetSettingError{Module: module, Setting: key,
|
||||
where: fmt.Sprintf("a container %v whose groups", resource["id"]), setToday: orNoSettings(layers),
|
||||
said: fmt.Sprintf(
|
||||
"%s has a container %v whose groups say ${setting:%s}, and nothing sets %q for it — an operator's "+
|
||||
"value is the assignment's, never the definition's (novox/hq ADR 0112): "+
|
||||
"`settings set %s <file>` with {%q: …}%s",
|
||||
module, resource["id"], key, key, module, key, orNoSettings(layers))}
|
||||
}
|
||||
v := plainly(value)
|
||||
id, whole := wholeID(v)
|
||||
if !whole || id == 0 {
|
||||
return fmt.Errorf("%s: the setting %q is %q, which cannot be a group of the container %v: a supplementary "+
|
||||
"group is a whole number above zero, never a name or other text (novox/hq ADR 0480 clause 7, ADR 0328 §15)",
|
||||
module, key, v, resource["id"])
|
||||
}
|
||||
filled = append(filled, int(id))
|
||||
}
|
||||
resource["groups"] = filled
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -126,44 +126,3 @@ func StateProblems(m Manifest) []string {
|
||||
}
|
||||
return problems
|
||||
}
|
||||
|
||||
// ControllerMirrors are the controller's record mirrors a module may read, each `mesh-controller.<name>`
|
||||
// (novox/hq ADR 0289, design 54 §5, design 57 F3). **The controller keeps them, not its manifest's `state`**:
|
||||
// they are asserted with the controller's own buckets, with options of their own, and written by the
|
||||
// controller's account alone (the writers table); declared as a module's state, they would be asserted a
|
||||
// second way and granted to the module's own principal as well. Named here so a read of one is a read of
|
||||
// kept state. The bus's list is broker.ControllerMirrors; a test keeps the two agreeing.
|
||||
var ControllerMirrors = []string{"assignments", "declared-config", "predefined", "builds", "rollouts", "checks-running"}
|
||||
|
||||
// StateReadsNothingDeclares is every read across a catalogue whose owner is present and declares no
|
||||
// such state. An absent owner says nothing — a module may be installed long before the one whose
|
||||
// state it reads, as a consumer may before its emitter (design 32 §1).
|
||||
func StateReadsNothingDeclares(manifests []Manifest) []string {
|
||||
declared := map[string]map[string]bool{}
|
||||
for _, m := range manifests {
|
||||
own := map[string]bool{}
|
||||
for _, s := range m.State {
|
||||
own[s.Name] = true
|
||||
}
|
||||
declared[m.Module] = own
|
||||
}
|
||||
if own, present := declared[ControllerSeatName]; present {
|
||||
for _, name := range ControllerMirrors {
|
||||
own[name] = true
|
||||
}
|
||||
}
|
||||
var problems []string
|
||||
for _, m := range manifests {
|
||||
for _, r := range m.Reads {
|
||||
module, local, err := ReadState(r)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if own, present := declared[module]; present && !own[local] {
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s reads %q, and %s keeps no state called %q", m.Module, r, module, local))
|
||||
}
|
||||
}
|
||||
}
|
||||
return problems
|
||||
}
|
||||
|
||||
@@ -52,59 +52,29 @@ func TestAStateNameIsLocalAndOneToken(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// A read whose owner is present must name a state that owner keeps; an absent owner says nothing,
|
||||
// because a module may be installed before the one whose state it reads.
|
||||
func TestAReadNamesStateItsOwnerKeeps(t *testing.T) {
|
||||
owner := Manifest{Module: "licence-manager", State: []StateDeclaration{{Name: "bindings"}}}
|
||||
good := Manifest{Module: "claude-code", Reads: []string{"licence-manager.bindings", "absent.anything"}}
|
||||
bad := Manifest{Module: "other", Reads: []string{"licence-manager.tokens"}}
|
||||
if p := StateReadsNothingDeclares([]Manifest{owner, good}); len(p) != 0 {
|
||||
t.Fatalf("a read of declared state was refused: %v", p)
|
||||
}
|
||||
p := StateReadsNothingDeclares([]Manifest{owner, bad})
|
||||
if len(p) != 1 || !strings.Contains(p[0], `licence-manager keeps no state called "tokens"`) {
|
||||
t.Fatalf("a read of state nobody keeps was not named: %v", p)
|
||||
}
|
||||
}
|
||||
|
||||
// **The controller's record mirrors are kept state** (design 54 §5, design 57 F3): a module may read
|
||||
// `mesh-controller.assignments` and `mesh-controller.declared-config` although the controller's manifest
|
||||
// declares no `state`; a read of anything else of the controller's is still named.
|
||||
func TestAReadOfTheControllersMirrorsIsAReadOfKeptState(t *testing.T) {
|
||||
controller := Manifest{Module: ControllerSeatName}
|
||||
reader := Manifest{Module: "mesh-config", Reads: []string{"mesh-controller.assignments", "mesh-controller.declared-config"}}
|
||||
if p := StateReadsNothingDeclares([]Manifest{controller, reader}); len(p) != 0 {
|
||||
t.Fatalf("a read of the controller's mirrors was refused: %v", p)
|
||||
}
|
||||
other := Manifest{Module: "other", Reads: []string{"mesh-controller.calls"}}
|
||||
if p := StateReadsNothingDeclares([]Manifest{controller, other}); len(p) != 1 {
|
||||
t.Fatalf("a read of the controller's own calls was not named: %v", p)
|
||||
}
|
||||
}
|
||||
|
||||
// **Across the whole catalogue**: every state name is local, and every read whose owner is present
|
||||
// names state that owner keeps.
|
||||
// **Across the whole catalogue**: every state name is local and every read names a module's state.
|
||||
func TestEveryManifestsStateIsLocalAndEveryReadIsKept(t *testing.T) {
|
||||
manifests := theCatalogue(t)
|
||||
var problems []string
|
||||
for _, m := range manifests {
|
||||
problems = append(problems, StateProblems(m)...)
|
||||
}
|
||||
problems = append(problems, StateReadsNothingDeclares(manifests)...)
|
||||
if len(problems) > 0 {
|
||||
t.Fatalf("the catalogue's state is not what ADR 0201 says:\n %s", strings.Join(problems, "\n "))
|
||||
}
|
||||
}
|
||||
|
||||
// `module check` says it too: the cross-catalogue pass names a read nothing on the shelf keeps.
|
||||
func TestTheCataloguePassNamesAReadItsOwnerDoesNotKeep(t *testing.T) {
|
||||
// The catalogue pass no longer judges one module's reads against another's manifest (novox/hq ADR 0495 clauses 1 and 5,
|
||||
// issue 1790): what an owner keeps is what it announces on the bus, and each module's own announcement test holds that.
|
||||
func TestTheCataloguePassJudgesNoReadAgainstTheOwnersManifest(t *testing.T) {
|
||||
shelf := Shelf{
|
||||
"licence-manager": {Module: "licence-manager", State: []StateDeclaration{{Name: "bindings"}}},
|
||||
"claude-code": {Module: "claude-code", Reads: []string{"licence-manager.tokens"}},
|
||||
}
|
||||
problems := CatalogueProblems(shelf)
|
||||
if len(problems) != 1 || !strings.Contains(problems[0], `keeps no state called "tokens"`) {
|
||||
t.Fatalf("the catalogue pass said %v", problems)
|
||||
for _, p := range CatalogueProblems(shelf) {
|
||||
if strings.Contains(p, "keeps no state") {
|
||||
t.Fatalf("the catalogue pass judged a read against the owner's manifest: %s", p)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,249 @@
|
||||
{
|
||||
"module": "claude-code",
|
||||
"version": "1",
|
||||
"moved": true,
|
||||
"slug": "agent",
|
||||
"capabilities": [
|
||||
"package-manager"
|
||||
],
|
||||
"requires": [
|
||||
"mcp-endpoint",
|
||||
{
|
||||
"name": "gitea-read",
|
||||
"optional": true
|
||||
}
|
||||
],
|
||||
"binds": {
|
||||
"mcp-endpoint": "${dir:state}/mcp-endpoint.json"
|
||||
},
|
||||
"secrets": {
|
||||
"gitea-read": "${dir:state}/gitea-read.secret"
|
||||
},
|
||||
"uses": [
|
||||
"operator-channel"
|
||||
],
|
||||
"state": [
|
||||
"servers",
|
||||
"holdings",
|
||||
"config",
|
||||
"proposals",
|
||||
"questions",
|
||||
"memory",
|
||||
"waits"
|
||||
],
|
||||
"reads": [
|
||||
"claude-licence-manager.bindings"
|
||||
],
|
||||
"tools": [
|
||||
"claude_code_status",
|
||||
"claude_code_render",
|
||||
"claude_code_guard",
|
||||
"claude_code_pull",
|
||||
"claude_code_grant",
|
||||
"claude_code_add_api_key",
|
||||
"claude_code_registrations",
|
||||
"claude_code_memory",
|
||||
"claude_code_mcp_list",
|
||||
"claude_code_mcp_register",
|
||||
"claude_code_mcp_unregister",
|
||||
"claude_code_skill_list",
|
||||
"claude_code_skill_register",
|
||||
"claude_code_skill_unregister",
|
||||
"claude_code_agent_list",
|
||||
"claude_code_agent_register",
|
||||
"claude_code_agent_unregister",
|
||||
"claude_code_command_list",
|
||||
"claude_code_command_register",
|
||||
"claude_code_command_unregister",
|
||||
"claude_code_hook_list",
|
||||
"claude_code_hook_register",
|
||||
"claude_code_hook_unregister",
|
||||
"claude_code_output_style_list",
|
||||
"claude_code_output_style_register",
|
||||
"claude_code_output_style_unregister",
|
||||
"claude_code_instructions_list",
|
||||
"claude_code_instructions_register",
|
||||
"claude_code_instructions_unregister",
|
||||
"claude_code_instructions_propose",
|
||||
"claude_code_proposals",
|
||||
"claude_code_ask",
|
||||
"claude_code_answer",
|
||||
"claude_code_wait",
|
||||
"claude_code_settings_get",
|
||||
"claude_code_settings_set",
|
||||
"claude_code_settings_clear",
|
||||
"claude_code_permission_add",
|
||||
"claude_code_permission_remove",
|
||||
"claude_code_config_list",
|
||||
"claude_code_config_show",
|
||||
"claude_code_config_status",
|
||||
"claude_code_config_import",
|
||||
"claude_code_home_show",
|
||||
"claude_code_home_remove",
|
||||
"claude_code_home_removed",
|
||||
"claude_code_home_restore",
|
||||
"claude_code_judge",
|
||||
"claude_code_judged"
|
||||
],
|
||||
"data": {
|
||||
"own": [
|
||||
{
|
||||
"id": "agent-home",
|
||||
"path": "${dir:agent-home}",
|
||||
"class": "valuable",
|
||||
"why": "the operator's agent's own files: its memory, history and projects"
|
||||
}
|
||||
]
|
||||
},
|
||||
"config": {
|
||||
"role": {
|
||||
"type": "string",
|
||||
"description": "What this machine is for, in a sentence, written into the instructions every Claude Code session on it reads; empty says nothing.",
|
||||
"trusted": true,
|
||||
"default": ""
|
||||
},
|
||||
"mcp_servers": {
|
||||
"type": "object",
|
||||
"description": "The tool servers every session on this machine is given beside the mesh's own, by name, each as Claude Code's managed MCP configuration takes it; empty adds none.",
|
||||
"trusted": true,
|
||||
"default": {}
|
||||
},
|
||||
"managed_settings": {
|
||||
"type": "object",
|
||||
"description": "Keys of Claude Code's managed settings every session on this machine obeys, such as spinnerTipsEnabled, laid over the mesh's own; empty adds none. Falls back to agent-managed-settings, the mesh's or this node's.",
|
||||
"trusted": true,
|
||||
"predefined": "agent-managed-settings",
|
||||
"default": {}
|
||||
},
|
||||
"instructions": {
|
||||
"type": "object",
|
||||
"description": "The instructions every session on this machine reads, by name, each a text as one line or a list of its lines; empty adds none.",
|
||||
"trusted": true,
|
||||
"default": {}
|
||||
},
|
||||
"output_styles": {
|
||||
"type": "object",
|
||||
"description": "The output styles offered to every session on this machine, by name, each a text as one line or a list of its lines; empty adds none.",
|
||||
"trusted": true,
|
||||
"default": {}
|
||||
},
|
||||
"skills": {
|
||||
"type": "object",
|
||||
"description": "The skills offered to every session on this machine, by name, each its files by path, a text as one line or a list of its lines; a skill without its SKILL.md is left out; empty adds none.",
|
||||
"trusted": true,
|
||||
"default": {}
|
||||
},
|
||||
"commands": {
|
||||
"type": "object",
|
||||
"description": "The slash commands offered to every session on this machine, by name, each a text as one line or a list of its lines; empty adds none.",
|
||||
"trusted": true,
|
||||
"default": {}
|
||||
},
|
||||
"agents": {
|
||||
"type": "object",
|
||||
"description": "The agents offered to every session on this machine, by name, each a text as one line or a list of its lines; empty adds none.",
|
||||
"trusted": true,
|
||||
"default": {}
|
||||
}
|
||||
},
|
||||
"resources": [
|
||||
{
|
||||
"id": "package",
|
||||
"type": "package",
|
||||
"package": "claude-code"
|
||||
},
|
||||
{
|
||||
"id": "managed",
|
||||
"type": "directory",
|
||||
"path": "/etc/claude-code",
|
||||
"mode": "0755",
|
||||
"unknown": "program"
|
||||
},
|
||||
{
|
||||
"id": "start",
|
||||
"type": "file",
|
||||
"path": "/usr/local/bin/claude-agent",
|
||||
"mode": "0755",
|
||||
"template": "files/claude-agent"
|
||||
},
|
||||
{
|
||||
"id": "agent-account-name",
|
||||
"type": "file",
|
||||
"path": "/etc/claude-code/agent-account",
|
||||
"mode": "0644",
|
||||
"template": "files/agent-account"
|
||||
},
|
||||
{
|
||||
"id": "agent-account",
|
||||
"type": "user",
|
||||
"name": "${machine:agent-account}",
|
||||
"home": "${machine:agent-home}",
|
||||
"root": "${machine:agent-root}"
|
||||
},
|
||||
{
|
||||
"id": "agent-home",
|
||||
"type": "directory",
|
||||
"path": "${machine:agent-home}/.claude",
|
||||
"mode": "0700",
|
||||
"owner": "${machine:agent-account}",
|
||||
"unknown": "module"
|
||||
},
|
||||
{
|
||||
"id": "state",
|
||||
"type": "directory",
|
||||
"mode": "0700",
|
||||
"owner": "${machine:account}",
|
||||
"place": ".",
|
||||
"unknown": "program"
|
||||
},
|
||||
{
|
||||
"id": "facts",
|
||||
"type": "file",
|
||||
"path": "${dir:state}/facts.json",
|
||||
"mode": "0600",
|
||||
"owner": "${machine:account}",
|
||||
"template": "files/facts.json"
|
||||
},
|
||||
{
|
||||
"id": "settings",
|
||||
"type": "file",
|
||||
"path": "${dir:state}/settings.json",
|
||||
"mode": "0600",
|
||||
"owner": "${machine:account}",
|
||||
"takes-config": ["role", "mcp_servers", "managed_settings", "instructions", "output_styles", "skills", "commands", "agents"],
|
||||
"merge": "json",
|
||||
"template": "files/settings.json"
|
||||
}
|
||||
],
|
||||
"items": [
|
||||
{
|
||||
"collection": "shell:zsh:normal",
|
||||
"template": "files/zsh-normal"
|
||||
}
|
||||
],
|
||||
"build": {
|
||||
"artifacts": [
|
||||
{
|
||||
"name": "tools",
|
||||
"kind": "bundle",
|
||||
"language": "go",
|
||||
"system": "arch",
|
||||
"from": "cmd/claude-code",
|
||||
"binary": "claude-code",
|
||||
"loads": [
|
||||
"claude-code"
|
||||
],
|
||||
"env": {
|
||||
"MESH_CLAUDE_CODE_STATE": "${dir:state}",
|
||||
"MESH_CLAUDE_CODE_FACTS": "${dir:state}/facts.json",
|
||||
"MESH_CLAUDE_CODE_SETTINGS": "${dir:state}/settings.json"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "files",
|
||||
"kind": "archive",
|
||||
"from": "files"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,123 @@
|
||||
{
|
||||
"module": "dunst",
|
||||
"version": "1",
|
||||
"moved": true,
|
||||
"capabilities": [
|
||||
"package-manager"
|
||||
],
|
||||
"claims": [
|
||||
{
|
||||
"name": "node-notifier",
|
||||
"scope": "node",
|
||||
"serves": [
|
||||
"send",
|
||||
"history"
|
||||
]
|
||||
}
|
||||
],
|
||||
"tools": [
|
||||
"dunst_pause",
|
||||
"dunst_resume",
|
||||
"dunst_close_all",
|
||||
"dunst_rules",
|
||||
"dunst_count",
|
||||
"dunst_reload"
|
||||
],
|
||||
"config": {
|
||||
"font-size": {
|
||||
"type": "number",
|
||||
"description": "The size of a notification's text, in points of Inter: 10 reads well on a screen of about 100 DPI, and a denser screen needs a larger size.",
|
||||
"minimum": 6,
|
||||
"maximum": 72,
|
||||
"predefined": "font-size",
|
||||
"default": 10
|
||||
},
|
||||
"width": {
|
||||
"type": "number",
|
||||
"description": "The width of a notification, in pixels: 250 fits a title of about forty characters at 10 points, and a larger font needs a wider notification.",
|
||||
"minimum": 50,
|
||||
"maximum": 4000,
|
||||
"default": 250
|
||||
}
|
||||
},
|
||||
"resources": [
|
||||
{
|
||||
"id": "package",
|
||||
"type": "package",
|
||||
"package": "dunst"
|
||||
},
|
||||
{
|
||||
"id": "client",
|
||||
"type": "package",
|
||||
"package": "libnotify"
|
||||
},
|
||||
{
|
||||
"id": "configuration-dir",
|
||||
"type": "directory",
|
||||
"path": "${machine:account-home}/.config/dunst",
|
||||
"owner": "${machine:account}",
|
||||
"mode": "0755",
|
||||
"unknown": "move-out"
|
||||
},
|
||||
{
|
||||
"id": "dropins",
|
||||
"type": "directory",
|
||||
"path": "${machine:account-home}/.config/dunst/dunstrc.d",
|
||||
"owner": "${machine:account}",
|
||||
"mode": "0755",
|
||||
"unknown": "move-out",
|
||||
"knows": ["50-slack.conf"]
|
||||
},
|
||||
{
|
||||
"id": "configuration",
|
||||
"trusted": false,
|
||||
"type": "file",
|
||||
"path": "${machine:account-home}/.config/dunst/dunstrc",
|
||||
"owner": "${machine:account}",
|
||||
"mode": "0644",
|
||||
"template": "files/dunstrc"
|
||||
},
|
||||
{
|
||||
"id": "notifier",
|
||||
"type": "service",
|
||||
"unit": "dunst.service",
|
||||
"scope": "user",
|
||||
"user": "${machine:account}",
|
||||
"reload-on": [
|
||||
"configuration"
|
||||
]
|
||||
}
|
||||
],
|
||||
"build": {
|
||||
"artifacts": [
|
||||
{
|
||||
"name": "tools",
|
||||
"kind": "bundle",
|
||||
"language": "go",
|
||||
"system": "arch",
|
||||
"from": "cmd/dunst-tools",
|
||||
"binary": "dunst-tools",
|
||||
"loads": [
|
||||
"dunst-tools"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "files",
|
||||
"kind": "archive",
|
||||
"from": "files"
|
||||
}
|
||||
]
|
||||
},
|
||||
"emits": [
|
||||
"action-chosen",
|
||||
"started",
|
||||
"paused"
|
||||
],
|
||||
"items": [
|
||||
{
|
||||
"collection": "contribution:node-display-session:config",
|
||||
"name": "display-session",
|
||||
"template": "files/i3/50-dunst.conf"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,236 @@
|
||||
{
|
||||
"module": "xterm",
|
||||
"version": "1",
|
||||
"moved": true,
|
||||
"capabilities": [
|
||||
"package-manager"
|
||||
],
|
||||
"requires": [
|
||||
"x11-display"
|
||||
],
|
||||
"claims": [
|
||||
{
|
||||
"name": "node-terminal-emulator",
|
||||
"scope": "node",
|
||||
"serves": [
|
||||
"open"
|
||||
]
|
||||
}
|
||||
],
|
||||
"tools": [
|
||||
"xterm_font",
|
||||
"xterm_colours"
|
||||
],
|
||||
"config": {
|
||||
"face": {
|
||||
"type": "string",
|
||||
"description": "The font family xterm draws with, as fc-list names it: JetBrainsMono Nerd Font, because research 026/04 chose it for its glyphs the prompt uses. A family the machine does not have falls back to the nearest installed face (xterm_font says which).",
|
||||
"pattern": "[A-Za-z0-9 ._-]{1,64}",
|
||||
"default": "JetBrainsMono Nerd Font"
|
||||
},
|
||||
"size": {
|
||||
"type": "number",
|
||||
"description": "The size of xterm's text, in points: 12 reads well on a screen of about 100 DPI, and a denser screen needs a larger size.",
|
||||
"minimum": 6,
|
||||
"maximum": 72,
|
||||
"predefined": "font-size",
|
||||
"default": 12
|
||||
},
|
||||
"save-lines": {
|
||||
"type": "number",
|
||||
"description": "How many lines of scrollback xterm keeps per terminal: 10000 holds a long build log, and each line costs memory in every open terminal.",
|
||||
"minimum": 0,
|
||||
"maximum": 1000000,
|
||||
"default": 10000
|
||||
},
|
||||
"term-name": {
|
||||
"type": "string",
|
||||
"description": "The TERM value xterm sets for the programs it runs: xterm-256color, because the palette has 256 colours and every program knows the name.",
|
||||
"pattern": "[A-Za-z0-9._-]{1,64}",
|
||||
"default": "xterm-256color"
|
||||
},
|
||||
"render-font": {
|
||||
"type": "boolean",
|
||||
"description": "Whether xterm draws with fontconfig faces (true) or the old X core fonts (false): true, because the face is a fontconfig family and core fonts cannot show it.",
|
||||
"default": true
|
||||
},
|
||||
"bold-mode": {
|
||||
"type": "boolean",
|
||||
"description": "Whether xterm draws bold text by overstriking the normal face (true): false, because the face has a real bold style and overstriking blurs it.",
|
||||
"default": false
|
||||
},
|
||||
"always-bold-mode": {
|
||||
"type": "boolean",
|
||||
"description": "Whether xterm overstrikes bold even where the face has a bold style (true): false, for the same reason as bold-mode.",
|
||||
"default": false
|
||||
},
|
||||
"allow-bold-fonts": {
|
||||
"type": "boolean",
|
||||
"description": "Whether xterm may use the face's bold style for bold text (true): true, so bold text is drawn with the real bold weight.",
|
||||
"default": true
|
||||
},
|
||||
"background": {
|
||||
"type": "string",
|
||||
"description": "The terminal's background colour, as #rrggbb: black, as both workstations ran it.",
|
||||
"pattern": "#[0-9a-fA-F]{6}",
|
||||
"default": "#000000"
|
||||
},
|
||||
"foreground": {
|
||||
"type": "string",
|
||||
"description": "The terminal's text colour, as #rrggbb: a light grey, easier on the eyes than white on black.",
|
||||
"pattern": "#[0-9a-fA-F]{6}",
|
||||
"default": "#a8a8a8"
|
||||
},
|
||||
"color0": {
|
||||
"type": "string",
|
||||
"description": "Palette entry 0, the normal black, as #rrggbb: #000000, as both workstations ran it.",
|
||||
"pattern": "#[0-9a-fA-F]{6}",
|
||||
"default": "#000000"
|
||||
},
|
||||
"color1": {
|
||||
"type": "string",
|
||||
"description": "Palette entry 1, the normal red, as #rrggbb: #9e1828, as both workstations ran it.",
|
||||
"pattern": "#[0-9a-fA-F]{6}",
|
||||
"default": "#9e1828"
|
||||
},
|
||||
"color2": {
|
||||
"type": "string",
|
||||
"description": "Palette entry 2, the normal green, as #rrggbb: #008800, as both workstations ran it.",
|
||||
"pattern": "#[0-9a-fA-F]{6}",
|
||||
"default": "#008800"
|
||||
},
|
||||
"color3": {
|
||||
"type": "string",
|
||||
"description": "Palette entry 3, the normal yellow, as #rrggbb: #d2bb4b, as both workstations ran it.",
|
||||
"pattern": "#[0-9a-fA-F]{6}",
|
||||
"default": "#d2bb4b"
|
||||
},
|
||||
"color4": {
|
||||
"type": "string",
|
||||
"description": "Palette entry 4, the normal blue, as #rrggbb: #414171, as both workstations ran it.",
|
||||
"pattern": "#[0-9a-fA-F]{6}",
|
||||
"default": "#414171"
|
||||
},
|
||||
"color5": {
|
||||
"type": "string",
|
||||
"description": "Palette entry 5, the normal magenta, as #rrggbb: #963c59, as both workstations ran it.",
|
||||
"pattern": "#[0-9a-fA-F]{6}",
|
||||
"default": "#963c59"
|
||||
},
|
||||
"color6": {
|
||||
"type": "string",
|
||||
"description": "Palette entry 6, the normal cyan, as #rrggbb: #418179, as both workstations ran it.",
|
||||
"pattern": "#[0-9a-fA-F]{6}",
|
||||
"default": "#418179"
|
||||
},
|
||||
"color7": {
|
||||
"type": "string",
|
||||
"description": "Palette entry 7, the normal white, as #rrggbb: #bebebe, as both workstations ran it.",
|
||||
"pattern": "#[0-9a-fA-F]{6}",
|
||||
"default": "#bebebe"
|
||||
},
|
||||
"color8": {
|
||||
"type": "string",
|
||||
"description": "Palette entry 8, the bright black, as #rrggbb: #666666, as both workstations ran it.",
|
||||
"pattern": "#[0-9a-fA-F]{6}",
|
||||
"default": "#666666"
|
||||
},
|
||||
"color9": {
|
||||
"type": "string",
|
||||
"description": "Palette entry 9, the bright red, as #rrggbb: #bc5766, as both workstations ran it.",
|
||||
"pattern": "#[0-9a-fA-F]{6}",
|
||||
"default": "#bc5766"
|
||||
},
|
||||
"color10": {
|
||||
"type": "string",
|
||||
"description": "Palette entry 10, the bright green, as #rrggbb: #61a171, as both workstations ran it.",
|
||||
"pattern": "#[0-9a-fA-F]{6}",
|
||||
"default": "#61a171"
|
||||
},
|
||||
"color11": {
|
||||
"type": "string",
|
||||
"description": "Palette entry 11, the bright yellow, as #rrggbb: #e7db52, as both workstations ran it.",
|
||||
"pattern": "#[0-9a-fA-F]{6}",
|
||||
"default": "#e7db52"
|
||||
},
|
||||
"color12": {
|
||||
"type": "string",
|
||||
"description": "Palette entry 12, the bright blue, as #rrggbb: #5085af, as both workstations ran it.",
|
||||
"pattern": "#[0-9a-fA-F]{6}",
|
||||
"default": "#5085af"
|
||||
},
|
||||
"color13": {
|
||||
"type": "string",
|
||||
"description": "Palette entry 13, the bright magenta, as #rrggbb: #a97a99, as both workstations ran it.",
|
||||
"pattern": "#[0-9a-fA-F]{6}",
|
||||
"default": "#a97a99"
|
||||
},
|
||||
"color14": {
|
||||
"type": "string",
|
||||
"description": "Palette entry 14, the bright cyan, as #rrggbb: #6ba4a4, as both workstations ran it.",
|
||||
"pattern": "#[0-9a-fA-F]{6}",
|
||||
"default": "#6ba4a4"
|
||||
},
|
||||
"color15": {
|
||||
"type": "string",
|
||||
"description": "Palette entry 15, the bright white, as #rrggbb: #ffffff, as both workstations ran it.",
|
||||
"pattern": "#[0-9a-fA-F]{6}",
|
||||
"default": "#ffffff"
|
||||
},
|
||||
"copy-binding": {
|
||||
"type": "string",
|
||||
"description": "The key combination that copies the selection to the clipboard, in X translation syntax: Ctrl Shift ~Alt <Key> C, so a plain Ctrl+C still interrupts the program. It holds no colon, backslash or line break, which would end the resource.",
|
||||
"pattern": "[^:\\\\\\n]+",
|
||||
"default": "Ctrl Shift ~Alt <Key> C"
|
||||
},
|
||||
"paste-binding": {
|
||||
"type": "string",
|
||||
"description": "The key combination that pastes the clipboard, in X translation syntax: Ctrl Shift ~Alt <Key> V, the pair of copy-binding. It holds no colon, backslash or line break, which would end the resource.",
|
||||
"pattern": "[^:\\\\\\n]+",
|
||||
"default": "Ctrl Shift ~Alt <Key> V"
|
||||
}
|
||||
},
|
||||
"items": [
|
||||
{
|
||||
"collection": "environment",
|
||||
"name": "terminal",
|
||||
"attributes": {
|
||||
"kind": "variable",
|
||||
"name": "TERMINAL"
|
||||
},
|
||||
"template": "files/env-terminal"
|
||||
},
|
||||
{
|
||||
"collection": "shell:xresources:normal",
|
||||
"name": "xresources-normal",
|
||||
"template": "files/xresources-normal"
|
||||
}
|
||||
],
|
||||
"resources": [
|
||||
{
|
||||
"id": "package",
|
||||
"type": "package",
|
||||
"package": "xterm"
|
||||
}
|
||||
],
|
||||
"build": {
|
||||
"artifacts": [
|
||||
{
|
||||
"name": "tools",
|
||||
"kind": "bundle",
|
||||
"language": "go",
|
||||
"system": "arch",
|
||||
"from": "cmd/xterm-tools",
|
||||
"binary": "xterm-tools",
|
||||
"loads": [
|
||||
"xterm-tools"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "files",
|
||||
"kind": "archive",
|
||||
"from": "files"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,90 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// Defends novox/hq ADR 0477 §4 (issue 1573): the go and nodejs modules own the toolchain packages and provide
|
||||
// them with the machine's reach, so a node without one refuses the module that needs it at resolution, naming
|
||||
// the provision, and does not find out at its first build. The manifests are the catalogue's, as they are.
|
||||
|
||||
// builderNode is a machine that can run a build module: the package manager and user namespaces it asks of
|
||||
// the machine, and the forge account (mesh-scope, provided by the forge) and the artifact store the build
|
||||
// modules publish to (ADR 0513, provided by distribution) stood in for, and node-tools, the tool runner they pull their
|
||||
// asks through (mesh-catalog issue 1911; mesh-tools' module, not in the catalogue).
|
||||
func builderNode() Node {
|
||||
return Node{Name: "builder", Site: "house",
|
||||
Capabilities: map[string]bool{"package-manager": true, "user-namespaces": true}}
|
||||
}
|
||||
|
||||
func toolchainShelf(t *testing.T) map[string]Manifest {
|
||||
t.Helper()
|
||||
m := map[string]Manifest{
|
||||
"forge": {Module: "forge", Version: "1", Provides: []Offer{{Name: "forge-account"}, {Name: "npm-package-registry"}, {Name: "artifact-store"}, {Name: "node-tools"}}},
|
||||
}
|
||||
for _, name := range []string{"go", "nodejs", "mesh-build-go", "mesh-build-node", "dev-tools"} {
|
||||
m[name] = catalogueManifest(t, name)
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
func TestAModuleNeedingAToolchainIsRefusedOnANodeWithoutIt(t *testing.T) {
|
||||
for _, c := range []struct{ module, provision string }{
|
||||
{"mesh-build-go", "go-toolchain"},
|
||||
{"mesh-build-node", "node-toolchain"},
|
||||
{"dev-tools", "go-toolchain"},
|
||||
} {
|
||||
_, err := Resolve(toolchainShelf(t), []string{"forge", c.module}, builderNode(), World{})
|
||||
if err == nil {
|
||||
t.Fatalf("%s composed on a node with no toolchain module", c.module)
|
||||
}
|
||||
for _, want := range []string{`"` + c.provision + `" is wanted by ` + c.module,
|
||||
"usable only on the machine that provides it"} {
|
||||
if !strings.Contains(err.Error(), want) {
|
||||
t.Errorf("%s: the refusal does not say %q:\n%v", c.module, want, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAModuleNeedingAToolchainComposesOnANodeWithIt(t *testing.T) {
|
||||
for _, c := range []struct{ module, owner string }{
|
||||
{"mesh-build-go", "go"},
|
||||
{"mesh-build-node", "nodejs"},
|
||||
{"dev-tools", "go"},
|
||||
} {
|
||||
assigned := []string{"forge", c.owner, c.module}
|
||||
if c.module == "dev-tools" {
|
||||
assigned = append(assigned, "nodejs")
|
||||
}
|
||||
got, err := Resolve(toolchainShelf(t), assigned, builderNode(), World{})
|
||||
if err != nil {
|
||||
t.Fatalf("%s with %s did not compose: %v", c.module, c.owner, err)
|
||||
}
|
||||
in := map[string]bool{}
|
||||
for _, n := range names(got) {
|
||||
in[n] = true
|
||||
}
|
||||
if !in[c.module] || !in[c.owner] {
|
||||
t.Errorf("%s with %s composed %v", c.module, c.owner, names(got))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Only the owners declare the packages: a build module that declared go again would compose without the
|
||||
// owner's provision being met by anything, which is the gap the provision closes.
|
||||
func TestOnlyTheToolchainModulesDeclareTheToolchainPackages(t *testing.T) {
|
||||
owner := map[string]string{"go": "go", "nodejs": "nodejs", "npm": "nodejs"}
|
||||
for name, m := range toolchainShelf(t) {
|
||||
for _, r := range m.Resources {
|
||||
if r["type"] != "package" {
|
||||
continue
|
||||
}
|
||||
pkg, _ := r["package"].(string)
|
||||
if o, ok := owner[pkg]; ok && o != name {
|
||||
t.Errorf("%s declares the package %s, which only %s may", name, pkg, o)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -56,7 +56,7 @@ type filler struct {
|
||||
// seatInto, machineInto and contributionsInto. A pass that comes to read another field adds it here,
|
||||
// or the sweep refuses the placeholder it would have filled.
|
||||
var fillers = []filler{
|
||||
{namespace: "setting", patterns: []*regexp.Regexp{settingRef}, fields: map[string][]string{"file": {"content"}, "service": {"unit"}, "directory": {"owner"}}},
|
||||
{namespace: "setting", patterns: []*regexp.Regexp{settingRef}, fields: map[string][]string{"file": {"content"}, "service": {"unit"}, "directory": {"owner"}, "container": {"groups"}}},
|
||||
{namespace: "dir", patterns: []*regexp.Regexp{dirRef}, fields: map[string][]string{"*": {"path", "content", "volumes", "env", "env-file"}}},
|
||||
{namespace: "access", patterns: []*regexp.Regexp{accessRef}, fields: map[string][]string{"*": {"path", "content", "volumes", "env", "env-file"}}},
|
||||
{namespace: "secret", patterns: []*regexp.Regexp{placeholder}, fields: map[string][]string{"file": {"content"}},
|
||||
|
||||
+80
-186
@@ -74,6 +74,29 @@ func VerbNames(verbs []Verb) []string {
|
||||
return out
|
||||
}
|
||||
|
||||
// RemovedControllerVerbs are the controller seat's verbs release 6 removed (novox/hq ADR 0539, ADR 0429 §8): no
|
||||
// delivery plan, batch, tier or push exists, so nothing starts, stops, retries, follows or reads one. Each stays in
|
||||
// ControllerVerbs as a row with no input, so a caller is answered by name with its replacement, given here.
|
||||
var RemovedControllerVerbs = map[string]string{
|
||||
"plans": "the delivery view (nox-delivery): each machine's want and have, read from the assigner's wants and the " +
|
||||
"node-engine's haves",
|
||||
"plan": "the node's compositions: mesh-controller_composed holds each module's digest and judgement per node, and " +
|
||||
"mesh-hand-compose prints one module's composition for one node",
|
||||
"push": "a want: the assigner writes want.<node>.<module> by its rules, and by hand the operator's credential writes " +
|
||||
"it, naming a kept composition (ADR 0539 clause 10)",
|
||||
"deliver": "nothing to start: a merged change reaches each machine by its want",
|
||||
"delivery-stop": "a bad fingerprint is held and its canary set back by the assigner (ADR 0429 §6, ADR 0539 clause 18)",
|
||||
"delivery-backups-again": "the assigner's backup ask, answered on the phone or in the delivery view (ADR 0539 clause 16)",
|
||||
"delivery-walks": "the delivery view (nox-delivery) and the wants, haves and compositions on the bus",
|
||||
"delivery-order": "no order to keep: each module reaches its machines by its own want, its providers first by rule 5",
|
||||
}
|
||||
|
||||
// removedInR6 is the row of a verb release 6 removed from the controller's seat: no input, refused by name.
|
||||
func removedInR6(name string) Verb {
|
||||
return Verb{Name: name, Description: "Removed in R6 (ADR 0539): refused by name, naming its replacement: " +
|
||||
RemovedControllerVerbs[name], Input: schema(nil, nil)}
|
||||
}
|
||||
|
||||
// ControllerSeatName is the seat the control plane holds, whose tools are the mesh's own verbs.
|
||||
const ControllerSeatName = "mesh-controller"
|
||||
|
||||
@@ -115,95 +138,23 @@ var ControllerVerbs = []Verb{
|
||||
"log": "a build's id (as `builds` lists it): print what the build machine said, line by line",
|
||||
"limit": "how many builds to list (default 20); not with log",
|
||||
}, nil)},
|
||||
{Name: "plans", Description: "What the last merges produced and where each stands (novox/hq ADR 0162): " +
|
||||
"the tiers, the tier a plan is at, what it waits for and since when; one plan whole, given its id; " +
|
||||
"or, given wait, one plan followed until it moves.",
|
||||
Input: schema(map[string]string{
|
||||
"id": "a plan's id (as `plans` lists them): that plan, tier by tier",
|
||||
"wait": "a plan's id: wait until it moves, then answer it as data — the plan, and what ended the wait " +
|
||||
"(moved, tier, done, or bound); ask again to follow it further (novox/hq issue 229)",
|
||||
"until": "with wait: moved (any change, the default), tier (it reaches another tier) or done (it ends)",
|
||||
"bound": "with wait: how long to wait at most, as 15s or seconds — 20s, the default, is the longest, " +
|
||||
"inside what a call is answered within",
|
||||
"stop": "a plan's id: stop it — what was asked still builds, nothing further is asked",
|
||||
"close": "a plan's id: close a plan that will not move again, as failed by hand (novox/hq issue 254)",
|
||||
"retry": "a failed plan's id: ask its failed builds again under new ids, and carry the plan on from that tier (novox/hq ADR 0219)",
|
||||
"go": "a walk's id that waits for its delivery's word: start it by hand, in place of mesh-delivery — with why, " +
|
||||
"recorded in the hand-act log (novox/hq ADR 0239)",
|
||||
"repository": "owner/repository: the plan a merge there would produce, saving nothing (what-if); with paths or modules",
|
||||
"paths": "with repository: the files the merge would change, comma-separated, from the repository's root",
|
||||
"modules": "with repository: or the modules it would change, comma-separated",
|
||||
"module-dirs": "with repository and paths: the directories holding a module.json at the commit, comma-separated, " +
|
||||
"as the forge's announcer says them (novox/hq issue 278); without it, a directory is a module only when the paths hold its manifest",
|
||||
"data": "with repository and paths: true answers the change plan as data, the document delivery-plan answers " +
|
||||
"(moved, dependents, new, tiers, machines with what each receives and waits for, steps, summary, class, head, base); " +
|
||||
"without it the answer is text (novox/hq ADR 0450 §2)",
|
||||
"head": "with data: the commit at hand",
|
||||
"base": "with data: the branch it merges into (default main)",
|
||||
"removed": "with data: the files among paths the change deletes, comma-separated",
|
||||
"limit": "how many plans to list (default 10); only when listing",
|
||||
"why": "with stop, close or go: why it is done by hand — required, and recorded in the hand-act log (novox/hq to-be 45 §7)",
|
||||
"cause": "with stop, close or go: the cause in a word, or a condition's kind (optional)",
|
||||
}, nil, "data")},
|
||||
removedInR6("plans"),
|
||||
// The verbs the delivery's owner asks with (novox/hq ADR 0239): the planner's answer, a group's order and
|
||||
// composed check, and the start and end of a walk. Callable by a person too; mesh-delivery is who needs them.
|
||||
{Name: "delivery-plan", Description: "The delivery plan of a diffset, as the planner computes it and keeping " +
|
||||
"nothing (novox/hq ADR 0239): what it moves and builds after them in tiers, what each machine receives and " +
|
||||
"what waits there, the steps that are not an ordinary send, and whether the gate would run.",
|
||||
Input: schema(map[string]string{"repository": "owner/repository", "head": "the commit at hand",
|
||||
"base": "the branch it merges into (default main)",
|
||||
"paths": "the files it changes, comma-separated, from the repository's root",
|
||||
"module-dirs": "the directories holding a module.json at the head, comma-separated, as the forge says them",
|
||||
"removed": "the files among paths it deletes, comma-separated"}, []string{"repository", "paths"})},
|
||||
{Name: "delivery-order", Description: "A delivery group's order (novox/hq ADR 0419): its members in the order " +
|
||||
"they are delivered, every pair and why — built by, verb before caller, version skew, engine before " +
|
||||
"controller — read from the module graph and the manifests' change alone; no `after:` line is read. The " +
|
||||
"cycle, with every module and edge of it, is the one refusal.",
|
||||
Input: schema(map[string]string{"members": "the members, as JSON: [{id, repository, base, head, number, paths, " +
|
||||
"module_dirs, module_dirs_said, removed}]"}, []string{"members"})},
|
||||
{Name: "delivery-check", Description: "Ask the build seat for a delivery group's composed check (novox/hq ADR " +
|
||||
"0239): every member's head laid over the mesh in turn and judged as one future state. Answers the ask's id; " +
|
||||
"the verdict is said as `checked` with the group's id. With one member and no group, that head is checked " +
|
||||
"again as its pull request is.",
|
||||
{Name: "delivery-plan", Description: "Removed in R4e (ADR 0501 §5 / ADR 0503 §3): refused by name, naming its replacement",
|
||||
Input: schema(nil, nil)},
|
||||
removedInR6("delivery-order"),
|
||||
{Name: "delivery-check", Description: "Answers that the checker (mesh-delivery-checker) judges a group (novox/hq " +
|
||||
"ADR 0503 §3); asks nothing. Its inputs stay, since hq's repository check still names it (ADR 0513 §4).",
|
||||
Input: schema(map[string]string{"group": "the group's id (none for one head checked again)",
|
||||
"members": "the members, as JSON, as delivery-order takes them",
|
||||
"ready": "\"true\" when the pull request is marked ready: its check is queued before every other check (novox/hq ADR 0406 §3)"},
|
||||
[]string{"members"})},
|
||||
{Name: "deliver", Description: "The delivery's word that a walk waiting for it may start (novox/hq ADR 0239): its " +
|
||||
"first tier is asked at the controller's next pass. Refused for a walk that waits for nobody.",
|
||||
Input: schema(map[string]string{"plan": "the walk's id", "why": "what lets it go: its turn, a person's release"},
|
||||
[]string{"plan"})},
|
||||
{Name: "delivery-stop", Description: "End a walk on its delivery's word (novox/hq ADR 0239): failed, said as stopped " +
|
||||
"by whom and why; what it asked still builds and registers, nothing further is asked or sent. Given merges to " +
|
||||
"leave out, every other merge the walk answered is walked again — in the open batch of its kind, or a new one — " +
|
||||
"without them, without each later merge of their repository, which contains one, and without each merge moving " +
|
||||
"a module one moves, each left out too and named (novox/hq ADR 0299); a batch not yet cut drops the merges. The " +
|
||||
"answer names the batch, the merges walked again, the merges left out, the modules the stopped walk sent that " +
|
||||
"stay on their machines, and the merges elsewhere that will still deliver a stopped change.",
|
||||
Input: schema(map[string]string{"plan": "the walk's id", "why": "why", "by": "who stopped the delivery",
|
||||
"without": "merges to leave out, <repository>@<commit>, comma-separated"},
|
||||
[]string{"plan", "why"})},
|
||||
{Name: "delivery-backups-again", Description: "Ask again the backups a walk was held for (novox/hq ADR 0404 " +
|
||||
"decision 4): a walk whose backup before its send failed or did not answer is held, nothing asked or sent; this " +
|
||||
"clears those failed backups and the next look at the plans asks them again, and only them. Refused for a walk " +
|
||||
"with no failed backup. The delivery's release calls it. The answer names the walk and the backups asked again.",
|
||||
Input: schema(map[string]string{"plan": "the held walk's id", "why": "a person's reason, as release passes it",
|
||||
"by": "who released the delivery"}, []string{"plan", "why"})},
|
||||
{Name: "delivery-walks", Description: "The walks the controller keeps (novox/hq ADR 0239): every open one and the " +
|
||||
"last ended ones, each whole — its tiers, each module's state, first machines and first-node gate with its readings, and its " +
|
||||
"phases from its merge to every machine running it (novox/hq ADR 0282) — and whether the delivery seat has a " +
|
||||
"holder on record. Given a plan, that one.",
|
||||
Input: schema(map[string]string{"plan": "one walk's id", "limit": "how many ended walks beside the open ones (default 50)"},
|
||||
nil)},
|
||||
{Name: "plan", Description: "What one machine would run, and why: the declaration the mesh would send it — " +
|
||||
"or, with files, the files it would be given; or, with diff, what a push would change there against what " +
|
||||
"it was last sent: resources added, removed and changed, a container's changed fields (it is recreated), " +
|
||||
"and any mount whose directory on the machine would move (novox/hq ADR 0217).",
|
||||
Input: schema(map[string]string{
|
||||
"node": "the machine's name",
|
||||
"files": "\"true\": the files this machine would be given, instead of the declaration as JSON",
|
||||
"diff": "\"true\": what a push would change on this machine, against what it was last sent; not with files",
|
||||
}, []string{"node"}, "files", "diff")},
|
||||
removedInR6("deliver"),
|
||||
removedInR6("delivery-stop"),
|
||||
removedInR6("delivery-backups-again"),
|
||||
removedInR6("delivery-walks"),
|
||||
removedInR6("plan"),
|
||||
{Name: "assign", Description: "Put a module on a machine. Refused with the mesh's own words when it cannot resolve there, " +
|
||||
"or when a seat its resources are applied through is held by nothing on the machine (novox/hq ADR 0207). " +
|
||||
"A module not registered yet is looked for (novox/hq issue 325, ADR 0261): while its build runs — a merge " +
|
||||
@@ -229,22 +180,7 @@ var ControllerVerbs = []Verb{
|
||||
}, []string{"node", "provision", "from", "module"})},
|
||||
{Name: "unpin", Description: "Take that choice back, putting the question to the mesh again.",
|
||||
Input: schema(map[string]string{"node": "the machine's name", "provision": "the provision"}, []string{"node", "provision"})},
|
||||
{Name: "push", Description: "Send one machine everything it should be. With no machine named it is a push of the " +
|
||||
"WHOLE mesh — every machine that is behind — and the answer says so first; behind says that outright. " +
|
||||
"Answers at once that it is running, with a call id: `calls` with that id says what it sent and, for a " +
|
||||
"machine named, how that machine applied it — applied, refused with its reason, or each failed resource — " +
|
||||
"once it reports within wait (a push can reload the bus, which then refuses any answer still to come). A push by hand is a repair, " +
|
||||
"and says why: recorded in the hand-act log (novox/hq to-be 45 §7). A machine whose push would give a " +
|
||||
"running module's data another directory is held and named, the others sent (novox/hq ADR 0217): read " +
|
||||
"`plan` with diff, and name the machine with move to send it.",
|
||||
Input: schema(map[string]string{
|
||||
"node": "the machine's name; without it, every machine that is behind",
|
||||
"move": "with node: the modules whose data may move with this push, comma-separated (optional)",
|
||||
"wait": "with node: how long the call waits for the machine to report how it applied what it was sent, as \"90s\" or \"2m\"; \"0\" waits for nothing; without it, the controller's setting push-wait (optional)",
|
||||
"behind": "\"true\": every machine that is behind, the whole mesh — the same as naming none, said outright; not with node",
|
||||
"why": "why this is pushed by hand: recorded in the hand-act log",
|
||||
"cause": "the cause in a word, or a condition's kind — the word a second push for the same reason uses (optional)",
|
||||
}, []string{"why"}, "behind")},
|
||||
removedInR6("push"),
|
||||
{Name: "rotate", Description: "Replace a credential. A pair credential, by provision (and a consuming machine and module, " +
|
||||
"else every holder): both ends are re-sent together. Or a module's own secret, by machine, module and " +
|
||||
"name: made anew and the machine sent, so the module starts again on it — for a secret its definition " +
|
||||
@@ -358,73 +294,40 @@ var ControllerVerbs = []Verb{
|
||||
"its shell — `node show ace`, `module list`, `plans`, `conditions show <key>` — and answer what it " +
|
||||
"printed. The generic verb beside the named ones (novox/hq ADR 0154), and since ADR 0266 it only reads: " +
|
||||
"status, version, help, seats, healers, hand-acts, durations, collection, images, artifacts, data, builds, " +
|
||||
"queue, plan, plans (not stop/close/go), doctor (not run), conditions list/show/history, node list/show, " +
|
||||
"plan, plans (not stop/close/go), doctor (not run), conditions list/show/history, node list/show, " +
|
||||
"module list, settings show/preferences, retire list, cleanup list, delivery plan/walks, bus, mirrors (not " +
|
||||
"--record/--confirm). What writes has its named verb; what sets a key, issues a credential or a token, or " +
|
||||
"accepts, recovers or exports a secret is the controller's terminal's alone.",
|
||||
Input: schema(map[string]string{
|
||||
"command": "the command line, as the controller's binary takes it; quotes group a word with spaces",
|
||||
}, []string{"command"})},
|
||||
// The build queue, controlled by hand (novox/hq ADR 0219). Every verb that drops an ask leaves a
|
||||
// failed outcome for it, so a plan waiting on it fails visibly instead of hanging.
|
||||
{Name: "queue", Description: "Every ask in the build queue: waiting, in flight (on which machine, for how long), " +
|
||||
"and dead (handed out as often as allowed and never settled) — each with its id, repository, path, ref and when it was asked. " +
|
||||
"Answers the table as text and, as data, each ask's state, kind (check or build), repository, commit, machine, since " +
|
||||
"and position among the waiting (novox/hq issue 455).",
|
||||
// The build queue went with build-agent (novox/hq ADR 0525 §9): asks are the build seats' streams, their backlog the
|
||||
// streams' own state, shown in the delivery view. The verbs are refused by name for one release, naming that.
|
||||
{Name: "queue", Description: "Removed in R4g (ADR 0525 §9): refused by name, naming its replacement: the build seats' backlog in the delivery view",
|
||||
Input: schema(nil, nil)},
|
||||
{Name: "cancel", Description: "Drop one waiting or dead ask from the build queue; its outcome is recorded failed, " +
|
||||
"cancelled by hand, and a plan that asked for it fails. One in flight is refused: `kill` ends it where it runs.",
|
||||
Input: schema(map[string]string{"id": "the ask's build id, as `queue` lists it"}, []string{"id"})},
|
||||
{Name: "clear", Description: "Cancel every waiting ask in the build queue — and with dead, every dead one too — each " +
|
||||
"recorded failed, cancelled by hand. Never touches one in flight.",
|
||||
Input: schema(map[string]string{"dead": "\"true\" to cancel the dead asks as well"}, nil, "dead")},
|
||||
{Name: "rebuild", Description: "Ask a module's current source again under a new id — the branch it follows — or, given a " +
|
||||
"build's id, that build's repository, path and ref. A module a plan holds unbuilt or failed joins that plan. Answers the new id.",
|
||||
Input: schema(map[string]string{"what": "a module's name, or a build's id"}, []string{"what"})},
|
||||
{Name: "replay", Description: "Ask a recorded build's repository and path again at the commit it built, under a new id. " +
|
||||
"A dry run unless register: nothing recorded or registered. Registering is refused when a newer build of the module " +
|
||||
"is registered — it would roll the older commit out (novox/hq issue 207) — unless older says so.",
|
||||
Input: schema(map[string]string{
|
||||
"id": "the build's id",
|
||||
"register": "\"true\" to register what it builds",
|
||||
"older": "\"true\", with register: even though a newer build of the module is registered",
|
||||
}, []string{"id"}, "register", "older")},
|
||||
{Name: "cancel", Description: "Removed in R4g (ADR 0525 §9): refused by name, naming its replacement", Input: schema(nil, nil)},
|
||||
{Name: "clear", Description: "Removed in R4g (ADR 0525 §9): refused by name, naming its replacement", Input: schema(nil, nil)},
|
||||
{Name: "replay", Description: "Removed in R4g (ADR 0525 §9): refused by name, naming its replacement", Input: schema(nil, nil)},
|
||||
// novox/hq issue 686: a builder ran a branch's store tests for the first time in its pull request's merge check.
|
||||
{Name: "try", Description: "Try a pushed branch or commit before its pull request (novox/hq issue 686): the build " +
|
||||
"seat merges it into the repository's trunk and runs the repository's own merge-check.sh on the merge, in the " +
|
||||
"toolchain containers and against the throwaway store and bus a pull request's merge check gets. Answers at " +
|
||||
{Name: "try", Description: "Try a pushed branch or commit before its pull request (novox/hq issue 686): the " +
|
||||
"checker (mesh-delivery-checker) merges the head onto the repository's trunk and runs its levels on the merge, " +
|
||||
"as it judges a pull request (ADR 0503 §3), its levels asks on the build seats (ADR 0525 §1); no merge-check.sh is run. Answers at " +
|
||||
"once with an id; asked again with that id it waits up to 20 s and answers pass, fail or error, how long it " +
|
||||
"took and the lines naming what failed — or that it is still waiting or running, to ask again. Sets no " +
|
||||
"status on the forge, registers and builds nothing, runs no merge gate, counts as a check in the build queue " +
|
||||
"(after deliveries and after every pull request's check, however long it waits) and is bounded as one. One " +
|
||||
"try of a commit at a time — a second ask answers the first's id — and at most four waiting or running in " +
|
||||
"all: a fifth is refused. Only for a repository of the mesh; the ref must be on the forge.",
|
||||
"status on the forge, registers and builds nothing for delivery. One try of a commit at a time — a second " +
|
||||
"ask answers the first's id. Only for a repository of the mesh; the ref must be on the forge.",
|
||||
Input: schema(map[string]string{
|
||||
"repository": "the repository on the forge, owner/name; with ref",
|
||||
"ref": "the branch or commit to try, as pushed to the forge",
|
||||
"base": "the branch it is merged into (the repository's default branch when none is named). The " +
|
||||
"merge-check.sh that runs is that branch's, never the ref's own: the same trust as a pull request's base",
|
||||
"tests": "only the tests this `go test -run` pattern names, where the repository's merge-check.sh reads " +
|
||||
"MESH_CHECK_TESTS; a script that does not runs whole",
|
||||
"store-only": "\"true\": only the repository's store tests, where its merge-check.sh reads " +
|
||||
"MESH_CHECK_STORE_ONLY",
|
||||
"id": "instead of a repository: the id a try answered, to read what came of it",
|
||||
"base": "the branch it is merged into (the repository's default branch when none is named)",
|
||||
"tests": "only the tests this `go test -run` pattern names",
|
||||
"store-only": "\"true\": only the integration level",
|
||||
"id": "instead of a repository: the id a try answered, to read what came of it",
|
||||
}, nil, "store-only"),
|
||||
Replaces: []string{"sh merge-check.sh", "docker run postgres"}},
|
||||
// novox/hq ADR 0350 §5: asked by the delivery's owner when the forge says a pull request closed unmerged.
|
||||
{Name: "forget-checks", Description: "Let go of every build a pull request's merge checks made for its merge in " +
|
||||
"the artifact store's scratch namespace (novox/hq ADR 0350 §5): asked when it closed unmerged; a merged one's go " +
|
||||
"when the walk answering its merge ends, and any check build goes after a week.",
|
||||
Input: schema(map[string]string{"owner": "the repository's owner", "repo": "the repository",
|
||||
"number": "the pull request's number"}, []string{"owner", "repo", "number"})},
|
||||
{Name: "kill", Description: "End a build where it runs: the machine that took it stops its commands and containers and " +
|
||||
"announces it failed, killed by hand — settled, never handed to another machine.",
|
||||
Input: schema(map[string]string{"id": "the build's id"}, []string{"id"})},
|
||||
{Name: "pause", Description: "The build seat's holder on one machine — or every holder — takes no new build until resumed; " +
|
||||
"a build running finishes. Kept across a restart of the holder. A plan waiting on a paused seat says so and is not late.",
|
||||
Input: schema(map[string]string{"node": "one machine; every holder when absent"}, nil)},
|
||||
{Name: "resume", Description: "The build seat's holder on one machine — or every holder — takes builds again.",
|
||||
Input: schema(map[string]string{"node": "one machine; every holder when absent"}, nil)},
|
||||
{Name: "kill", Description: "Removed in R4g (ADR 0525 §9): refused by name, naming its replacement: a build seat's ask ends where it runs", Input: schema(nil, nil)},
|
||||
{Name: "pause", Description: "Removed in R4g (ADR 0525 §9): refused by name, naming its replacement: <node>/node-build-<language>.pause", Input: schema(nil, nil)},
|
||||
{Name: "resume", Description: "Removed in R4g (ADR 0525 §9): refused by name, naming its replacement: <node>/node-build-<language>.resume", Input: schema(nil, nil)},
|
||||
// Acts done by hand, and what the bounds are set from (novox/hq to-be 45 §7, Phase 0).
|
||||
{Name: "hand-act", Description: "Record an act done by hand outside the mesh — a container restarted, a file " +
|
||||
"edited, a service started on a machine — with why and its cause, in the hand-act log beside the pushes and " +
|
||||
@@ -498,26 +401,22 @@ var ControllerVerbs = []Verb{
|
||||
"probe": "one probe's id (D14): its verdict alone, with every finding — the verdict shows at most twenty a probe",
|
||||
}, nil, "run", "probes", "signals")},
|
||||
// How a module's new builds reach its machines, and the bus's planned step (novox/hq ADR 0236).
|
||||
{Name: "upgrade", Description: "How each module's new builds reach its machines (novox/hq ADR 0236): rolled " +
|
||||
"out one machine first and judged there at the gate, then the rest — or recorded, waiting for a person's " +
|
||||
"push — with where that comes from (a person, the module, the bus, its irreplaceable data, the default) and " +
|
||||
"why. With module, that one; with policy, a person's choice for it — roll-out, record (with why) or default " +
|
||||
"to take the choice back. The bus is never rolled out.",
|
||||
{Name: "upgrade", Description: "Whether each module's new builds wait for a person (novox/hq ADR 0236, ADR 0539 clause 14): " +
|
||||
"for a module of policy record, the assigner writes no want naming a composition judged to recreate a container or move " +
|
||||
"data until the operator answers its ask; with where the policy comes from (a person, the module, the bus, its " +
|
||||
"irreplaceable data, the default) and why. With module, that one; with policy, a person's choice for it — roll-out, " +
|
||||
"record (with why) or default to take the choice back. The bus is never moved by the assigner (ADR 0539 clause 15).",
|
||||
Input: schema(map[string]string{
|
||||
"module": "one module",
|
||||
"policy": "with module: roll-out, record or default",
|
||||
"together": "\"true\": with roll-out, every machine at once instead of one machine first",
|
||||
"why": "with policy or release-backlog: why — required for record and for a release, kept",
|
||||
"backlog": "\"true\": every build waiting for a gate, per machine — what a release plan would send",
|
||||
"release-backlog": "\"true\": release what waits now, one machine at a time, each judged — after a release " +
|
||||
"plan failed, the mesh waits for this; with why",
|
||||
"cause": "with release-backlog: the cause in a word (optional)",
|
||||
}, nil, "together", "backlog", "release-backlog")},
|
||||
{Name: "bus", Description: "The bus as a planned step (novox/hq to-be 45 §8, ADR 0236): what a bus upgrade " +
|
||||
"would do — the bus's build on each machine against the one the mesh holds — and how the last step went. " +
|
||||
"With upgrade, start one: a person's act with why, after the streams are snapshotted (snapshot-taken says " +
|
||||
"where, while the mesh takes none itself), saying first whether it can be reverted; bus-maintenance is open " +
|
||||
"while it runs and every stream, consumer and a round trip are checked after.",
|
||||
"module": "one module",
|
||||
"policy": "with module: roll-out, record or default",
|
||||
"why": "with policy: why — required for record, kept",
|
||||
}, nil)},
|
||||
{Name: "bus", Description: "The bus as a person's step (novox/hq to-be 45 §8, ADR 0236, ADR 0539 clause 15): what a bus " +
|
||||
"upgrade would do — the bus's build on each machine against the one the mesh holds — and how the last step went. " +
|
||||
"With upgrade, check one: a person's act with why, after the streams are snapshotted (snapshot-taken says where, " +
|
||||
"while the mesh takes none itself), saying first whether it can be reverted; then it answers the want the operator " +
|
||||
"writes with the operator's credential, want.<node>.<bus module>, recorded — nothing is sent, and the assigner never " +
|
||||
"moves the bus.",
|
||||
Input: schema(map[string]string{
|
||||
"upgrade": "\"true\": start the bus's upgrade",
|
||||
"why": "with upgrade: why — required, recorded in the hand-act log",
|
||||
@@ -595,22 +494,8 @@ var ControllerVerbs = []Verb{
|
||||
"machine": "one machine (optional)",
|
||||
"retired": "\"true\": only what is retired",
|
||||
}, nil, "retired")},
|
||||
// Three shapes, as the command has them: one repository, every module standing on a base (on), or
|
||||
// every module behind its source (behind). One per call; a second is refused as passed over.
|
||||
{Name: "build", Description: "Have the build machine build a repository — or, instead, every module that " +
|
||||
"stands on a base (on), or every module behind its source (behind). Answers at once with each build's id: " +
|
||||
"`builds` with an id follows it line by line, and a module is registered when its outcome comes.",
|
||||
Input: schema(map[string]string{
|
||||
"repository": "the repository's URL, or its path on the forge holding the git seat (owner/name); " +
|
||||
"needed unless on or behind is given",
|
||||
"path": "with repository: the module's directory inside it (optional)",
|
||||
"ref": "with repository: the branch, tag or commit to build (optional)",
|
||||
"on": "instead of a repository: a module's name. Every module the mesh holds that was built against " +
|
||||
"its artifacts is built again from its own source, bases among them asked first — the rebuild a " +
|
||||
"changed base needs, which behind does not see because their sources did not move",
|
||||
"behind": "\"true\", instead of a repository: build every module the mesh holds whose source has moved " +
|
||||
"past the commit it was built from, bases asked first",
|
||||
}, nil, "behind")},
|
||||
{Name: "build", Description: "Removed in R4e (ADR 0501 §5 / ADR 0503 §3): refused by name, naming its replacement",
|
||||
Input: schema(nil, nil)},
|
||||
// What the records say the registries may keep, asked on demand (novox/hq ADR 0251, to-be 51).
|
||||
{Name: "artifacts", Description: "Every artifact the mesh recorded making that lives in its artifact store, " +
|
||||
"each with its repository, digest, kind (image or archive) and state: kept — and why: a definition names " +
|
||||
@@ -621,6 +506,15 @@ var ControllerVerbs = []Verb{
|
||||
"repository": "only this repository's references, as <module>/<artifact> (the counts stay whole)",
|
||||
"collected": "\"true\": list the collected references too (they only grow; counted always)",
|
||||
}, nil, "collected")},
|
||||
// Whether the artifact store holds what a builder's done key names, before the builder reuses it (novox/hq issue
|
||||
// 1937): the store's collector deletes a blob nothing holds, and a key the controller never took in holds nothing.
|
||||
{Name: "held", Description: "Whether the artifact store holds each reference given, asked with one HEAD each: " +
|
||||
"held, gone (the store answered 404) or unknown with why. The builder asks it before it reuses a done key, and " +
|
||||
"builds a key whose artifact is gone again. Reads only: nothing is held or written (novox/hq issue 1937).",
|
||||
Input: listed(schema(map[string]string{
|
||||
"references": "the artifacts' references, artifact-store://<module>/<artifact>/blobs/sha256:<hex>: a list, " +
|
||||
"or one text separated by commas",
|
||||
}, []string{"references"}), "references")},
|
||||
{Name: "collect", Description: "The artifact store's sweep, now: what the mesh made and keeps for no reason " +
|
||||
"is let go of. Without confirm a dry run — every kept archive asked about with HEADs, what would be let go " +
|
||||
"listed, nothing held or deleted. With confirm and why: every kept archive held first, then each eligible " +
|
||||
|
||||
@@ -0,0 +1,31 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// 1887/A9b: the build queue's verbs are refused by name, naming what replaced them, and the verbs R4e removed are gone.
|
||||
func TestTheBuildQueueVerbsAreRefusedByName(t *testing.T) {
|
||||
rows := map[string]Verb{}
|
||||
for _, v := range ControllerVerbs {
|
||||
rows[v.Name] = v
|
||||
}
|
||||
for _, gone := range []string{"rebuild", "forget-checks"} {
|
||||
if _, ok := rows[gone]; ok {
|
||||
t.Errorf("the controller still lists %s", gone)
|
||||
}
|
||||
}
|
||||
for _, refused := range []string{"queue", "cancel", "clear", "kill", "pause", "resume", "replay"} {
|
||||
v, ok := rows[refused]
|
||||
if !ok || !strings.HasPrefix(v.Description, "Removed in R4g (ADR 0525 §9): refused by name") {
|
||||
t.Errorf("%s = %q, want refused by name", refused, v.Description)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(rows["queue"].Description, "delivery view") || !strings.Contains(rows["pause"].Description, "node-build-") {
|
||||
t.Errorf("queue and pause do not name their replacements: %q, %q", rows["queue"].Description, rows["pause"].Description)
|
||||
}
|
||||
if strings.Contains(rows["try"].Description, "build-agent") {
|
||||
t.Errorf("try still names build-agent: %q", rows["try"].Description)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,115 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// 1943/AC4: the controller's seat no longer offers a verb that starts, stops, retries, follows or reads a delivery
|
||||
// plan, nor a push: each is a row with no input whose description refuses it by name and names its replacement, and
|
||||
// each is in the removed list the holder answers from. upgrade keeps its policy and drops the plan's backlog.
|
||||
func TestTheControllerSeatHasNoPlanVerbs(t *testing.T) {
|
||||
rows := map[string]Verb{}
|
||||
for _, v := range ControllerVerbs {
|
||||
rows[v.Name] = v
|
||||
}
|
||||
gone := []string{"plans", "plan", "push", "deliver", "delivery-stop", "delivery-backups-again", "delivery-walks", "delivery-order"}
|
||||
for _, name := range gone {
|
||||
v, ok := rows[name]
|
||||
if !ok {
|
||||
t.Errorf("%s is not in the table, so a caller is not answered by name", name)
|
||||
continue
|
||||
}
|
||||
replacement, removed := RemovedControllerVerbs[name]
|
||||
if !removed || replacement == "" {
|
||||
t.Errorf("%s is not in the removed list with its replacement", name)
|
||||
}
|
||||
if !strings.HasPrefix(v.Description, "Removed in R6 (ADR 0539): refused by name") || !strings.Contains(v.Description, replacement) {
|
||||
t.Errorf("%s = %q, want refused by name, naming %q", name, v.Description, replacement)
|
||||
}
|
||||
if props, _ := v.Input["properties"].(map[string]any); len(props) > 0 {
|
||||
t.Errorf("%s still takes %v", name, props)
|
||||
}
|
||||
}
|
||||
props, _ := rows["upgrade"].Input["properties"].(map[string]any)
|
||||
for _, arg := range []string{"together", "backlog", "release-backlog", "cause"} {
|
||||
if _, has := props[arg]; has {
|
||||
t.Errorf("upgrade still takes %s, a plan's", arg)
|
||||
}
|
||||
}
|
||||
for _, arg := range []string{"module", "policy", "why"} {
|
||||
if _, has := props[arg]; !has {
|
||||
t.Errorf("upgrade no longer takes %s", arg)
|
||||
}
|
||||
}
|
||||
for _, word := range []string{"rolled", "gate", "backlog", "push"} {
|
||||
if strings.Contains(rows["upgrade"].Description, word) {
|
||||
t.Errorf("upgrade's description still speaks of %q: %q", word, rows["upgrade"].Description)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(rows["bus"].Description, "want") {
|
||||
t.Errorf("bus does not say the operator writes the bus's want: %q", rows["bus"].Description)
|
||||
}
|
||||
for name := range RemovedControllerVerbs {
|
||||
if v := rows[name]; !strings.HasPrefix(v.Description, "Removed in R6") {
|
||||
t.Errorf("%s is in the removed list and its row says %q", name, v.Description)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1943/AC4: the delivery seat keeps the merge's verbs and no longer offers times, what-if, release, stop or
|
||||
// retire-history: each is a row with no input, optional so the holder of before and the holder of release 6 both hold
|
||||
// the seat, refusing it by name; mark-ready, served by nobody since issue 1022, is gone.
|
||||
func TestTheDeliverySeatHasNoPlanVerbs(t *testing.T) {
|
||||
seat, ok := SeatNamed(DeliverySeat)
|
||||
if !ok {
|
||||
t.Fatal("the delivery seat is not in the set")
|
||||
}
|
||||
rows := map[string]Verb{}
|
||||
for _, v := range seat.Serves {
|
||||
rows[v.Name] = v
|
||||
}
|
||||
for _, name := range []string{"times", "what-if", "release", "stop", "retire-history"} {
|
||||
v, ok := rows[name]
|
||||
replacement, removed := RemovedDeliveryVerbs[name]
|
||||
if !ok || !removed || replacement == "" {
|
||||
t.Errorf("%s: in the table %v, in the removed list %v", name, ok, removed)
|
||||
continue
|
||||
}
|
||||
if !strings.HasPrefix(v.Description, "Removed in R6 (ADR 0539): refused by name") || !strings.Contains(v.Description, replacement) || !v.Optional {
|
||||
t.Errorf("%s = %q (optional %v), want an optional row refusing it by name", name, v.Description, v.Optional)
|
||||
}
|
||||
if props, _ := v.Input["properties"].(map[string]any); len(props) > 0 {
|
||||
t.Errorf("%s still takes %v", name, props)
|
||||
}
|
||||
}
|
||||
if _, ok := rows["mark-ready"]; ok {
|
||||
t.Errorf("mark-ready is still promised")
|
||||
}
|
||||
for _, kept := range []string{"deliveries", "show", "groups", "checks", "table", "recheck", "close", "stalled"} {
|
||||
if v, ok := rows[kept]; !ok || strings.HasPrefix(v.Description, "Removed") {
|
||||
t.Errorf("%s is not kept (ADR 0539 clause 17)", kept)
|
||||
}
|
||||
}
|
||||
gate := Manifest{Module: "mesh-delivery", Claims: []Claim{{Name: DeliverySeat, Scope: ScopeMesh,
|
||||
Serves: []string{"deliveries", "show", "groups", "checks", "table", "stalled", "recheck", "close"}}}}
|
||||
if err := CanHold(gate, seat); err != nil {
|
||||
t.Errorf("the gate of release 6 cannot hold the seat: %v", err)
|
||||
}
|
||||
gate.Claims[0].Serves = append(gate.Claims[0].Serves, "times", "what-if", "release", "stop", "retire-history")
|
||||
if err := CanHold(gate, seat); err != nil {
|
||||
t.Errorf("the gate of release 5 cannot hold the seat: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// 1943 (for 1945/AC14): the controller's seat says a bucket it deleted at its first start as its event, bucket-deleted
|
||||
// (ADR 0539 clause 13).
|
||||
func TestTheControllerSaysABucketItDeleted(t *testing.T) {
|
||||
seat, ok := SeatNamed(ControllerSeatName)
|
||||
if !ok {
|
||||
t.Fatal("no controller seat")
|
||||
}
|
||||
if !SeatSays(seat.Emits, "bucket-deleted") {
|
||||
t.Errorf("the controller's seat does not emit bucket-deleted: %v", seat.Emits)
|
||||
}
|
||||
}
|
||||
@@ -106,7 +106,7 @@ func TestAFailedWriteWaitsNoLongerThanTheDeadline(t *testing.T) {
|
||||
default:
|
||||
}
|
||||
})
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond)
|
||||
defer cancel()
|
||||
start := time.Now()
|
||||
_, err = Ask(ctx, socket, aLineTheSocketCannotHold())
|
||||
@@ -117,7 +117,7 @@ func TestAFailedWriteWaitsNoLongerThanTheDeadline(t *testing.T) {
|
||||
if !errors.Is(err, os.ErrDeadlineExceeded) {
|
||||
t.Fatalf("the error is not the deadline: %v", err)
|
||||
}
|
||||
if waited > 3*time.Second {
|
||||
t.Fatalf("waited %s with a deadline of 1s", waited)
|
||||
if waited > time.Second {
|
||||
t.Fatalf("waited %s with a deadline of 50ms", waited)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,163 @@
|
||||
// mesh-compose is mesh-sdk's composer command (novox/hq ADR 0482 clause 4, issue 1621): it reads a composer.Request on
|
||||
// standard input, validates every manifest with this mesh-sdk's reader, composes the machines named in each case and
|
||||
// writes one composer.Verdict on standard output. Built at any commit of mesh-sdk (or at the mesh-sdk a module pins),
|
||||
// it answers what that commit's composition answers, so a program that links one mesh-sdk can have another's judged.
|
||||
//
|
||||
// mesh-compose < request.json > verdict.json
|
||||
// mesh-compose -request request.json > verdict.json
|
||||
// mesh-compose -catalogue <dir> [-node <name>]... [-depth resolve|compose] [-declarations] [< request.json]
|
||||
//
|
||||
// -catalogue reads <dir>/modules/*/module.json and a module.json at <dir>'s root as the first case's manifests (a
|
||||
// checkout or `git archive` of the catalogue repository at a ref: the command fetches nothing). -node adds a machine to
|
||||
// compose; -declarations answers each machine's declared paths with their owners and modes.
|
||||
//
|
||||
// It opens no network connection, bus, store or file beyond its standard input, the -request file and the -catalogue
|
||||
// directory, and reads
|
||||
// no environment. Exit codes: 0 the verdict is written (a machine that does not compose is a verdict, not an error);
|
||||
// 1 the request cannot be read or run, said on standard error; 2 the flags are wrong.
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/composer"
|
||||
)
|
||||
|
||||
func main() {
|
||||
os.Exit(run(os.Args[1:], os.Stdin, os.Stdout, os.Stderr))
|
||||
}
|
||||
|
||||
type nodes []string
|
||||
|
||||
func (n *nodes) String() string { return strings.Join(*n, ",") }
|
||||
func (n *nodes) Set(v string) error { *n = append(*n, v); return nil }
|
||||
|
||||
// run is the command: its arguments, standard input, output and error; it answers the exit code.
|
||||
func run(args []string, stdin io.Reader, stdout, stderr io.Writer) int {
|
||||
fs := flag.NewFlagSet("mesh-compose", flag.ContinueOnError)
|
||||
fs.SetOutput(stderr)
|
||||
catalogueDir := fs.String("catalogue", "", "a catalogue checkout: <dir>/modules/*/module.json are the first case's manifests")
|
||||
var machines nodes
|
||||
fs.Var(&machines, "node", "a machine to compose (repeatable)")
|
||||
depth := fs.String("depth", "", "resolve or compose (default: the request's, else compose)")
|
||||
declarations := fs.Bool("declarations", false, "answer each composed machine's declared paths, owners and modes")
|
||||
requestFile := fs.String("request", "", "read the request from this file instead of standard input (a run step that hands no standard input)")
|
||||
if err := fs.Parse(args); err != nil {
|
||||
return 2
|
||||
}
|
||||
if fs.NArg() > 0 {
|
||||
fmt.Fprintf(stderr, "mesh-compose: unexpected arguments %q\n", fs.Args())
|
||||
return 2
|
||||
}
|
||||
var raw []byte
|
||||
var err error
|
||||
if *requestFile != "" {
|
||||
raw, err = os.ReadFile(*requestFile)
|
||||
} else {
|
||||
raw, err = io.ReadAll(stdin)
|
||||
}
|
||||
if err != nil {
|
||||
fmt.Fprintln(stderr, "mesh-compose: the request cannot be read:", err)
|
||||
return 1
|
||||
}
|
||||
var req composer.Request
|
||||
if strings.TrimSpace(string(raw)) == "" {
|
||||
if *catalogueDir == "" {
|
||||
fmt.Fprintln(stderr, "mesh-compose: no request on standard input and no -catalogue")
|
||||
return 1
|
||||
}
|
||||
req = composer.Request{V: composer.Version}
|
||||
} else if req, err = composer.Decode(raw); err != nil {
|
||||
fmt.Fprintln(stderr, "mesh-compose:", err)
|
||||
return 1
|
||||
}
|
||||
if *catalogueDir != "" {
|
||||
read, err := readCatalogue(*catalogueDir)
|
||||
if err != nil {
|
||||
fmt.Fprintln(stderr, "mesh-compose:", err)
|
||||
return 1
|
||||
}
|
||||
if len(req.Cases) == 0 {
|
||||
req.Cases = []composer.Case{{Name: "catalogue", Manifests: map[string]composer.Manifest{}}}
|
||||
}
|
||||
if req.Cases[0].Manifests == nil {
|
||||
req.Cases[0].Manifests = map[string]composer.Manifest{}
|
||||
}
|
||||
for name, m := range read {
|
||||
if _, given := req.Cases[0].Manifests[name]; !given {
|
||||
req.Cases[0].Manifests[name] = m
|
||||
}
|
||||
}
|
||||
// The catalogue's list of predefined keys is the one its manifests' fallbacks are judged against (ADR 0348 §7).
|
||||
if req.Predefined == nil {
|
||||
list, err := catalogue.ReadPredefined(*catalogueDir)
|
||||
if err != nil {
|
||||
fmt.Fprintln(stderr, "mesh-compose:", err)
|
||||
return 1
|
||||
}
|
||||
req.Predefined = list
|
||||
}
|
||||
}
|
||||
req.Machines = append(req.Machines, machines...)
|
||||
if *depth != "" {
|
||||
req.Depth = *depth
|
||||
}
|
||||
if *declarations {
|
||||
req.Declarations = true
|
||||
}
|
||||
v, err := composer.Run(req)
|
||||
if err != nil {
|
||||
fmt.Fprintln(stderr, "mesh-compose:", err)
|
||||
return 1
|
||||
}
|
||||
enc := json.NewEncoder(stdout)
|
||||
if err := enc.Encode(v); err != nil {
|
||||
fmt.Fprintln(stderr, "mesh-compose: the verdict cannot be written:", err)
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// readCatalogue reads every module.json of a catalogue checkout by its module's name: the root's and each of
|
||||
// modules/*/. A manifest whose name cannot be read is kept under its directory's name, so the reader's refusal is said.
|
||||
func readCatalogue(dir string) (map[string]composer.Manifest, error) {
|
||||
paths, err := filepath.Glob(filepath.Join(dir, "modules", "*", "module.json"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(dir, "module.json")); err == nil {
|
||||
paths = append(paths, filepath.Join(dir, "module.json"))
|
||||
}
|
||||
if len(paths) == 0 {
|
||||
return nil, fmt.Errorf("the catalogue %s holds no module.json", dir)
|
||||
}
|
||||
sort.Strings(paths)
|
||||
out := map[string]composer.Manifest{}
|
||||
for _, p := range paths {
|
||||
raw, err := os.ReadFile(p)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var probe struct {
|
||||
Module string `json:"module"`
|
||||
}
|
||||
name := filepath.Base(filepath.Dir(p))
|
||||
if json.Unmarshal(raw, &probe) == nil && probe.Module != "" {
|
||||
name = probe.Module
|
||||
}
|
||||
if _, twice := out[name]; twice {
|
||||
return nil, errors.New("the catalogue " + dir + " defines " + name + " twice")
|
||||
}
|
||||
out[name] = composer.Manifest{Raw: raw}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
@@ -0,0 +1,347 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"go/build"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"runtime"
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/compose"
|
||||
"git.novox.be/novox/mesh-sdk/go/composer"
|
||||
"git.novox.be/novox/mesh-sdk/go/internal/beside"
|
||||
)
|
||||
|
||||
// captured is every manifest of the catalogue captured in testdata/beside (internal/beside), raw, by module name.
|
||||
func captured(t *testing.T) map[string]composer.Manifest {
|
||||
t.Helper()
|
||||
read, err := readCatalogue(filepath.Dir(beside.Catalogue(t)))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(read) < 100 {
|
||||
t.Fatalf("the captured catalogue holds %d manifests, fewer than the catalogue's", len(read))
|
||||
}
|
||||
return read
|
||||
}
|
||||
|
||||
// meshOf is a mesh of three machines over the captured catalogue, each assigned real modules, the input the assigner
|
||||
// composes with (nodes and their assignments, holdings from the claims): anchor runs the bus and the store, laptop a
|
||||
// desktop's modules, ace a module whose requirement is provided elsewhere.
|
||||
func meshOf() compose.Input {
|
||||
return compose.Input{Nodes: []compose.Node{
|
||||
{Name: "anchor", Assigned: []string{"nats", "postgres"}},
|
||||
{Name: "laptop", Assigned: []string{"sudo", "zsh"}},
|
||||
{Name: "ace", Assigned: []string{"mesh-reads", "nats"}},
|
||||
}}
|
||||
}
|
||||
|
||||
func request(t *testing.T, depth string) composer.Request {
|
||||
t.Helper()
|
||||
return composer.Request{V: composer.Version, Input: meshOf(), Depth: depth, HoldingsFromClaims: true,
|
||||
Machines: []string{"anchor", "laptop", "ace"},
|
||||
Cases: []composer.Case{{Name: "trunk", Manifests: captured(t)}}}
|
||||
}
|
||||
|
||||
func runJSON(t *testing.T, args []string, in []byte) (int, composer.Verdict, string) {
|
||||
t.Helper()
|
||||
var out, errOut bytes.Buffer
|
||||
code := run(args, bytes.NewReader(in), &out, &errOut)
|
||||
var v composer.Verdict
|
||||
if code == 0 {
|
||||
if err := json.Unmarshal(out.Bytes(), &v); err != nil {
|
||||
t.Fatalf("the verdict is not JSON: %v\n%s", err, out.String())
|
||||
}
|
||||
}
|
||||
return code, v, errOut.String()
|
||||
}
|
||||
|
||||
func encode(t *testing.T, r composer.Request) []byte {
|
||||
t.Helper()
|
||||
raw, err := json.Marshal(r)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return raw
|
||||
}
|
||||
|
||||
// 1621/AC1: the command reads the plain input and the manifests on standard input, validates every manifest, composes
|
||||
// the machines named and writes one verdict: per machine composes or the reason, per manifest accepted or the field
|
||||
// refused.
|
||||
func TestTheCommandWritesOneVerdictPerManifestAndMachine(t *testing.T) {
|
||||
r := request(t, composer.DepthResolve)
|
||||
r.Cases[0].Manifests["broken"] = composer.Manifest{Raw: json.RawMessage(`{"module":"broken","version":"1","shares":["x"]}`)}
|
||||
code, v, stderr := runJSON(t, nil, encode(t, r))
|
||||
if code != 0 {
|
||||
t.Fatalf("exit %d: %s", code, stderr)
|
||||
}
|
||||
if len(v.Cases) != 1 || v.Cases[0].Name != "trunk" || v.SDK == "" {
|
||||
t.Fatalf("verdict = %+v", v)
|
||||
}
|
||||
c := v.Cases[0]
|
||||
if got := c.Manifests["broken"]; got.Accepted || !strings.Contains(got.Refused, `"shares"`) {
|
||||
t.Errorf("the manifest with an unknown field reads %+v, want refused naming the field", got)
|
||||
}
|
||||
if got := c.Manifests["nats"]; !got.Accepted {
|
||||
t.Errorf("nats reads %+v, want accepted", got)
|
||||
}
|
||||
if len(c.Machines) != 3 {
|
||||
t.Fatalf("machines = %v, want the three named", c.Machines)
|
||||
}
|
||||
for name, m := range c.Machines {
|
||||
if !m.Composes && m.Reason == "" {
|
||||
t.Errorf("%s does not compose and says no reason", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1621/AC3: over the captured catalogue and a mesh of three machines, the command's verdict equals, machine for
|
||||
// machine and manifest for manifest, what the linked reader, resolution and composition answer, at both depths.
|
||||
func TestTheCommandAnswersWhatTheLinkedCompositionAnswers(t *testing.T) {
|
||||
for _, depth := range []string{composer.DepthResolve, composer.DepthCompose} {
|
||||
r := request(t, depth)
|
||||
code, v, stderr := runJSON(t, nil, encode(t, r))
|
||||
if code != 0 {
|
||||
t.Fatalf("%s: exit %d: %s", depth, code, stderr)
|
||||
}
|
||||
in := meshOf()
|
||||
in.Catalogue = map[string]catalogue.Manifest{}
|
||||
for name, m := range r.Cases[0].Manifests {
|
||||
// A request without a list of predefined keys reads a fallback as the indexer does (Request.Predefined).
|
||||
parsed, err := catalogue.ParseManifestForInputs(m.Raw)
|
||||
got := v.Cases[0].Manifests[name]
|
||||
if (err == nil) != got.Accepted || (err != nil && err.Error() != got.Refused) {
|
||||
t.Errorf("%s: the manifest of %s: linked %v, command %+v", depth, name, err, got)
|
||||
}
|
||||
if err == nil {
|
||||
in.Catalogue[name] = parsed
|
||||
}
|
||||
}
|
||||
in.Holdings = composer.HoldingsOf(in.Catalogue, in.Nodes)
|
||||
composed := 0
|
||||
for _, node := range r.Machines {
|
||||
got := v.Cases[0].Machines[node]
|
||||
var want composer.Machine
|
||||
if depth == composer.DepthResolve {
|
||||
if _, err := compose.Resolve(in, node); err != nil {
|
||||
want = composer.Machine{Reason: err.Error()}
|
||||
} else {
|
||||
want = composer.Machine{Composes: true}
|
||||
}
|
||||
} else {
|
||||
d, problems, err := compose.Compose(in, node)
|
||||
switch {
|
||||
case err != nil:
|
||||
want = composer.Machine{Reason: err.Error()}
|
||||
default:
|
||||
want = composer.Machine{Composes: len(problems) == 0, Problems: problems, Refusals: d.Refusals()}
|
||||
if len(problems) > 0 {
|
||||
want.Reason = "the node-engine's validator refuses the declaration"
|
||||
}
|
||||
}
|
||||
}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Errorf("%s: %s: command %+v, linked %+v", depth, node, got, want)
|
||||
}
|
||||
if got.Composes {
|
||||
composed++
|
||||
} else {
|
||||
t.Logf("%s: %s does not compose: %s %v", depth, node, got.Reason, got.Problems)
|
||||
}
|
||||
}
|
||||
t.Logf("%s: %d of %d machines compose", depth, composed, len(r.Machines))
|
||||
}
|
||||
}
|
||||
|
||||
// 1621 note 6: a person composes a catalogue checkout (repository@ref checked out or archived: the command fetches
|
||||
// nothing) for a node and reads its declaration: each path with its kind, owner and mode.
|
||||
func TestACatalogueCheckoutIsComposedForANode(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
if err := os.MkdirAll(filepath.Join(dir, "modules", "demo"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
manifest := `{"module": "demo", "version": "1", "resources": [{"id": "state", "type": "directory", ` +
|
||||
`"path": "/var/lib/demo", "mode": "0750", "owner": "root"}]}`
|
||||
if err := os.WriteFile(filepath.Join(dir, "modules", "demo", "module.json"), []byte(manifest), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
in := encode(t, composer.Request{V: composer.Version, Input: compose.Input{Nodes: []compose.Node{{Name: "laptop", Assigned: []string{"demo"}}}}})
|
||||
code, v, stderr := runJSON(t, []string{"-catalogue", dir, "-node", "laptop", "-declarations"}, in)
|
||||
if code != 0 {
|
||||
t.Fatalf("exit %d: %s", code, stderr)
|
||||
}
|
||||
m := v.Cases[0].Machines["laptop"]
|
||||
want := []composer.Path{{Path: "/var/lib/demo", Kind: "directory", Owner: "root", Mode: "0750", Module: ""}}
|
||||
if !m.Composes || len(m.Declared) != 1 || m.Declared[0].Path != want[0].Path || m.Declared[0].Mode != "0750" ||
|
||||
m.Declared[0].Owner != "root" || m.Declared[0].Kind != "directory" {
|
||||
t.Fatalf("laptop = %+v, want it composed declaring %+v", m, want)
|
||||
}
|
||||
}
|
||||
|
||||
// A run step hands the command no standard input: the request is read from the file -request names, and a file that is
|
||||
// not there is a request it cannot read.
|
||||
func TestTheRequestIsReadFromAFile(t *testing.T) {
|
||||
file := filepath.Join(t.TempDir(), "request.json")
|
||||
r := composer.Request{V: composer.Version, Cases: []composer.Case{{Name: "only", Manifests: map[string]composer.Manifest{
|
||||
"demo": {Raw: json.RawMessage(`{"module":"demo","version":"1"}`)}}}}}
|
||||
if err := os.WriteFile(file, encode(t, r), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
code, v, stderr := runJSON(t, []string{"-request", file}, nil)
|
||||
if code != 0 || len(v.Cases) != 1 || !v.Cases[0].Manifests["demo"].Accepted {
|
||||
t.Fatalf("exit %d (%s), verdict %+v", code, stderr, v)
|
||||
}
|
||||
if code, _, stderr := runJSON(t, []string{"-request", file + ".gone"}, nil); code != 1 || !strings.Contains(stderr, "cannot be read") {
|
||||
t.Errorf("a request file that is not there: exit %d, %q", code, stderr)
|
||||
}
|
||||
}
|
||||
|
||||
// The exit codes: 1 for a request that cannot be read or run, 2 for wrong flags, each said on standard error.
|
||||
func TestTheExitCodesSayWhatFailed(t *testing.T) {
|
||||
unfilled := encode(t, composer.Request{V: composer.Version, Cases: []composer.Case{{Name: "c", Manifests: map[string]composer.Manifest{"x": {Digest: "abc"}}}}})
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
args []string
|
||||
in string
|
||||
code int
|
||||
says string
|
||||
}{
|
||||
{"not JSON", nil, "{", 1, "not JSON"},
|
||||
{"another version", nil, `{"v":2}`, 1, "version 2"},
|
||||
{"a digest left unfilled", nil, string(unfilled), 1, "digest"},
|
||||
{"an unknown depth", nil, `{"v":1,"depth":"deep"}`, 1, "depth"},
|
||||
{"nothing to read", nil, "", 1, "no request"},
|
||||
{"a wrong flag", []string{"-nope"}, "", 2, "nope"},
|
||||
{"an argument", []string{"extra"}, "", 2, "unexpected"},
|
||||
} {
|
||||
var out, errOut bytes.Buffer
|
||||
if got := run(c.args, strings.NewReader(c.in), &out, &errOut); got != c.code || !strings.Contains(errOut.String(), c.says) {
|
||||
t.Errorf("%s: exit %d saying %q, want %d saying %q", c.name, got, errOut.String(), c.code, c.says)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// sdkPackages is every package of this module the command imports, directly or not, with the packages each imports.
|
||||
func sdkPackages(t *testing.T) map[string][]string {
|
||||
t.Helper()
|
||||
_, file, _, _ := runtime.Caller(0)
|
||||
root := filepath.Join(filepath.Dir(file), "..", "..")
|
||||
const module = "git.novox.be/novox/mesh-sdk/go"
|
||||
out := map[string][]string{}
|
||||
var walk func(path string)
|
||||
walk = func(path string) {
|
||||
if _, done := out[path]; done {
|
||||
return
|
||||
}
|
||||
rel := strings.TrimPrefix(strings.TrimPrefix(path, module), "/")
|
||||
pkg, err := build.Default.ImportDir(filepath.Join(root, rel), 0)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", path, err)
|
||||
}
|
||||
out[path] = pkg.Imports
|
||||
for _, imp := range pkg.Imports {
|
||||
if strings.HasPrefix(imp, module+"/") {
|
||||
walk(imp)
|
||||
}
|
||||
}
|
||||
}
|
||||
walk(composer.Package)
|
||||
return out
|
||||
}
|
||||
|
||||
// 1621/AC2 and AC4: the command imports, through every package of mesh-sdk it reaches, only the standard library and
|
||||
// mesh-sdk's own packages (so `go build` from mesh-sdk's module needs nothing beyond its lock), and none that reaches
|
||||
// a bus, a store, a network service or another process.
|
||||
func TestTheCommandImportsNoBusStoreNetworkOrProcess(t *testing.T) {
|
||||
forbidden := []string{"net/http", "net/rpc", "net/smtp", "os/exec", "database/sql", "plugin", "syscall"}
|
||||
for path, imports := range sdkPackages(t) {
|
||||
for _, imp := range imports {
|
||||
std := !strings.Contains(strings.SplitN(imp, "/", 2)[0], ".")
|
||||
if !std && !strings.HasPrefix(imp, "git.novox.be/novox/mesh-sdk/go/") {
|
||||
t.Errorf("%s imports %s, which is neither the standard library nor mesh-sdk", path, imp)
|
||||
}
|
||||
for _, f := range forbidden {
|
||||
if imp == f {
|
||||
t.Errorf("%s imports %s", path, imp)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1621/AC2: run with an empty network namespace, the command still writes its verdict: it needs no network. The test
|
||||
// runs its own binary again under `unshare --net` as an unprivileged user's namespace; where the kernel or the sandbox
|
||||
// allows no new namespace (a build module's run step is already one with no network), it says so and skips.
|
||||
func TestTheCommandRunsWithNoNetwork(t *testing.T) {
|
||||
if os.Getenv("MESH_COMPOSE_HELPER") == "1" {
|
||||
os.Exit(run(nil, os.Stdin, os.Stdout, os.Stderr))
|
||||
}
|
||||
unshare, err := exec.LookPath("unshare")
|
||||
if err != nil {
|
||||
t.Skip("no unshare on this machine, so no empty network namespace can be made here")
|
||||
}
|
||||
if out, err := exec.Command(unshare, "--net", "--map-root-user", "true").CombinedOutput(); err != nil {
|
||||
t.Skipf("this machine makes no unprivileged network namespace (%v: %s); a run step's own has no network", err, out)
|
||||
}
|
||||
r := request(t, composer.DepthResolve)
|
||||
cmd := exec.Command(unshare, "--net", "--map-root-user", os.Args[0], "-test.run=^TestTheCommandRunsWithNoNetwork$")
|
||||
cmd.Env = []string{"MESH_COMPOSE_HELPER=1"}
|
||||
cmd.Stdin = bytes.NewReader(encode(t, r))
|
||||
var out, errOut bytes.Buffer
|
||||
cmd.Stdout, cmd.Stderr = &out, &errOut
|
||||
if err := cmd.Run(); err != nil {
|
||||
t.Fatalf("the command in an empty network namespace: %v: %s", err, errOut.String())
|
||||
}
|
||||
var v composer.Verdict
|
||||
if err := json.Unmarshal(out.Bytes(), &v); err != nil || len(v.Cases) != 1 || len(v.Cases[0].Machines) != 3 {
|
||||
t.Fatalf("no verdict from the command with no network: %v\n%s", err, out.String())
|
||||
}
|
||||
names := make([]string, 0, len(v.Cases[0].Machines))
|
||||
for n := range v.Cases[0].Machines {
|
||||
names = append(names, n)
|
||||
}
|
||||
sort.Strings(names)
|
||||
t.Logf("composed with no network: %v", names)
|
||||
}
|
||||
|
||||
// -catalogue judges a fallback to a predefined key against the predefined.json at the checkout's root (ADR 0348 §7):
|
||||
// xterm (mesh-catalog at 835bf22b) falls back to "font-size", accepted where the root lists it and refused where the
|
||||
// root's list does not.
|
||||
func TestACatalogueCheckoutJudgesFallbacksAgainstItsPredefinedKeys(t *testing.T) {
|
||||
raw, err := os.ReadFile("../../catalogue/testdata/catalogue-835bf22b-xterm.module.json")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, c := range []struct {
|
||||
list string
|
||||
want bool
|
||||
}{
|
||||
{`{"font-size": {"type": "number", "description": "the desktop's font size"}}`, true},
|
||||
{`{"font-family": {"type": "string", "description": "the desktop's font"}}`, false},
|
||||
} {
|
||||
dir := t.TempDir()
|
||||
if err := os.MkdirAll(filepath.Join(dir, "modules", "xterm"), 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(dir, "modules", "xterm", "module.json"), raw, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(dir, catalogue.PredefinedFile), []byte(c.list), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
code, v, stderr := runJSON(t, []string{"-catalogue", dir}, nil)
|
||||
if code != 0 {
|
||||
t.Fatalf("exit %d: %s", code, stderr)
|
||||
}
|
||||
if got := v.Cases[0].Manifests["xterm"]; got.Accepted != c.want {
|
||||
t.Errorf("predefined.json %s: xterm %+v, want accepted %v", c.list, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935).
|
||||
@@ -0,0 +1,209 @@
|
||||
// mesh-hand-compose composes one module for one node by hand, from the inputs the controller keeps, and signs it with the
|
||||
// operator's key, for the hand path of novox/hq ADR 0539 clause 10: a fingerprint the controller never composed (a
|
||||
// hand build), while the controller cannot compose its own fix. It composes with mesh-sdk's composition, the one the
|
||||
// controller writes with (compose.Compositions), and prints the composition, its key and its digest; only with -write
|
||||
// does it write them, with the operator's credential: the composition into the node's bucket composed-<node>, each
|
||||
// file over 64 KiB into the node's object bucket, and the digest into mesh-controller_composed. The want is the
|
||||
// operator's next act, naming that digest; each act is recorded in the hand-act log.
|
||||
//
|
||||
// mesh-hand-compose -input <inputs.json> -node <node> -module <module> -fp <src2:...> -key <operator key>
|
||||
// [-write -nats <url> -creds <operator credential>]
|
||||
//
|
||||
// The inputs are a compose.Input as JSON. The key is the operator's ed25519 private key, PKCS#8 in PEM; the
|
||||
// node-engine trusts it beside the controller's (ADR 0539 clause 3).
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/ed25519"
|
||||
"crypto/x509"
|
||||
"encoding/json"
|
||||
"encoding/pem"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/compose"
|
||||
"git.novox.be/novox/mesh-sdk/go/declaration"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
func main() {
|
||||
if err := run(os.Args[1:], os.Stdout); err != nil {
|
||||
fmt.Fprintln(os.Stderr, "mesh-hand-compose:", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
// printed is what mesh-hand-compose prints: where the composition goes, its digest, the composition, and whether it was
|
||||
// written.
|
||||
type printed struct {
|
||||
Bucket string `json:"bucket"`
|
||||
Key string `json:"key"`
|
||||
Digest string `json:"digest"`
|
||||
Composition json.RawMessage `json:"composition"`
|
||||
Objects []string `json:"objects,omitempty"`
|
||||
Written bool `json:"written"`
|
||||
Next string `json:"next"`
|
||||
}
|
||||
|
||||
func run(args []string, stdout io.Writer) error {
|
||||
fs := flag.NewFlagSet("mesh-hand-compose", flag.ContinueOnError)
|
||||
input := fs.String("input", "", "the controller's kept inputs, a compose.Input as JSON")
|
||||
node := fs.String("node", "", "the node to compose for")
|
||||
module := fs.String("module", "", "the module to compose")
|
||||
fp := fs.String("fp", "", "the module's fingerprint, src2: and 64 hex digits")
|
||||
keyFile := fs.String("key", "", "the operator's ed25519 private key, PKCS#8 in PEM")
|
||||
write := fs.Bool("write", false, "write the composition, its files and its digest, with the operator's credential")
|
||||
url := fs.String("nats", nats.DefaultURL, "with -write: the bus")
|
||||
creds := fs.String("creds", "", "with -write: the operator's credential file")
|
||||
if err := fs.Parse(args); err != nil {
|
||||
return err
|
||||
}
|
||||
switch {
|
||||
case *input == "" || *node == "" || *module == "" || *fp == "":
|
||||
return errors.New("-input, -node, -module and -fp are each required")
|
||||
case *keyFile == "":
|
||||
return errors.New("-key is required: a composition made by hand is signed by the operator's key, and the node-engine applies no other")
|
||||
case *write && *creds == "":
|
||||
return errors.New("-write needs -creds: the operator's credential is the only hand writer of a composition")
|
||||
}
|
||||
key, err := readKey(*keyFile)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
key2, err := delivery.ComposedKey(*node, *module, *fp)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
bucket, err := delivery.ComposedBucketFor(*node)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
raw, err := os.ReadFile(*input)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var in compose.Input
|
||||
if err := json.Unmarshal(raw, &in); err != nil {
|
||||
return fmt.Errorf("the inputs in %s are not a compose.Input: %w", *input, err)
|
||||
}
|
||||
all, _, err := compose.Compositions(in, *node, map[string]string{*module: *fp}, compose.Reading)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
c, ok := all[key2]
|
||||
if !ok {
|
||||
return fmt.Errorf("%s does not run %s: its composition holds none of it", *node, *module)
|
||||
}
|
||||
c, objects, err := c.ByDigest()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if c, err = delivery.SignComposed(c, key); err != nil {
|
||||
return err
|
||||
}
|
||||
encoded, err := delivery.EncodeComposed(c)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
digest, err := delivery.ComposedDigestOf(c)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
out := printed{Bucket: bucket, Key: key2, Digest: digest, Composition: encoded,
|
||||
Next: fmt.Sprintf("write want.%s.%s naming fp %s and composed %s with the operator's credential, and record each act in the hand-act log", *node, *module, *fp, digest)}
|
||||
for d := range objects {
|
||||
out.Objects = append(out.Objects, d)
|
||||
}
|
||||
if *write {
|
||||
if err := writeIt(*url, *creds, *node, bucket, key2, encoded, digest, objects); err != nil {
|
||||
return err
|
||||
}
|
||||
out.Written = true
|
||||
}
|
||||
enc := json.NewEncoder(stdout)
|
||||
enc.SetIndent("", " ")
|
||||
return enc.Encode(out)
|
||||
}
|
||||
|
||||
func readKey(path string) (ed25519.PrivateKey, error) {
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
block, _ := pem.Decode(raw)
|
||||
if block == nil {
|
||||
return nil, fmt.Errorf("%s holds no PEM block", path)
|
||||
}
|
||||
parsed, err := x509.ParsePKCS8PrivateKey(block.Bytes)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s: %w", path, err)
|
||||
}
|
||||
key, ok := parsed.(ed25519.PrivateKey)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("%s is not an ed25519 key", path)
|
||||
}
|
||||
return key, nil
|
||||
}
|
||||
|
||||
// writeIt writes the files by digest, then the composition, then its digest: a reader that finds the digest finds the
|
||||
// composition and its files already there.
|
||||
func writeIt(url, creds, node, bucket, key string, encoded []byte, digest string, objects map[string][]byte) error {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
|
||||
defer cancel()
|
||||
nc, err := nats.Connect(url, nats.UserCredentials(creds), nats.Name("mesh-hand-compose"))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer nc.Close()
|
||||
js, err := jetstream.New(nc)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(objects) > 0 {
|
||||
filesBucket, err := declaration.FileBucketFor(node)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
store, err := js.ObjectStore(ctx, filesBucket)
|
||||
if err != nil {
|
||||
return fmt.Errorf("the node's object bucket %s: %w", filesBucket, err)
|
||||
}
|
||||
for d, body := range objects {
|
||||
name, err := declaration.ObjectName(d)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := store.PutBytes(ctx, name, body); err != nil {
|
||||
return fmt.Errorf("the file %s: %w", d, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
composed, err := js.KeyValue(ctx, bucket)
|
||||
if err != nil {
|
||||
return fmt.Errorf("the bucket %s: %w", bucket, err)
|
||||
}
|
||||
if _, err := composed.Put(ctx, key, encoded); err != nil {
|
||||
return fmt.Errorf("%s in %s: %w", key, bucket, err)
|
||||
}
|
||||
value, err := delivery.EncodeComposedDigest(delivery.ComposedDigest{Digest: digest, Judgement: delivery.JudgedCannot,
|
||||
Words: "composed by hand with mesh-hand-compose (ADR 0539 clause 10): the operator writes the want"})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
digests, err := js.KeyValue(ctx, delivery.ControllerComposedBucket)
|
||||
if err != nil {
|
||||
return fmt.Errorf("the bucket %s: %w", delivery.ControllerComposedBucket, err)
|
||||
}
|
||||
if _, err := digests.Put(ctx, key, value); err != nil {
|
||||
return fmt.Errorf("%s in %s: %w", key, delivery.ControllerComposedBucket, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,145 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/ed25519"
|
||||
"crypto/rand"
|
||||
"crypto/x509"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"encoding/pem"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/compose"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/overlay"
|
||||
)
|
||||
|
||||
// aMesh is the Input of mesh-controller's aMesh, as mesh-sdk's compose tests make it (compose/fixture_test.go): two
|
||||
// machines on the private network, the module the mesh provides for itself assigned to both. The manifest is the one
|
||||
// the controller provides (overlay.Manifest), read back as the catalogue reads it.
|
||||
func aMesh(t *testing.T) compose.Input {
|
||||
t.Helper()
|
||||
var m catalogue.Manifest
|
||||
b, _ := json.Marshal(overlay.Manifest())
|
||||
if err := json.Unmarshal(b, &m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
in := compose.Input{Catalogue: map[string]catalogue.Manifest{overlay.Name: m}}
|
||||
for i, name := range []string{"anchor", "laptop"} {
|
||||
pub, _, _ := ed25519.GenerateKey(rand.Reader)
|
||||
in.Nodes = append(in.Nodes, compose.Node{Name: name,
|
||||
Place: compose.Place{Key: base64.StdEncoding.EncodeToString(pub), Endpoint: name + ".example:51820", Site: "here", Hub: name == "anchor", Address: []string{"10.77.0.1", "10.77.0.2"}[i]},
|
||||
Assigned: []string{overlay.Name}, Capabilities: map[string]bool{"container-runtime": true, "wireguard": true, "systemd": true}})
|
||||
}
|
||||
return in
|
||||
}
|
||||
|
||||
func writeKey(t *testing.T, dir string) ed25519.PublicKey {
|
||||
t.Helper()
|
||||
pub, priv, _ := ed25519.GenerateKey(rand.Reader)
|
||||
der, err := x509.MarshalPKCS8PrivateKey(priv)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(dir, "operator.key"), pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: der}), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return pub
|
||||
}
|
||||
|
||||
// 1943/AC5: mesh-hand-compose, given the kept inputs, a node, a module and its fingerprint, prints that module's
|
||||
// composition for that node signed by the operator's key and its digest: its resources are the controller's for that
|
||||
// module on that node (compose.Compose), and it verifies against the operator's key. Without --write it writes nothing.
|
||||
func TestMeshComposeGivesTheControllersResourcesSignedByTheOperator(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
in := aMesh(t)
|
||||
raw, err := json.Marshal(in)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
input := filepath.Join(dir, "input.json")
|
||||
if err := os.WriteFile(input, raw, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
operator := writeKey(t, dir)
|
||||
fp := delivery.SourcePrefix2 + strings.Repeat("a", 64)
|
||||
|
||||
var out bytes.Buffer
|
||||
if err := run([]string{"-input", input, "-node", "laptop", "-module", overlay.Name, "-fp", fp,
|
||||
"-key", filepath.Join(dir, "operator.key")}, &out); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var printed struct {
|
||||
Key string `json:"key"`
|
||||
Bucket string `json:"bucket"`
|
||||
Digest string `json:"digest"`
|
||||
Composition json.RawMessage `json:"composition"`
|
||||
Written bool `json:"written"`
|
||||
}
|
||||
if err := json.Unmarshal(out.Bytes(), &printed); err != nil {
|
||||
t.Fatalf("%s: %v", out.String(), err)
|
||||
}
|
||||
c, err := delivery.DecodeComposed(printed.Composition)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if digest, err := delivery.VerifyComposed(c, []ed25519.PublicKey{operator}); err != nil || digest != printed.Digest {
|
||||
t.Errorf("the composition verifies %q, %v against the operator's key; printed %s", digest, err, printed.Digest)
|
||||
}
|
||||
wantKey, _ := delivery.ComposedKey("laptop", overlay.Name, fp)
|
||||
if printed.Key != wantKey || printed.Bucket != "composed-laptop" || printed.Written {
|
||||
t.Errorf("printed key %s in %s, written %v", printed.Key, printed.Bucket, printed.Written)
|
||||
}
|
||||
|
||||
d, _, err := compose.Compose(in, "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var want, got []string
|
||||
for _, r := range d.Resources {
|
||||
want = append(want, r["id"].(string))
|
||||
}
|
||||
for _, r := range c.Resources {
|
||||
var head struct{ ID string }
|
||||
_ = json.Unmarshal(r, &head)
|
||||
got = append(got, head.ID)
|
||||
}
|
||||
sort.Strings(want)
|
||||
sort.Strings(got)
|
||||
if len(got) == 0 || strings.Join(got, " ") != strings.Join(want, " ") {
|
||||
t.Errorf("mesh-hand-compose gives %v, the controller's composition %v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
// 1943/AC5: mesh-hand-compose refuses what it cannot compose, naming why: no key, a fingerprint not of the new form, a
|
||||
// module the node does not run, a node the mesh does not have.
|
||||
func TestMeshComposeRefuses(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
raw, _ := json.Marshal(aMesh(t))
|
||||
input := filepath.Join(dir, "input.json")
|
||||
_ = os.WriteFile(input, raw, 0o600)
|
||||
writeKey(t, dir)
|
||||
key := filepath.Join(dir, "operator.key")
|
||||
fp := delivery.SourcePrefix2 + strings.Repeat("a", 64)
|
||||
for _, c := range []struct {
|
||||
args []string
|
||||
why string
|
||||
}{
|
||||
{[]string{"-input", input, "-node", "laptop", "-module", overlay.Name, "-fp", fp}, "-key"},
|
||||
{[]string{"-input", input, "-node", "laptop", "-module", overlay.Name, "-fp", "src2:1", "-key", key}, "fingerprint"},
|
||||
{[]string{"-input", input, "-node", "laptop", "-module", "mesh-mail", "-fp", fp, "-key", key}, "does not run mesh-mail"},
|
||||
{[]string{"-input", input, "-node", "nowhere", "-module", overlay.Name, "-fp", fp, "-key", key}, "nowhere"},
|
||||
{[]string{"-input", input, "-node", "laptop", "-module", overlay.Name, "-fp", fp, "-key", key, "-write"}, "-creds"},
|
||||
} {
|
||||
var out bytes.Buffer
|
||||
if err := run(c.args, &out); err == nil || !strings.Contains(err.Error(), c.why) {
|
||||
t.Errorf("mesh-hand-compose %v = %v, want a refusal saying %q", c.args, err, c.why)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,111 @@
|
||||
// mesh-pack builds a Go bundle by hand with the recipe of the mesh-build-go module and packs it with mesh-sdk's own
|
||||
// buildstep.Pack, printing the digest, so a hand build has the form of a built artifact and, with the same Go version,
|
||||
// its digest (novox/hq ADR 0526; fallback record F1; issue 1499 for the Go version).
|
||||
//
|
||||
// mesh-pack -dir <module dir> -from <package> -binary <name> [-system arch] [-out <archive file>]
|
||||
// mesh-pack -pack <directory> [-out <archive file>] (a `files` archive: pack a tree as it is)
|
||||
//
|
||||
// The recipe is mesh-catalog modules/mesh-build-go cmd/mesh-build-go/build.go (recipeFlags, recipeLinkerFlags,
|
||||
// recipeSystemStamp, buildEnv): CGO_ENABLED=0, GOFLAGS "-trimpath -buildvcs=false -mod=mod" (-mod=vendor when the
|
||||
// directory vendors), GOTOOLCHAIN=local, GOPROXY=off, and -ldflags "-s -w -X main.builtFor=<system>".
|
||||
package main
|
||||
|
||||
import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
)
|
||||
|
||||
const (
|
||||
recipeFlags = "-trimpath -buildvcs=false"
|
||||
recipeLinkerFlags = "-s -w"
|
||||
recipeSystemStamp = "main.builtFor"
|
||||
)
|
||||
|
||||
// Env is the build module's environment for the compile of dir, added to the caller's own.
|
||||
func Env(dir string) []string {
|
||||
mod := "-mod=mod"
|
||||
if _, err := os.Stat(filepath.Join(dir, "vendor", "modules.txt")); err == nil {
|
||||
mod = "-mod=vendor"
|
||||
}
|
||||
return append(os.Environ(),
|
||||
"CGO_ENABLED=0",
|
||||
"GOFLAGS="+recipeFlags+" "+mod,
|
||||
"GOTOOLCHAIN=local",
|
||||
"GOPROXY=off",
|
||||
)
|
||||
}
|
||||
|
||||
// Args is the arguments of the go command of the recipe.
|
||||
func Args(dir, from, out, binary, system string) []string {
|
||||
linker := recipeLinkerFlags
|
||||
if s := strings.TrimSpace(system); s != "" {
|
||||
linker += " -X " + recipeSystemStamp + "=" + s
|
||||
}
|
||||
return []string{"-C", dir, "build", "-ldflags", linker, "-o", filepath.Join(out, binary), "./" + strings.Trim(from, "./")}
|
||||
}
|
||||
|
||||
// Build compiles the package into a fresh directory and answers it.
|
||||
func Build(dir, from, binary, system string) (string, error) {
|
||||
out, err := os.MkdirTemp("", "mesh-pack-")
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
cmd := exec.Command("go", Args(dir, from, out, binary, system)...)
|
||||
cmd.Env = Env(dir)
|
||||
cmd.Stdout, cmd.Stderr = os.Stderr, os.Stderr
|
||||
if err := cmd.Run(); err != nil {
|
||||
os.RemoveAll(out)
|
||||
return "", fmt.Errorf("go build of %s: %w", from, err)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func run(args []string) (string, error) {
|
||||
fs := flag.NewFlagSet("mesh-pack", flag.ContinueOnError)
|
||||
dir := fs.String("dir", ".", "the Go module's directory")
|
||||
from := fs.String("from", "", "the package to build, relative to -dir (the artifact's from)")
|
||||
binary := fs.String("binary", "", "the binary's name in the archive")
|
||||
system := fs.String("system", "", "the system stamped into main.builtFor (an artifact's system, e.g. arch)")
|
||||
pack := fs.String("pack", "", "pack this directory as it is (a files archive) instead of building")
|
||||
outFile := fs.String("out", "", "write the archive here")
|
||||
if err := fs.Parse(args); err != nil {
|
||||
return "", err
|
||||
}
|
||||
root := *pack
|
||||
if root == "" {
|
||||
if *from == "" || *binary == "" {
|
||||
return "", fmt.Errorf("-from and -binary are needed (or -pack <directory>)")
|
||||
}
|
||||
out, err := Build(*dir, *from, *binary, *system)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer os.RemoveAll(out)
|
||||
root = out
|
||||
}
|
||||
archive, digest, err := buildstep.Pack(root)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if *outFile != "" {
|
||||
if err := os.WriteFile(*outFile, archive, 0o644); err != nil {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
return digest, nil
|
||||
}
|
||||
|
||||
func main() {
|
||||
digest, err := run(os.Args[1:])
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, "mesh-pack:", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
fmt.Println(digest)
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
)
|
||||
|
||||
func fixture(t *testing.T) string {
|
||||
t.Helper()
|
||||
d := t.TempDir()
|
||||
os.MkdirAll(filepath.Join(d, "sub"), 0o755)
|
||||
os.WriteFile(filepath.Join(d, "a.txt"), []byte("alpha"), 0o644)
|
||||
os.WriteFile(filepath.Join(d, "sub", "run"), []byte("#!/bin/sh\n"), 0o755)
|
||||
return d
|
||||
}
|
||||
|
||||
func TestPackFixtureTwiceGivesBuildstepsDigest(t *testing.T) {
|
||||
d := fixture(t)
|
||||
_, want, err := buildstep.Pack(d)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for i := 0; i < 2; i++ {
|
||||
got, err := run([]string{"-pack", d, "-out", filepath.Join(t.TempDir(), "x.tgz")})
|
||||
if err != nil || got != want {
|
||||
t.Fatalf("run %d: %q, %v; want %q", i, got, err, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildTwiceGivesOneDigestAndRecipeArgs(t *testing.T) {
|
||||
m := t.TempDir()
|
||||
os.WriteFile(filepath.Join(m, "go.mod"), []byte("module x\n\ngo 1.21\n"), 0o644)
|
||||
os.MkdirAll(filepath.Join(m, "cmd", "p"), 0o755)
|
||||
os.WriteFile(filepath.Join(m, "cmd", "p", "main.go"), []byte("package main\n\nvar builtFor string\n\nfunc main() { println(builtFor) }\n"), 0o644)
|
||||
args := []string{"-dir", m, "-from", "cmd/p", "-binary", "p", "-system", "arch"}
|
||||
a, err := run(args)
|
||||
if err != nil {
|
||||
t.Skipf("no go toolchain that builds offline: %v", err)
|
||||
}
|
||||
b, err := run(args)
|
||||
if err != nil || a != b {
|
||||
t.Fatalf("two builds: %q, %q, %v", a, b, err)
|
||||
}
|
||||
got := Args("/d", "./cmd/p/", "/o", "p", "arch")
|
||||
if got[len(got)-1] != "./cmd/p" || got[4] != "-s -w -X main.builtFor=arch" {
|
||||
t.Fatalf("args %v", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,94 @@
|
||||
package compose
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sort"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/declaration"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// Compositions is one node's compositions (novox/hq ADR 0539 clauses 1 and 2): its declaration composed over in, as
|
||||
// the controller composes it, split into each module's composition at the fingerprint fps gives it and the node's own
|
||||
// (`<node>.node`), by key (delivery.ComposedKey). Each resource says its module, whatever the node reported it reads.
|
||||
// A module fps gives no fingerprint for is not composed and is named in skipped, so a caller composing one module
|
||||
// (mesh-hand-compose) gets that one. Each module's Data says whether its manifest declares data. The compositions are not
|
||||
// signed and not yet by digest: the writer does both (delivery.SignComposed, Composed.ByDigest).
|
||||
func Compositions(in Input, node string, fps map[string]string, mode Mode) (out map[string]delivery.Composed, skipped []string, err error) {
|
||||
reading := in
|
||||
reading.Nodes = append([]Node(nil), in.Nodes...)
|
||||
found := false
|
||||
for i := range reading.Nodes {
|
||||
if reading.Nodes[i].Name == node {
|
||||
reading.Nodes[i].ReadsModules = true
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
return nil, nil, fmt.Errorf("compose: the mesh has no node %q", node)
|
||||
}
|
||||
d, problems, err := ComposeIn(reading, node, mode)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
if len(problems) > 0 {
|
||||
return nil, nil, fmt.Errorf("compose: %s's declaration is refused by the node-engine's validator: %v", node, problems)
|
||||
}
|
||||
body, err := d.Body()
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
var envelope struct {
|
||||
Resources []json.RawMessage `json:"resources"`
|
||||
Adoption *declaration.Adoption `json:"adoption"`
|
||||
Filled []string `json:"filled_files"`
|
||||
Provisioner string `json:"provisioner_account"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &envelope); err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
var kept []json.RawMessage
|
||||
keptPaths := map[string]bool{}
|
||||
left := map[string]bool{}
|
||||
for _, raw := range envelope.Resources {
|
||||
var head struct {
|
||||
Module string `json:"module"`
|
||||
Type string `json:"type"`
|
||||
Path string `json:"path"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &head); err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
if _, ok := fps[head.Module]; head.Module != "" && !ok {
|
||||
left[head.Module] = true
|
||||
continue
|
||||
}
|
||||
if head.Type == string(declaration.TypeFile) {
|
||||
keptPaths[head.Path] = true
|
||||
}
|
||||
kept = append(kept, raw)
|
||||
}
|
||||
var filled []string
|
||||
for _, p := range envelope.Filled {
|
||||
if keptPaths[p] {
|
||||
filled = append(filled, p)
|
||||
}
|
||||
}
|
||||
out, err = delivery.SplitComposed(node, kept, fps, filled,
|
||||
delivery.NodeOwn{Adoption: envelope.Adoption, Provisioner: envelope.Provisioner})
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
for key, c := range out {
|
||||
if m, ok := in.Catalogue[c.Module]; ok && c.Module != "" {
|
||||
c.Data = len(m.DataItems()) > 0
|
||||
out[key] = c
|
||||
}
|
||||
}
|
||||
for m := range left {
|
||||
skipped = append(skipped, m)
|
||||
}
|
||||
sort.Strings(skipped)
|
||||
return out, skipped, nil
|
||||
}
|
||||
@@ -0,0 +1,88 @@
|
||||
package compose
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
func fpFor(c byte) string { return delivery.SourcePrefix2 + strings.Repeat(string(c), 64) }
|
||||
|
||||
// 1943/AC5: a node's compositions are its declaration as Compose gives it, split by the module each resource belongs
|
||||
// to: every resource once, each in its module's composition at the fingerprint given, the module's ownership said on
|
||||
// it; a module given no fingerprint is named as skipped and composed nowhere.
|
||||
func TestCompositionsAreTheDeclarationSplitByModule(t *testing.T) {
|
||||
in := aSmallMesh(t)
|
||||
d, _, err := Compose(in, "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var want []string
|
||||
for _, r := range d.Resources {
|
||||
want = append(want, r["type"].(string)+":"+r["id"].(string))
|
||||
}
|
||||
sort.Strings(want)
|
||||
|
||||
fps := map[string]string{"mesh-wireguard": fpFor('a'), "files": fpFor('b')}
|
||||
out, skipped, err := Compositions(in, "laptop", fps, Foreseeing)
|
||||
if err != nil || len(skipped) != 0 {
|
||||
t.Fatalf("Compositions = %v, skipped %v", err, skipped)
|
||||
}
|
||||
var got []string
|
||||
for key, c := range out {
|
||||
node, module, fp, isNode, err := delivery.ParseComposedKey(key)
|
||||
if err != nil || node != "laptop" || module != c.Module || fp != c.FP || (isNode != (module == "")) {
|
||||
t.Errorf("key %s holds %s at %s (%v)", key, c.Module, c.FP, err)
|
||||
}
|
||||
if !isNode && fp != fps[module] {
|
||||
t.Errorf("%s is composed at %s, given %s", module, fp, fps[module])
|
||||
}
|
||||
if _, err := delivery.EncodeComposed(c); err != nil {
|
||||
t.Errorf("%s: %v", key, err)
|
||||
}
|
||||
for _, raw := range c.Resources {
|
||||
var r struct{ ID, Type, Module string }
|
||||
_ = json.Unmarshal(raw, &r)
|
||||
if r.Module != module {
|
||||
t.Errorf("%s holds %s of %q", key, r.ID, r.Module)
|
||||
}
|
||||
got = append(got, r.Type+":"+r.ID)
|
||||
}
|
||||
}
|
||||
sort.Strings(got)
|
||||
if strings.Join(got, " ") != strings.Join(want, " ") {
|
||||
t.Errorf("the compositions hold %v, the declaration %v", got, want)
|
||||
}
|
||||
|
||||
one, skipped, err := Compositions(in, "laptop", map[string]string{"files": fpFor('b')}, Foreseeing)
|
||||
if err != nil || strings.Join(skipped, ",") != "mesh-wireguard" {
|
||||
t.Fatalf("one module: %v, skipped %v", err, skipped)
|
||||
}
|
||||
for _, c := range one {
|
||||
if c.Module == "mesh-wireguard" {
|
||||
t.Errorf("a module given no fingerprint was composed")
|
||||
}
|
||||
}
|
||||
if _, _, err := Compositions(in, "nowhere", fps, Foreseeing); err == nil {
|
||||
t.Errorf("a node the mesh does not have was composed")
|
||||
}
|
||||
}
|
||||
|
||||
// 1943/AC1: the upgrade policies and sources a composition's value may name are the catalogue's own words.
|
||||
func TestTheComposedPoliciesAreTheCatalogues(t *testing.T) {
|
||||
for _, p := range []string{catalogue.PolicyRoll, catalogue.PolicyTogether, catalogue.PolicyRecord} {
|
||||
for _, from := range []string{catalogue.FromPerson, catalogue.FromModule, catalogue.FromBus, catalogue.FromData, catalogue.FromDefault} {
|
||||
v := delivery.ComposedDigest{Digest: "sha256:" + strings.Repeat("a", 64), Judgement: delivery.JudgedNeither, Policy: p, PolicyFrom: from}
|
||||
if _, err := delivery.EncodeComposedDigest(v); err != nil {
|
||||
t.Errorf("%s from %s: %v", p, from, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
if delivery.PolicyRecord != catalogue.PolicyRecord {
|
||||
t.Errorf("delivery.PolicyRecord = %q, the catalogue's %q", delivery.PolicyRecord, catalogue.PolicyRecord)
|
||||
}
|
||||
}
|
||||
+20
-3
@@ -74,7 +74,10 @@ func (c *composer) overlayRange() string {
|
||||
}
|
||||
|
||||
// network builds the private network over the machines that resolved the module for it. A machine is on it because it
|
||||
// was given the module, or because it is joining with a key and an address (novox/hq ADR 0169).
|
||||
// was given the module, or because it is joining with a key and an address (novox/hq ADR 0169), or because its own set
|
||||
// could not be resolved and it holds a place (a key and an address): that place is its last good one on the network, and
|
||||
// it does not change because its modules failed, so every other machine keeps its path to it -- the hub above all -- and
|
||||
// only its own declaration is refused (issue 1833). A machine refused with no place is left off.
|
||||
func (c *composer) network(on map[string]bool, refused map[string]string) (*overlay.Generator, error) {
|
||||
tunnels := c.in.Tunnels
|
||||
carried := c.in.CarriedPeers
|
||||
@@ -85,9 +88,15 @@ func (c *composer) network(on map[string]bool, refused map[string]string) (*over
|
||||
var nodes []overlay.Node
|
||||
for _, node := range c.in.sorted() {
|
||||
p := node.Place
|
||||
if !on[node.Name] && !(isJoining[node.Name] && p.Key != "" && p.Address != "") {
|
||||
placed := p.Key != "" && p.Address != ""
|
||||
_, wasRefused := refused[node.Name]
|
||||
if !on[node.Name] && !((isJoining[node.Name] || wasRefused) && placed) {
|
||||
continue
|
||||
}
|
||||
if !on[node.Name] && wasRefused && !isJoining[node.Name] {
|
||||
c.say("%s could not be resolved and keeps its place on the private network at %s, so the others keep their path to it",
|
||||
node.Name, p.Address)
|
||||
}
|
||||
n := overlay.Node{
|
||||
Name: node.Name, Key: p.Key, Endpoint: p.Endpoint,
|
||||
Site: p.Site, Hub: p.Hub, Address: p.Address,
|
||||
@@ -233,7 +242,9 @@ func (c *composer) whereEveryoneIs() (map[string]string, error) {
|
||||
}
|
||||
|
||||
// onTheNetwork is every placed machine that resolves the private network: has an address AND runs what puts it there
|
||||
// (novox/hq issue 079). A machine that does not resolve is not counted, and said.
|
||||
// (novox/hq issue 079). A machine that does not resolve keeps its place when it holds one (a key and an address), by the
|
||||
// same rule as network(), so its name and its address stay where the tunnel to it does (issue 1833); one without is not
|
||||
// counted, and said.
|
||||
func (c *composer) onTheNetwork() []Node {
|
||||
var out []Node
|
||||
for _, n := range c.in.sorted() {
|
||||
@@ -247,6 +258,12 @@ func (c *composer) onTheNetwork() []Node {
|
||||
catalogue.Node{Name: n.Name, Site: n.Place.Site, Capabilities: n.Capabilities},
|
||||
catalogue.World{Unchecked: true, Holdings: c.in.Holdings})
|
||||
if err != nil {
|
||||
if n.Place.Key != "" {
|
||||
c.say("%s could not be resolved and keeps its place on the private network at %s, so the others keep their path to it",
|
||||
n.Name, n.Place.Address)
|
||||
out = append(out, n)
|
||||
continue
|
||||
}
|
||||
c.say("%s is not counted as on the network: it does not resolve: %v", n.Name, err)
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -159,9 +159,11 @@ func TestANodeNotOnThePrivateNetworkGetsNoAddressAndNoNetwork(t *testing.T) {
|
||||
}
|
||||
|
||||
// 1360/AC3: a node whose resolution is refused is said with today's words: the network missing its hub names the
|
||||
// machine that could not be resolved, which is almost always why.
|
||||
// machine that could not be resolved, which is almost always why. The hub here holds no complete place (no key): a refused
|
||||
// machine with one keeps it on the network (issue 1833), so only one without is missing from it.
|
||||
func TestARefusedNodeIsSaidWhenTheNetworkCannotBeComposed(t *testing.T) {
|
||||
in := aMesh(t)
|
||||
nodeOf(t, &in, "anchor").Place.Key = ""
|
||||
one := catalogue.Manifest{Module: "rival-one", Version: "1", Claims: []catalogue.Claim{{Name: "the-seat", Scope: catalogue.ScopeNode}}}
|
||||
two := catalogue.Manifest{Module: "rival-two", Version: "1", Claims: []catalogue.Claim{{Name: "the-seat", Scope: catalogue.ScopeNode}}}
|
||||
in.Catalogue["rival-one"], in.Catalogue["rival-two"] = one, two
|
||||
@@ -210,9 +212,10 @@ func TestEachFieldOfTheInputReachesTheGenerator(t *testing.T) {
|
||||
if len(base) != 2 {
|
||||
t.Fatalf("the base mesh has %v", base)
|
||||
}
|
||||
t.Run("Catalogue: without the module nobody is on the network", func(t *testing.T) {
|
||||
t.Run("Catalogue: without the module offering it nobody is on the network", func(t *testing.T) {
|
||||
in := aMesh(t)
|
||||
in.Catalogue = map[string]catalogue.Manifest{}
|
||||
// The module stays known, so every machine still resolves: a machine that does not resolve keeps its place (issue 1833).
|
||||
in.Catalogue = map[string]catalogue.Manifest{overlay.Name: {Module: overlay.Name, Version: "1"}}
|
||||
if got := addressesOf(in); len(got) != 0 {
|
||||
t.Errorf("got %v", got)
|
||||
}
|
||||
@@ -290,6 +293,8 @@ func TestEachFieldOfTheInputReachesTheGenerator(t *testing.T) {
|
||||
}
|
||||
laptop := nodeOf(t, &in, "laptop")
|
||||
laptop.Assigned = append(laptop.Assigned, "holder-one", "holder-two")
|
||||
// Without a key: a machine that does not resolve and holds a complete place keeps it on the network (issue 1833).
|
||||
laptop.Place.Key = ""
|
||||
if got := addressesOf(in); got["laptop"] != "" {
|
||||
t.Fatalf("a machine whose set claims one seat twice is on the network: %v", got)
|
||||
}
|
||||
|
||||
@@ -387,9 +387,11 @@ func TestAnotherNodesKeptBuildDoesNotChangeTheRestOfTheMeshOrWhoIsOnTheNetwork(t
|
||||
|
||||
func overlayName() string { return overlay.Name }
|
||||
|
||||
// Round 1, finding 6: the nodes the composition did not count as on the network are returned, said, not lost.
|
||||
// Round 1, finding 6: the nodes the composition did not count as on the network are returned, said, not lost. laptop holds no
|
||||
// complete place (no key): one that does keeps it on the network (issue 1833) and is counted.
|
||||
func TestANodeNotCountedAsOnTheNetworkIsSaidInTheResult(t *testing.T) {
|
||||
in := aMesh(t)
|
||||
nodeOf(t, &in, "laptop").Place.Key = ""
|
||||
in.Catalogue["rival-one"] = catalogue.Manifest{Module: "rival-one", Version: "1", Claims: []catalogue.Claim{{Name: "the-seat", Scope: catalogue.ScopeNode}}}
|
||||
in.Catalogue["rival-two"] = catalogue.Manifest{Module: "rival-two", Version: "1", Claims: []catalogue.Claim{{Name: "the-seat", Scope: catalogue.ScopeNode}}}
|
||||
laptop := nodeOf(t, &in, "laptop")
|
||||
|
||||
@@ -0,0 +1,167 @@
|
||||
package compose
|
||||
|
||||
import (
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/overlay"
|
||||
)
|
||||
|
||||
// fourOnTheNetwork is aMesh and two more machines on the private network at the site `there`: `ace`, dialled at
|
||||
// ace.example:51820, at 10.77.0.3, and `box`, behind no endpoint, at 10.77.0.4. A module `office` requires
|
||||
// `document-server`, which nothing in the catalogue provides: the shape of issue 1833 (nextcloud on the hub wanting a
|
||||
// document server nobody offered).
|
||||
func fourOnTheNetwork(t *testing.T) Input {
|
||||
t.Helper()
|
||||
in := aMesh(t)
|
||||
in.Catalogue["office"] = catalogue.Manifest{Module: "office", Version: "1", Requires: []string{"document-server"}}
|
||||
caps := map[string]bool{"container-runtime": true, "wireguard": true, "systemd": true}
|
||||
in.Nodes = append(in.Nodes,
|
||||
Node{Name: "ace", Place: Place{Key: keyOf(t, "ace"), Endpoint: "ace.example:51820", Site: "there", Address: "10.77.0.3"},
|
||||
Assigned: []string{overlay.Name}, Capabilities: caps},
|
||||
Node{Name: "box", Place: Place{Key: keyOf(t, "box"), Site: "there", Address: "10.77.0.4"},
|
||||
Assigned: []string{overlay.Name}, Capabilities: caps})
|
||||
return in
|
||||
}
|
||||
|
||||
// unresolvableBy makes one machine's own set fail to resolve: it is also given `office`, whose requirement nothing provides.
|
||||
func unresolvableBy(t *testing.T, in *Input, name string) {
|
||||
t.Helper()
|
||||
n := nodeOf(t, in, name)
|
||||
n.Assigned = append(n.Assigned, "office")
|
||||
}
|
||||
|
||||
// networkOf is the generator of an Input's private network, failing the test when it is not composed.
|
||||
func networkOf(t *testing.T, in Input) *overlay.Generator {
|
||||
t.Helper()
|
||||
gens, err := Generators(in)
|
||||
if err != nil {
|
||||
t.Fatalf("the private network was not composed: %v", err)
|
||||
}
|
||||
return gens[overlay.Name].(*overlay.Generator)
|
||||
}
|
||||
|
||||
func placesOf(g *overlay.Generator) map[string]overlay.Node {
|
||||
out := map[string]overlay.Node{}
|
||||
for _, n := range g.Nodes() {
|
||||
out[n.Name] = n
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// 1833/AC1: four machines, the hub's own set does not resolve; the other three compose their declarations, and the hub alone is
|
||||
// refused, with an error that is its own set failing and names what could not be resolved.
|
||||
func TestOneMachineThatDoesNotResolveIsRefusedAlone(t *testing.T) {
|
||||
in := fourOnTheNetwork(t)
|
||||
unresolvableBy(t, &in, "anchor")
|
||||
for _, node := range []string{"laptop", "ace", "box"} {
|
||||
d, problems, err := Compose(in, node)
|
||||
if err != nil || len(problems) != 0 {
|
||||
t.Errorf("%s does not compose because another machine does not resolve: %v %v", node, err, problems)
|
||||
continue
|
||||
}
|
||||
if !has(resourceIDs(d), "file:mesh-wireguard.overlay-config") {
|
||||
t.Errorf("%s is given no private network: %v", node, resourceIDs(d))
|
||||
}
|
||||
}
|
||||
_, _, err := Compose(in, "anchor")
|
||||
if err == nil || !IsUnresolvable(err) {
|
||||
t.Fatalf("the machine that does not resolve is not refused as its own set failing: %v", err)
|
||||
}
|
||||
if !strings.Contains(err.Error(), `nothing provides "document-server", wanted by office`) {
|
||||
t.Errorf("the refusal does not name what could not be resolved: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// 1833/AC2: the private network is composed with every machine on it: the hub that does not resolve keeps its place (key,
|
||||
// address, endpoint, site, hub role), so every other machine keeps its path through it.
|
||||
func TestAMachineThatDoesNotResolveKeepsItsPlaceOnTheNetwork(t *testing.T) {
|
||||
in := fourOnTheNetwork(t)
|
||||
unresolvableBy(t, &in, "anchor")
|
||||
got := placesOf(networkOf(t, in))
|
||||
if len(got) != 4 {
|
||||
t.Fatalf("the network is over %v, want all four machines", got)
|
||||
}
|
||||
hub := got["anchor"]
|
||||
if !hub.Hub || hub.Key != keyOf(t, "anchor") || hub.Address != "10.77.0.1" || hub.Endpoint != "anchor.example:51820" || hub.Site != "here" {
|
||||
t.Errorf("the hub did not keep its place: %+v", hub)
|
||||
}
|
||||
nodes, graph, err := Graph(in)
|
||||
if err != nil || len(nodes) != 4 {
|
||||
t.Fatalf("the graph is not shown with every machine: %v %v", nodes, err)
|
||||
}
|
||||
for _, node := range []string{"ace", "box"} {
|
||||
through := false
|
||||
for _, p := range graph[node] {
|
||||
through = through || (p.Name == "anchor" && p.Endpoint == "anchor.example:51820")
|
||||
}
|
||||
if !through {
|
||||
t.Errorf("%s has no path through the hub: %+v", node, graph[node])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1833/AC3: a machine that does not resolve and was never placed (a new machine: no key, no address) is left off the network,
|
||||
// which is composed from the rest -- here with the hub that does not resolve on its last place -- and that machine alone is refused.
|
||||
func TestANewMachineThatDoesNotResolveIsLeftOffAndRefusedAlone(t *testing.T) {
|
||||
in := fourOnTheNetwork(t)
|
||||
unresolvableBy(t, &in, "anchor")
|
||||
in.Nodes = append(in.Nodes, Node{Name: "fresh", Place: Place{Site: "there"},
|
||||
Assigned: []string{overlay.Name, "office"}, Capabilities: map[string]bool{"wireguard": true, "systemd": true}})
|
||||
got := placesOf(networkOf(t, in))
|
||||
if _, on := got["fresh"]; on || len(got) != 4 {
|
||||
t.Fatalf("the network is over %v, want the four placed machines without fresh", got)
|
||||
}
|
||||
if _, _, err := Compose(in, "fresh"); err == nil || !IsUnresolvable(err) {
|
||||
t.Errorf("the new machine is not refused as its own set failing: %v", err)
|
||||
}
|
||||
if _, problems, err := Compose(in, "box"); err != nil || len(problems) != 0 {
|
||||
t.Errorf("another machine does not compose: %v %v", err, problems)
|
||||
}
|
||||
}
|
||||
|
||||
// 1833/AC4: a network that truly cannot be composed from what remains still fails: the hub does not resolve and has no
|
||||
// complete place (no key), so nothing is the hub; the refusal is today's, naming the machine that could not be resolved.
|
||||
func TestANetworkThatCannotBeComposedFromWhatRemainsStillFails(t *testing.T) {
|
||||
in := fourOnTheNetwork(t)
|
||||
unresolvableBy(t, &in, "anchor")
|
||||
nodeOf(t, &in, "anchor").Place.Key = ""
|
||||
_, err := Generators(in)
|
||||
if err == nil {
|
||||
t.Fatal("a network without a hub was composed")
|
||||
}
|
||||
for _, want := range []string{overlay.ErrNoHub.Error(), "and 1 node(s) could not be resolved at all, which is probably why:", " anchor: "} {
|
||||
if !strings.Contains(err.Error(), want) {
|
||||
t.Errorf("the refusal does not say %q:\n%v", want, err)
|
||||
}
|
||||
}
|
||||
if _, _, err := Compose(in, "laptop"); err == nil {
|
||||
t.Error("a declaration was composed over a network that cannot be")
|
||||
}
|
||||
}
|
||||
|
||||
// 1833/AC2, a failure the composition's unchecked pass refuses too (two modules claiming one seat on the hub, the 1360/AC3
|
||||
// shape): the hub keeps its place in what every other machine is sent besides the tunnel -- its internal name and its address
|
||||
// among the sources "from the mesh" -- and another machine still composes.
|
||||
func TestAMachineThatDoesNotResolveKeepsItsNameAndItsAddressInTheMesh(t *testing.T) {
|
||||
in := fourOnTheNetwork(t)
|
||||
in.Catalogue["rival-one"] = catalogue.Manifest{Module: "rival-one", Version: "1", Claims: []catalogue.Claim{{Name: "the-seat", Scope: catalogue.ScopeNode}}}
|
||||
in.Catalogue["rival-two"] = catalogue.Manifest{Module: "rival-two", Version: "1", Claims: []catalogue.Claim{{Name: "the-seat", Scope: catalogue.ScopeNode}}}
|
||||
hub := nodeOf(t, &in, "anchor")
|
||||
hub.Assigned = append(hub.Assigned, "rival-one", "rival-two")
|
||||
if _, err := Resolve(in, "anchor"); !IsUnresolvable(err) {
|
||||
t.Fatalf("the hub is meant not to resolve: %v", err)
|
||||
}
|
||||
c := newComposer(in)
|
||||
if got := c.namesInTheMesh()["anchor."+c.suffix()]; got != "10.77.0.1" {
|
||||
t.Errorf("the hub's name is not in the mesh's names: %v", c.namesInTheMesh())
|
||||
}
|
||||
if got := c.onThePrivateNetwork(); !slices.Contains(got, "10.77.0.1") || len(got) != 4 {
|
||||
t.Errorf("the sources from the mesh are %v, want all four machines with the hub", got)
|
||||
}
|
||||
if _, problems, err := Compose(in, "ace"); err != nil || len(problems) != 0 {
|
||||
t.Errorf("another machine does not compose: %v %v", err, problems)
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user