From b6ed18197b316d78a2d0fc5cd692c97d9eb5a150 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 22 Oct 2026 18:34:44 +0200 Subject: [PATCH] Vendor the R4e mesh-sdk member's head with the states announcement (issue 1790; re-pinned to the final head in E) --- go.mod | 2 +- go.sum | 2 + internal/broker/states_agreement_test.go | 67 ------- testdata/beside/CAPTURED | 2 +- .../novox/mesh-sdk/go/announce/announce.go | 126 +++++++++++++ .../novox/mesh-sdk/go/buildstep/blob.go | 170 +++++++++++++++++ .../novox/mesh-sdk/go/buildstep/isolate.go | 175 ++++++++++++++++++ .../novox/mesh-sdk/go/buildstep/request.go | 115 ++++++++++++ .../novox/mesh-sdk/go/buildstep/trees.go | 58 +++++- .../novox/mesh-sdk/go/catalogue/build.go | 27 +++ .../mesh-sdk/go/catalogue/delivery_seat.go | 3 +- .../go/catalogue/predefined_inputs.go | 41 ++++ .../mesh-sdk/go/catalogue/seats_declared.go | 7 - .../novox/mesh-sdk/go/catalogue/state.go | 43 ----- .../novox/mesh-sdk/go/catalogue/verbs.go | 78 +++----- .../novox/mesh-sdk/go/delivery/builder.go | 117 +++++++++++- .../novox/mesh-sdk/go/delivery/gate.go | 4 +- .../novox/mesh-sdk/go/delivery/namedtree.go | 47 +++++ .../novox/mesh-sdk/go/delivery/preview.go | 8 +- .../novox/mesh-sdk/go/delivery/source.go | 97 ++++++++-- .../git.novox.be/novox/mesh-sdk/go/keeps.go | 80 ++++++++ .../novox/mesh-sdk/go/merge-check.sh | 9 +- .../novox/mesh-sdk/go/process/conn.go | 6 + .../novox/mesh-sdk/go/process/keeps.go | 71 +++++++ .../novox/mesh-sdk/go/process/state.go | 3 + .../git.novox.be/novox/mesh-sdk/go/state.go | 3 + .../git.novox.be/novox/mesh-sdk/go/stdio.go | 3 + vendor/modules.txt | 3 +- 28 files changed, 1166 insertions(+), 201 deletions(-) delete mode 100644 internal/broker/states_agreement_test.go create mode 100644 vendor/git.novox.be/novox/mesh-sdk/go/announce/announce.go create mode 100644 vendor/git.novox.be/novox/mesh-sdk/go/buildstep/blob.go create mode 100644 vendor/git.novox.be/novox/mesh-sdk/go/buildstep/isolate.go create mode 100644 vendor/git.novox.be/novox/mesh-sdk/go/buildstep/request.go create mode 100644 vendor/git.novox.be/novox/mesh-sdk/go/catalogue/predefined_inputs.go create mode 100644 vendor/git.novox.be/novox/mesh-sdk/go/delivery/namedtree.go create mode 100644 vendor/git.novox.be/novox/mesh-sdk/go/keeps.go create mode 100644 vendor/git.novox.be/novox/mesh-sdk/go/process/keeps.go diff --git a/go.mod b/go.mod index fde4f61c..23a119b3 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/novox/mesh-controller go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022152521-dfc2c858d94b + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022161855-e76a5a31a07b github.com/jackc/pgx/v5 v5.10.0 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 diff --git a/go.sum b/go.sum index c8522d52..de414370 100644 --- a/go.sum +++ b/go.sum @@ -4,6 +4,8 @@ git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022075101-c61212eba77c h1:YdSbkI42 git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022075101-c61212eba77c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022152521-dfc2c858d94b h1:cfJ2LK/etUiV1eL58pnaEjmSi34+EVmO4ud9J7pXKOs= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022152521-dfc2c858d94b/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022161855-e76a5a31a07b h1:OjlhfiGr+wKLWxafV+BRjZx3TbBtwRYcm9mgoam/jv8= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022161855-e76a5a31a07b/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= diff --git a/internal/broker/states_agreement_test.go b/internal/broker/states_agreement_test.go deleted file mode 100644 index 87ad8f70..00000000 --- a/internal/broker/states_agreement_test.go +++ /dev/null @@ -1,67 +0,0 @@ -package broker_test - -import ( - "slices" - "testing" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/broker" - "github.com/novox/mesh-controller/internal/conditions" - "github.com/novox/mesh-controller/internal/link" -) - -// The facts the control plane states are named twice — in the grant that permits them and in the code -// that states them — because `link` imports `broker` and the dependency cannot go the other way. So a -// test keeps them agreeing: a subject the grant omits is refused at the moment the mesh has something -// to say, and one the grant adds that nothing states is authority nobody uses. -// -// An external test package, because it may import both while neither imports the other. -func TestTheFactsTheGrantPermitsAreTheFactsTheMeshStates(t *testing.T) { - if broker.ControllerSeat != link.MeshControllerSeat { - t.Fatalf("the grant is written for the %q seat and the mesh states its facts under %q", - broker.ControllerSeat, link.MeshControllerSeat) - } - // And what is wrong, as it changes, and the self-check's heartbeat (novox/hq to-be 45 §2, §4). - states := append([]string{link.KeyApplied, link.KeyRefused, link.KeyBuiltBefore}, conditions.Events...) - states = append(states, conditions.HeartbeatEvent) - // And a value given by hand, replaced after its module's first good start (novox/hq ADR 0228). - states = append(states, link.KeySecretReplaced) - // And every act a healer takes (novox/hq to-be 45 §7). - states = append(states, link.KeyHealerActed) - // And a build put back after its gate failed (novox/hq ADR 0236). - states = append(states, link.KeyRolledBack) - // And a pull request's merge check, judged (novox/hq to-be 45 §9). - states = append(states, link.KeyChecked) - // And a walk kept in a new state, for the delivery it walks (novox/hq ADR 0239). - states = append(states, link.KeyPlanMoved) - // And a tier sent to a machine, once the send is kept (novox/hq ADR 0406 §5, issue 974). - states = append(states, link.KeySent) - // And a key of a record mirror written or removed (novox/hq ADR 0289, design 54 §3; design 57 F3). - states = append(states, link.KeyAssignmentChanged, link.KeyDeclaredConfigChanged, link.KeyPredefinedChanged) - for _, event := range states { - if !slices.Contains(broker.ControllerStates, event) { - t.Errorf("the mesh states %q and its account may not publish it", event) - } - } - if len(broker.ControllerStates) != len(states) { - t.Errorf("the grant permits %v, which is more than the mesh states", broker.ControllerStates) - } - // The record mirrors a module may read are the ones the bus holds (design 54 §5, design 57 F3). - if !slices.Equal(catalogue.ControllerMirrors, broker.ControllerMirrors()) { - t.Errorf("the catalogue lets a module read the controller's mirrors %v and the bus holds %v", - catalogue.ControllerMirrors, broker.ControllerMirrors()) - } - // **And the seat says it.** A seat carries the protocol of its role (novox/hq ADR 0129), so the - // facts the control plane states are the seat's `emits` — which is what lets anything else declare - // that it consumes them, and what the subject-agreement check reads to know they have an owner. - var declared []string - for _, seat := range catalogue.SeatsWithAProtocol() { - if seat.Name == broker.ControllerSeat { - declared = seat.Emits - } - } - if !slices.Equal(declared, broker.ControllerStates) { - t.Errorf("the %s seat emits %v and the grant permits %v", broker.ControllerSeat, - declared, broker.ControllerStates) - } -} diff --git a/testdata/beside/CAPTURED b/testdata/beside/CAPTURED index d14a5a7b..e8914254 100644 --- a/testdata/beside/CAPTURED +++ b/testdata/beside/CAPTURED @@ -9,7 +9,7 @@ a second place to clean besides the catalogue itself. mesh-catalog e0bc919d4945bb5f13f971f74643a0b714cf45da modules/*/module.json, modules/nats/Dockerfile mesh-host 9d9e3a46c4f5943cfffb413d962ce48b75e2fd39 examples/foundation-first-node.lock -mesh-sdk dfc2c858d94ba877caa96b5e0b84200590c1391f conformance/events/module-event.json +mesh-sdk e76a5a31a07bedb0cfa1e1244e54b391a33f9b75 conformance/events/module-event.json To move them, from this repository's root, with the two repositories checked out beside it: diff --git a/vendor/git.novox.be/novox/mesh-sdk/go/announce/announce.go b/vendor/git.novox.be/novox/mesh-sdk/go/announce/announce.go new file mode 100644 index 00000000..37dfd21d --- /dev/null +++ b/vendor/git.novox.be/novox/mesh-sdk/go/announce/announce.go @@ -0,0 +1,126 @@ +// Package announce is how a module says on the bus which states it keeps and which of them others may read (novox/hq +// ADR 0495 clauses 2 and 3; issue 1790): the subject it says it on, the value it says, and where the controller keeps +// the last one said by each module. What a module announced is the truth of what it keeps; a reader is granted a read +// from it, never from a list or from the owner's manifest. +// +// A module announces from its code (process.Conn.Keeps, stdio.Keeps, the TypeScript SDK's keeps), never from its +// manifest, which only the core reads (clause 1). A module that keeps no state on the bus announces nothing. +package announce + +import ( + "encoding/json" + "fmt" + "regexp" + "sort" +) + +// Version is the version of the value this package writes. +const Version = 1 + +// Event is the event type of an announcement, under the module's own namespace. +const Event = "states" + +// Subject is where module announces what it keeps: its own event subject, mesh.mod..event.states, which its +// account may already publish to and the events stream keeps. +func Subject(module string) string { return "mesh.mod." + module + ".event." + Event } + +// SubjectAll is every module's announcement, what the controller consumes. +const SubjectAll = "mesh.mod.*.event." + Event + +// KeptBucket is the controller's bucket of the last announcement of each module, by Key. +const KeptBucket = "mesh-controller_states-announced" + +var name = regexp.MustCompile(`^[a-z0-9][a-z0-9-]*$`) + +// Key is module's key in KeptBucket: the module itself, which must be one plain name, since a module's name is part of +// its buckets' names (catalogue.StateProblems). +func Key(module string) (string, error) { + if !name.MatchString(module) { + return "", fmt.Errorf("announce: %q is not a module that keeps state: lower-case letters, digits and hyphens", module) + } + return module, nil +} + +// State is one state a module keeps, by its local name, and whether others may read it. Others never write it: a +// state another module writes is the owner's own code's business. +type State struct { + Name string `json:"name"` + Readable bool `json:"readable,omitempty"` +} + +// States is an announcement: the module, the machine whose instance said it, and every state the module keeps, sorted +// by name. An empty list says the module keeps nothing. +type States struct { + V int `json:"v"` + Module string `json:"module"` + Node string `json:"node,omitempty"` + States []State `json:"states"` +} + +// Keeps says whether the module announced keeping the state. +func (s States) Keeps(state string) bool { + for _, k := range s.States { + if k.Name == state { + return true + } + } + return false +} + +// Readable says whether the module announced the state readable by others. +func (s States) Readable(state string) bool { + for _, k := range s.States { + if k.Name == state { + return k.Readable + } + } + return false +} + +func (s *States) check() error { + if s.Module == "" { + return fmt.Errorf("announce: an announcement names no module") + } + if _, err := Key(s.Module); err != nil { + return err + } + seen := map[string]bool{} + for _, k := range s.States { + if !name.MatchString(k.Name) { + return fmt.Errorf("announce: %s announces the state %q, and a state is named locally: lower-case letters, digits and hyphens", s.Module, k.Name) + } + if seen[k.Name] { + return fmt.Errorf("announce: %s announces the state %q twice", s.Module, k.Name) + } + seen[k.Name] = true + } + return nil +} + +// Encode is s as JSON at Version, its states sorted; it refuses a module or a state no bucket could be named by and a +// state announced twice, naming it. +func Encode(s States) ([]byte, error) { + if err := s.check(); err != nil { + return nil, err + } + s.V = Version + s.States = append([]State{}, s.States...) + sort.Slice(s.States, func(i, j int) bool { return s.States[i].Name < s.States[j].Name }) + return json.Marshal(s) +} + +// Decode reads an announcement; it refuses another version and what Encode refuses. +func Decode(data []byte) (States, error) { + var s States + if err := json.Unmarshal(data, &s); err != nil { + return States{}, fmt.Errorf("announce: the announcement is not JSON: %w", err) + } + if s.V != Version { + return States{}, fmt.Errorf("announce: the announcement is version %d, and this reads version %d", s.V, Version) + } + if err := s.check(); err != nil { + return States{}, err + } + sort.Slice(s.States, func(i, j int) bool { return s.States[i].Name < s.States[j].Name }) + return s, nil +} diff --git a/vendor/git.novox.be/novox/mesh-sdk/go/buildstep/blob.go b/vendor/git.novox.be/novox/mesh-sdk/go/buildstep/blob.go new file mode 100644 index 00000000..127c6ace --- /dev/null +++ b/vendor/git.novox.be/novox/mesh-sdk/go/buildstep/blob.go @@ -0,0 +1,170 @@ +package buildstep + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "syscall" + "time" +) + +// RegistryWait is how long a push waits for an artifact store that refuses connections, longer than its nightly +// collection holds it still (about a minute and a half, issue 457); RegistryFirstPause is the first pause, doubling +// to RegistryMostPause. Variables so a test shortens them. +var ( + RegistryWait = 5 * time.Minute + RegistryFirstPause = time.Second + RegistryMostPause = 10 * time.Second +) + +// BlobReference is where a pushed bundle or archive is, as a done key names it and the controller delivers from +// (novox/hq ADR 0498 clause 1): artifact-store:////blobs/, the store's address left out +// (issue 102). +func BlobReference(module, artifact, digest string) string { + return "artifact-store://" + module + "/" + artifact + "/blobs/" + digest +} + +// PushBlob puts body into the artifact store at registry (host:port) as a blob of repository (/) by +// its digest (`sha256:` and hex), from the build module's own process, after the build step ended (novox/hq ADR 0498 +// clauses 1 and 2): a blob the store already holds is done without an upload; otherwise it is uploaded (POST +// blobs/uploads/, then PUT with the digest). A store refusing connections is waited out up to RegistryWait; any other +// refusal fails, naming the store's status and its own words, for the caller's `not stored:` error. Bytes that are not +// their digest are refused before anything is sent. +func PushBlob(ctx context.Context, client *http.Client, registry, repository, digest string, body []byte) error { + sum := sha256.Sum256(body) + if got := "sha256:" + hex.EncodeToString(sum[:]); got != digest { + return fmt.Errorf("the blob is %s, not the digest %s it was to be pushed as", got, digest) + } + if client == nil { + client = http.DefaultClient + } + base := "http://" + registry + "/v2/" + repository + "/blobs/" + var present bool + if err := waitForStore(ctx, registry, func() error { + var err error + present, err = blobPresent(ctx, client, base+digest) + return err + }); err != nil { + return err + } + if present { + return nil + } + var location string + if err := waitForStore(ctx, registry, func() error { + var err error + location, err = startUpload(ctx, client, base+"uploads/") + return err + }); err != nil { + return err + } + target, err := url.Parse(base) + if err != nil { + return err + } + loc, err := target.Parse(location) + if err != nil { + return fmt.Errorf("the store answered the upload's location %q, which is not a URL: %w", location, err) + } + q := loc.Query() + q.Set("digest", digest) + loc.RawQuery = q.Encode() + req, err := http.NewRequestWithContext(ctx, http.MethodPut, loc.String(), bytes.NewReader(body)) + if err != nil { + return err + } + req.Header.Set("Content-Type", "application/octet-stream") + resp, err := client.Do(req) + if err != nil { + return fmt.Errorf("putting %s of %s: %w", digest, repository, err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusCreated { + return storeSaid(resp, "putting "+digest+" of "+repository) + } + return nil +} + +func blobPresent(ctx context.Context, client *http.Client, at string) (bool, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodHead, at, nil) + if err != nil { + return false, err + } + resp, err := client.Do(req) + if err != nil { + return false, err + } + resp.Body.Close() + switch resp.StatusCode { + case http.StatusOK: + return true, nil + case http.StatusNotFound: + return false, nil + } + return false, fmt.Errorf("asking the store for %s: it answered %s", at, resp.Status) +} + +func startUpload(ctx context.Context, client *http.Client, at string) (string, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodPost, at, nil) + if err != nil { + return "", err + } + resp, err := client.Do(req) + if err != nil { + return "", err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusAccepted { + return "", storeSaid(resp, "starting an upload at "+at) + } + location := resp.Header.Get("Location") + if location == "" { + return "", fmt.Errorf("starting an upload at %s: the store named no location", at) + } + return location, nil +} + +// storeSaid is the store's refusal: its status and the first of its own words. +func storeSaid(resp *http.Response, what string) error { + words, _ := io.ReadAll(io.LimitReader(resp.Body, 2048)) + return fmt.Errorf("%s: the store answered %s: %s", what, resp.Status, strings.TrimSpace(string(words))) +} + +// refused is whether an error is a connection refused. +func refused(err error) bool { + return err != nil && (errors.Is(err, syscall.ECONNREFUSED) || strings.Contains(err.Error(), "connection refused")) +} + +// waitForStore runs try, and while it fails because the store refuses connections, tries again with a growing pause +// until RegistryWait has passed. A refusal is the one error waited for: nothing was sent, so trying again does nothing +// twice. +func waitForStore(ctx context.Context, registry string, try func() error) error { + err := try() + if !refused(err) { + return err + } + started, pause := time.Now(), RegistryFirstPause + for { + left := RegistryWait - time.Since(started) + if left <= 0 { + return fmt.Errorf("the artifact store at %s refused every connection for %s, longer than its nightly collection holds it still, so it is down, not paused: %w", + registry, time.Since(started).Round(time.Millisecond), err) + } + select { + case <-ctx.Done(): + return fmt.Errorf("stopped while waiting for the artifact store at %s: %w (last: %v)", registry, ctx.Err(), err) + case <-time.After(min(pause, left)): + } + pause = min(pause*2, RegistryMostPause) + if err = try(); !refused(err) { + return err + } + } +} diff --git a/vendor/git.novox.be/novox/mesh-sdk/go/buildstep/isolate.go b/vendor/git.novox.be/novox/mesh-sdk/go/buildstep/isolate.go new file mode 100644 index 00000000..2efa739e --- /dev/null +++ b/vendor/git.novox.be/novox/mesh-sdk/go/buildstep/isolate.go @@ -0,0 +1,175 @@ +package buildstep + +import ( + "context" + "fmt" + "os" + "path" + "path/filepath" + "strings" + + "git.novox.be/novox/mesh-sdk/go/forgekey" +) + +// Bubblewrap is the program a run step runs in (novox/hq ADR 0448): a build module declares its package, which needs +// unprivileged user namespaces (issue 1308). +const Bubblewrap = "bwrap" + +// Isolation is how a run step's command is shut off (novox/hq ADR 0448; issue 1309, the lead's decision L2 of R4e: one +// runner in the SDK for every build module). The fetch step holds the forge key and the network and runs none of the +// change's code; the run step runs the change's build and tests in bubblewrap with: +// +// - no network: its own network namespace, with only a loopback, so a test may start a bus or a store of its own but +// resolves no name and reaches no socket of the machine; +// - none of the account: not its home, not the forge key, not a bus login, not the runtime directory (/run), since +// none of them is bound; HOME is a directory in the step's own /tmp unless Env names one; +// - an environment of exactly Env and the runner's env: nothing of the build module's own passes; +// - the system's programs and libraries read-only, and of /etc only what a program needs to run (users, libraries, +// certificates, time zone), with a hosts file naming only the loopback; +// - the Workspace and Writable directories read-write, the ReadOnly ones (the toolchain, the modules the fetch step +// downloaded) read-only; +// - the shared warm Caches read-only under a throwaway writable layer (S18), or read-write for a trunk build alone +// (WriteCaches), so a candidate's code never changes what a later build reads. +type Isolation struct { + Workspace string + Writable []string + ReadOnly []string + Caches []string + WriteCaches bool + Env []string + // Hosts is the file bound as /etc/hosts; Isolated writes one naming only the loopback. + Hosts string +} + +// etcNeeded is what of /etc a program in the step may read: who the users are, where the libraries are, the +// certificates, the time zone. Not resolv.conf, not hosts: the step resolves no name. +var etcNeeded = []string{"/etc/passwd", "/etc/group", "/etc/nsswitch.conf", "/etc/ld.so.cache", "/etc/ld.so.conf", + "/etc/ld.so.conf.d", "/etc/ssl", "/etc/ca-certificates", "/etc/pki", "/etc/localtime", "/etc/os-release", "/etc/alternatives"} + +// systemArgs binds the system's programs and libraries read-only: /usr, and /bin, /sbin, /lib, /lib64 as the symlinks +// they are on a merged-usr system, or bound when they are directories. A variable so a unit test fixes it. +var systemArgs = func() []string { + args := []string{"--ro-bind", "/usr", "/usr"} + for _, d := range []string{"/bin", "/sbin", "/lib", "/lib64"} { + info, err := os.Lstat(d) + switch { + case err != nil: + case info.Mode()&os.ModeSymlink != 0: + if target, err := os.Readlink(d); err == nil { + args = append(args, "--symlink", target, d) + } + case info.IsDir(): + args = append(args, "--ro-bind", d, d) + } + } + for _, e := range etcNeeded { + args = append(args, "--ro-bind-try", e, e) + } + return args +} + +// stepHome is the HOME of a step that is given none: inside its own /tmp, gone with it. +const stepHome = "/tmp/home" + +func checkDir(what, dir string) error { + if dir == "" || !strings.HasPrefix(dir, "/") || path.Clean(dir) != dir || dir == "/" { + return fmt.Errorf("buildstep: the %s %q is not an absolute, clean directory other than the root", what, dir) + } + return nil +} + +// Command is the bubblewrap command line that runs name with args shut off as iso says, env added to iso.Env. It +// refuses a directory that is not absolute and clean, or is the root, and an environment word without "=". +func (iso Isolation) Command(name string, args ...string) ([]string, error) { + return iso.command(nil, name, args...) +} + +func (iso Isolation) command(env []string, name string, args ...string) ([]string, error) { + if err := checkDir("workspace", iso.Workspace); err != nil { + return nil, err + } + for _, group := range []struct { + what string + dirs []string + }{{"writable directory", iso.Writable}, {"read-only directory", iso.ReadOnly}, {"cache", iso.Caches}} { + for _, d := range group.dirs { + if err := checkDir(group.what, d); err != nil { + return nil, err + } + } + } + words := append(append([]string{}, iso.Env...), env...) + home := false + for _, w := range words { + k, _, ok := strings.Cut(w, "=") + if !ok || k == "" { + return nil, fmt.Errorf("buildstep: the environment word %q is not NAME=value", w) + } + home = home || k == "HOME" + } + argv := []string{"--unshare-all", "--die-with-parent", "--new-session", "--clearenv", + "--proc", "/proc", "--dev", "/dev", "--tmpfs", "/tmp", "--dir", stepHome} + argv = append(argv, systemArgs()...) + for _, d := range iso.ReadOnly { + argv = append(argv, "--ro-bind", d, d) + } + argv = append(argv, cacheArgs(iso.Caches, iso.WriteCaches)...) + argv = append(argv, "--bind", iso.Workspace, iso.Workspace) + for _, d := range iso.Writable { + argv = append(argv, "--bind", d, d) + } + if iso.Hosts != "" { + argv = append(argv, "--ro-bind", iso.Hosts, "/etc/hosts") + } + for _, w := range words { + k, v, _ := strings.Cut(w, "=") + argv = append(argv, "--setenv", k, v) + } + if !home { + argv = append(argv, "--setenv", "HOME", stepHome) + } + argv = append(argv, "--chdir", iso.Workspace, "--", name) + return append(argv, args...), nil +} + +// cacheArgs binds the shared warm caches (S18): for a trunk build read-write, so the next build is warm; for any other +// read-only beneath an overlay whose writes go to a tmpfs of the step's own, so the step writes as a build does and +// nothing it wrote outlives it (bubblewrap's --tmp-overlay, an overlay mount in the step's user namespace). +func cacheArgs(caches []string, write bool) []string { + var argv []string + for _, c := range caches { + if write { + argv = append(argv, "--bind", c, c) + continue + } + argv = append(argv, "--overlay-src", c, "--tmp-overlay", c) + } + return argv +} + +// loopbackHosts is the hosts file of a step: the loopback and nothing else. +const loopbackHosts = "127.0.0.1 localhost\n::1 localhost\n" + +// Isolated is run with every command shut off as iso says (Command), run's env going inside the step: the build modules' +// run steps call their test, build and digest commands through it, never run itself. Each command gets a hosts file of +// its own, naming only the loopback, removed when it ends. +func Isolated(run forgekey.Runner, iso Isolation) forgekey.Runner { + return func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran { + dir, err := os.MkdirTemp("", "step-hosts-") + if err != nil { + return forgekey.Ran{Status: 1, Err: "the step's hosts file: " + err.Error()} + } + defer os.RemoveAll(dir) + hosts := filepath.Join(dir, "hosts") + if err := os.WriteFile(hosts, []byte(loopbackHosts), 0o644); err != nil { + return forgekey.Ran{Status: 1, Err: "the step's hosts file: " + err.Error()} + } + iso := iso + iso.Hosts = hosts + argv, err := iso.command(env, name, args...) + if err != nil { + return forgekey.Ran{Status: 1, Err: err.Error()} + } + return run(ctx, nil, Bubblewrap, argv...) + } +} diff --git a/vendor/git.novox.be/novox/mesh-sdk/go/buildstep/request.go b/vendor/git.novox.be/novox/mesh-sdk/go/buildstep/request.go new file mode 100644 index 00000000..c0189dcc --- /dev/null +++ b/vendor/git.novox.be/novox/mesh-sdk/go/buildstep/request.go @@ -0,0 +1,115 @@ +package buildstep + +import ( + "fmt" + "path" + "regexp" + "strings" +) + +var ( + requestCommitOK = regexp.MustCompile(`^[0-9a-f]{40}$`) + requestRepositoryOK = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]*/[A-Za-z0-9][A-Za-z0-9._-]*$`) +) + +// FetchRequest is what a build module's fetch tool is asked: the repository and the full commit, the pull request's +// ref when only its head carries the commit, the trunk commit to merge onto for a merge preview, and, for a build of a +// module whose every artifact names its paths, that module's directory and its named paths (novox/hq ADR 0499 clause 4). +// +// **The paths come from the builder, not from a manifest** (ADR 0495 clause 1; the lead's decision L8 of R4e): the +// builder fills Dir and Paths from the indexer's Module value (Path, and KeptInputs2.Paths' keys), and the build module +// gives the build and test steps a workspace holding those paths, the manifest and its lock files. A request without +// them builds from the module's directory, as before. +type FetchRequest struct { + Repository string `json:"repository"` + Commit string `json:"commit"` + Ref string `json:"ref,omitempty"` + Base string `json:"base,omitempty"` + Dir string `json:"dir,omitempty"` + Paths []string `json:"paths,omitempty"` +} + +// FetchRequestProperties are the JSON schema properties of FetchRequest's dir and paths, for a fetch tool's Input. +func FetchRequestProperties() map[string]any { + return map[string]any{ + "dir": map[string]any{"type": "string", "description": "the module's directory in the repository (\".\" for its root), with paths"}, + "paths": map[string]any{"type": "array", "items": map[string]any{"type": "string"}, + "description": "the paths the module names, relative to dir, from the indexer's kept inputs: the build sees only these, the manifest and its lock files"}, + } +} + +// Args is r as a tool call's arguments. +func (r FetchRequest) Args() map[string]any { + args := map[string]any{"repository": r.Repository, "commit": r.Commit} + for k, v := range map[string]string{"ref": r.Ref, "base": r.Base, "dir": r.Dir} { + if v != "" { + args[k] = v + } + } + if len(r.Paths) > 0 { + args["paths"] = append([]string(nil), r.Paths...) + } + return args +} + +// ParseFetchRequest reads a fetch tool's arguments; arguments it does not know (a build module's own) are left to the +// caller. It refuses a repository that is not /, a commit or a base that is not 40 hex digits, paths with no +// module directory or a directory with no path, a directory leaving the repository and a path leaving the directory, +// naming each. +func ParseFetchRequest(args map[string]any) (FetchRequest, error) { + var r FetchRequest + r.Repository, _ = args["repository"].(string) + r.Commit, _ = args["commit"].(string) + r.Ref, _ = args["ref"].(string) + r.Base, _ = args["base"].(string) + r.Dir, _ = args["dir"].(string) + switch v := args["paths"].(type) { + case nil: + case []string: + r.Paths = append([]string(nil), v...) + case []any: + for _, p := range v { + s, ok := p.(string) + if !ok { + return FetchRequest{}, fmt.Errorf("paths is a list of strings") + } + r.Paths = append(r.Paths, s) + } + default: + return FetchRequest{}, fmt.Errorf("paths is a list of strings") + } + return r, r.Check() +} + +// Check is what ParseFetchRequest refuses. +func (r FetchRequest) Check() error { + if !requestRepositoryOK.MatchString(r.Repository) || strings.Contains(r.Repository, "..") { + return fmt.Errorf("%q is not a repository, /", r.Repository) + } + if !requestCommitOK.MatchString(r.Commit) { + return fmt.Errorf("%q is not a full commit (40 hex digits)", r.Commit) + } + if r.Base != "" && !requestCommitOK.MatchString(r.Base) { + return fmt.Errorf("%q is not a trunk commit (40 hex digits)", r.Base) + } + switch { + case len(r.Paths) > 0 && r.Dir == "": + return fmt.Errorf("the request names paths and names no module directory (dir) they are relative to") + case len(r.Paths) == 0 && r.Dir != "": + return fmt.Errorf("the request names a module directory and no path") + } + if r.Dir != "" && (outside(r.Dir) || strings.HasPrefix(r.Dir, "/")) { + return fmt.Errorf("the module directory %q leaves the repository", r.Dir) + } + for _, p := range r.Paths { + if p == "" || strings.HasPrefix(p, "/") || outside(p) || path.Clean(p) == "." { + return fmt.Errorf("the path %q leaves the module's directory or is not a path in it", p) + } + } + return nil +} + +func outside(p string) bool { + c := path.Clean(p) + return c == ".." || strings.HasPrefix(c, "../") +} diff --git a/vendor/git.novox.be/novox/mesh-sdk/go/buildstep/trees.go b/vendor/git.novox.be/novox/mesh-sdk/go/buildstep/trees.go index 58b2ccd8..fc31c62d 100644 --- a/vendor/git.novox.be/novox/mesh-sdk/go/buildstep/trees.go +++ b/vendor/git.novox.be/novox/mesh-sdk/go/buildstep/trees.go @@ -6,9 +6,12 @@ import ( "encoding/hex" "fmt" "os" + "path" "path/filepath" "strings" + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/delivery" "git.novox.be/novox/mesh-sdk/go/forgekey" ) @@ -19,12 +22,16 @@ type ModuleTree struct { Tree string `json:"tree"` Manifest string `json:"manifest"` Locks map[string]string `json:"locks,omitempty"` + // Paths are, of a module whose every artifact names its paths, each named path's git object id at the commit; Tree + // is then their delivery.NamedTree with Manifest (novox/hq ADR 0499 clause 2). Absent otherwise. + Paths map[string]string `json:"paths,omitempty"` } var lockFiles = []string{"go.sum", "package-lock.json"} // ModuleTrees are the trees, in commit, of every directory of the workspace dir that holds a module.json, by directory -// (the repository's root is "."). The workspace is at commit, so the manifests and the lock files are read from it. env is +// (the repository's root is "."); for a module whose every artifact names its paths, the named tree of those paths and +// their objects (novox/hq ADR 0499 clause 2). The workspace is at commit, so the manifests and the lock files are read from it. env is // the environment git runs with (nil for none). Both build modules' fetch steps answer it for a merge. func ModuleTrees(ctx context.Context, run forgekey.Runner, env []string, dir, commit string) (map[string]ModuleTree, error) { r := run(ctx, env, "git", "-C", dir, "ls-files", "--", "module.json", "*/module.json") @@ -51,6 +58,9 @@ func ModuleTrees(ctx context.Context, run forgekey.Runner, env []string, dir, co return nil, err } mt.Manifest = sha256Hex(raw) + if err := narrow(ctx, run, env, dir, commit, mdir, raw, &mt); err != nil { + return nil, err + } for _, lock := range lockFiles { if raw, err := os.ReadFile(filepath.Join(dir, filepath.FromSlash(mdir), lock)); err == nil { if mt.Locks == nil { @@ -64,4 +74,50 @@ func ModuleTrees(ctx context.Context, run forgekey.Runner, env []string, dir, co return trees, nil } +// narrow sets mt's Paths and its named Tree when the module's manifest names paths on every artifact (novox/hq ADR 0499 +// clause 2): one `git ls-tree -z` gives each named path's object at commit. The merge preview reads the manifest here, as +// the indexer does, and no other build step reads paths from one (ADR 0495 clause 1). A manifest that does not parse +// is not narrowed: the indexer gives it none. A named path absent at commit fails, naming the module and the path. +func narrow(ctx context.Context, run forgekey.Runner, env []string, dir, commit, mdir string, raw []byte, mt *ModuleTree) error { + m, err := catalogue.ParseManifest(raw) + if err != nil { + return nil + } + paths, every := m.NamedPaths() + if !every { + return nil + } + full := make([]string, len(paths)) + for i, p := range paths { + full[i] = path.Join(mdir, p) + } + r := run(ctx, env, "git", append([]string{"-C", dir, "ls-tree", "-z", commit, "--"}, full...)...) + if r.Status != 0 || r.Err != "" { + return fmt.Errorf("git ls-tree %s of the module %s: %s", commit, mdir, firstLine(r.Err+" "+r.Stderr)) + } + objects := map[string]string{} + for _, l := range strings.Split(r.Stdout, "\x00") { + meta, name, ok := strings.Cut(l, "\t") + fields := strings.Fields(meta) + if !ok || len(fields) != 3 { + continue + } + objects[name] = fields[2] + } + mt.Paths = map[string]string{} + for i, p := range paths { + object := objects[full[i]] + if object == "" { + return fmt.Errorf("the module %s names the path %q, which is not in %s", mdir, p, commit) + } + mt.Paths[p] = object + } + named, err := delivery.NamedTree(mt.Manifest, mt.Paths) + if err != nil { + return fmt.Errorf("the module %s: %w", mdir, err) + } + mt.Tree = named + return nil +} + func sha256Hex(b []byte) string { h := sha256.Sum256(b); return hex.EncodeToString(h[:]) } diff --git a/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/build.go b/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/build.go index f4fcd25b..91e34601 100644 --- a/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/build.go +++ b/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/build.go @@ -399,6 +399,33 @@ func (b *Build) problems(module string) []string { return problems } +// NamedPaths are the paths a module's fingerprint takes its tree from (novox/hq ADR 0499 clause 2): the union of the +// paths its artifacts name, cleaned and sorted, and every true only when the module builds something and every artifact +// names paths; otherwise none, and the module keeps its directory's tree. +// +// **Read by the core alone** (ADR 0495 clause 1, the lead's decision L8 of R4e): the indexer and its merge preview +// (buildstep.ModuleTrees) call it. A build module's fetch step is handed the paths by the builder, from the indexer's +// kept inputs (delivery.KeptInputs2.Paths), and reads no manifest for them. +func (m Manifest) NamedPaths() (paths []string, every bool) { + if m.Build == nil || len(m.Build.Artifacts) == 0 { + return nil, false + } + seen := map[string]bool{} + for _, a := range m.Build.Artifacts { + if len(a.Paths) == 0 { + return nil, false + } + for _, p := range a.Paths { + seen[path.Clean(p)] = true + } + } + for p := range seen { + paths = append(paths, p) + } + sort.Strings(paths) + return paths, true +} + // pathsProblems is what is wrong with an artifact's `paths` (novox/hq ADR 0499 clause 1): named on a kind whose build // source is derived or named otherwise, or a path that leaves the module's directory, is its whole directory, or is a // pattern. diff --git a/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/delivery_seat.go b/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/delivery_seat.go index dbc80faf..7df4d868 100644 --- a/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/delivery_seat.go +++ b/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/delivery_seat.go @@ -42,7 +42,8 @@ func deliveryVerbs() []Verb { "paths": "the files it changes, comma-separated, from the repository's root", "base": "the branch it merges into (default main)"}, []string{"repository", "paths"})}, {Name: "checks", Description: "What the mesh's checks said of a pull request's head or of one commit: each " + - "of the commit's mesh statuses (mesh/merge-gate, mesh/repo-check, mesh/delivery, …) with its state, " + + "of the commit's mesh statuses (mesh/check, the one a main requires from R4e, hq's mesh/repo-check, " + + "mesh/delivery, …) with its state, " + "description and when it was set; the merge check's full verdict as the controller said it — each " + "layer's summary, the machine that ran it, when, its build id and its report; and whether the branch's " + "protection would let it merge, every required status being success.", diff --git a/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/predefined_inputs.go b/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/predefined_inputs.go new file mode 100644 index 00000000..1552cf93 --- /dev/null +++ b/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/predefined_inputs.go @@ -0,0 +1,41 @@ +package catalogue + +import "encoding/json" + +// ParseManifestForInputs reads a manifest for the inputs of its build and its fingerprint: the indexer's and the +// builder's read (novox/hq issue 1751). It is never a judge of config values. A value falling back to a predefined key +// (ADR 0348 §7) is judged against the list the mesh published when this process holds one (PublishedPredefined); +// where it holds none, as in the indexer and the builder, each key a value names is taken as the value declares it (its +// type, trusted when it is), since the list is read where a manifest is judged: the merge gate from the tree's root, +// the controller from what it published. No input of a fingerprint or a build comes from the list. Everything else +// ParseManifest refuses is refused here too. +func ParseManifestForInputs(raw []byte) (Manifest, error) { + list := PublishedPredefined() + if list == nil { + list = fallbacksOf(raw) + } + return parseManifest(raw, list) +} + +// fallbacksOf is the list of the predefined keys a manifest's config values fall back to, each as the value declares +// it. A manifest that is not JSON gives an empty list, and the parser says what is wrong with it. +func fallbacksOf(raw []byte) PredefinedList { + var m struct { + Config map[string]struct { + Type string `json:"type"` + Description string `json:"description"` + Predefined string `json:"predefined"` + Trusted bool `json:"trusted"` + } `json:"config"` + } + list := PredefinedList{} + if json.Unmarshal(raw, &m) != nil { + return list + } + for _, d := range m.Config { + if d.Predefined != "" { + list[d.Predefined] = PredefinedKey{Type: d.Type, Description: d.Description, Trusted: d.Trusted} + } + } + return list +} diff --git a/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/seats_declared.go b/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/seats_declared.go index 2a9ad3fe..457f87e6 100644 --- a/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/seats_declared.go +++ b/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/seats_declared.go @@ -423,13 +423,6 @@ func CatalogueProblems(shelf Shelf) []string { "operator's account, which every agent runs as (novox/hq ADR 0259)", module, why)) } } - // A read of a module's state that module does not keep (novox/hq ADR 0201) — said only where the - // owner is on the shelf, as a consumer may be installed before its emitter. - var manifests []Manifest - for _, module := range shelfOrder(shelf) { - manifests = append(manifests, shelf[module]) - } - problems = append(problems, StateReadsNothingDeclares(manifests)...) sort.Strings(problems) return problems } diff --git a/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/state.go b/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/state.go index 87e402f1..9229d3d3 100644 --- a/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/state.go +++ b/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/state.go @@ -126,46 +126,3 @@ func StateProblems(m Manifest) []string { } return problems } - -// ControllerMirrors are the controller's record mirrors a module may read, each `mesh-controller.` -// (novox/hq ADR 0289, design 54 §5, design 57 F3). **The controller keeps them, not its manifest's `state`**: -// they are asserted with the controller's own buckets, with options of their own, and written by the -// controller's account alone (the writers table); declared as a module's state, they would be asserted a -// second way and granted to the module's own principal as well. Named here so a read of one is a read of -// kept state. The bus's list is broker.ControllerMirrors; a test keeps the two agreeing. -// plans-building is the plans in their build phase, read by the checker to hold checks while a delivery builds (ADR 0483 -// clause 2, issue 1687). -var ControllerMirrors = []string{"assignments", "declared-config", "predefined", "builds", "rollouts", "checks-running", "plans-building"} - -// StateReadsNothingDeclares is every read across a catalogue whose owner is present and declares no -// such state. An absent owner says nothing — a module may be installed long before the one whose -// state it reads, as a consumer may before its emitter (design 32 §1). -func StateReadsNothingDeclares(manifests []Manifest) []string { - declared := map[string]map[string]bool{} - for _, m := range manifests { - own := map[string]bool{} - for _, s := range m.State { - own[s.Name] = true - } - declared[m.Module] = own - } - if own, present := declared[ControllerSeatName]; present { - for _, name := range ControllerMirrors { - own[name] = true - } - } - var problems []string - for _, m := range manifests { - for _, r := range m.Reads { - module, local, err := ReadState(r) - if err != nil { - continue - } - if own, present := declared[module]; present && !own[local] { - problems = append(problems, fmt.Sprintf( - "%s reads %q, and %s keeps no state called %q", m.Module, r, module, local)) - } - } - } - return problems -} diff --git a/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/verbs.go b/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/verbs.go index 17e417a0..266bac6f 100644 --- a/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/verbs.go +++ b/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/verbs.go @@ -147,24 +147,16 @@ var ControllerVerbs = []Verb{ }, nil, "data")}, // The verbs the delivery's owner asks with (novox/hq ADR 0239): the planner's answer, a group's order and // composed check, and the start and end of a walk. Callable by a person too; mesh-delivery is who needs them. - {Name: "delivery-plan", Description: "The delivery plan of a diffset, as the planner computes it and keeping " + - "nothing (novox/hq ADR 0239): what it moves and builds after them in tiers, what each machine receives and " + - "what waits there, the steps that are not an ordinary send, and whether the gate would run.", - Input: schema(map[string]string{"repository": "owner/repository", "head": "the commit at hand", - "base": "the branch it merges into (default main)", - "paths": "the files it changes, comma-separated, from the repository's root", - "module-dirs": "the directories holding a module.json at the head, comma-separated, as the forge says them", - "removed": "the files among paths it deletes, comma-separated"}, []string{"repository", "paths"})}, + {Name: "delivery-plan", Description: "Removed in R4e (ADR 0501 §5 / ADR 0503 §3): refused by name, naming its replacement", + Input: schema(nil, nil)}, {Name: "delivery-order", Description: "A delivery group's order (novox/hq ADR 0419): its members in the order " + "they are delivered, every pair and why — built by, verb before caller, version skew, engine before " + "controller — read from the module graph and the manifests' change alone; no `after:` line is read. The " + "cycle, with every module and edge of it, is the one refusal.", Input: schema(map[string]string{"members": "the members, as JSON: [{id, repository, base, head, number, paths, " + "module_dirs, module_dirs_said, removed}]"}, []string{"members"})}, - {Name: "delivery-check", Description: "Ask the build seat for a delivery group's composed check (novox/hq ADR " + - "0239): every member's head laid over the mesh in turn and judged as one future state. Answers the ask's id; " + - "the verdict is said as `checked` with the group's id. With one member and no group, that head is checked " + - "again as its pull request is.", + {Name: "delivery-check", Description: "Answers that the checker (mesh-delivery-checker) judges a group (novox/hq " + + "ADR 0503 §3); asks nothing. Its inputs stay, since hq's repository check still names it (ADR 0513 §4).", Input: schema(map[string]string{"group": "the group's id (none for one head checked again)", "members": "the members, as JSON, as delivery-order takes them", "ready": "\"true\" when the pull request is marked ready: its check is queued before every other check (novox/hq ADR 0406 §3)"}, @@ -378,45 +370,33 @@ var ControllerVerbs = []Verb{ {Name: "clear", Description: "Cancel every waiting ask in the build queue — and with dead, every dead one too — each " + "recorded failed, cancelled by hand. Never touches one in flight.", Input: schema(map[string]string{"dead": "\"true\" to cancel the dead asks as well"}, nil, "dead")}, - {Name: "rebuild", Description: "Ask a module's current source again under a new id — the branch it follows — or, given a " + - "build's id, that build's repository, path and ref. A module a plan holds unbuilt or failed joins that plan. Answers the new id.", - Input: schema(map[string]string{"what": "a module's name, or a build's id"}, []string{"what"})}, - {Name: "replay", Description: "Ask a recorded build's repository and path again at the commit it built, under a new id. " + - "A dry run unless register: nothing recorded or registered. Registering is refused when a newer build of the module " + - "is registered — it would roll the older commit out (novox/hq issue 207) — unless older says so.", - Input: schema(map[string]string{ - "id": "the build's id", - "register": "\"true\" to register what it builds", - "older": "\"true\", with register: even though a newer build of the module is registered", - }, []string{"id"}, "register", "older")}, + {Name: "rebuild", Description: "Removed in R4e (ADR 0501 §5 / ADR 0503 §3): refused by name, naming its replacement", + Input: schema(nil, nil)}, + {Name: "replay", Description: "Ask a recorded build's repository and path again at the commit it built, under a new id: " + + "a dry run only, nothing recorded or registered (novox/hq ADR 0429 §18: a fingerprint's key is built again by the " + + "builder, never registered by hand).", + Input: schema(map[string]string{"id": "the build's id"}, []string{"id"})}, // novox/hq issue 686: a builder ran a branch's store tests for the first time in its pull request's merge check. - {Name: "try", Description: "Try a pushed branch or commit before its pull request (novox/hq issue 686): the build " + - "seat merges it into the repository's trunk and runs the repository's own merge-check.sh on the merge, in the " + - "toolchain containers and against the throwaway store and bus a pull request's merge check gets. Answers at " + + {Name: "try", Description: "Try a pushed branch or commit before its pull request (novox/hq issue 686): the " + + "checker (mesh-delivery-checker) merges the head onto the repository's trunk and runs its levels on the merge, " + + "as it judges a pull request (ADR 0503 §3); no merge-check.sh is run and build-agent is not asked. Answers at " + "once with an id; asked again with that id it waits up to 20 s and answers pass, fail or error, how long it " + "took and the lines naming what failed — or that it is still waiting or running, to ask again. Sets no " + - "status on the forge, registers and builds nothing, runs no merge gate, counts as a check in the build queue " + - "(after deliveries and after every pull request's check, however long it waits) and is bounded as one. One " + - "try of a commit at a time — a second ask answers the first's id — and at most four waiting or running in " + - "all: a fifth is refused. Only for a repository of the mesh; the ref must be on the forge.", + "status on the forge, registers and builds nothing for delivery. One try of a commit at a time — a second " + + "ask answers the first's id. Only for a repository of the mesh; the ref must be on the forge.", Input: schema(map[string]string{ "repository": "the repository on the forge, owner/name; with ref", "ref": "the branch or commit to try, as pushed to the forge", - "base": "the branch it is merged into (the repository's default branch when none is named). The " + - "merge-check.sh that runs is that branch's, never the ref's own: the same trust as a pull request's base", + "base": "the branch it is merged into (the repository's default branch when none is named)", "tests": "only the tests this `go test -run` pattern names, where the repository's merge-check.sh reads " + "MESH_CHECK_TESTS; a script that does not runs whole", - "store-only": "\"true\": only the repository's store tests, where its merge-check.sh reads " + - "MESH_CHECK_STORE_ONLY", - "id": "instead of a repository: the id a try answered, to read what came of it", + "store-only": "\"true\": only the integration level", + "id": "instead of a repository: the id a try answered, to read what came of it", }, nil, "store-only"), Replaces: []string{"sh merge-check.sh", "docker run postgres"}}, // novox/hq ADR 0350 §5: asked by the delivery's owner when the forge says a pull request closed unmerged. - {Name: "forget-checks", Description: "Let go of every build a pull request's merge checks made for its merge in " + - "the artifact store's scratch namespace (novox/hq ADR 0350 §5): asked when it closed unmerged; a merged one's go " + - "when the walk answering its merge ends, and any check build goes after a week.", - Input: schema(map[string]string{"owner": "the repository's owner", "repo": "the repository", - "number": "the pull request's number"}, []string{"owner", "repo", "number"})}, + {Name: "forget-checks", Description: "Removed in R4e (ADR 0501 §5 / ADR 0503 §3): refused by name, naming its replacement", + Input: schema(nil, nil)}, {Name: "kill", Description: "End a build where it runs: the machine that took it stops its commands and containers and " + "announces it failed, killed by hand — settled, never handed to another machine.", Input: schema(map[string]string{"id": "the build's id"}, []string{"id"})}, @@ -595,22 +575,8 @@ var ControllerVerbs = []Verb{ "machine": "one machine (optional)", "retired": "\"true\": only what is retired", }, nil, "retired")}, - // Three shapes, as the command has them: one repository, every module standing on a base (on), or - // every module behind its source (behind). One per call; a second is refused as passed over. - {Name: "build", Description: "Have the build machine build a repository — or, instead, every module that " + - "stands on a base (on), or every module behind its source (behind). Answers at once with each build's id: " + - "`builds` with an id follows it line by line, and a module is registered when its outcome comes.", - Input: schema(map[string]string{ - "repository": "the repository's URL, or its path on the forge holding the git seat (owner/name); " + - "needed unless on or behind is given", - "path": "with repository: the module's directory inside it (optional)", - "ref": "with repository: the branch, tag or commit to build (optional)", - "on": "instead of a repository: a module's name. Every module the mesh holds that was built against " + - "its artifacts is built again from its own source, bases among them asked first — the rebuild a " + - "changed base needs, which behind does not see because their sources did not move", - "behind": "\"true\", instead of a repository: build every module the mesh holds whose source has moved " + - "past the commit it was built from, bases asked first", - }, nil, "behind")}, + {Name: "build", Description: "Removed in R4e (ADR 0501 §5 / ADR 0503 §3): refused by name, naming its replacement", + Input: schema(nil, nil)}, // What the records say the registries may keep, asked on demand (novox/hq ADR 0251, to-be 51). {Name: "artifacts", Description: "Every artifact the mesh recorded making that lives in its artifact store, " + "each with its repository, digest, kind (image or archive) and state: kept — and why: a definition names " + diff --git a/vendor/git.novox.be/novox/mesh-sdk/go/delivery/builder.go b/vendor/git.novox.be/novox/mesh-sdk/go/delivery/builder.go index 205e5d56..47bcab76 100644 --- a/vendor/git.novox.be/novox/mesh-sdk/go/delivery/builder.go +++ b/vendor/git.novox.be/novox/mesh-sdk/go/delivery/builder.go @@ -63,8 +63,18 @@ func ParseBuilderBuildKey(key string) (module, fingerprint string, err error) { return module, fingerprint, nil } -// BuiltArtifact is one artifact of a done build: its name in the manifest, its kind, its content digest and, for an -// image, the reference it was pushed to. +// BuiltArtifact is one artifact of a done build: its name in the manifest, its kind, its content digest (`sha256:` and +// 64 hex digits) and the reference that names where it is (novox/hq ADR 0498 clauses 1, 5 and 6; the lead's decision +// L1 of R4e), by kind: +// +// bundle, archive artifact-store:////blobs/ the blob pushed to the artifact store +// image, upstream //@ the image registry's reference, which +// artifact-store:///@ catalogue.Recorded keeps without its address +// package @ the version published from the trunk +// +// is Name and is Digest. A reference is empty for a candidate's image or package, which is never +// pushed or published (ADR 0498 clause 6), and in a key done before R4e; the controller delivers only from a key whose +// every artifact names one (Referenced, clause 8). A kind left empty (a key written before kinds) is not checked. type BuiltArtifact struct { Name string `json:"name"` Kind string `json:"kind,omitempty"` @@ -72,6 +82,83 @@ type BuiltArtifact struct { Reference string `json:"reference,omitempty"` } +const artifactStore = "artifact-store://" + +// referenceProblem says what is wrong with a's reference for its kind, or "". +func (a BuiltArtifact) referenceProblem() string { + r := a.Reference + if r == "" || a.Kind == "" { + return "" + } + switch a.Kind { + case "bundle", "archive": + path, ok := strings.CutPrefix(r, artifactStore) + module, rest, _ := strings.Cut(path, "/") + if !ok || module == "" || rest != a.Name+"/blobs/"+a.Digest { + return "is not artifact-store:///" + a.Name + "/blobs/" + a.Digest + } + case "image", "upstream": + path, digest, ok := strings.Cut(r, "@") + if !ok || digest != a.Digest { + return "does not name its digest " + a.Digest + } + var host, module, name string + if rest, kept := strings.CutPrefix(path, artifactStore); kept { + module, name, _ = strings.Cut(rest, "/") + host = "kept" + } else { + parts := strings.Split(path, "/") + if len(parts) == 3 { + host, module, name = parts[0], parts[1], parts[2] + } + } + if host == "" || !(host == "kept" || host == "localhost" || strings.ContainsAny(host, ".:")) || module == "" || name != a.Name { + return "is not //" + a.Name + "@" + a.Digest + } + case "package": + at := strings.LastIndex(r, "@") + if at <= 0 || at == len(r)-1 || r[:at] == "@" { + return "is not @" + } + } + return "" +} + +// Referenced says whether b is done and every artifact names where it is, which the controller delivers from (novox/hq +// ADR 0498 clause 8), and names the artifacts that name no reference. A done key with no artifact (a module that builds +// none, issue 1613) is referenced. +func (b BuilderBuild) Referenced() (ok bool, missing []string) { + for _, a := range b.Artifacts { + if a.Reference == "" { + missing = append(missing, a.Name) + } + } + return b.State == BuildDone && len(missing) == 0, missing +} + +// The prefixes of a failed key's error that a build module answers (novox/hq ADR 0498 clauses 2, 4 and 5; ADR 0478 +// clause 2), each followed by the artifact and the words of what refused it. +const ( + // NotStored: the push of a bundle, an archive or an upstream copy to the artifact store failed; built again. + NotStored = "not stored: " + // NotPinned: an upstream artifact names no digest. + NotPinned = "not pinned: " + // VersionTaken: the package registry holds the package's version with another packed file. + VersionTaken = "version taken: " + // RefusedKind: no build module builds this kind of artifact in its language. + RefusedKind = "refused kind: " + // WaitsFor: a fetch found the module's lock naming a package version the registry does not hold yet; the builder + // writes the key queued with Waits (novox/hq ADR 0500 clause 6), and it is built once the package is published. + WaitsFor = "waits for: " +) + +// FailedUntilChanged says whether a failed key's error is one that is not built again until the module's fingerprint +// changes (not pinned, version taken, refused kind: the same source fails the same way); every other failed key, a +// push that was not stored included, is built again as a new revision (ADR 0429 clause 18). +func FailedUntilChanged(err string) bool { + return strings.HasPrefix(err, NotPinned) || strings.HasPrefix(err, VersionTaken) || strings.HasPrefix(err, RefusedKind) +} + // ToolchainUsed is what a build used for one language (ADR 0405 clauses 2 and 7): the installed toolchain's version and // the build module's recipe version. type ToolchainUsed struct { @@ -82,7 +169,8 @@ type ToolchainUsed struct { // BuilderBuild is the value of a builder build key: its State (queued, building, done or failed), By the builder // instance that holds it, Step what it is doing or did last, Manifest the digest of the module's manifest (the key of // the indexer's manifests bucket), Artifacts when done, Toolchains by language when it built or is building (a done -// build names them, so two builders that differ are seen: ADR 0405 clause 2), and Error when failed. +// build names them, so two builders that differ are seen: ADR 0405 clause 2), Error when failed, and Waits when queued for a +// package version. type BuilderBuild struct { V int `json:"v"` State string `json:"state"` @@ -92,6 +180,10 @@ type BuilderBuild struct { Artifacts []BuiltArtifact `json:"artifacts,omitempty"` Toolchains map[string]ToolchainUsed `json:"toolchains,omitempty"` Error string `json:"error,omitempty"` + // Waits is, on a queued key only, the package version `@` the module's lock names and the registry + // does not yet hold: the key is not claimed until that package's trunk key is done and published (novox/hq ADR 0500 + // clause 6). A reader that does not know the field reads the key as queued. + Waits string `json:"waits,omitempty"` } func (b BuilderBuild) check() error { @@ -120,6 +212,17 @@ func (b BuilderBuild) check() error { if a.Digest == "" { return fmt.Errorf("delivery: the artifact %q of the build has no digest", a.Name) } + if p := a.referenceProblem(); p != "" { + return fmt.Errorf("delivery: the reference %q of the %s %q %s", a.Reference, a.Kind, a.Name, p) + } + } + if b.Waits != "" { + if b.State != BuildQueued { + return fmt.Errorf("delivery: a %s build waits for %q; only a queued key waits for a package", b.State, b.Waits) + } + if p := (BuiltArtifact{Kind: "package", Reference: b.Waits}).referenceProblem(); p != "" { + return fmt.Errorf("delivery: the package a key waits for, %q, %s", b.Waits, p) + } } switch b.State { case BuildQueued, BuildBuilding: @@ -127,9 +230,7 @@ func (b BuilderBuild) check() error { return fmt.Errorf("delivery: a %s build carries no artifact and no error", b.State) } case BuildDone: - if len(b.Artifacts) == 0 { - return fmt.Errorf("delivery: a done build names no artifact") - } + // A module that builds no artifact gets a done key too, and is delivered from it (issue 1613). if b.Manifest == "" { return fmt.Errorf("delivery: a done build names no manifest digest") } @@ -144,8 +245,8 @@ func (b BuilderBuild) check() error { return nil } -// EncodeBuilderBuild is b as JSON at Version; it refuses a build its state does not allow (done without artifacts or a -// manifest digest, failed without an error, an artifact without a name or digest, a toolchain without a version or a +// EncodeBuilderBuild is b as JSON at Version; it refuses a build its state does not allow (done without a manifest +// digest, failed without an error, an artifact without a name or digest, a toolchain without a version or a // recipe version). func EncodeBuilderBuild(b BuilderBuild) ([]byte, error) { if err := b.check(); err != nil { diff --git a/vendor/git.novox.be/novox/mesh-sdk/go/delivery/gate.go b/vendor/git.novox.be/novox/mesh-sdk/go/delivery/gate.go index 18444c57..5fbe9fb9 100644 --- a/vendor/git.novox.be/novox/mesh-sdk/go/delivery/gate.go +++ b/vendor/git.novox.be/novox/mesh-sdk/go/delivery/gate.go @@ -29,7 +29,9 @@ const ( ) // The contexts of the statuses the mesh owes the forge: the gate's, the delivery's view of a head and of a group -// (design 47), and the checker's, which is not required until R4f (ADR 0484 clause 2, ADR 0453 clause 4). +// (design 47), and the checker's. From R4e the checker's mesh/check is the one status a repository's main requires, +// in place of mesh/merge-gate and mesh/repo-check (ADR 0503 clauses 3 and 5); a main that requires only +// mesh/repo-check (hq) keeps it, and the controller writes it for a repository the checker writes no status for. const ( StatusContextMergeGate = "mesh/merge-gate" StatusContextDelivery = "mesh/delivery" diff --git a/vendor/git.novox.be/novox/mesh-sdk/go/delivery/namedtree.go b/vendor/git.novox.be/novox/mesh-sdk/go/delivery/namedtree.go new file mode 100644 index 00000000..cc2cbad1 --- /dev/null +++ b/vendor/git.novox.be/novox/mesh-sdk/go/delivery/namedtree.go @@ -0,0 +1,47 @@ +package delivery + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + "path" + "sort" + "strings" +) + +// NamedTreePrefix opens the tree input of a module whose every artifact names its paths, so such a value is never read +// as a git tree id and a reader of a kept Tree can tell the two apart. +const NamedTreePrefix = "named:" + +// NamedTree is the tree input of the new-form fingerprint of a module whose every artifact names the paths it is built +// from (novox/hq ADR 0499 clause 2): `named:` and the hex sha256 of the lines "manifest " and, sorted, +// "path ", joined by a newline. manifest is the module's ManifestDigest (the sha256 of its module.json's +// exact bytes, as Module.Manifest and the merge preview carry it); paths are the named paths, each clean and relative to +// the module's directory as the manifest names them (catalogue.Manifest.NamedPaths), with the git object id each has at +// the commit (a blob for a file, a tree for a directory). +// +// **One function computes it** (clause 2): the indexer, its merge preview (buildstep.ModuleTrees) and the build modules +// call this one, and no second computation exists. It refuses no path, a named path with no object (absent at the +// commit), a path that is not clean, and a manifest that is not a digest, naming them: the indexer then gives the +// module none, with that reason (clause 3). +func NamedTree(manifest string, paths map[string]string) (string, error) { + if _, err := ManifestsKey(manifest); err != nil { + return "", err + } + if len(paths) == 0 { + return "", fmt.Errorf("delivery: the module names no path, so it has no named tree") + } + lines := make([]string, 0, len(paths)) + for p, object := range paths { + if p == "" || p != path.Clean(p) || strings.HasPrefix(p, "/") || p == ".." || strings.HasPrefix(p, "../") { + return "", fmt.Errorf("delivery: the named path %q is not a clean path inside the module's directory", p) + } + if object == "" { + return "", fmt.Errorf("delivery: the named path %q has no git object at the commit", p) + } + lines = append(lines, "path "+p+" "+object) + } + sort.Strings(lines) + sum := sha256.Sum256([]byte("manifest " + manifest + "\n" + strings.Join(lines, "\n"))) + return NamedTreePrefix + hex.EncodeToString(sum[:]), nil +} diff --git a/vendor/git.novox.be/novox/mesh-sdk/go/delivery/preview.go b/vendor/git.novox.be/novox/mesh-sdk/go/delivery/preview.go index dd90cb76..36791460 100644 --- a/vendor/git.novox.be/novox/mesh-sdk/go/delivery/preview.go +++ b/vendor/git.novox.be/novox/mesh-sdk/go/delivery/preview.go @@ -98,12 +98,15 @@ func ParsePreviewKey(key string) (repository, head, trunk string, err error) { return repository, parts[1], parts[2], nil } -// PreviewModule is what the preview holds of one module directory: the tree sha of the directory, the digest of its +// PreviewModule is what the preview holds of one module directory: the tree sha of the directory (or its named tree), the digest of its // manifest (the key in the manifests bucket) and the hex sha256 of its lock files by path relative to it. type PreviewModule struct { Tree string `json:"tree"` Manifest string `json:"manifest"` Locks map[string]string `json:"locks,omitempty"` + // Paths are, of a module whose every artifact names its paths, each named path's git object id at the merge; Tree + // is then their NamedTree (novox/hq ADR 0499 clause 2). Absent otherwise. + Paths map[string]string `json:"paths,omitempty"` } // Preview is the value of a preview key: the repository, the head and the trunk commit it was merged onto, the merge @@ -152,6 +155,9 @@ func (p Preview) check() error { return fmt.Errorf("delivery: the lock %q of the preview's module %q has an empty path or hash", path, dir) } } + if err := checkNamed(m.Tree, m.Manifest, m.Paths, fmt.Sprintf("the preview's module %q", dir)); err != nil { + return err + } } return nil } diff --git a/vendor/git.novox.be/novox/mesh-sdk/go/delivery/source.go b/vendor/git.novox.be/novox/mesh-sdk/go/delivery/source.go index 8d85b64a..145b4e25 100644 --- a/vendor/git.novox.be/novox/mesh-sdk/go/delivery/source.go +++ b/vendor/git.novox.be/novox/mesh-sdk/go/delivery/source.go @@ -5,6 +5,7 @@ import ( "encoding/hex" "encoding/json" "fmt" + "path" "reflect" "sort" "strings" @@ -276,6 +277,9 @@ type CandidateModule struct { FP2 string `json:"fp2,omitempty"` None2 string `json:"none2,omitempty"` Manifest string `json:"manifest,omitempty"` + // Paths are, of a module whose every artifact names its paths, each named path's git object id at the preview + // (novox/hq ADR 0499 clause 2), so the builder's fetch request for the candidate names them. Absent otherwise. + Paths map[string]string `json:"paths,omitempty"` } func (m CandidateModule) check() error { @@ -285,6 +289,9 @@ func (m CandidateModule) check() error { case m.FP2 == "" && m.None2 == "": return fmt.Errorf("delivery: a module has neither fp2 nor none2") } + if err := checkPathObjects(m.Paths); err != nil { + return err + } return checkNewForm(m.FP2, m.Manifest) } @@ -356,7 +363,7 @@ func (c Candidate) check() error { if err := m.check(); err != nil { return fmt.Errorf("delivery: the candidate's module %q: %w", module, err) } - if before, ok := seen[module]; ok && before != m { + if before, ok := seen[module]; ok && !reflect.DeepEqual(before, m) { return fmt.Errorf("delivery: two pull requests of the candidate give the module %q different fingerprints", module) } seen[module] = m @@ -370,12 +377,13 @@ func (c Candidate) check() error { // Module is the value of a modules key: the module's source fingerprint, or none and why (a build that resolves // packages when built gets none, said here), and where it was computed: Repository and Commit are the trunk merge, -// Src the source revision then (0 before the gate writes source.main). Exactly one of FP and None is set. +// Src the source revision then (0 before the gate writes source.main). At most one of FP and None is set. // // From R4c the indexer writes the new form beside it (novox/hq ADR 0453 clause 6): FP2, the fingerprint of the new // form (`src2:` and 64 hex digits), or None2, why it has none; one of them whenever either is, never both; and // Manifest, the digest of the module's manifest, the key of the manifests bucket. A value written before has none of -// the three, and a reader of today's shape ignores them. +// the three, and a reader of today's shape ignores them. From R4e a value may carry the new form alone (ADR 0453 clause +// 6; issue 1851): one of FP, None, FP2 and None2 is set, and the old form stays readable. type Module struct { V int `json:"v"` FP string `json:"fp,omitempty"` @@ -398,6 +406,11 @@ type Module struct { // builds the fingerprint only with these. Empty when FP2 is empty or the module has no toolchain; a value written // before has none (issue 1615). Toolchains map[string]ToolchainUsed `json:"toolchains,omitempty"` + // ArtifactPaths are, of each artifact that names the paths it is built from, those paths as its manifest names + // them, by artifact (novox/hq ADR 0499 clause 1): the indexer fills them from each catalogue.Artifact.Paths it reads, + // so the checker tells whether a change touches an artifact's build source without reading a manifest (the lead's + // decision L8 of R4e). Absent for an artifact that names none, and in a value written before. + ArtifactPaths map[string][]string `json:"paths,omitempty"` } // RootPath is Module.Path of a module whose manifest is its repository's root. @@ -411,6 +424,39 @@ type KeptInputs2 struct { Builder string `json:"builder,omitempty"` Contexts map[string]string `json:"contexts,omitempty"` Locks map[string]string `json:"locks,omitempty"` + // Paths are, of a module whose every artifact names its paths, each named path's git object id at the merge, by + // path relative to the module's directory; Tree is then their NamedTree with the module's manifest digest, so a + // computation from kept inputs gives the merge's value (novox/hq ADR 0499 clause 2). Absent for every other module, + // whose Tree is its directory's tree, and in a value written before. + Paths map[string]string `json:"paths,omitempty"` +} + +// checkPathObjects refuses a named path that is not clean and relative to the module's directory, or names no object. +func checkPathObjects(paths map[string]string) error { + for p, object := range paths { + if p == "" || p != path.Clean(p) || strings.HasPrefix(p, "/") || p == ".." || strings.HasPrefix(p, "../") { + return fmt.Errorf("delivery: the named path %q is not a clean path inside the module's directory", p) + } + if object == "" { + return fmt.Errorf("delivery: the named path %q names no git object", p) + } + } + return nil +} + +// checkNamed refuses kept named paths that do not compute to the kept tree with the module's manifest digest. +func checkNamed(tree, manifest string, paths map[string]string, of string) error { + if len(paths) == 0 { + return nil + } + named, err := NamedTree(manifest, paths) + if err != nil { + return fmt.Errorf("delivery: %s: %w", of, err) + } + if named != tree { + return fmt.Errorf("delivery: %s keeps the tree %q, and its named paths give the named tree %q", of, tree, named) + } + return nil } // With is the full inputs of module m: the kept ones and the given toolchains. @@ -423,8 +469,8 @@ func (m Module) check() error { switch { case m.FP != "" && m.None != "": return fmt.Errorf("delivery: a module has both a fingerprint and a reason for none") - case m.FP == "" && m.None == "": - return fmt.Errorf("delivery: a module has neither a fingerprint nor a reason for none") + case m.FP == "" && m.None == "" && m.FP2 == "" && m.None2 == "": + return fmt.Errorf("delivery: a module has neither a fingerprint nor a reason for none, of either form") case m.FP != "" && !strings.HasPrefix(m.FP, SourcePrefix): return fmt.Errorf("delivery: the module's fingerprint %q does not start with %s", m.FP, SourcePrefix) case m.FP2 != "" && m.None2 != "": @@ -438,6 +484,21 @@ func (m Module) check() error { if m.Inputs2 != nil && m.Manifest == "" { return fmt.Errorf("delivery: a module keeps new-form inputs but has no manifest digest") } + for artifact, paths := range m.ArtifactPaths { + if artifact == "" || len(paths) == 0 { + return fmt.Errorf("delivery: the module's named paths of the artifact %q are empty or name no artifact", artifact) + } + for _, p := range paths { + if p == "" || p != path.Clean(p) || strings.HasPrefix(p, "/") || p == ".." || strings.HasPrefix(p, "../") { + return fmt.Errorf("delivery: the artifact %q names the path %q, not a clean path inside the module's directory", artifact, p) + } + } + } + if m.Inputs2 != nil { + if err := checkNamed(m.Inputs2.Tree, m.Manifest, m.Inputs2.Paths, "the module's kept inputs"); err != nil { + return err + } + } return checkNewForm(m.FP2, m.Manifest) } @@ -465,9 +526,13 @@ type Forge struct { // relative to it (novox/hq ADR 0465 clause 1). Older writers send neither. Manifests map[string]string `json:"manifests,omitempty"` Locks map[string]map[string]string `json:"locks,omitempty"` - Act string `json:"act,omitempty"` - Words string `json:"words,omitempty"` - Open []OpenPull `json:"open,omitempty"` + // Paths are, of each module directory whose manifest names paths on every artifact, each named path's git object + // id at the same commit as its manifest, by path relative to the directory (novox/hq ADR 0499 clause 3). The + // indexer names the paths; a message without them gives such a module none. Older writers send none. + Paths map[string]map[string]string `json:"paths,omitempty"` + Act string `json:"act,omitempty"` + Words string `json:"words,omitempty"` + Open []OpenPull `json:"open,omitempty"` } func (f Forge) check() error { @@ -537,8 +602,8 @@ func (f Forge) check() error { if len(f.Dirs) > 0 || f.DirsKnown { return fmt.Errorf("delivery: a refused message carries no module directories") } - if len(f.Manifests) > 0 || len(f.Locks) > 0 { - return fmt.Errorf("delivery: a refused message carries no manifests or lock hashes") + if len(f.Manifests) > 0 || len(f.Locks) > 0 || len(f.Paths) > 0 { + return fmt.Errorf("delivery: a refused message carries no manifests, lock hashes or named paths") } if f.Act == "" { return fmt.Errorf("delivery: a refused message needs the act it answers") @@ -571,6 +636,16 @@ func (f Forge) checkInputs(known func(dir string) bool, of string) error { } } } + for dir, paths := range f.Paths { + if !known(dir) { + return fmt.Errorf("delivery: the named paths of the directory %q are not of %s", dir, of) + } + for path, object := range paths { + if path == "" || object == "" { + return fmt.Errorf("delivery: the named path %q of the directory %q has an empty path or object", path, dir) + } + } + } return nil } @@ -578,7 +653,7 @@ func (f Forge) checkInputs(known func(dir string) bool, of string) error { // an open pull request that is not `/` and a number. func (f Forge) checkOpen() error { if f.Repository != "" || f.Branch != "" || f.Number != 0 || f.Head != "" || f.Action != "" || len(f.Dirs) > 0 || f.DirsKnown || - f.Merge != "" || len(f.Trees) > 0 || len(f.Manifests) > 0 || len(f.Locks) > 0 || f.Root != "" || f.Act != "" || f.Words != "" { + f.Merge != "" || len(f.Trees) > 0 || len(f.Manifests) > 0 || len(f.Locks) > 0 || len(f.Paths) > 0 || f.Root != "" || f.Act != "" || f.Words != "" { return fmt.Errorf("delivery: an open message carries only the open pull requests it names") } for _, p := range f.Open { diff --git a/vendor/git.novox.be/novox/mesh-sdk/go/keeps.go b/vendor/git.novox.be/novox/mesh-sdk/go/keeps.go new file mode 100644 index 00000000..cffdf9c7 --- /dev/null +++ b/vendor/git.novox.be/novox/mesh-sdk/go/keeps.go @@ -0,0 +1,80 @@ +package stdio + +import ( + "encoding/json" + "fmt" + "os" + "strings" + "sync" + + "git.novox.be/novox/mesh-sdk/go/announce" +) + +// What this bundle said it keeps (novox/hq ADR 0495 clause 2; issue 1790). +var ( + keepsMu sync.Mutex + kept map[string]bool // nil until Keeps: no state is refused then + announced []byte // the announcement, published once the bundle serves +) + +// Keeps says on the bus, as this bundle's module, every state it keeps and which of them others may read (novox/hq +// ADR 0495 clause 2): what a reader is granted is what the owner announced (clause 3). Call it before Serve; the +// announcement is published through the runtime once the bundle serves, or at once when it already does. From then +// on a state of the module's own that it did not announce is refused at the call, naming it, so code and announcement +// cannot drift (clause 5). It refuses a state no bucket could be named by and a state given twice. +func Keeps(states ...announce.State) error { + module := os.Getenv("MESH_MODULE") + if module == "" { + module = os.Getenv(ServedAs) + } + data, err := announce.Encode(announce.States{Module: module, Node: os.Getenv("MESH_NODE"), States: states}) + if err != nil { + return err + } + names := map[string]bool{} + for _, s := range states { + names[s.Name] = true + } + keepsMu.Lock() + kept, announced = names, data + keepsMu.Unlock() + currentMu.Lock() + serving := current != nil + currentMu.Unlock() + if serving { + return announce1() + } + return nil +} + +// announce1 publishes the announcement Keeps made, if any. +func announce1() error { + keepsMu.Lock() + data := announced + keepsMu.Unlock() + if data == nil { + return nil + } + return Emit(announce.Event, json.RawMessage(data)) +} + +// refusedUnannounced is the error of a call on a state of the module's own that Keeps did not name, or nil. A read of +// another module's state (`.`) is the grant's business. +func refusedUnannounced(name string) error { + if strings.Contains(name, ".") { + return nil + } + keepsMu.Lock() + defer keepsMu.Unlock() + if kept != nil && !kept[name] { + return fmt.Errorf("the state %q is not among the states this module announced it keeps (stdio.Keeps)", name) + } + return nil +} + +// forgetKeeps is for a test: nothing announced. +func forgetKeeps() { + keepsMu.Lock() + kept, announced = nil, nil + keepsMu.Unlock() +} diff --git a/vendor/git.novox.be/novox/mesh-sdk/go/merge-check.sh b/vendor/git.novox.be/novox/mesh-sdk/go/merge-check.sh index 92e9d9cd..00587326 100644 --- a/vendor/git.novox.be/novox/mesh-sdk/go/merge-check.sh +++ b/vendor/git.novox.be/novox/mesh-sdk/go/merge-check.sh @@ -10,13 +10,17 @@ # 1. the unit level: `go test ./...` untagged, no bus or other process, timed and held to a bound # (unit_bound below); over it the check fails and names the slowest tests (issue 895); # 2. the integration level: the whole suite with `-tags integration`, each test against a bus of its -# own (internal/testbus, which issue 1171 brings with the first test that needs a bus) so nothing is +# own (testbus, which issue 1171 brings with the first test that needs a bus) so nothing is # serialised. With no tagged test yet, the level runs the unit tests again and says their count. # # **A test that starts a bus or another process carries `//go:build integration`** as the first line of # its file; a file that held both kinds is two files (`_integration_test.go` beside `_test.go`). # Each level says its test count and its time. # +# **A test that needs bubblewrap carries `//go:build integration && isolation`** (issue 1309's proof that a run step +# is shut off): this script vets it but does not run it, since the toolchain it runs in has no bubblewrap; the checker +# runs the `isolation` tag on the build nodes, which have it (R4e). +# # No dependency is committed (no vendor directory: .gitignore; the operator, 2026-10-19; issue 1304). Every dependency # is resolved by the version and checksum go.sum names (novox/hq ADR 0405 §5), from its public source: this binding # requires nothing of the mesh's own forge. `go mod verify` fails by module name when a downloaded dependency does not @@ -98,6 +102,7 @@ fi go vet ./... # The files tagged `integration` are vetted too, or nothing would read them. go vet -tags integration ./... +go vet -tags integration,isolation ./... # NOT RACE-CHECKED here: the SDK's suite is small and carries no store or other shared state between # packages, so the race detector is not required of this check (novox/hq issue 1175); a module that needs @@ -117,7 +122,7 @@ if [ "$status" -ne 0 ]; then fi judge_unit_run "$work/unit.out" "$unit_bound" -# The integration level: the whole suite with the tag, each test against its own bus (internal/testbus). +# The integration level: the whole suite with the tag, each test against its own bus (testbus). integration_start=$(date +%s) status=0 go test -tags integration -count=1 -v ./... >"$work/integration.out" 2>&1 || status=$? diff --git a/vendor/git.novox.be/novox/mesh-sdk/go/process/conn.go b/vendor/git.novox.be/novox/mesh-sdk/go/process/conn.go index 58efb40c..ca469221 100644 --- a/vendor/git.novox.be/novox/mesh-sdk/go/process/conn.go +++ b/vendor/git.novox.be/novox/mesh-sdk/go/process/conn.go @@ -24,6 +24,12 @@ type Conn struct { mu sync.Mutex next uint64 pending map[string]map[uint64]context.CancelCauseFunc + + // What Keeps announced (keeps.go): the states' names (nil before Keeps), the announcement, and whether a + // reconnect announces it again. + kept map[string]bool + announced []byte + reannounces bool } // errPermissionRefused marks a context cancelled because the bus refused this account's publish of the diff --git a/vendor/git.novox.be/novox/mesh-sdk/go/process/keeps.go b/vendor/git.novox.be/novox/mesh-sdk/go/process/keeps.go new file mode 100644 index 00000000..22a1df79 --- /dev/null +++ b/vendor/git.novox.be/novox/mesh-sdk/go/process/keeps.go @@ -0,0 +1,71 @@ +package process + +import ( + "encoding/json" + "fmt" + + "github.com/nats-io/nats.go" + + "git.novox.be/novox/mesh-sdk/go/announce" +) + +// Keeps says on the bus, as this process's module, every state it keeps and which of them others may read (novox/hq +// ADR 0495 clause 2; issue 1790), on announce.Subject; what a reader is granted is what the owner announced (clause 3). +// It publishes at once and again after every reconnect, so the controller hears it whenever it missed it. From then on +// a state of the module that it did not announce is refused at the call, naming it, so code and announcement cannot +// drift (clause 5). It refuses a state no bucket could be named by and a state given twice. +func (c *Conn) Keeps(states ...announce.State) error { + if err := c.remember(states); err != nil { + return err + } + c.mu.Lock() + first := !c.reannounces + c.reannounces = true + c.mu.Unlock() + if first { + before := c.nc.Opts.ReconnectedCB + c.nc.SetReconnectHandler(func(nc *nats.Conn) { + if before != nil { + before(nc) + } + _ = c.publishKeeps() + }) + } + return c.publishKeeps() +} + +// remember makes the announcement of states and binds only them from now on. +func (c *Conn) remember(states []announce.State) error { + data, err := announce.Encode(announce.States{Module: c.account.Module, Node: c.account.Node, States: states}) + if err != nil { + return err + } + names := map[string]bool{} + for _, s := range states { + names[s.Name] = true + } + c.mu.Lock() + c.kept, c.announced = names, data + c.mu.Unlock() + return nil +} + +func (c *Conn) publishKeeps() error { + c.mu.Lock() + data := c.announced + c.mu.Unlock() + if err := c.Emit(announce.Event, json.RawMessage(data)); err != nil { + return fmt.Errorf("announcing what %s keeps: %w", c.account.Module, err) + } + return nil +} + +// refusedUnannounced is the error of a bind of a state Keeps did not name, or nil (also before Keeps). +func (c *Conn) refusedUnannounced(name string) error { + c.mu.Lock() + defer c.mu.Unlock() + if c.kept != nil && !c.kept[name] { + return fmt.Errorf("the state %q is not among the states %s announced it keeps (Conn.Keeps)", name, c.account.Module) + } + return nil +} diff --git a/vendor/git.novox.be/novox/mesh-sdk/go/process/state.go b/vendor/git.novox.be/novox/mesh-sdk/go/process/state.go index 43221fac..8e294e31 100644 --- a/vendor/git.novox.be/novox/mesh-sdk/go/process/state.go +++ b/vendor/git.novox.be/novox/mesh-sdk/go/process/state.go @@ -39,6 +39,9 @@ func (s State) bucket() string { return s.Module + "_" + s.Name } func (c *Conn) State(name string) State { bucket := c.account.Module + "_" + name return State{Module: c.account.Module, Name: name, kv: func(ctx context.Context) (jetstream.KeyValue, error) { + if err := c.refusedUnannounced(name); err != nil { + return nil, err + } js, err := jetstream.New(c.nc) if err != nil { return nil, err diff --git a/vendor/git.novox.be/novox/mesh-sdk/go/state.go b/vendor/git.novox.be/novox/mesh-sdk/go/state.go index 70e1b35c..56490b6e 100644 --- a/vendor/git.novox.be/novox/mesh-sdk/go/state.go +++ b/vendor/git.novox.be/novox/mesh-sdk/go/state.go @@ -50,6 +50,9 @@ func State(name string) KeptState { return KeptState{name: name} } const stateTimeout = 30 * time.Second func (s KeptState) ask(verb string, params map[string]any) (json.RawMessage, error) { + if err := refusedUnannounced(s.name); err != nil { + return nil, err + } c, err := live() if err != nil { return nil, err diff --git a/vendor/git.novox.be/novox/mesh-sdk/go/stdio.go b/vendor/git.novox.be/novox/mesh-sdk/go/stdio.go index a61f4dec..a072f40d 100644 --- a/vendor/git.novox.be/novox/mesh-sdk/go/stdio.go +++ b/vendor/git.novox.be/novox/mesh-sdk/go/stdio.go @@ -180,6 +180,9 @@ func ServeOn(name string, tools []Tool, in io.Reader, out io.Writer) error { currentMu.Lock() current = c currentMu.Unlock() + // What the bundle said it keeps is announced once it serves: the runtime answers the publish on this channel, so + // it is asked from beside the loop that reads the answer. + go func() { _ = announce1() }() scanner := bufio.NewScanner(in) scanner.Buffer(make([]byte, 1<<20), 1<<24) for scanner.Scan() { diff --git a/vendor/modules.txt b/vendor/modules.txt index 26ad86b3..7ba7a3f9 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1,6 +1,7 @@ -# git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022152521-dfc2c858d94b +# git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022161855-e76a5a31a07b ## explicit; go 1.26.0 git.novox.be/novox/mesh-sdk/go +git.novox.be/novox/mesh-sdk/go/announce git.novox.be/novox/mesh-sdk/go/asks git.novox.be/novox/mesh-sdk/go/buildstep git.novox.be/novox/mesh-sdk/go/catalogue