delivery: the bus node hands the loaded list's hashes to the vault, sealed, in the bus module's have (ADR 0547 clause 9, issue 1945)

The vault holds no value for the controller's own account, the node accounts and the enrolled devices, so it cannot
hash them; the one part that has their hashes is the node-engine of the bus node, in the list the bus loaded. The have
of the bus module now carries that list's hashes sealed to the vault's key (UserListWritten.Handed), which every reader
of the have bucket sees as a count, a list digest and a key fingerprint, and only the vault opens. HandedAccountsOf
reads them from a composed list, HandAccounts seals, OpenHanded opens and refuses a box sealed to another key.

go/buildstep/askrun.go

@@ -208,6 +208,9 @@ func (s AskSteps) level(ctx context.Context, job delivery.CheckJob, level string
 		_ = json.Unmarshal(r, &t)
 		return t.Passed, nil
 	}
+	// Every test step of the level runs, also after one that did not pass, so one check names every failing test of the
+	// level and not only the first (issue 1921); the level as a whole passes only when each of its steps did.
+	all := true
 	if s.Digest == nil {
 		// A module with no digest (Node) tests each module directory of the tree, those whose tests changed first.
 		for _, priority := range []bool{true, false} {
@@ -219,14 +222,12 @@ func (s AskSteps) level(ctx context.Context, job delivery.CheckJob, level string
 				if err != nil {
 					return false, fmt.Errorf("the %s level of %s: %w", level, dir, err)
 				}
-				if !passed {
-					return false, nil
-				}
+				all = all && passed
 			}
 		}
-		return true, nil
+		return all, nil
 	}
-	todo, all, keepRunner := s.plan(ctx, job, level, f, runner, withRunner, la)
+	todo, whole, keepRunner := s.plan(ctx, job, level, f, runner, withRunner, la)
 	if !keepRunner {
 		runner = "" // an installed module that ignores the runner judges the level as it judges any other (ADR 0535 §2)
 	}
@@ -235,7 +236,7 @@ func (s AskSteps) level(ctx context.Context, job delivery.CheckJob, level string
 		if job.Run != "" {
 			args["run"] = job.Run
 		}
-		if all {
+		if whole {
 			if priority {
 				continue // nothing to put first: the level runs whole after
 			}
@@ -255,11 +256,9 @@ func (s AskSteps) level(ctx context.Context, job delivery.CheckJob, level string
 		if err != nil {
 			return false, fmt.Errorf("the %s level: %w", level, err)
 		}
-		if !passed {
-			return false, nil
-		}
+		all = all && passed // the packages left still run (issue 1921)
 	}
-	return true, nil
+	return all, nil
 }
 
 // release removes an ask's workspace, saying a failure: the module's sweep removes it later.

go/buildstep/askrun_test.go

@@ -109,9 +109,9 @@ func TestRunAskRefusesAnotherToolchain(t *testing.T) {
 	}
 }
 
-// 1887/A3, A5a, issue 1887 design limit 1: a check ask runs the levels of one check after one fetch: it digests the unit
-// level, takes the kept verdicts, tests the packages whose tests changed first, and stops at the first failure, running
-// no later level.
+// 1887/A3, A5a: a check ask runs the levels of one check after one fetch: it digests the unit level, takes the kept
+// verdicts, tests the packages whose tests changed first, then the rest also after a failure (issue 1921), and runs no
+// level after one that did not pass (ADR 0484 clause 13).
 func TestRunAskChecksTheLevelsOfOneCheck(t *testing.T) {
 	r := &recorder{
 		fetch:   map[string]any{"workspace": "/w", "changed": []string{"pkg/b/b_test.go"}},
@@ -126,7 +126,8 @@ func TestRunAskChecksTheLevelsOfOneCheck(t *testing.T) {
 	got := strings.Join(r.calls, "\n")
 	want := `fetch {"base":"` + runCommit + `","commit":"` + runCommit + `","repository":"novox/x"}
 digest {"level":"unit","workspace":"/w"}
-test {"level":"unit","packages":["x/pkg/b"],"workspace":"/w"}`
+test {"level":"unit","packages":["x/pkg/b"],"workspace":"/w"}
+test {"level":"unit","packages":["x/pkg/a"],"workspace":"/w"}`
 	if got != want {
 		t.Fatalf("calls:\n%s\nwant:\n%s", got, want)
 	}
@@ -135,7 +136,7 @@ test {"level":"unit","packages":["x/pkg/b"],"workspace":"/w"}`
 		t.Fatalf("levels %+v, want the unit level alone", a.Levels)
 	}
 	u := a.Levels[0]
-	if u.Level != delivery.LevelUnit || len(u.Kept) != 1 || u.Kept[0].Package != "x/pkg/k" || u.Kept[0].Tests != 7 || len(u.Tests) != 1 || u.Digest == nil {
+	if u.Level != delivery.LevelUnit || len(u.Kept) != 1 || u.Kept[0].Package != "x/pkg/k" || u.Kept[0].Tests != 7 || len(u.Tests) != 2 || u.Digest == nil {
 		t.Fatalf("the unit level %+v", u)
 	}
 }
@@ -203,3 +204,42 @@ func TestRunAskChecksNodeModulesAndConflicts(t *testing.T) {
 		t.Fatalf("a conflict: %v %v", err, r.calls)
 	}
 }
+
+// Issue 1921: a failure in the packages whose tests changed does not stop the level: the other packages still run, so two
+// failing packages, one changed and one not, are two test results of the one level, and the level does not pass. A Node
+// check runs every module directory after one fails too.
+func TestRunAskRunsEveryPackageAfterAFailure(t *testing.T) {
+	r := &recorder{
+		fetch:   map[string]any{"workspace": "/w", "changed": []string{"pkg/b/b_test.go"}},
+		digests: map[string]string{"x/pkg/a": "d-a", "x/pkg/b": "d-b"},
+		tests:   map[string]bool{"x/pkg/a": false, "x/pkg/b": false},
+	}
+	out, err := RunAsk(context.Background(), r.steps(), ask(delivery.CheckJob{Levels: []string{delivery.LevelIntegration, delivery.LevelUnit},
+		Fetch: map[string]any{"repository": "novox/x", "commit": runCommit}}))
+	if err != nil {
+		t.Fatal(err)
+	}
+	a := out.(delivery.CheckAnswer)
+	if len(a.Levels) != 1 || len(a.Levels[0].Tests) != 2 {
+		t.Fatalf("levels %+v, want the one level with both packages' results", a.Levels)
+	}
+	for i, raw := range a.Levels[0].Tests {
+		var res struct{ Passed bool }
+		if err := json.Unmarshal(raw, &res); err != nil || res.Passed {
+			t.Errorf("result %d %s, want a failure", i, raw)
+		}
+	}
+
+	r = &recorder{fetch: map[string]any{"workspace": "/w", "modules": []string{"a", "b"}, "changed": []string{"b/test/x.test.ts"}},
+		tests: map[string]bool{"a": true, "b": false}}
+	s := r.steps()
+	s.Digest = nil
+	out, err = RunAsk(context.Background(), s, ask(delivery.CheckJob{Levels: []string{delivery.LevelUnit, delivery.LevelIntegration},
+		Fetch: map[string]any{"repository": "novox/x", "commit": runCommit}}))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if a := out.(delivery.CheckAnswer); len(r.calls) != 3 || len(a.Levels) != 1 || len(a.Levels[0].Tests) != 2 {
+		t.Fatalf("calls %v, levels %+v: want both directories of the unit level and no integration level", r.calls, a.Levels)
+	}
+}

go/delivery/checker.go

@@ -139,16 +139,25 @@ type FailedFirst struct {
 	Words   string `json:"words,omitempty"`
 }
 
+// MaxFailures is the most entries a verdict's Failures holds (issue 1921): fifty tests named one by one, and an entry
+// after them saying how many more failed.
+const MaxFailures = 50
+
 // Verdict is the value of a verdict key: the group and its heads hash, the trunk head of each repository it was judged
 // against (repository to full sha), the State (running, pass or fail), what failed first, the tests run and kept per
 // level, each part's time, the machines composed and the mesh-sdk version the composition ran with (ADR 0446 clause 5).
 type Verdict struct {
-	V        int                    `json:"v"`
-	Group    string                 `json:"group"`
-	Heads    string                 `json:"heads"`
-	Trunk    map[string]string      `json:"trunk,omitempty"`
-	State    string                 `json:"state"`
-	Failed   FailedFirst            `json:"failed,omitzero"`
+	V      int               `json:"v"`
+	Group  string            `json:"group"`
+	Heads  string            `json:"heads"`
+	Trunk  map[string]string `json:"trunk,omitempty"`
+	State  string            `json:"state"`
+	Failed FailedFirst       `json:"failed,omitzero"`
+	// Failures names every test that failed, in the order judged, each with the first line of its words (issue 1921):
+	// Failed keeps the first one's words whole, and a check that ran every test says all it found here. At most
+	// MaxFailures entries, the last of a longer list saying how many more failed. Absent in a value written before it,
+	// and from a verdict whose failure is no test.
+	Failures []FailedFirst          `json:"failures,omitempty"`
 	Levels   map[string]LevelResult `json:"levels,omitempty"`
 	Took     Took                   `json:"took,omitzero"`
 	Machines []string               `json:"machines,omitempty"`
@@ -222,6 +231,12 @@ func (v Verdict) check() error {
 		}
 	}
 	named := v.Failed.Test != "" || v.Failed.Machine != ""
+	if len(v.Failures) > 0 && v.State != VerdictFail {
+		return fmt.Errorf("delivery: a %s verdict names no failures", v.State)
+	}
+	if len(v.Failures) > MaxFailures+1 {
+		return fmt.Errorf("delivery: the verdict names %d failures, more than %d and the count of the rest", len(v.Failures), MaxFailures)
+	}
 	switch v.State {
 	case VerdictRunning:
 		if v.Failed != (FailedFirst{}) {

go/delivery/checker_test.go

@@ -3,6 +3,7 @@ package delivery
 import (
 	"encoding/json"
 	"errors"
+	"strconv"
 	"strings"
 	"testing"
 	"time"
@@ -600,3 +601,38 @@ func TestPreviewOfAHeadInConflict(t *testing.T) {
 		t.Errorf("the old reader: %+v %v", old, err)
 	}
 }
+
+// Issue 1921: a failed verdict names every failed test in Failures and reads back the same; an older reader's value
+// without it reads too; Failures on a verdict that is no fail, and more than MaxFailures and the count of the rest, are
+// refused.
+func TestVerdictNamesEveryFailure(t *testing.T) {
+	v := passVerdict()
+	v.State = VerdictFail
+	v.Failed = FailedFirst{Test: "integration: x/a.TestA", Words: "a_test.go:9: want 1, got 2\nmore"}
+	v.Failures = []FailedFirst{{Test: "integration: x/a.TestA", Words: "a_test.go:9: want 1, got 2"}, {Test: "integration: x/b.TestB", Words: "b_test.go:3: boom"}}
+	data, err := EncodeVerdict(v)
+	if err != nil {
+		t.Fatal(err)
+	}
+	got, err := DecodeVerdict(data)
+	v.V = Version
+	if err != nil || !SameVerdict(got, v) || !strings.Contains(string(data), `"failures":[{"test":"integration: x/a.TestA"`) {
+		t.Fatalf("%s: %+v %v", data, got, err)
+	}
+	old := strings.Replace(string(data), string(data[strings.Index(string(data), `,"failures":`):strings.Index(string(data), `,"levels":`)]), "", 1)
+	if got, err := DecodeVerdict([]byte(old)); err != nil || len(got.Failures) != 0 || got.Failed != v.Failed {
+		t.Fatalf("a value without failures: %+v %v (%s)", got, err, old)
+	}
+	p := passVerdict()
+	p.Failures = v.Failures
+	if _, err := EncodeVerdict(p); err == nil {
+		t.Error("a pass with failures was written")
+	}
+	v.Failures = make([]FailedFirst, MaxFailures+2)
+	for i := range v.Failures {
+		v.Failures[i] = FailedFirst{Test: "unit: x.T" + strconv.Itoa(i)}
+	}
+	if _, err := EncodeVerdict(v); err == nil {
+		t.Error("a verdict naming more than MaxFailures and the rest was written")
+	}
+}

go/delivery/delivery.go

@@ -34,9 +34,12 @@
 // **The release 4 layouts** (issues 1356 to 1359; ADR 0453 clauses 4, 6 and 16): the new fingerprint form (fingerprint2.go),
 // the indexer's manifests by digest and the new fields of its modules and candidates (manifests.go, source.go), the
 // builder's build keys (builder.go) and the checker's verdicts and kept verdicts (checker.go). A manifests value is the
-// one exception to the size above: a manifest is up to about 21 KB, whole, byte for byte.
+// one exception to the size above: a manifest is up to about 21 KB, whole, byte for byte. The bus module's have on the
+// node holding mesh-broker is the other: its user list's handing carries a sealed box of every loaded user's hash, some
+// kilobytes (handed.go).
 //
-// Standard library only, and this module's own asks and declaration packages (a composition holds declaration resources).
+// Standard library only, and this module's own asks, declaration and seal packages (a composition holds declaration
+// resources; the adoption handing is a sealed box, handed.go).
 package delivery
 
 import (
@@ -352,6 +355,9 @@ type UserListWritten struct {
 	Users  int       `json:"users"`
 	At     time.Time `json:"at"`
 	Error  string    `json:"error,omitempty"`
+	// Handed is the hashes of the users of the list on the machine, sealed to the vault, for its adoption of the
+	// accounts it holds no value for (ADR 0547 clause 9, handed.go); nil while the vault's key is not known.
+	Handed *AccountsHanded `json:"handed,omitempty"`
 }
 
 // HaveNode is a node's own facts, written under HaveNodeKey. Each field is the node-engine's own JSON, carried whole:
@@ -502,7 +508,8 @@ func SameHave(a, b Have) bool {
 	}
 	x, y := *ua, *ub
 	x.At, y.At = time.Time{}, time.Time{}
-	return x == y && ua.At.Equal(ub.At)
+	x.Handed, y.Handed = nil, nil
+	return x == y && ua.At.Equal(ub.At) && sameHanded(ua.Handed, ub.Handed)
 }
 
 // SameHaveNode says whether two node facts are the same value, comparing each carried JSON without its spacing.

go/delivery/delivery_test.go

@@ -7,6 +7,8 @@ import (
 	"strings"
 	"testing"
 	"time"
+
+	"git.novox.be/novox/mesh-sdk/go/seal"
 )
 
 // 1131/AC1: the have key forms and parses back; bad names are refused, naming them.
@@ -167,6 +169,47 @@ func TestRoundTrip(t *testing.T) {
 		if !reflect.DeepEqual(out, in) || !out.UserList.At.Equal(at) {
 			t.Fatalf("got %+v want %+v", out, in)
 		}
+		// And with the hashes handed to the vault (ADR 0547 clause 9): sealed to its key, opened by it alone, every
+		// listed user with a bcrypt hash once, and refused to another key.
+		hash := "$2a$11$" + strings.Repeat("a", 53)
+		list := []byte("{ user: \"controller\", password: \"" + hash + "\", permissions: {\n" +
+			"{ user: \"node.ace\", password: \"" + hash + "\", permissions: {\n" +
+			"{ user: \"controller\", password: \"" + hash + "\" }\n{ user: \"nkey\", password: \"plain\" }\n")
+		accounts := HandedAccountsOf(list)
+		if len(accounts) != 2 || accounts[0].User != "controller" || accounts[1].User != "node.ace" || accounts[1].Hash != hash {
+			t.Fatalf("handed accounts: %+v", accounts)
+		}
+		vault, err := seal.GenerateKeyPair()
+		if err != nil {
+			t.Fatal(err)
+		}
+		other, _ := seal.GenerateKeyPair()
+		handed, err := HandAccounts(accounts, vault.PublicKey, "sha256:ab", at)
+		if err != nil {
+			t.Fatal(err)
+		}
+		in.UserList.Handed = &handed
+		b, err = EncodeHave(in)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if strings.Contains(string(b), "node.ace") || strings.Contains(string(b), hash) {
+			t.Fatalf("a user or a hash is in the clear: %s", b)
+		}
+		out, err = DecodeHave(b)
+		if err != nil || !SameHave(out, in) {
+			t.Fatalf("handed round trip: %v %+v", err, out.UserList)
+		}
+		opened, err := OpenHanded(*out.UserList.Handed, vault.PublicKey, vault.PrivateKey)
+		if err != nil || !reflect.DeepEqual(opened, accounts) {
+			t.Fatalf("opened %+v, %v", opened, err)
+		}
+		if _, err := OpenHanded(*out.UserList.Handed, other.PublicKey, other.PrivateKey); err == nil {
+			t.Fatal("a handing opened for another key")
+		}
+		if _, err := HandAccounts(accounts, "", "sha256:ab", at); err == nil {
+			t.Fatal("handed with no vault key")
+		}
 	})
 	t.Run("have node", func(t *testing.T) {
 		in := HaveNode{Windows: json.RawMessage(`{"a":1}`), Busy: json.RawMessage(`true`)}
@@ -306,6 +349,20 @@ func TestSame(t *testing.T) {
 	if SameHave(ua, ub) {
 		t.Error("different user lists same")
 	}
+	// A handing is compared by what it says: one said and one not differ, as do two to other keys.
+	ub.UserList.Revision = 1
+	ua.UserList.Handed = &AccountsHanded{From: "sha256:a", To: "sha256:k", Users: 2, At: at}
+	if SameHave(ua, ub) {
+		t.Error("a handing said and one not are the same")
+	}
+	ub.UserList.Handed = &AccountsHanded{From: "sha256:a", To: "sha256:k", Users: 2, At: at.In(time.Local)}
+	if !SameHave(ua, ub) {
+		t.Error("equal handings differ")
+	}
+	ub.UserList.Handed.To = "sha256:j"
+	if SameHave(ua, ub) {
+		t.Error("handings to other keys same")
+	}
 	n := HaveNode{V: 1, Busy: json.RawMessage(`true`)}
 	if !SameHaveNode(n, HaveNode{V: 1, Busy: json.RawMessage(`true`)}) || SameHaveNode(n, HaveNode{V: 1}) {
 		t.Error("have node")

go/delivery/handed.go

@@ -0,0 +1,125 @@
+package delivery
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"regexp"
+	"strings"
+	"time"
+
+	"git.novox.be/novox/mesh-sdk/go/seal"
+)
+
+// The adoption handing of novox/hq ADR 0547 clause 9 (issue 1945): the vault adopts, once and before `take-over`, the
+// bcrypt hashes of the bus accounts it holds no value for (the controller's own, the node accounts, the enrolled
+// devices) from the list the bus has loaded. The node-engine of the node holding mesh-broker is the one part that has
+// that list, so it hands the hashes in the bus module's have, beside what it says of the list it wrote: sealed to the
+// vault's own key, which it learns from the vault's answer to `node-key`, so that of every reader of the have bucket
+// the vault alone can open them. The have says in the clear only how many users it hands, from which list and to which
+// key; the users' names stay sealed, as the list itself is (ADR 0547 clause 7: it names every account).
+//
+// It hands every user of the loaded list; the vault takes from it only the users it holds no value for, and never
+// replaces a hash it made from a value it holds, so the node-engine never needs to read the vault's state (which the
+// grants part alone is to read, clause 9).
+
+// AccountsHanded is the hashes of the loaded list's users, sealed to the vault, in UserListWritten.
+type AccountsHanded struct {
+	// From is the digest of the list on the machine the hashes are taken from, `sha256:<hex>` (UserListWritten.Digest).
+	From string `json:"from"`
+	// To is the fingerprint of the vault's key the box is sealed to (KeyFingerprint).
+	To string `json:"to"`
+	// Users is how many users the box holds.
+	Users int `json:"users"`
+	// Box is the JSON of []HandedAccount, sealed to the vault's key (ADR 0206).
+	Box seal.Box  `json:"box"`
+	At  time.Time `json:"at"`
+}
+
+// HandedAccount is one user of an AccountsHanded box, opened: the bus user and the bcrypt hash the loaded list
+// carries for it.
+type HandedAccount struct {
+	User string `json:"user"`
+	Hash string `json:"hash"`
+}
+
+// KeyFingerprint is a public key's fingerprint as a handing names it: `sha256:<hex>` of its PEM, without the space
+// around it.
+func KeyFingerprint(publicPEM string) string {
+	sum := sha256.Sum256([]byte(strings.TrimSpace(publicPEM)))
+	return "sha256:" + hex.EncodeToString(sum[:])
+}
+
+// listedAccount is one user of a bus user list as the composer writes it (buscompose), with its password's hash.
+var listedAccount = regexp.MustCompile(`\{\s*user:\s*"([^"]+)",\s*password:\s*"([^"]+)"`)
+
+// HandedAccountsOf is every user of a bus user list that carries a bcrypt hash, in the list's order. A user listed
+// twice is handed once, as the list first names it.
+func HandedAccountsOf(list []byte) []HandedAccount {
+	var out []HandedAccount
+	seen := map[string]bool{}
+	for _, m := range listedAccount.FindAllSubmatch(list, -1) {
+		user, hash := string(m[1]), string(m[2])
+		if seen[user] || !IsBcrypt(hash) {
+			continue
+		}
+		seen[user] = true
+		out = append(out, HandedAccount{User: user, Hash: hash})
+	}
+	return out
+}
+
+// bcryptHash is the form of a bcrypt hash: version, cost, and 53 characters of salt and digest.
+var bcryptHash = regexp.MustCompile(`^\$2[abxy]?\$[0-9]{2}\$[./A-Za-z0-9]{53}$`)
+
+// IsBcrypt says whether s has the form of a bcrypt hash.
+func IsBcrypt(s string) bool { return bcryptHash.MatchString(s) }
+
+// HandAccounts seals the accounts to the vault's public key (SPKI PEM) as the handing of the list whose digest is from.
+func HandAccounts(accounts []HandedAccount, vaultPEM, from string, at time.Time) (AccountsHanded, error) {
+	if strings.TrimSpace(vaultPEM) == "" {
+		return AccountsHanded{}, errors.New("delivery: no vault key to seal the handed accounts to")
+	}
+	body, err := json.Marshal(accounts)
+	if err != nil {
+		return AccountsHanded{}, err
+	}
+	box, err := seal.Seal(string(body), vaultPEM)
+	if err != nil {
+		return AccountsHanded{}, fmt.Errorf("delivery: the handed accounts could not be sealed to the vault's key: %w", err)
+	}
+	return AccountsHanded{From: from, To: KeyFingerprint(vaultPEM), Users: len(accounts), Box: box, At: at.UTC()}, nil
+}
+
+// OpenHanded opens a handing with the vault's private key (PEM), refusing one sealed to another key than publicPEM's
+// and a box that does not hold as many users as it says.
+func OpenHanded(h AccountsHanded, publicPEM, privatePEM string) ([]HandedAccount, error) {
+	if h.To != KeyFingerprint(publicPEM) {
+		return nil, fmt.Errorf("delivery: the handing is sealed to the key %s, not to this one (%s)", h.To,
+			KeyFingerprint(publicPEM))
+	}
+	plain, err := seal.Open(h.Box, privatePEM)
+	if err != nil {
+		return nil, fmt.Errorf("delivery: the handing does not open: %w", err)
+	}
+	var out []HandedAccount
+	if err := json.Unmarshal([]byte(plain), &out); err != nil {
+		return nil, fmt.Errorf("delivery: the handing holds no accounts: %w", err)
+	}
+	if len(out) != h.Users {
+		return nil, fmt.Errorf("delivery: the handing says %d users and holds %d", h.Users, len(out))
+	}
+	return out, nil
+}
+
+// sameHanded says whether two handings say the same: the same list, key, count and box, whatever zone their time is in.
+func sameHanded(a, b *AccountsHanded) bool {
+	if a == nil || b == nil {
+		return a == b
+	}
+	x, y := *a, *b
+	x.At, y.At = time.Time{}, time.Time{}
+	return x == y && a.At.Equal(b.At)
+}