diff --git a/go/buscompose/derived.go b/go/buscompose/derived.go index 1885a933..b61d3336 100644 --- a/go/buscompose/derived.go +++ b/go/buscompose/derived.go @@ -38,6 +38,10 @@ type Consumer struct { // and a merge that came back rebuilt what it had just built, five times over on 2026-09-30. // With one outstanding, the server holds the rest, and the heartbeat is keeping the message. MaxAckPending int + // InProgressSeconds is how often a holder working a delivery says it is still in progress, so its ack wait does + // not pass while it works (design 47 H2 §777; H-A-340 (1)): the holder's to keep, never sent to the server. Zero + // says nothing. + InProgressSeconds int // FromNow makes a consumer that does not exist yet start at the stream's end rather than its // beginning (novox/hq issue 248). For a consumer that reacts to announcements — a merge, a build's // outcome — on a stream that keeps a week of them: the server's default, everything the stream diff --git a/go/buscompose/names_test.go b/go/buscompose/names_test.go index ba22fde5..75510257 100644 --- a/go/buscompose/names_test.go +++ b/go/buscompose/names_test.go @@ -49,4 +49,23 @@ func TestTheBusNamesAgree(t *testing.T) { t.Errorf("the shared events stream keeps %d s, a seat's definition says %d days", st.MaxAge, seatdef.SharedEventsMaxAgeDays) } } + // Design 47 H2 §777, H-A-362: a seat's work queue and its one worker are derived from its accepted kind's settings; + // absent settings take the defaults, two kinds naming one setting differently are refused, no accept is no queue. + st, w, ok, err := SeatQueue("mesh-builds", []seatdef.Message{{Name: "step", MaxAge: "24h", DuplicatesWindow: "2m", + AckWait: "60s", InProgress: "20s", MaxMsgs: 10000, MaxAckPending: 64, MaxDeliveries: 3}}) + if err != nil || !ok || st.Name != "SEAT_MESH_BUILDS" || st.Retention != RetentionWorkQueue || st.MaxAge != 86400 || + st.MaxMsgs != 10000 || st.DuplicatesSeconds != 120 || st.Subjects[0] != "mesh.seat.mesh-builds.accept.>" || + w.Name != "SEAT_MESH_BUILDS_worker" || w.Stream != st.Name || w.AckWaitSeconds != 60 || w.MaxDeliver != 3 || + w.MaxAckPending != 64 || w.InProgressSeconds != 20 { + t.Errorf("derived %+v and %+v, %v", st, w, err) + } + if st, w, _, _ := SeatQueue("mesh-wants", []seatdef.Message{{Name: "step"}}); st.MaxAge != 7*86400 || w.AckWaitSeconds != 60 || w.MaxDeliver != 5 { + t.Errorf("the defaults: %+v, %+v", st, w) + } + if _, _, _, err := SeatQueue("s", []seatdef.Message{{Name: "a", AckWait: "60s"}, {Name: "b", AckWait: "2m"}}); err == nil { + t.Error("two ack waits for one worker were derived") + } + if _, _, ok, _ := SeatQueue("s", nil); ok { + t.Error("a seat that accepts nothing has a queue") + } } diff --git a/go/buscompose/seatqueue.go b/go/buscompose/seatqueue.go new file mode 100644 index 00000000..be4fdcc7 --- /dev/null +++ b/go/buscompose/seatqueue.go @@ -0,0 +1,97 @@ +package buscompose + +import ( + "fmt" + "strings" + "time" + + "git.novox.be/novox/mesh-sdk/go/seatdef" +) + +// The work queue's settings where a seat's accepted kinds name none: a week of asks, a delivery given up after five +// handings, a minute before an unacknowledged one is handed again. +const ( + SeatQueueDefaultMaxAge = 7 * 24 * time.Hour + SeatQueueDefaultAckWait = time.Minute + SeatQueueDefaultDelivers = 5 +) + +// SeatQueueStreamName is the stream holding a seat's inbound work, named after the seat, never its holder. +func SeatQueueStreamName(seat string) string { return "SEAT_" + seatUpper(seat) } + +// SeatQueueWorkerName is the one worker a seat's holders share on its work queue. +func SeatQueueWorkerName(seat string) string { return "SEAT_" + seatUpper(seat) + "_worker" } + +func seatUpper(seat string) string { + return strings.ToUpper(strings.NewReplacer("-", "_", ".", "_").Replace(seat)) +} + +// SeatQueue derives a seat's work-queue stream and its one shared worker from its accepted kinds' work-queue settings +// (design 47 H2 §777; H-A-350 DMC-1, H-A-362), the one derivation every part takes: max-age, max-msgs and +// duplicates-window bound the stream; ack-wait, max-deliveries, max-ack-pending and in-progress the worker. A setting +// named by two accepted kinds with two values is refused: one stream and one worker carry one value. A seat that +// accepts nothing has no queue (ok false). +func SeatQueue(seat string, accepts []seatdef.Message) (stream Stream, worker Consumer, ok bool, err error) { + if len(accepts) == 0 { + return Stream{}, Consumer{}, false, nil + } + durations := map[string]string{} + counts := map[string]int{} + for _, a := range accepts { + if ps := a.QueueProblems("accepts " + a.Name); len(ps) > 0 { + return Stream{}, Consumer{}, false, fmt.Errorf("the seat %s's work queue: %w", seat, ps) + } + for name, v := range map[string]string{"max-age": a.MaxAge, "duplicates-window": a.DuplicatesWindow, + "ack-wait": a.AckWait, "in-progress": a.InProgress} { + if v == "" { + continue + } + if was, seen := durations[name]; seen && was != v { + return Stream{}, Consumer{}, false, fmt.Errorf("the seat %s's work queue: %s is %s and %s in two accepted kinds", seat, name, was, v) + } + durations[name] = v + } + for name, v := range map[string]int{"max-msgs": a.MaxMsgs, "max-ack-pending": a.MaxAckPending, "max-deliveries": a.MaxDeliveries} { + if v == 0 { + continue + } + if was, seen := counts[name]; seen && was != v { + return Stream{}, Consumer{}, false, fmt.Errorf("the seat %s's work queue: %s is %d and %d in two accepted kinds", seat, name, was, v) + } + counts[name] = v + } + } + seconds := func(name string, def time.Duration) int { + if v, set := durations[name]; set { + d, _ := time.ParseDuration(v) // QueueProblems read it + return int(d / time.Second) + } + return int(def / time.Second) + } + delivers := SeatQueueDefaultDelivers + if v, set := counts["max-deliveries"]; set { + delivers = v + } + subject := "mesh.seat." + seat + ".accept.>" + stream = Stream{ + Name: SeatQueueStreamName(seat), + Subjects: []string{subject}, + Retention: RetentionWorkQueue, + MaxAge: seconds("max-age", SeatQueueDefaultMaxAge), + MaxMsgs: counts["max-msgs"], + DuplicatesSeconds: seconds("duplicates-window", 0), + Why: fmt.Sprintf("work submitted to the %s seat; one holder consumes it, and it queues while nobody does", seat), + } + worker = Consumer{ + Name: SeatQueueWorkerName(seat), + Stream: stream.Name, + Filters: []string{subject}, + AckWaitSeconds: seconds("ack-wait", SeatQueueDefaultAckWait), + MaxDeliver: delivers, + MaxAckPending: counts["max-ack-pending"], + InProgressSeconds: seconds("in-progress", 0), + Why: fmt.Sprintf("every holder of %s pulls its asks from this one worker and acknowledges after the work is "+ + "done, so a crash mid-work redelivers rather than loses", seat), + } + return stream, worker, true, nil +} diff --git a/go/buscompose/streams.go b/go/buscompose/streams.go index 637793d6..627ec0a2 100644 --- a/go/buscompose/streams.go +++ b/go/buscompose/streams.go @@ -32,6 +32,8 @@ type Stream struct { // another's events out of a shared stream. Verified: with a cap of 3, ten messages on one // subject and one on another leave four in the stream, not three. MaxMsgsPerSubject int + // MaxMsgs caps the stream's messages, zero for unbounded: a seat's work queue's max-msgs (design 47 H2 §777). + MaxMsgs int // Why is carried into the assertion so an operator reading the server's own state finds the // reason there, rather than only in a repository they may not have. Why string diff --git a/go/busretry/busretry.go b/go/busretry/busretry.go index e4814ab2..52daccef 100644 --- a/go/busretry/busretry.go +++ b/go/busretry/busretry.go @@ -15,10 +15,15 @@ package busretry import ( "context" + "encoding/json" "errors" "fmt" + "math/rand/v2" "os" + "sort" "strings" + "sync" + "sync/atomic" "time" "github.com/nats-io/nats.go" @@ -36,10 +41,286 @@ var ( // in a process shares, so a test sets them only through it, as t.Cleanup(busretry.Waits(...)): a test that set them // and left them set shortened the next test's bound to milliseconds, and a loaded check machine overran it (the check // of release sdk1, 2026-11-02). A bound a test waits on to succeed is seconds: no slow machine reaches it. +// It sets the kept-call pace (KeptPace) to first and most too, so a test of a kept call waits milliseconds. func Waits(bound, first, most time.Duration) (restore func()) { was := [3]time.Duration{Bound, FirstWait, MostWait} + wasPace := keptPace.Swap(&Pace{First: first, Max: most}) Bound, FirstWait, MostWait = bound, first, most - return func() { Bound, FirstWait, MostWait = was[0], was[1], was[2] } + return func() { Bound, FirstWait, MostWait = was[0], was[1], was[2]; keptPace.Store(wasPace) } +} + +// Pace is the kept-call form's pace (design 47 H2 §775): on a transport fault the call is made again after First, +// the wait doubling to at most Max, each wait drawn within 20 % of its value so many callers do not call in step. +type Pace struct{ First, Max time.Duration } + +// The settings every module using the kept-call form declares, with DefaultPace's values as their defaults (§775). +const ( + PaceFirstSetting = "kept-call-pace-first" + PaceMaxSetting = "kept-call-pace-max" +) + +// DefaultPace is the pace a module's settings name when they name none (§775): 1 s doubling to 30 s, so a holder +// between two lives (a restart takes 20 to 50 s) is met within one minute without a call a second. +var DefaultPace = Pace{First: time.Second, Max: 30 * time.Second} + +// keptPace is the pace ReadPace read, nil until it read one. It is set once and read by every kept call at once, so it +// is held atomically: a kept call on one goroutine never races a Connect or a Serve on another (§782). +var keptPace atomic.Pointer[Pace] + +// KeptPace is the pace of every kept call this process makes: DefaultPace until the process reads its settings +// (ReadPace, at process.Connect and at a bundle's stdio.Serve). Safe for concurrent use. +func KeptPace() Pace { + if p := keptPace.Load(); p != nil { + return *p + } + return DefaultPace +} + +// ReadPace sets KeptPace from the module's settings PaceFirstSetting and PaceMaxSetting, read through read (the one +// settings reader, stdio.SettingDuration, which answers its default for a setting not composed), so a module's +// declared pace paces its kept calls (§775). A setting that is no duration, or a most below the first, is an error +// naming it, and KeptPace stays. The pace is read once, at the process's start: a later call reading the same pace +// changes nothing, and one reading another pace is refused, naming both, and KeptPace stays (§782). Safe for +// concurrent use. +func ReadPace(read func(key string, def time.Duration) (time.Duration, error)) error { + held := KeptPace() + first, err := read(PaceFirstSetting, held.First) + if err != nil { + return err + } + most, err := read(PaceMaxSetting, held.Max) + if err != nil { + return err + } + if most < first { + return fmt.Errorf("the setting %s (%s) is below %s (%s)", PaceMaxSetting, most, PaceFirstSetting, first) + } + got := Pace{First: first, Max: most} + if keptPace.CompareAndSwap(nil, &got) { + return nil + } + if held = KeptPace(); got != held { + return fmt.Errorf("the kept-call pace is read once, at the start: it is %s to %s, and the settings now say %s to %s", + held.First, held.Max, got.First, got.Max) + } + return nil +} + +// The clock and the sleep the paced form waits on; a test of this package replaces both, so nothing sleeps. +var ( + now = time.Now + sleep = time.Sleep +) + +// jittered is w drawn within 20 % of its value. +func jittered(w time.Duration) time.Duration { + return time.Duration(float64(w) * (0.8 + 0.4*rand.Float64())) +} + +// Kept names one kept call of the paced form (design 47 H2 §775): the seat and the verb it asks (named in the journal +// line), the ask's id (named among the asks waiting), the caller's bound (zero is Bound) and the pace (zero is +// KeptPace). Open, when given, makes it asked again only while its key is open (DoKept). Logf is the journal; nil +// writes to standard error. +type Kept struct { + Seat, Verb, Ask string + Bound time.Duration + Pace Pace + Open *Open + Logf func(format string, args ...any) +} + +// Try is one call of a kept call that met a transport fault: when it was made and the words it met. +type Try struct { + At time.Time + Words string +} + +// Unanswered is a kept call's end past its caller's bound (§775): every call's time and words, so the caller's +// could-not-run or could-not-be-performed names them. It wraps the last fault, so Undelivered and TimedOut read it. +type Unanswered struct { + Seat, Verb, Ask string + Bound time.Duration + Tries []Try + Last error +} + +func (u *Unanswered) Error() string { + var b strings.Builder + fmt.Fprintf(&b, "%s.%s", u.Seat, u.Verb) + if u.Ask != "" { + fmt.Fprintf(&b, " (ask %s)", u.Ask) + } + fmt.Fprintf(&b, " unanswered within its bound of %s, called %d times:", u.Bound, len(u.Tries)) + for i, t := range u.Tries { + if i > 0 { + b.WriteString(";") + } + fmt.Fprintf(&b, " %s %s", t.At.UTC().Format("15:04:05.000"), t.Words) + } + return b.String() +} + +func (u *Unanswered) Unwrap() error { return u.Last } + +// Fault says err is a transport fault of a kept call (§775): a request that reached no one, a call's own deadline +// passing, or a holder's answer of the kind away (§776); a callee's refusal never is. +func Fault(err error) bool { + var away *Away + return TimedOut(err) || Undelivered(err) || errors.As(err, &away) +} + +// ErrorKindAway is the seat error kind a holder answers when a part it depends on is away (design 47 H2 §776): its +// answer's `error-kind`, beside `part` (what did not answer) and `words`. A caller reads it as a transport fault. +const ErrorKindAway = "away" + +// Away is a holder's answer of the kind away, read as an error: the part that did not answer and the holder's words. +type Away struct { + Part, Words string +} + +func (a *Away) Error() string { + return fmt.Sprintf("away: %s does not answer: %s", a.Part, a.Words) +} + +// AwayAnswer is the seat error answer of the kind away a holder gives (§776): `error` and `error-kind` as each seat's +// error answer has them, with `part` and `words`. +func AwayAnswer(part, words string) map[string]string { + return map[string]string{"error": (&Away{Part: part, Words: words}).Error(), "error-kind": ErrorKindAway, + "part": part, "words": words} +} + +// AwayIn is the *Away a verb's answer says when its `error-kind` is away, else nil: an answer that is no object, or +// of another kind, is the caller's to read. +func AwayIn(answer json.RawMessage) *Away { + var a struct { + Error string `json:"error"` + Kind string `json:"error-kind"` + Part string `json:"part"` + Words string `json:"words"` + } + if json.Unmarshal(answer, &a) != nil || a.Kind != ErrorKindAway { + return nil + } + if a.Words == "" { + a.Words = a.Error + } + return &Away{Part: a.Part, Words: a.Words} +} + +// DoPaced makes a kept call through try (design 47 H2 §775): on a transport fault (Fault) it calls again at k's pace, +// each wait jittered, until the caller's bound, writing one journal line per seat at the first fault and then at most +// once a minute (the seat, the verb, the asks waiting and the count of calls). Past the bound it returns the last +// answer with an *Unanswered naming every call; a refusal of what was asked returns at once. +func DoPaced[T any](k Kept, try func() (T, error)) (T, error) { + bound, pace := k.Bound, k.Pace + if bound <= 0 { + bound = Bound + } + if pace.First <= 0 || pace.Max <= 0 { + held := KeptPace() + if pace.First <= 0 { + pace.First = held.First + } + if pace.Max <= 0 { + pace.Max = held.Max + } + } + logf := k.Logf + if logf == nil { + logf = func(f string, a ...any) { fmt.Fprintf(os.Stderr, "[mesh-sdk] "+f+"\n", a...) } + } + start, wait := now(), pace.First + var tries []Try + waiting := false + defer func() { + if waiting { + journal.done(k.Seat, k.Ask) + } + }() + for { + at := now() + v, err := try() + if !Fault(err) || (k.Open != nil && !k.Open.stillOpen()) { + return v, err + } + tries = append(tries, Try{At: at, Words: err.Error()}) + if !waiting { + journal.wait(k.Seat, k.Ask) + waiting = true + } + journal.fault(k.Seat, k.Verb, logf) + left := bound - now().Sub(start) + if left <= 0 { + return v, &Unanswered{Seat: k.Seat, Verb: k.Verb, Ask: k.Ask, Bound: bound, Tries: tries, Last: err} + } + sleep(min(jittered(wait), left)) + wait = min(wait*2, pace.Max) + } +} + +// journal holds, per seat, the asks of this process waiting on it through a fault, the calls made since its last +// line and when that line was written: one line per seat at the first fault, then at most once a minute (§775). +var journal = &faults{seats: map[string]*seatFaults{}} + +type faults struct { + mu sync.Mutex + seats map[string]*seatFaults +} + +type seatFaults struct { + asks map[string]int + calls int + last time.Time +} + +func (f *faults) of(seat string) *seatFaults { + s := f.seats[seat] + if s == nil { + s = &seatFaults{asks: map[string]int{}} + f.seats[seat] = s + } + return s +} + +func (f *faults) wait(seat, ask string) { + f.mu.Lock() + defer f.mu.Unlock() + f.of(seat).asks[ask]++ +} + +func (f *faults) done(seat, ask string) { + f.mu.Lock() + defer f.mu.Unlock() + s := f.of(seat) + if s.asks[ask]--; s.asks[ask] <= 0 { + delete(s.asks, ask) + } +} + +func (f *faults) fault(seat, verb string, logf func(string, ...any)) { + f.mu.Lock() + s := f.of(seat) + s.calls++ + t := now() + if !s.last.IsZero() && t.Sub(s.last) < time.Minute { + f.mu.Unlock() + return + } + s.last = t + var asks []string + n := 0 + for a, c := range s.asks { + n += c + if a != "" { + asks = append(asks, a) + } + } + sort.Strings(asks) + calls := s.calls + s.calls = 0 + f.mu.Unlock() + logf("kept call: the seat %s does not answer (%s): %d asks waiting %v, %d calls since the last line; called "+ + "again at the kept-call pace until each caller's bound", seat, verb, n, asks, calls) } // Undelivered says err means the request reached no one: no responders, JetStream not enabled (for the account), or diff --git a/go/busretry/busretry_test.go b/go/busretry/busretry_test.go index 53883f6f..d805b115 100644 --- a/go/busretry/busretry_test.go +++ b/go/busretry/busretry_test.go @@ -2,9 +2,11 @@ package busretry import ( "context" + "encoding/json" "errors" "fmt" "strings" + "sync" "testing" "time" @@ -110,8 +112,157 @@ func subAKeptCallIsAskedAgainOnlyWhileItsKeyIsOpen(t *testing.T) { } } +// fakeClock replaces the paced form's clock and sleep for one test: a sleep moves the clock, nothing sleeps. +func fakeClock(t *testing.T) (slept *[]time.Duration) { + at := time.Date(2026, 11, 21, 22, 0, 0, 0, time.UTC) + waits := []time.Duration{} + wasNow, wasSleep, wasJournal := now, sleep, journal + now = func() time.Time { return at } + sleep = func(d time.Duration) { waits = append(waits, d); at = at.Add(d) } + journal = &faults{seats: map[string]*seatFaults{}} + t.Cleanup(func() { now, sleep, journal = wasNow, wasSleep, wasJournal }) + return &waits +} + +// Design 47 H2 §775: the kept call is paced from 1 s doubling to 30 s, each wait within 20 %; met by no responders twice +// and then answered it returns the answer and writes one journal line; past its caller's bound it returns every try; +// a refusal returns at once; the journal says a seat at most once a minute. +func subAKeptCallIsPaced(t *testing.T) { + t.Run("no responders twice, then answered: the answer and one line", func(t *testing.T) { + slept := fakeClock(t) + tries, logs := 0, []string{} + v, err := DoPaced(Kept{Seat: "mesh-builds", Verb: "build", Ask: "a1", Bound: time.Minute, + Logf: func(f string, a ...any) { logs = append(logs, fmt.Sprintf(f, a...)) }}, func() (int, error) { + if tries++; tries <= 2 { + return 0, nats.ErrNoResponders + } + return 7, nil + }) + if v != 7 || err != nil || tries != 3 || len(logs) != 1 || !strings.Contains(logs[0], "mesh-builds") || + !strings.Contains(logs[0], "build") || !strings.Contains(logs[0], "[a1]") { + t.Fatalf("%d, %v after %d tries, logs %q", v, err, tries, logs) + } + for i, base := range []time.Duration{time.Second, 2 * time.Second} { + if w := (*slept)[i]; w < base*8/10 || w > base*12/10 { + t.Errorf("wait %d is %s, not within 20 %% of %s", i, w, base) + } + } + }) + t.Run("past its bound: every try", func(t *testing.T) { + slept := fakeClock(t) + tries := 0 + _, err := DoPaced(Kept{Seat: "mesh-wants", Verb: "compose-candidate", Ask: "a2", Bound: 3 * time.Minute, + Logf: func(string, ...any) {}}, func() (int, error) { tries++; return 0, context.DeadlineExceeded }) + var u *Unanswered + if !errors.As(err, &u) || len(u.Tries) != tries || tries < 5 || !TimedOut(err) || + !strings.Contains(err.Error(), "context deadline exceeded") { + t.Fatalf("%d tries: %v", tries, err) + } + for _, w := range *slept { + if w > 30*time.Second*12/10 { + t.Errorf("a wait of %s passes the pace's most", w) + } + } + }) + t.Run("a refusal returns at once", func(t *testing.T) { + slept := fakeClock(t) + tries := 0 + _, err := DoPaced(Kept{Seat: "git", Verb: "merge"}, func() (int, error) { tries++; return 0, &Answered{Words: "refused"} }) + if tries != 1 || len(*slept) != 0 || err == nil { + t.Fatalf("%d tries, %v", tries, err) + } + }) + t.Run("away is a transport fault; its answer reads back", func(t *testing.T) { + raw, _ := json.Marshal(AwayAnswer("the forge's store", "401 and the version read failed")) + a := AwayIn(raw) + if a == nil || a.Part != "the forge's store" || a.Words != "401 and the version read failed" || !Fault(a) || + !strings.Contains(string(raw), `"error-kind":"away"`) { + t.Fatalf("%s read as %+v", raw, a) + } + if AwayIn([]byte(`{"error":"no","error-kind":"refused"}`)) != nil || AwayIn([]byte(`[1]`)) != nil || Fault(&Answered{Words: "away"}) { + t.Fatal("a refusal reads as away") + } + if a := AwayIn([]byte(`{"error":"the store is down","error-kind":"away"}`)); a == nil || a.Words != "the store is down" { + t.Fatalf("an away answer with no words reads %+v", a) + } + }) + t.Run("the pace is the module's settings", func(t *testing.T) { + was := keptPace.Load() + t.Cleanup(func() { keptPace.Store(was) }) + reader := func(set map[string]time.Duration) func(string, time.Duration) (time.Duration, error) { + return func(key string, def time.Duration) (time.Duration, error) { + if d, ok := set[key]; ok { + if d < 0 { + return 0, fmt.Errorf("the setting %s is no duration", key) + } + return d, nil + } + return def, nil + } + } + two := map[string]time.Duration{PaceFirstSetting: 2 * time.Second, PaceMaxSetting: time.Minute} + for _, c := range []struct { + name string + held *Pace // the pace read before, nil none + set map[string]time.Duration + bad string + want Pace + }{ + {name: "none set", want: DefaultPace}, + {name: "both set", set: two, want: Pace{First: 2 * time.Second, Max: time.Minute}}, + {name: "most below first", set: map[string]time.Duration{PaceFirstSetting: time.Minute, PaceMaxSetting: time.Second}, bad: "is below", want: DefaultPace}, + {name: "no duration", set: map[string]time.Duration{PaceFirstSetting: -1}, bad: "no duration", want: DefaultPace}, + // Design 47 H2 §782: the pace is read once; a later read of the same pace changes nothing, of another is refused. + {name: "read again, the same", held: &Pace{First: 2 * time.Second, Max: time.Minute}, set: two, want: Pace{First: 2 * time.Second, Max: time.Minute}}, + {name: "read again, none set", held: &Pace{First: 2 * time.Second, Max: time.Minute}, want: Pace{First: 2 * time.Second, Max: time.Minute}}, + {name: "read again, another", held: &DefaultPace, set: two, bad: "read once", want: DefaultPace}, + } { + keptPace.Store(c.held) + err := ReadPace(reader(c.set)) + if (err != nil) != (c.bad != "") || (err != nil && !strings.Contains(err.Error(), c.bad)) || KeptPace() != c.want { + t.Errorf("%s: pace %+v, %v", c.name, KeptPace(), err) + } + } + // Design 47 H2 §782, under -race: a Connect or a Serve reading the pace while kept calls read it is no data race. + keptPace.Store(nil) + var wg sync.WaitGroup + for i := range 8 { + wg.Add(2) + go func() { defer wg.Done(); _ = ReadPace(reader(two)) }() + go func() { + defer wg.Done() + if _, err := DoPaced(Kept{Seat: "s", Verb: fmt.Sprint(i), Logf: func(string, ...any) {}}, func() (int, error) { return 1, nil }); err != nil { + t.Error(err) + } + _ = KeptPace() + }() + } + wg.Wait() + if KeptPace() != (Pace{First: 2 * time.Second, Max: time.Minute}) { + t.Errorf("after concurrent reads the pace is %+v", KeptPace()) + } + }) + t.Run("one line per seat a minute", func(t *testing.T) { + fakeClock(t) + logs := 0 + k := Kept{Seat: "mesh-builds", Verb: "build", Bound: 10 * time.Second, Logf: func(string, ...any) { logs++ }} + fail := func() (int, error) { return 0, nats.ErrNoResponders } + _, _ = DoPaced(k, fail) + _, _ = DoPaced(k, fail) + if logs != 1 { + t.Fatalf("%d lines within a minute", logs) + } + k.Bound = 2 * time.Minute + _, _ = DoPaced(k, fail) + if logs < 2 { + t.Fatalf("%d lines past a minute", logs) + } + }) +} + // TestABusThatAnswersNothingIsWaitedOut runs the tests of this package that were merged to keep the module within its test budget (design 47 G §419), each as a subtest. func TestABusThatAnswersNothingIsWaitedOut(t *testing.T) { t.Run("ABusThatAnswersNothingIsWaitedOut", subABusThatAnswersNothingIsWaitedOut) t.Run("AKeptCallIsAskedAgainOnlyWhileItsKeyIsOpen", subAKeptCallIsAskedAgainOnlyWhileItsKeyIsOpen) + t.Run("AKeptCallIsPaced", subAKeptCallIsPaced) } diff --git a/go/busstate/busstate.go b/go/busstate/busstate.go index 6abf2e8e..71ff9b6c 100644 --- a/go/busstate/busstate.go +++ b/go/busstate/busstate.go @@ -78,6 +78,13 @@ func (o Objects) Delete(name string) error { return err } +// Purge removes what the deleted object name left behind in the store, its chunks, through the runtime's delete, which +// purges them again when asked again (H-A-366): a bundle never opens a connection of its own for it. +func (o Objects) Purge(name string) error { + _, err := call[buscalls.Done](buscalls.ObjectsDelete, buscalls.ObjectsNameRequest{Objects: o.local, Name: name}) + return err +} + // Fill is how much the store holds against its cap. func (o Objects) Fill() (buscalls.Fill, error) { return call[buscalls.Fill](buscalls.ObjectsInfo, buscalls.ObjectsRequest{Objects: o.local}) diff --git a/go/busstate/busstate_test.go b/go/busstate/busstate_test.go index 549d9725..e557892b 100644 --- a/go/busstate/busstate_test.go +++ b/go/busstate/busstate_test.go @@ -35,6 +35,7 @@ func subABundleKeepsObjectsThroughTheRuntime(t *testing.T) { list, err := in.List() results <- result{"list", err, len(list)} results <- result{"delete", in.Delete("t1.part0"), nil} + results <- result{"purge", in.Purge("t1.part0"), nil} // H-A-366 fill, err := in.Fill() results <- result{"fill", err, fill.MaxBytes} _, err = stdio.BusState("mesh/state.get", nil) @@ -81,8 +82,8 @@ func subABundleKeepsObjectsThroughTheRuntime(t *testing.T) { asked[method] = p send(map[string]any{"jsonrpc": "2.0", "id": m["id"], "result": answers[method]}) } - want := map[string]any{"put": "SHA-256=x", "get": "hello", "list": 1, "delete": nil, "fill": uint64(20 << 30)} - for range 5 { + want := map[string]any{"put": "SHA-256=x", "get": "hello", "list": 1, "delete": nil, "purge": nil, "fill": uint64(20 << 30)} + for range 6 { r := <-results if r.err != nil || r.got != want[r.what] { t.Errorf("%s answered %v, %v; want %v", r.what, r.got, r.err, want[r.what]) @@ -96,6 +97,9 @@ func subABundleKeepsObjectsThroughTheRuntime(t *testing.T) { t.Errorf("%s named the store %v, not its local name", method, p["objects"]) } } + if p := asked[buscalls.ObjectsDelete]; p["name"] != "t1.part0" { + t.Errorf("delete and purge asked %v", p) + } if p := asked[buscalls.ObjectsPut]; p["name"] != "t1.part0" || p["data"] != "aGVsbG8=" || p["description"] != "a part" { t.Errorf("put asked %v", p) } diff --git a/go/caller.go b/go/caller.go index d503c586..94de4fa1 100644 --- a/go/caller.go +++ b/go/caller.go @@ -28,11 +28,19 @@ var busUser = regexp.MustCompile(`^[A-Za-z0-9_-]+(\.[A-Za-z0-9_-]+)*$`) // buscalls.MetaCall, written here too so this package imports nothing of buscalls. const MetaCall = "mesh/call" +// AskHeader is the bus header a kept call carries its ask's id in (design 47 H2 §775; H-A-351): a holder in a process +// reads it with process.AskOf, and the tool runner copies it into a `tools/call`'s `_meta` under MetaAsk, read with +// AskIDOf. +const AskHeader = "x-ask" + +// MetaAsk is where in a `tools/call`'s `_meta` the runtime names the ask id the call carried in AskHeader (H-A-351). +const MetaAsk = "mesh/ask" + // callID is the shape of a call's id: one subject-safe token, as the runtime makes it. Anything else reads as none. var callID = regexp.MustCompile(`^[A-Za-z0-9_-]{1,128}$`) // inFlight is what the runtime said of one call while its tool runs. -type inFlight struct{ caller, call string } +type inFlight struct{ caller, call, ask string } // calls holds the caller and the id of every call while its tool runs, by the arguments map that call was handed. var calls sync.Map // unsafe.Pointer → inFlight @@ -75,6 +83,27 @@ func callIn(meta map[string]any) string { return "" } +// AskIDOf is the ask id the call that was handed args carried (design 47 H2 §775; H-A-351): what the kept call set in +// AskHeader and the runtime named under MetaAsk, by which a holder takes the ask once. Empty when the call +// carried none. Valid while the tool runs. +func AskIDOf(args map[string]any) string { + if args == nil { + return "" + } + if c, ok := calls.Load(argsKey(args)); ok { + return c.(inFlight).ask + } + return "" +} + +// askIn is the ask id a `tools/call`'s `_meta` names, or empty: one subject-safe token, as a call's id is. +func askIn(meta map[string]any) string { + if s, ok := meta[MetaAsk].(string); ok && callID.MatchString(s) { + return s + } + return "" +} + // callerIn is the caller a `tools/call`'s `_meta` names, or Unproven. func callerIn(meta map[string]any) string { if s, ok := meta[MetaCaller].(string); ok && busUser.MatchString(s) { @@ -84,9 +113,9 @@ func callerIn(meta map[string]any) string { } // runAs runs a tool with the caller and the call's id held for its arguments for as long as it runs. -func runAs(t Tool, caller, call string, args map[string]any) (any, error) { +func runAs(t Tool, f inFlight, args map[string]any) (any, error) { key := argsKey(args) - calls.Store(key, inFlight{caller, call}) + calls.Store(key, f) defer calls.Delete(key) return t.Run(args) } diff --git a/go/caller_test.go b/go/caller_test.go index c816933c..0a724cac 100644 --- a/go/caller_test.go +++ b/go/caller_test.go @@ -4,6 +4,7 @@ import ( "bufio" "encoding/json" "io" + "strings" "testing" "time" @@ -13,26 +14,26 @@ import ( // callWith serves one tool that answers the caller CallerOf reads from its arguments, sends it one tools/call // with the params given, and returns the caller it answered. func callWith(t *testing.T, params map[string]any) string { - caller, _ := callAnswering(t, params) + caller, _, _ := callAnswering(t, params) return caller } // callAnswering is callWith answering the call's id CallIDOf read too. -func callAnswering(t *testing.T, params map[string]any) (string, string) { +func callAnswering(t *testing.T, params map[string]any) (string, string, string) { servesAlone(t) t.Helper() inR, inW := io.Pipe() outR, outW := io.Pipe() go func() { _ = ServeOn("tracker", []Tool{{Name: "who", Run: func(args map[string]any) (any, error) { - return map[string]any{"caller": CallerOf(args), "call": CallIDOf(args)}, nil + return map[string]any{"caller": CallerOf(args), "call": CallIDOf(args), "ask": AskIDOf(args)}, nil }}}, inR, outW) }() defer inW.Close() b, _ := json.Marshal(map[string]any{"jsonrpc": "2.0", "id": 1, "method": "tools/call", "params": params}) _, _ = inW.Write(append(b, '\n')) lines := bufio.NewScanner(outR) - done := make(chan [2]string, 1) + done := make(chan [3]string, 1) go func() { if lines.Scan() { var m struct { @@ -46,19 +47,20 @@ func callAnswering(t *testing.T, params map[string]any) (string, string) { var answer struct { Caller string `json:"caller"` Call string `json:"call"` + Ask string `json:"ask"` } if len(m.Result.Content) > 0 { _ = json.Unmarshal([]byte(m.Result.Content[0].Text), &answer) } - done <- [2]string{answer.Caller, answer.Call} + done <- [3]string{answer.Caller, answer.Call, answer.Ask} } }() select { case got := <-done: - return got[0], got[1] + return got[0], got[1], got[2] case <-time.After(3 * time.Second): t.Fatal("the bundle did not answer the call") - return "", "" + return "", "", "" } } @@ -83,13 +85,46 @@ func TestAToolReadsTheCallerTheRuntimeNamed(t *testing.T) { {map[string]any{}, map[string]any{MetaCall: "a b"}, ""}, {map[string]any{}, map[string]any{MetaCall: 7}, ""}, } { - if _, call := callAnswering(t, map[string]any{"name": "who", "arguments": c.args, "_meta": c.meta}); call != c.want { + if _, call, _ := callAnswering(t, map[string]any{"name": "who", "arguments": c.args, "_meta": c.meta}); call != c.want { t.Errorf("args %v, meta %v: the tool read call %q, want %q", c.args, c.meta, call, c.want) } } + // Design 47 H2 §775, H-A-351: the ask id a kept call carried is the runtime's to name under mesh/ask, never the + // arguments'. + for _, c := range []struct { + args, meta map[string]any + want string + }{ + {map[string]any{}, map[string]any{MetaAsk: "build-7Hq2"}, "build-7Hq2"}, + {map[string]any{MetaAsk: "forged"}, map[string]any{}, ""}, + {map[string]any{}, map[string]any{MetaAsk: "a.b"}, ""}, + } { + if _, _, ask := callAnswering(t, map[string]any{"name": "who", "arguments": c.args, "_meta": c.meta}); ask != c.want { + t.Errorf("args %v, meta %v: the tool read ask %q, want %q", c.args, c.meta, ask, c.want) + } + } if CallIDOf(map[string]any{"x": 1}) != "" || CallIDOf(nil) != "" { t.Error("arguments no call was handed have a call id") } + // Design 47 H2 §781, H-A-370: a process and a bundle read a setting by its key from the word the node-engine composes. + t.Setenv("MESH_CONFIG_ASK_UNANSWERED_BOUND", "10m") + t.Setenv("MESH_CONFIG_KEPT_CALL_PACE_FIRST", "soon") + if v, ok := Setting("ask-unanswered-bound"); !ok || v != "10m" { + t.Errorf("the setting read %q, %v", v, ok) + } + if d, err := SettingDuration("ask-unanswered-bound", time.Minute); err != nil || d != 10*time.Minute { + t.Errorf("the duration read %s, %v", d, err) + } + if d, err := SettingDuration("compose-ask-bound", 10*time.Second); err != nil || d != 10*time.Second { + t.Errorf("an absent setting read %s, %v, not its default", d, err) + } + if _, err := SettingDuration("kept-call-pace-first", time.Second); err == nil || !strings.Contains(err.Error(), "kept-call-pace-first") { + t.Errorf("a value that is no duration read as %v", err) + } + // §775: a bundle reads its kept-call pace from the same settings at its start, and refuses one that is no duration. + if err := ServeOn("b", nil, strings.NewReader(""), io.Discard); err == nil || !strings.Contains(err.Error(), "kept-call-pace-first") { + t.Errorf("a bundle served with a pace that is no duration: %v", err) + } if MetaCall != buscalls.MetaCall { t.Errorf("the call id's place is %q here and %q in buscalls", MetaCall, buscalls.MetaCall) } diff --git a/go/catalogue/caps.go b/go/catalogue/caps.go new file mode 100644 index 00000000..8c0faa7b --- /dev/null +++ b/go/catalogue/caps.go @@ -0,0 +1,58 @@ +package catalogue + +import ( + "encoding/json" + "strconv" +) + +// Every kept store of a delivery module names its cap (design 47 H2 §812, §814): the manifest validator refuses a +// delivery module's state, log or objects entry with no cap. This is what it reads of a manifest; which module is a +// delivery module (one claiming a seat of the delivery system) and the foundation parts' old sections are its own to +// judge. + +// Uncapped is one state, log or objects entry of a manifest that names no cap. +type Uncapped struct { + // Name is the entry's local name. + Name string `json:"name"` + // Kind is state, log or objects (the template's, or the old section's). + Kind string `json:"kind"` + // OldSection says the entry is in the old section of its kind (`state`, `logs`, `objects`), not an entry of + // `requires`. + OldSection bool `json:"old-section,omitempty"` +} + +// UncappedStores is every state, log and objects entry of m that names no cap, the old sections' (max-mib) and the +// bus-state entries' (the option max-mib) alike, in the manifest's order. A log's default cap is not one it names. +func (m Manifest) UncappedStores() []Uncapped { + var out []Uncapped + for _, s := range m.State { + if s.MaxMiB == 0 { + out = append(out, Uncapped{Name: s.Name, Kind: TemplateState, OldSection: true}) + } + } + for _, l := range m.Logs { + if l.MaxMiB == 0 { + out = append(out, Uncapped{Name: l.Name, Kind: TemplateLog, OldSection: true}) + } + } + for _, o := range m.Objects { + if o.MaxMiB == 0 { + out = append(out, Uncapped{Name: o.Name, Kind: TemplateObjects, OldSection: true}) + } + } + for _, e := range m.BusState { + switch e.Template { + case TemplateState, TemplateLog, TemplateObjects: + if !namesCap(e.Options[OptionCapMiB]) { + out = append(out, Uncapped{Name: e.Name, Kind: e.Template}) + } + } + } + return out +} + +// namesCap is whether an option's value is a cap above zero. +func namesCap(raw json.RawMessage) bool { + n, err := strconv.ParseFloat(string(raw), 64) + return err == nil && n > 0 +} diff --git a/go/catalogue/config_test.go b/go/catalogue/config_test.go index 01e69253..b8825344 100644 --- a/go/catalogue/config_test.go +++ b/go/catalogue/config_test.go @@ -3,6 +3,7 @@ package catalogue import ( "bytes" "encoding/json" + "slices" "strings" "testing" ) @@ -135,10 +136,10 @@ func TestAConfigManifestIsWrittenBackInItsOwnForm(t *testing.T) { // The registry milestone's words round-trip the same (design 32): a config value's old name, an artifact's series // and contracts marker, the series a module accepts of a package; absent, a series is 0.1. - declared := []byte(`{"module":"c","version":"1","config":{"font":{"type":"string","renamed-from":"font-name"}}, + declared := []byte(`{"module":"c","version":"1","check-needs":["node-build-node"],"config":{"font":{"type":"string","renamed-from":"font-name"}}, "packages":{"mesh-sdk":"0.2"}, "build":{"artifacts":[{"name":"go","kind":"package","language":"go","paths":["go"],"series":"0.2", - "generated-from":"seats"},{"name":"lib","kind":"package","language":"typescript"}]}}`) + "generated-from":"seats","check-needs":["node-build-node"]},{"name":"lib","kind":"package","language":"typescript"}]}}`) m, err = decodeForTest(declared) if err != nil { t.Fatal(err) @@ -146,16 +147,21 @@ func TestAConfigManifestIsWrittenBackInItsOwnForm(t *testing.T) { if m.Settings["font"].RenamedFrom != "font-name" || m.AcceptedSeries("mesh-sdk") != "0.2" || m.AcceptedSeries("mesh-contracts") != DefaultSeries || m.Build.Artifacts[0].SeriesOf() != "0.2" || m.AcceptedSeries("mesh-sdk", "0.1") != "0.2" || m.AcceptedSeries("mesh-contracts", "0.10", "x", "0.3", "1.0") != "0.3" || - m.Build.Artifacts[0].GeneratedFrom != "seats" || m.Build.Artifacts[1].SeriesOf() != "0.1" { + m.Build.Artifacts[0].GeneratedFrom != "seats" || m.Build.Artifacts[1].SeriesOf() != "0.1" || + !slices.Equal(m.Build.Artifacts[0].CheckNeeds, []string{"node-build-node"}) || m.Build.Artifacts[1].CheckNeeds != nil || + !slices.Equal(m.CheckNeeds, []string{"node-build-node"}) { t.Fatalf("read %+v", m) } raw, _ = json.Marshal(m) for _, said := range []string{`"renamed-from":"font-name"`, `"packages":{"mesh-sdk":"0.2"}`, `"series":"0.2"`, - `"generated-from":"seats"`} { + `"generated-from":"seats"`, `"check-needs":["node-build-node"]`} { if !strings.Contains(string(raw), said) { t.Errorf("written back without %s: %s", said, raw) } } + if strings.Count(string(raw), `"check-needs":["node-build-node"]`) != 2 { // H-A-453 (2): the module's own and the artifact's + t.Errorf("the module's check-needs is not written back: %s", raw) + } if strings.Count(string(raw), `"series"`) != 1 { t.Errorf("a series nobody said is written: %s", raw) } diff --git a/go/catalogue/manifest.go b/go/catalogue/manifest.go index e4444846..9c4d5855 100644 --- a/go/catalogue/manifest.go +++ b/go/catalogue/manifest.go @@ -452,6 +452,15 @@ func (m Manifest) AnsweredByHelperFor(provision string) bool { return false } +// The classes a module is (Manifest.Class; design 47 H2 §810, H-A-394). +const ( + ClassCore = "core" + ClassLeaf = "leaf" +) + +// Leaf is whether m is a leaf: its class says so, or it names none. +func (m Manifest) Leaf() bool { return m.Class == "" || m.Class == ClassLeaf } + // Manifest is everything a module says about itself. type Manifest struct { Module string `json:"module"` @@ -737,6 +746,11 @@ type Manifest struct { // wrong the moment anybody edits anything. Build *Build `json:"build,omitempty"` + // CheckNeeds are the seats the integration tests of a package outside every artifact's `from` need the node to hold, by + // name, in an artifact's form (Artifact.CheckNeeds): the Go build module reads them for the packages no artifact's + // `from` holds (design 47 H2 H-A-453 (2)). Empty needs nothing. + CheckNeeds []string `json:"check-needs,omitempty"` + // Templates is the module's directory its template files are kept in, and its build packs into its templates // archive: files/ when it says none (DefaultTemplatesDir; novox/hq ADR 0349 §2, issue 731). A module whose files/ // is already an archive unpacked onto the machine names another, so its templates neither land beside what it @@ -823,6 +837,18 @@ type Manifest struct { // no module): the bus and the provisioner and its client say it. CoreCredentials bool `json:"core-credentials,omitempty"` + // DeclarationReads is the declaration Version (declaration.Version) the node-engine this module is reads, and + // DeclarationWrites the one the composer this module is writes (design 47 H2 §813 (12), H-A-388): the edge the delivery + // plan derives orders the writer of a Version after every node's engine that reads it. Said by the module; zero for + // one that is neither. + DeclarationReads int `json:"declaration-reads,omitempty"` + DeclarationWrites int `json:"declaration-writes,omitempty"` + + // Class is ClassCore for a module whose delivery carries a contract of the mesh (one that claims a seat) and + // ClassLeaf for one that claims none (design 47 H2 §810, H-A-394): the gate reads it from the module's catalog + // entry. Absent is a leaf (Leaf); which manifest must name it is the manifest check's. + Class string `json:"class,omitempty"` + // SecretsOwner is who the files holding this module's secrets belong to on the machine — // `uid:gid`, or a name — when its process is not root. // @@ -1263,6 +1289,11 @@ type Artifact struct { // module carrying it holds contracts only and claims no seat, which the judge refuses naming the seat; the build // modules' test step regenerates the package and fails on a file out of step. GeneratedFrom string `json:"generated-from,omitempty"` + + // CheckNeeds are the seats a Go artifact's integration tests need the node to hold, by name (`node-build-node`), as the + // marker file check-needs in a package's directory named them, which this field replaces (design 47 H2 §800; H-A-350 + // DMC-6, H-A-399 (3)); the Go build module's digest and the delivery SDK's step run read it here. Empty needs nothing. + CheckNeeds []string `json:"check-needs,omitempty"` } // AcceptedSeries is the series of the package of that name this module accepts: the one Packages names, or else the diff --git a/go/catalogue/manifest_reads_baseline.txt b/go/catalogue/manifest_reads_baseline.txt index e1f0f367..68c7239f 100644 --- a/go/catalogue/manifest_reads_baseline.txt +++ b/go/catalogue/manifest_reads_baseline.txt @@ -303,6 +303,7 @@ word:slug word:state word:state.history word:state.key-ttl +word:state.max-mib word:state.name word:state.per-machine word:state.ttl-seconds diff --git a/go/catalogue/operators_machine_test.go b/go/catalogue/operators_machine_test.go index 979bad03..4b9e50a3 100644 --- a/go/catalogue/operators_machine_test.go +++ b/go/catalogue/operators_machine_test.go @@ -7,7 +7,8 @@ import ( ) // The service manager is a seat of the mesh's own with the unit verbs as its contract (novox/hq -// ADR 0177): every verb described, with a schema, taking a scope. +// ADR 0177): every verb described, with a schema, taking a scope but kernel-log, the kernel's messages, which no user +// unit has (issue 1975). func TestTheServiceManagerSeatServesTheUnitVerbs(t *testing.T) { seat, ok := SeatNamed("node-service-manager") if !ok { @@ -16,7 +17,7 @@ func TestTheServiceManagerSeatServesTheUnitVerbs(t *testing.T) { if seat.Scope != ScopeNode { t.Fatalf("the service manager is a role each machine has once, and the seat is %s-scoped", seat.Scope) } - want := []string{"units", "status", "start", "stop", "restart", "enable", "disable", "journal", "failed", "reset-failed", "wanted-by", "unlink-dangling"} + want := []string{"units", "status", "start", "stop", "restart", "enable", "disable", "journal", "kernel-log", "failed", "reset-failed", "wanted-by", "unlink-dangling"} var got []string for _, v := range seat.Serves { got = append(got, v.Name) @@ -24,7 +25,7 @@ func TestTheServiceManagerSeatServesTheUnitVerbs(t *testing.T) { t.Fatalf("%s is promised without a description or a schema", v.Name) } props, _ := v.Input["properties"].(map[string]any) - if _, has := props["scope"]; !has { + if _, has := props["scope"]; !has && v.Name != "kernel-log" { t.Fatalf("%s takes no scope, and a user unit could not be asked for", v.Name) } } @@ -41,7 +42,7 @@ func TestFailedIsAnOptionalVerbOfTheServiceManager(t *testing.T) { holder := func(serves ...string) Manifest { return Manifest{Module: "systemd", Version: "1", Claims: []Claim{{Name: ServiceManagerSeat, Scope: ScopeNode, Serves: serves}}} } - eight := []string{"units", "status", "start", "stop", "restart", "enable", "disable", "journal", + eight := []string{"units", "status", "start", "stop", "restart", "enable", "disable", "journal", "kernel-log", "reset-failed", "wanted-by", "unlink-dangling"} if err := CanHold(holder(eight...), seat); err != nil { t.Fatalf("a holder without failed is refused: %v", err) @@ -69,7 +70,7 @@ func TestFailedIsAnOptionalVerbOfTheServiceManager(t *testing.T) { // back from the store's row they stay required. func TestResetFailedWantedByAndUnlinkDanglingAreRequiredVerbsOfTheServiceManager(t *testing.T) { defer UseSeats(DefaultSeats()) - nine := []string{"units", "status", "start", "stop", "restart", "enable", "disable", "journal", "failed"} + nine := []string{"units", "status", "start", "stop", "restart", "enable", "disable", "journal", "kernel-log", "failed"} all := append(slices.Clone(nine), "reset-failed", "wanted-by", "unlink-dangling") holder := func(serves ...string) Manifest { return Manifest{Module: "systemd", Version: "1", Claims: []Claim{{Name: ServiceManagerSeat, Scope: ScopeNode, Serves: serves}}} diff --git a/go/catalogue/own_manifest_test.go b/go/catalogue/own_manifest_test.go index 5c86bd0b..b08babff 100644 --- a/go/catalogue/own_manifest_test.go +++ b/go/catalogue/own_manifest_test.go @@ -67,12 +67,18 @@ func TestOwnManifestUsesNoWordTheRunningControllerRefuses(t *testing.T) { } for _, want := range []string{"word:tool-runner", "word:core-credentials", "word:bus-state", "word:bus-state.storage", // The reference form a registry seat answers (ADR 0605 §9), and the seats the registry milestone adds. - ReadForm + ReferenceFormSeat, ReadForm + ReferenceFormSeatReach, "seat:go-package-registry", "seat:container-image-registry", "seat:oci-artifact-registry"} { + ReadForm + ReferenceFormSeat, ReadForm + ReferenceFormSeatReach, "seat:go-package-registry", "seat:container-image-registry", "seat:oci-artifact-registry", + // H-A-388: the declaration Version a node-engine reads and a composer writes, the delivery plan's edge. + "word:declaration-reads", "word:declaration-writes", + // H-A-394: a module's class, core or leaf. + "word:class"} { if !slices.Contains(added, want) { t.Errorf("%s is in the baseline, or not read: added since it are %v", want, added) } } - for _, want := range []string{"word:claims.serves", "word:settings.default", "form:config", "form:setting", "seat:mesh-catalog"} { + for _, want := range []string{"word:claims.serves", "word:settings.default", "form:config", "form:setting", "seat:mesh-catalog", + // H-A-383: every node-engine reads a state's max-mib, so a manifest naming it needs no newer engine. + "word:state.max-mib"} { if !slices.Contains(baseline, want) { t.Errorf("the baseline lacks %s", want) } diff --git a/go/catalogue/state_test.go b/go/catalogue/state_test.go index 8532ef3a..78d6671e 100644 --- a/go/catalogue/state_test.go +++ b/go/catalogue/state_test.go @@ -2,6 +2,7 @@ package catalogue import ( "encoding/json" + "reflect" "strings" "testing" @@ -28,6 +29,35 @@ func TestAManifestMaySayWhatStateItKeepsAndReads(t *testing.T) { if string(out) != `["servers",{"name":"seen","history":5,"ttl-seconds":3600}]` { t.Fatalf("written back as %s", out) } + // Design 47 H2 §812, §814: the validator reads every state, log and objects entry of a manifest that names no cap, in + // the old sections and in requires alike; a capped entry, and an entry of another template, is none of them. + t.Run("each kept store that names no cap", func(t *testing.T) { + m, err := decodeForTest([]byte(`{"module":"x","version":"1", + "state":["seen",{"name":"held","max-mib":8}], + "logs":[{"name":"acts"},{"name":"kept","max-mib":64}], + "objects":[{"name":"outputs","max-mib":256}], + "requires":[{"name":"performed","storage":"key-value","template":"state","options":{"history":1}}, + {"name":"taken","storage":"key-value","template":"state","options":{"max-mib":8,"key-ttl":true}}, + {"name":"archives","storage":"object-store","template":"objects"}, + {"name":"finding-events","storage":"stream","template":"events"}]}`)) + if err != nil { + t.Fatal(err) + } + want := []Uncapped{{Name: "seen", Kind: "state", OldSection: true}, {Name: "acts", Kind: "log", OldSection: true}, + {Name: "performed", Kind: "state"}, {Name: "archives", Kind: "objects"}} + if got := m.UncappedStores(); !reflect.DeepEqual(got, want) { + t.Fatalf("the uncapped stores are %+v, want %+v", got, want) + } + }) + // Design 47 H2 §810 (H-A-394): a manifest says its class, core or leaf; one that names none is a leaf. + t.Run("its class, none a leaf", func(t *testing.T) { + for raw, leaf := range map[string]bool{`"class":"core",`: false, `"class":"leaf",`: true, ``: true} { + m, err := decodeForTest([]byte(`{` + raw + `"module":"x","version":"1"}`)) + if err != nil || m.Leaf() != leaf || m.UnknownField() != "" { + t.Errorf("%s read as class %q, leaf %v (%v, unknown %q), want leaf %v", raw, m.Class, m.Leaf(), err, m.UnknownField(), leaf) + } + } + }) } // 1154/AC1: a state entry may say key-ttl; it is read, written back, and a bare name or an entry without it reads as before. diff --git a/go/declaration/declaration.go b/go/declaration/declaration.go index f7cb8832..aac5e67b 100644 --- a/go/declaration/declaration.go +++ b/go/declaration/declaration.go @@ -26,9 +26,10 @@ import ( "git.novox.be/novox/mesh-sdk/go/seal" ) -// Version is the vocabulary this host speaks. A declaration naming any other version is -// refused: an older host handed a newer vocabulary must not quietly do half of it. -const Version = 1 +// Version is the vocabulary this host speaks, the declaration's form. A declaration naming a higher version is +// refused: an older host handed a newer vocabulary must not quietly do half of it. A lower one is a part of this +// vocabulary and is read. Version 2 carries a process's drain-bound (design 47 H2 §813 (12), H-A-358, H-A-388). +const Version = 2 // Type names a kind of resource. Every addition widens what a compromised control plane can // express, so the list is a security artefact and grows deliberately. @@ -921,6 +922,11 @@ type Process struct { // runtime of a build seat's holder, whose builds must never slow the operator's desktop. Absent: the machine's // defaults. For a process that stays up. Yields *Yields `json:"yields,omitempty"` + + // DrainBound is how long, in seconds, the process finishes the calls in flight once stopped before it exits + // (design 47 H2 §813 (12); H-A-358): the node-engine writes the unit's stop so the stop signal reaches this process + // alone and waits that long and the time its children take. Absent: none. For a process that stays up. + DrainBound int `json:"drain-bound,omitempty"` } // YieldIdle is the one priority a process yields at: the lowest there is. @@ -1024,6 +1030,13 @@ func (d *Process) validate(where string, _ bool) []string { problems = append(problems, where+": a witness judges a process that stays up; a step or a "+ "scheduled run is not running between its runs") } + if d.DrainBound < 0 { + problems = append(problems, fmt.Sprintf("%s: drain-bound is %d: a bound is seconds, none or more", where, d.DrainBound)) + } + if d.DrainBound > 0 && (d.RunOnce || d.Schedule != "") { + problems = append(problems, where+": drain-bound is for a process that stays up; a step or a scheduled run "+ + "is not stopped while it serves calls") + } if strings.ContainsAny(d.NotReversible, "\n\r") { problems = append(problems, where+": not-reversible is one line") } @@ -2349,7 +2362,7 @@ func parse(raw []byte, allowActions bool) (*Declaration, error) { return nil, &RefusalError{Problems: []string{"not a declaration: " + err.Error()}} } - if env.Version != Version { + if env.Version < 1 || env.Version > Version { // Everything below assumes the vocabulary, so there is nothing further to say. return nil, &RefusalError{Problems: []string{fmt.Sprintf( "declaration version %d; this host speaks version %d. Refused whole rather than "+ diff --git a/go/declaration/declaration_test.go b/go/declaration/declaration_test.go index 8b3354ee..522ddde8 100644 --- a/go/declaration/declaration_test.go +++ b/go/declaration/declaration_test.go @@ -2,6 +2,7 @@ package declaration import ( "errors" + "fmt" "strings" "testing" ) @@ -92,9 +93,22 @@ func testAnUnknownVersionIsRefusedWhole(t *testing.T) { {"id":"a","type":"directory","path":"/etc/mesh"} ]}`) joined := strings.Join(refusal.Problems, "\n") - if !strings.Contains(joined, "99") || !strings.Contains(joined, "version 1") { + if !strings.Contains(joined, "99") || !strings.Contains(joined, "version 2") { t.Errorf("the version mismatch was not stated plainly: %v", refusal.Problems) } + // Design 47 H2 §813 (12), H-A-388: Version is the declaration's form, raised to 2 by drain-bound; a lower one is a + // part of this vocabulary and is read, so a node-engine that reads 2 takes what a composer of 1 still writes. + for _, v := range []int{1, Version} { + if _, err := Parse([]byte(fmt.Sprintf(`{"declaration":%d,"resources":[{"id":"a","type":"directory","path":"/etc/mesh"}]}`, v))); err != nil { + t.Errorf("a declaration of version %d was refused: %v", v, err) + } + } + if Version != 2 { + t.Errorf("the declaration's Version is %d, want 2 (H-A-388)", Version) + } + if r := refusalFor(t, `{"declaration":0,"resources":[]}`); !strings.Contains(strings.Join(r.Problems, " "), "version 0") { + t.Errorf("a declaration of version 0 was refused as %v", r.Problems) + } // Nothing else is reported, because everything else assumes a vocabulary this host does // not have — a list of complaints derived from the wrong grammar is noise. if len(refusal.Problems) != 1 { diff --git a/go/declaration/process_test.go b/go/declaration/process_test.go index 753771cd..754a5cb7 100644 --- a/go/declaration/process_test.go +++ b/go/declaration/process_test.go @@ -1,6 +1,7 @@ package declaration import ( + "encoding/json" "strings" "testing" ) @@ -99,6 +100,9 @@ func testTheModesAreExclusive(t *testing.T) { "an unknown priority": func(p *Process) { p.Yields = &Yields{Priority: "low"} }, "a share over the whole": func(p *Process) { p.Yields = &Yields{CPUShare: 150} }, "a yield saying nothing": func(p *Process) { p.Yields = &Yields{} }, + // Design 47 H2 §813 (12), H-A-358: a drain bound is seconds, for a process that stays up. + "a step with a drain bound": func(p *Process) { p.RunOnce = true; p.DrainBound = 30 }, + "a negative drain bound": func(p *Process) { p.DrainBound = -1 }, } { p := aProcess() bad(p) @@ -106,6 +110,18 @@ func testTheModesAreExclusive(t *testing.T) { t.Errorf("%s was accepted", name) } } + // It travels in the declaration under drain-bound, and so in its digest; absent says none. + up := aProcess() + up.DrainBound = 30 + if problems := up.validate("a process", false); len(problems) != 0 { + t.Errorf("a process that stays up with a drain bound was refused: %v", problems) + } + if b, _ := json.Marshal(up); !strings.Contains(string(b), `"drain-bound":30`) { + t.Errorf("the drain bound is not in the declaration: %s", b) + } + if b, _ := json.Marshal(aProcess()); strings.Contains(string(b), "drain-bound") { + t.Errorf("a process with no drain bound says one: %s", b) + } } // A cadence that is not a cadence is refused near its author, rather than by a machine at the far diff --git a/go/go.mod b/go/go.mod index 2b853831..fe822625 100644 --- a/go/go.mod +++ b/go/go.mod @@ -6,6 +6,7 @@ require ( github.com/klauspost/compress v1.20.0 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 + golang.org/x/crypto v0.57.0 ) require ( @@ -15,7 +16,6 @@ require ( github.com/nats-io/jwt/v2 v2.8.1 // indirect github.com/nats-io/nkeys v0.4.16 // indirect github.com/nats-io/nuid v1.0.1 // indirect - golang.org/x/crypto v0.57.0 // indirect golang.org/x/sys v0.48.0 // indirect golang.org/x/time v0.15.0 // indirect ) diff --git a/go/process/account_test.go b/go/process/account_test.go index 884b922f..0676b3d9 100644 --- a/go/process/account_test.go +++ b/go/process/account_test.go @@ -14,6 +14,9 @@ import ( "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" + + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/busretry" ) // 1171/AC2: given an account file broker- in the controller's form {url, fingerprint, node, @@ -342,6 +345,12 @@ func subTheOwnBuildIsTheFingerprintTheProcessWasStartedWith(t *testing.T) { t.Errorf("%s: the process's own build is not said at its start: %+v", name, said) } } + // Design 47 H2 §775, §781: Connect reads the module's kept-call pace from its settings, and refuses one that is no + // duration, naming it. + t.Setenv(stdio.SettingWord(busretry.PaceFirstSetting), "soon") + if _, err := Connect(Account{URL: "nats://127.0.0.1:1", User: "n1.m"}); err == nil || !strings.Contains(err.Error(), "kept-call-pace-first") { + t.Errorf("a pace that is no duration connected: %v", err) + } } // TestReadAccountYieldsEveryFieldAndToleratesClaims runs the tests of this package that were merged to keep the module within its test budget (design 47 G §419), each as a subtest. diff --git a/go/process/call.go b/go/process/call.go index 407ebd59..8cf84bd9 100644 --- a/go/process/call.go +++ b/go/process/call.go @@ -10,6 +10,7 @@ import ( "github.com/nats-io/nats.go" + stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-sdk/go/approval" "git.novox.be/novox/mesh-sdk/go/busretry" "git.novox.be/novox/mesh-sdk/go/calladdress" @@ -52,34 +53,51 @@ func (c *Conn) CallOn(node, key string, body any) (json.RawMessage, error) { } // CallKept is Call for a call whose answer the caller keeps and which is safe to make again (a read, or a write the -// callee deduplicates): it is asked again through a transport fault, a timeout or no responders, whether or not the bus -// is blacked out, until it is answered or busretry.Bound has passed (novox/hq ADR 0668, extending ADR 0615 §6; issue -// 1966). A callee that restarts answers no one for seconds; Call fails that at once. The caller keeps a call still -// unanswered past the bound as a wait of its own, and makes it again on the callee's event or its consumer's bind. +// callee deduplicates): mesh-sdk's kept-call form (design 47 H2 §775), asked again through a transport fault, a timeout +// or no responders at the kept-call pace (busretry.KeptPace(), jittered), whether or not the bus is blacked out, until it +// is answered or busretry.Bound has passed (novox/hq ADR 0668, extending ADR 0615 §6; issue 1966). A refusal returns at +// once. A caller with an ask id and a bound of its own uses CallKeptAsk. func (c *Conn) CallKept(key string, body any) (json.RawMessage, error) { - called, fallback, err := callSubjectsOn(key, "", c.callerToken()) - if err != nil { - return nil, err - } - return c.callKeptOn(called, fallback, body) + return c.CallKeptAsk(KeptAsk{}, key, body) } // CallKeptOn is CallKept to the tool or seat verb of one named machine, as CallOn is Call's. func (c *Conn) CallKeptOn(node, key string, body any) (json.RawMessage, error) { + return c.CallKeptAskOn(KeptAsk{}, node, key, body) +} + +// KeptAsk is what a caller names of its kept call (design 47 H2 §775): the ask's id, which the call carries in the +// header stdio.AskHeader for the holder to take it once by (H-A-351), and the caller's bound (zero is busretry.Bound). +type KeptAsk struct { + ID string + Bound time.Duration +} + +// CallKeptAsk is CallKept carrying ask's id and stopping at ask's bound: past it the error is a *busretry.Unanswered +// naming every call's time and words. +func (c *Conn) CallKeptAsk(ask KeptAsk, key string, body any) (json.RawMessage, error) { + return c.callKeptAskOn(ask, "", key, body, nil) +} + +// CallKeptAskOn is CallKeptAsk to the tool or seat verb of one named machine. +func (c *Conn) CallKeptAskOn(ask KeptAsk, node, key string, body any) (json.RawMessage, error) { if err := calladdress.CheckNode(node); err != nil { return nil, err } - called, fallback, err := callSubjectsOn(key, node, c.callerToken()) - if err != nil { - return nil, err - } - return c.callKeptOn(called, fallback, body) + return c.callKeptAskOn(ask, node, key, body, nil) } // CallKeptWhileOpen is CallKept for an ask the asker can close or replace: it is asked again only while open says its // key is still open (design 47 G §351, §460). func (c *Conn) CallKeptWhileOpen(open busretry.Open, key string, body any) (json.RawMessage, error) { - called, fallback, err := callSubjectsOn(key, "", c.callerToken()) + if open.Open == nil { + return nil, fmt.Errorf("the call on %s: a kept call names its key's open function", key) + } + return c.callKeptAskOn(KeptAsk{}, "", key, body, &open) +} + +func (c *Conn) callKeptAskOn(ask KeptAsk, node, key string, body any, open *busretry.Open) (json.RawMessage, error) { + called, fallback, err := callSubjectsOn(key, node, c.callerToken()) if err != nil { return nil, err } @@ -87,21 +105,35 @@ func (c *Conn) CallKeptWhileOpen(open busretry.Open, key string, body any) (json if err != nil { return nil, err } - return busretry.DoKept("the call on "+fallback, open, askBoth(c.request, called, fallback, data), nil) + return callKeptWith(keptOf(key, ask, open), c.requestAsking(ask.ID), called, fallback, data) } -func (c *Conn) callKeptOn(called, fallback string, body any) (json.RawMessage, error) { - data, err := json.Marshal(body) - if err != nil { - return nil, err +// keptOf is the paced form's naming of a call on key: its seat and verb, or its module and tool. +func keptOf(key string, ask KeptAsk, open *busretry.Open) busretry.Kept { + k := busretry.Kept{Ask: ask.ID, Bound: ask.Bound, Open: open} + if a, err := calladdress.Parse(key); err == nil { + k.Seat, k.Verb = a.Seat, a.Verb + if a.Seat == "" { + k.Seat, k.Verb = a.Module, a.Tool + } } - return callKeptWith(c.request, called, fallback, data) + return k } -// callKeptWith is callWith for a kept call: busretry.DoRepeatable with no blackout probe. -func callKeptWith(request func(string, []byte) (json.RawMessage, error), called, fallback string, +// callKeptWith is callWith for a kept call: busretry.DoPaced with no blackout probe. +func callKeptWith(k busretry.Kept, request func(string, []byte) (json.RawMessage, error), called, fallback string, data []byte) (json.RawMessage, error) { - return busretry.DoRepeatable("the call on "+fallback, askBoth(request, called, fallback, data), nil) + ask := askBoth(request, called, fallback, data) + return busretry.DoPaced(k, func() (json.RawMessage, error) { + result, err := ask() + if err == nil { + // A holder's answer of the kind away is a transport fault (§776): called again at the pace. + if away := busretry.AwayIn(result); away != nil { + return result, away + } + } + return result, err + }) } // askBoth asks called, then fallback when nothing answers it. @@ -209,10 +241,28 @@ func callSubjectsOn(key, node, token string) (called, fallback string, err error // request asks one subject once and reads its reply ({result, error, node}); a reply naming an error is // the call's failure, not a transport one. func (c *Conn) request(subject string, data []byte) (json.RawMessage, error) { - ctx, cancel := context.WithTimeout(context.Background(), CallTimeout) - defer cancel() + return c.requestAsk(subject, data, "") +} + +// requestAsking is request carrying an ask's id in the header stdio.AskHeader (H-A-351); empty carries none. +func (c *Conn) requestAsking(ask string) func(string, []byte) (json.RawMessage, error) { + return func(subject string, data []byte) (json.RawMessage, error) { return c.requestAsk(subject, data, ask) } +} + +// askMsg is the request a call publishes: its ask's id in stdio.AskHeader when it carries one (H-A-351). +func askMsg(subject string, data []byte, ask string) *nats.Msg { req := nats.NewMsg(subject) req.Data = data + if ask != "" { + req.Header.Set(stdio.AskHeader, ask) + } + return req +} + +func (c *Conn) requestAsk(subject string, data []byte, ask string) (json.RawMessage, error) { + ctx, cancel := context.WithTimeout(context.Background(), CallTimeout) + defer cancel() + req := askMsg(subject, data, ask) if strings.HasPrefix(subject, "mesh.seat.") { // Every seat call says it waits for a held call's end; a verb that is not marked ignores it (lead answer A-244). req.Header.Set(approval.WaitHeader, approval.WaitHeld) diff --git a/go/process/call_test.go b/go/process/call_test.go index dd1a1121..093ae36f 100644 --- a/go/process/call_test.go +++ b/go/process/call_test.go @@ -1,8 +1,11 @@ package process import ( + "bytes" "context" "encoding/json" + "errors" + "fmt" "git.novox.be/novox/mesh-sdk/go/seatdef" "strings" "testing" @@ -10,6 +13,9 @@ import ( "github.com/nats-io/nats.go" + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/approval" + "git.novox.be/novox/mesh-sdk/go/buscalls" "git.novox.be/novox/mesh-sdk/go/busretry" ) @@ -128,7 +134,7 @@ func TestACallIsAskedAgainOnlyWhenItReachedNoOne(t *testing.T) { } var err error if row.kept { - _, err = callKeptWith(request, "called", "fallback", nil) + _, err = callKeptWith(busretry.Kept{Seat: "s", Verb: "v"}, request, "called", "fallback", nil) } else { _, err = callWith(request, func() bool { return true }, "called", "fallback", nil) } @@ -136,6 +142,106 @@ func TestACallIsAskedAgainOnlyWhenItReachedNoOne(t *testing.T) { t.Errorf("%s: asked %d times (%v), want %d", row.name, asks, err, row.asks) } } + // Design 47 H2 §776: a holder's answer of the kind away is a transport fault, called again at the pace; a refusal + // of another kind is the caller's answer at once. + for _, row := range []struct { + name string + answers []string + asks int + }{ + {"away, then answered", []string{`{"error":"x","error-kind":"away","part":"the forge's store","words":"401"}`, `{"merged":true}`}, 2}, + {"refused: answered at once", []string{`{"error":"no","error-kind":"refused"}`}, 1}, + } { + asks := 0 + request := func(string, []byte) (json.RawMessage, error) { + a := row.answers[min(asks, len(row.answers)-1)] + asks++ + return json.RawMessage(a), nil + } + got, err := callKeptWith(busretry.Kept{Seat: "git", Verb: "merge-pull"}, request, "called", "fallback", nil) + if asks != row.asks || err != nil || string(got) != row.answers[len(row.answers)-1] { + t.Errorf("%s: asked %d times, %s, %v", row.name, asks, got, err) + } + } + // Design 47 H2 §775, H-A-351: a kept call carries its ask's id in the header, and a holder in a process reads it with + // AskOf while its handler runs; a call carrying none reads empty. + if m := askMsg("mesh.seat.mesh-builds.call.build", []byte(`{}`), "ask-9"); m.Header.Get(stdio.AskHeader) != "ask-9" { + t.Errorf("the ask's id is not in the header: %v", m.Header) + } + if m := askMsg("s", nil, ""); m.Header.Get(stdio.AskHeader) != "" { + t.Errorf("a call with no ask carries one: %v", m.Header) + } + c := &Conn{} + for _, ask := range []string{"ask-9", ""} { + msg := askMsg("mesh.seat.mesh-builds.tool.build", []byte(`{"x":1}`), ask) + read := "unread" + c.answerCall(msg, "mesh.seat.mesh-builds.tool.build", func(_ string, body json.RawMessage, _ bool) (any, *approval.Held, error) { + read = AskOf(body) + return nil, nil, nil + }) + if read != ask || AskOf(json.RawMessage(msg.Data)) != "" { + t.Errorf("the holder read the ask %q, want %q, and none once its handler ended", read, ask) + } + } + // Design 47 H2 §775, H-A-405: a kept chunked call (CallChunksKept, CallFileKept) asks its verb through the kept-call + // form, carries its one ask id on every chunk's acknowledgement, and past its bound ends with the kept call's words. + send := buscalls.SeatChunks{Seat: "node-build-go", Verb: "output", Call: "c1"} + for _, row := range []struct { + name string + answer []error // per ask; the last repeats + bound time.Duration + asks int + chunks int64 + unasked bool // ends unanswered past its bound + }{ + {"no responders twice, then answered with two chunks", []error{nats.ErrNoResponders, nats.ErrNoResponders, nil}, 0, 3, 2, false}, + {"answered at once with three chunks", []error{nil}, 0, 1, 3, false}, + {"no responders past its bound", []error{nats.ErrNoResponders}, time.Nanosecond, 2, 0, true}, + } { + asks := 0 + request := func(string, []byte) (json.RawMessage, error) { + e := row.answer[min(asks, len(row.answer)-1)] + asks++ + return json.RawMessage(fmt.Sprintf(`{"chunks":%d}`, row.chunks)), e + } + chunks := make(chan *nats.Msg, row.chunks) + var want bytes.Buffer + for n := int64(1); n <= row.chunks; n++ { + data := []byte(fmt.Sprintf("part %d;", n)) + want.Write(data) + body, _ := json.Marshal(buscalls.SeatChunk{N: n, Of: row.chunks, Data: data}) + chunks <- &nats.Msg{Subject: send.Chunk(n), Data: body} + } + var acks []string + ack := func(m *nats.Msg) error { + acks = append(acks, m.Subject+" "+m.Header.Get(stdio.AskHeader)) + return nil + } + ask := KeptAsk{ID: "ask-7", Bound: row.bound} + asking := func() (json.RawMessage, error) { + return callKeptWith(keptOf("seat:node-build-go.output@ace", ask, nil), request, "called", "fallback", nil) + } + var got bytes.Buffer + _, err := receiveChunks(context.Background(), "seat:node-build-go.output@ace", send, chunks, ask.ID, asking, nil, ack, &got) + var unanswered *busretry.Unanswered + if row.unasked != errors.As(err, &unanswered) || asks != row.asks || (!row.unasked && err != nil) { + t.Errorf("%s: asked %d times (want %d), %v", row.name, asks, row.asks, err) + } + if row.unasked && (unanswered.Ask != "ask-7" || !strings.Contains(err.Error(), "node-build-go.output (ask ask-7) unanswered within its bound")) { + t.Errorf("%s: ended with %v, not the kept call's words", row.name, err) + } + if got.String() != want.String() || int64(len(acks)) != row.chunks { + t.Errorf("%s: received %q and acknowledged %d chunks, want %q and %d", row.name, got.String(), len(acks), want.String(), row.chunks) + } + for n, a := range acks { + if a != send.Ack(int64(n+1))+" ask-7" { + t.Errorf("%s: acknowledgement %d is %q, carrying not the ask's id in its header", row.name, n+1, a) + } + } + } + if _, err := (&Conn{}).CallChunksKept(context.Background(), KeptAsk{}, "seat:node-build-go.output", "c1", nil, nil, &bytes.Buffer{}); err == nil { + t.Error("a kept chunked call naming no ask id was made") + } } // A-107: AnswerSeat reads the caller from a call's last token (dots written `~`), "" for anything that is not a diff --git a/go/process/chunks.go b/go/process/chunks.go index ee5c14e9..8cd25216 100644 --- a/go/process/chunks.go +++ b/go/process/chunks.go @@ -22,24 +22,77 @@ import ( // is taken), then receives the number of chunks the answer's member `chunks` names, in order, acknowledging each, into w. // It answers the verb's answer. A chunk that does not come within buscalls.SeatAckWithin, or ctx ending, ends the receive. func (c *Conn) CallChunks(ctx context.Context, key, call string, body any, accept func(answer json.RawMessage) error, w io.Writer) (json.RawMessage, error) { - a, err := calladdress.Parse(key) + return c.callChunks(ctx, key, call, "", func() (json.RawMessage, error) { return c.Call(key, body) }, accept, w) +} + +// CallChunksKept is CallChunks made as a kept call (design 47 H2 §775; H-A-405): the verb is asked through the kept-call +// form (busretry.DoPaced: called again at the kept-call pace on a transport fault, ADR 0664 §1's classes, the call's own +// deadline and the kind away; one journal line per seat at the first fault, then at most once a minute), carrying ask's id +// in the header stdio.AskHeader on the call and on every chunk's acknowledgement, and stopping at ask's bound with the +// words CallKeptAsk gives (a *busretry.Unanswered naming every call). A kept chunked call carries an ask id: one naming +// none is refused before anything is published. +func (c *Conn) CallChunksKept(ctx context.Context, ask KeptAsk, key, call string, body any, accept func(answer json.RawMessage) error, + w io.Writer) (json.RawMessage, error) { + if ask.ID == "" { + return nil, fmt.Errorf("the kept call of %s names no ask id", key) + } + called, fallback, err := callSubjectsOn(key, "", c.callerToken()) if err != nil { return nil, err } - if a.Seat == "" { - return nil, fmt.Errorf("%q names no seat verb, so it sends no chunks", key) + data, err := json.Marshal(body) + if err != nil { + return nil, err } - if call == "" { - return nil, fmt.Errorf("the call of %s names no call token for its chunks", key) + asking := func() (json.RawMessage, error) { + return callKeptWith(keptOf(key, ask, nil), c.requestAsking(ask.ID), called, fallback, data) + } + return c.callChunks(ctx, key, call, ask.ID, asking, accept, w) +} + +// callChunks hears the chunks of call before it asks through asking, then receives them (receiveChunks), acknowledging +// each with ask's id in stdio.AskHeader when it carries one. +func (c *Conn) callChunks(ctx context.Context, key, call, ask string, asking func() (json.RawMessage, error), + accept func(answer json.RawMessage) error, w io.Writer) (json.RawMessage, error) { + send, err := chunksOf(key, call) + if err != nil { + return nil, err } - send := buscalls.SeatChunks{Seat: a.Seat, Verb: a.Verb, Call: call} chunks := make(chan *nats.Msg, 4) sub, err := c.nc.ChanSubscribe(send.Chunks(), chunks) if err != nil { return nil, fmt.Errorf("the chunks of %s cannot be heard: %w", key, err) } defer func() { _ = sub.Unsubscribe() }() - answer, err := c.Call(key, body) + ack := func(m *nats.Msg) error { + _, err := c.nc.RequestMsg(m, buscalls.SeatAckWithin) + return err + } + return receiveChunks(ctx, key, send, chunks, ask, asking, accept, ack, w) +} + +// chunksOf is the chunks of call of the seat verb key names; a key naming no seat verb, or no call, is refused. +func chunksOf(key, call string) (buscalls.SeatChunks, error) { + a, err := calladdress.Parse(key) + if err != nil { + return buscalls.SeatChunks{}, err + } + if a.Seat == "" { + return buscalls.SeatChunks{}, fmt.Errorf("%q names no seat verb, so it sends no chunks", key) + } + if call == "" { + return buscalls.SeatChunks{}, fmt.Errorf("the call of %s names no call token for its chunks", key) + } + return buscalls.SeatChunks{Seat: a.Seat, Verb: a.Verb, Call: call}, nil +} + +// receiveChunks asks through asking, hands its answer to accept, then receives the number of chunks the answer's member +// `chunks` names from chunks, in order, acknowledging each through ack with a message carrying ask's id in +// stdio.AskHeader (none when empty), into w. +func receiveChunks(ctx context.Context, key string, send buscalls.SeatChunks, chunks <-chan *nats.Msg, ask string, + asking func() (json.RawMessage, error), accept func(answer json.RawMessage) error, ack func(*nats.Msg) error, + w io.Writer) (json.RawMessage, error) { + answer, err := asking() if err != nil { return nil, err } @@ -66,11 +119,8 @@ func (c *Conn) CallChunks(ctx context.Context, key, call string, body any, accep return "", nil, ctx.Err() } } - ack := func(subject string) error { - _, err := c.nc.Request(subject, []byte("{}"), buscalls.SeatAckWithin) - return err - } - if _, err := send.Receive(count.Chunks, next, ack, w); err != nil { + acked := func(subject string) error { return ack(askMsg(subject, []byte("{}"), ask)) } + if _, err := send.Receive(count.Chunks, next, acked, w); err != nil { return answer, fmt.Errorf("the answer of %s: %w", key, err) } return answer, nil @@ -80,18 +130,32 @@ func (c *Conn) CallChunks(ctx context.Context, key, call string, body any, accep // G A-249, A-253): the verb's answer with the whole file added as its member `file` (base64), none when no chunk was sent. The // call token is body's member `call`. func (c *Conn) CallFile(key string, body any) (json.RawMessage, error) { - call := callOf(body) var file bytes.Buffer - answer, err := c.CallChunks(context.Background(), key, call, body, nil, &file) + answer, err := c.CallChunks(context.Background(), key, callOf(body), body, nil, &file) if err != nil { return nil, err } + return withFile(key, answer, file.Bytes()) +} + +// CallFileKept is CallFile made as a kept call, as CallChunksKept is CallChunks's (design 47 H2 §775; H-A-405). +func (c *Conn) CallFileKept(ask KeptAsk, key string, body any) (json.RawMessage, error) { + var file bytes.Buffer + answer, err := c.CallChunksKept(context.Background(), ask, key, callOf(body), body, nil, &file) + if err != nil { + return nil, err + } + return withFile(key, answer, file.Bytes()) +} + +// withFile is a chunked verb's answer with file added as its member `file` (base64), none when file is empty. +func withFile(key string, answer json.RawMessage, file []byte) (json.RawMessage, error) { var whole map[string]any if err := json.Unmarshal(answer, &whole); err != nil { return nil, fmt.Errorf("%s answered what is no object: %w", key, err) } - if file.Len() > 0 { - whole["file"] = base64.StdEncoding.EncodeToString(file.Bytes()) + if len(file) > 0 { + whole["file"] = base64.StdEncoding.EncodeToString(file) } return json.Marshal(whole) } diff --git a/go/process/conn.go b/go/process/conn.go index 2b6ecaf5..bb4c0597 100644 --- a/go/process/conn.go +++ b/go/process/conn.go @@ -12,6 +12,9 @@ import ( "sync" "github.com/nats-io/nats.go" + + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/busretry" ) // Conn is one process's own connection to the bus (novox/hq ADR 0436 clause 4): dialled with the account @@ -54,6 +57,9 @@ var refusedPublish = regexp.MustCompile(`(?i)Publish to "(\S+)"`) // at once. func Connect(account Account, opts ...nats.Option) (*Conn, error) { OwnBuild() + if err := busretry.ReadPace(stdio.SettingDuration); err != nil { + return nil, fmt.Errorf("the kept-call pace of the process account %s: %w", account.User, err) + } c := &Conn{account: account, pending: map[string]map[uint64]context.CancelCauseFunc{}} nc, err := nats.Connect(account.URL, c.options(opts)...) if err != nil { diff --git a/go/process/prune.go b/go/process/prune.go new file mode 100644 index 00000000..042a39d1 --- /dev/null +++ b/go/process/prune.go @@ -0,0 +1,137 @@ +package process + +import ( + "context" + "encoding/json" + "errors" + "log" + "time" + + "github.com/nats-io/nats.go/jetstream" + + stdio "git.novox.be/novox/mesh-sdk/go" +) + +// sayPrune writes a prune's one journal line (design 47 H2 §812). +var sayPrune = func(line string) { log.Print(line) } + +// PrunedState is a state a process prunes at its own writes (design 47 H2 §812; H-A-380): the entry's figures are read +// once when it is bound and once after each prune, and between those a running sum of the bytes each write puts +// (stdio.PutBytes, an upper bound) is the judgement; past prune-mark of the cap the oldest keys by their current +// revision are purged down to prune-target, skipping a key kept under stdio.KeepKey. No read per write, no poll, no timer. +// Its reads are the state's own. +type PrunedState struct { + State + pruner *stdio.Pruner +} + +// Pruned binds the state for a prune by p, reading its figures once. +func (s State) Pruned(p stdio.Prune) (*PrunedState, error) { + ps := &PrunedState{State: s} + ps.pruner = &stdio.Pruner{State: s.Module + "'s " + s.Name, Prune: p, Say: func(line string) { sayPrune(line) }, + Figures: s.figures, Entries: s.entries, Remove: s.purge} + if err := ps.pruner.Bind(); err != nil { + return nil, err + } + return ps, nil +} + +// figures is the bucket's bytes and its cap, zero for none. +func (s State) figures() (uint64, uint64, error) { + ctx, cancel := context.WithTimeout(context.Background(), stateTimeout) + defer cancel() + kv, err := s.store(ctx) + if err != nil { + return 0, 0, err + } + st, err := kv.Status(ctx) + if err != nil { + return 0, 0, err + } + return st.Bytes(), uint64(max(st.Config().MaxBytes, 0)), nil +} + +// entries is every current value of the bucket with its revision, keeps included, read once. +func (s State) entries() ([]stdio.PruneEntry, error) { + ctx, cancel := context.WithTimeout(context.Background(), stateTimeout) + defer cancel() + kv, err := s.store(ctx) + if err != nil { + return nil, err + } + w, err := kv.WatchAll(ctx, jetstream.IgnoreDeletes()) + if err != nil { + return nil, err + } + defer w.Stop() + var out []stdio.PruneEntry + for { + select { + case e, open := <-w.Updates(): + if !open { + return nil, errors.New("the read of the current values ended") + } + if e == nil { + return out, nil + } + if e.Operation() == jetstream.KeyValuePut { + out = append(out, stdio.PruneEntry{Key: e.Key(), Revision: e.Revision(), Bytes: stdio.PutBytes(e.Key(), e.Value()), Value: e.Value()}) + } + case <-ctx.Done(): + return nil, ctx.Err() + } + } +} + +// purge removes a key with its past values, so its bytes leave the bucket. +func (s State) purge(key string) error { + ctx, cancel := context.WithTimeout(context.Background(), stateTimeout) + defer cancel() + kv, err := s.store(ctx) + if err != nil { + return err + } + return kv.Purge(ctx, key) +} + +// wrote counts a write and prunes past the mark. +func (s *PrunedState) wrote(key string, value any, rev uint64, err error) (uint64, error) { + if err != nil { + return rev, err + } + data, _ := json.Marshal(value) + if _, err := s.pruner.Wrote(stdio.PutBytes(key, data)); err != nil { + return rev, err + } + return rev, nil +} + +// Put writes a key whole, counted toward the prune. +func (s *PrunedState) Put(key string, value any) (uint64, error) { + rev, err := s.State.Put(key, value) + return s.wrote(key, value, rev, err) +} + +// Create writes a key only when it has none, counted toward the prune. +func (s *PrunedState) Create(key string, value any) (uint64, error) { + rev, err := s.State.Create(key, value) + return s.wrote(key, value, rev, err) +} + +// Update writes a key only when it is still at revision, counted toward the prune. +func (s *PrunedState) Update(key string, value any, revision uint64) (uint64, error) { + rev, err := s.State.Update(key, value, revision) + return s.wrote(key, value, rev, err) +} + +// PruneNow prunes the state to its target now. +func (s *PrunedState) PruneNow() (stdio.Pruned, error) { return s.pruner.PruneNow() } + +// Keep keeps key from the prune until until, written under stdio.KeepKey. +func (s *PrunedState) Keep(key string, until time.Time) error { + if err := stdio.KeepProblem(key, until); err != nil { + return err + } + _, err := s.Put(stdio.KeepKey(key), stdio.Keep{Until: until.UTC()}) + return err +} diff --git a/go/process/prune_test.go b/go/process/prune_test.go new file mode 100644 index 00000000..2ce60859 --- /dev/null +++ b/go/process/prune_test.go @@ -0,0 +1,62 @@ +package process + +import ( + "context" + + "github.com/nats-io/nats.go/jetstream" + + stdio "git.novox.be/novox/mesh-sdk/go" +) + +// pruneKV is a fakeKV with a cap: its bytes are what its current values put (stdio.PutBytes), its Status says them +// with the cap, WatchAll hands every current value, and Purge removes a key. +type pruneKV struct { + *fakeKV + cap int64 + statuss int +} + +type pruneStatus struct { + jetstream.KeyValueStatus + bytes uint64 + cap int64 +} + +func (s pruneStatus) Bytes() uint64 { return s.bytes } +func (s pruneStatus) Config() jetstream.KeyValueConfig { + return jetstream.KeyValueConfig{MaxBytes: s.cap} +} + +func (f *pruneKV) held() uint64 { + f.mu.Lock() + defer f.mu.Unlock() + var n uint64 + for k, v := range f.values { + n += stdio.PutBytes(k, v) + } + return n +} + +func (f *pruneKV) Status(context.Context) (jetstream.KeyValueStatus, error) { + f.statuss++ + return pruneStatus{bytes: f.held(), cap: f.cap}, nil +} + +func (f *pruneKV) WatchAll(context.Context, ...jetstream.WatchOpt) (jetstream.KeyWatcher, error) { + f.mu.Lock() + defer f.mu.Unlock() + w := &fakeWatcher{updates: make(chan jetstream.KeyValueEntry, len(f.values)+1)} + for k, v := range f.values { + w.updates <- putEntry{fakeEntry{key: k, value: v, rev: f.revs[k]}} + } + w.updates <- nil + return w, nil +} + +func (f *pruneKV) Purge(ctx context.Context, key string, _ ...jetstream.KVDeleteOpt) error { + return f.Delete(ctx, key) +} + +type putEntry struct{ fakeEntry } + +func (putEntry) Operation() jetstream.KeyValueOp { return jetstream.KeyValuePut } diff --git a/go/process/serve.go b/go/process/serve.go index 6234ddf3..4e2eabf1 100644 --- a/go/process/serve.go +++ b/go/process/serve.go @@ -6,9 +6,12 @@ import ( "regexp" "sort" "strings" + "sync" + "unsafe" "github.com/nats-io/nats.go" + stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-sdk/go/approval" "git.novox.be/novox/mesh-sdk/go/calladdress" "git.novox.be/novox/mesh-sdk/go/seatdef" @@ -157,6 +160,21 @@ func (c *Conn) admitted(seat, verb string, definition func() (seatdef.Definition } } +// asks holds the ask id each call in flight carried (stdio.AskHeader), by its body's bytes, while its handler runs. +var asks sync.Map // unsafe.Pointer → string + +// AskOf is the ask id the call whose body a SeatHandler was handed carried in stdio.AskHeader (design 47 H2 §775; +// H-A-351): what a holder in a process takes the ask once by. Empty when it carried none. Valid while the handler runs. +func AskOf(body json.RawMessage) string { + if len(body) == 0 { + return "" + } + if a, ok := asks.Load(unsafe.Pointer(unsafe.SliceData(body))); ok { + return a.(string) + } + return "" +} + // answerCall runs h for one call and replies {result, error, node}; a panic in h is the call's error, never silence. func (c *Conn) answerCall(msg *nats.Msg, pattern string, h heldHandler) { caller := "" @@ -177,7 +195,13 @@ func (c *Conn) answerCall(msg *nats.Msg, pattern string, h heldHandler) { reply.Error = fmt.Sprintf("%s failed: %v", msg.Subject, r) } }() - result, held, err := h(caller, json.RawMessage(msg.Data), wait) + body := json.RawMessage(msg.Data) + if ask := msg.Header.Get(stdio.AskHeader); ask != "" && len(body) > 0 { + key := unsafe.Pointer(unsafe.SliceData(body)) + asks.Store(key, ask) + defer asks.Delete(key) + } + result, held, err := h(caller, body, wait) switch { case err != nil: reply.Error = err.Error() diff --git a/go/process/state.go b/go/process/state.go index 4011ef92..d923260b 100644 --- a/go/process/state.go +++ b/go/process/state.go @@ -257,7 +257,7 @@ func (s State) put(key string, value any) (uint64, error) { return rev, nil } -// Keys is every key with a current value, sorted; none is an empty list. +// Keys is every key with a current value, sorted; none is an empty list. A keep (stdio.KeepPrefix) is not an entry and is left out. // Keys waits out a bus that answers nothing for a few seconds (busretry): only a request that reached no one is asked again. func (s State) Keys() ([]string, error) { return busretry.Do(s.what("keys"), func() ([]string, error) { return s.keys() }, nil) @@ -277,8 +277,14 @@ func (s State) keys() ([]string, error) { if err != nil { return nil, fmt.Errorf("%s's state %s, keys: %w", s.Module, s.Name, err) } - sort.Strings(keys) - return keys, nil + entries := keys[:0] + for _, k := range keys { + if !stdio.IsKeep(k) { + entries = append(entries, k) + } + } + sort.Strings(entries) + return entries, nil } // watchPattern is a key narrowing as the bus names it: `*` stays one name, `**` is the rest, empty is every key. @@ -334,6 +340,9 @@ func (s State) Watch(key string, handle func(stdio.StateChange) error) error { } } deliver := func(e jetstream.KeyValueEntry, current bool) error { + if stdio.IsKeep(e.Key()) { + return nil // a keep (stdio.KeepPrefix) is not an entry of the state: no watcher hears it + } c := stdio.StateChange{State: s.Name, Key: e.Key(), Op: "put", Revision: e.Revision(), Current: current} if e.Operation() == jetstream.KeyValuePut { c.Value = json.RawMessage(e.Value()) @@ -398,3 +407,31 @@ func (s State) delete(key string) error { } return nil } + +// StateOver is a module's state read and written through kv, a key-value store bound over a link of the caller's own +// rather than a Conn (H-A-364): the node-engine, which serves its seat over its own link, takes asks once on its own +// taken state with it. module and name name it in errors. +func StateOver(module, name string, kv jetstream.KeyValue) *State { + return &State{Module: module, Name: name, kv: func(context.Context) (jetstream.KeyValue, error) { return kv, nil }} +} + +// TakenState is the state a holder of a kept call keeps the ids of the asks it took in: stdio.TakenState. +const TakenState = stdio.TakenState + +// TakenKey is the key an ask taken from seat is kept under in TakenState: stdio.TakenKey. +func TakenKey(seat, ask string) string { return stdio.TakenKey(seat, ask) } + +// TakeOnce says whether this holder takes the ask ask of seat now (design 47 H2 §775; ADR 0668 §2), as +// stdio.KeptState.TakeOnce does for a bundle: the first time it is given, it is kept under TakenKey for the ask's bound +// and TakeOnce answers true; given again within the bound, false, and the holder acts once. s is the holder's +// TakenState, declared `key-ttl`. An ask with no id, or an id that is not one key, is refused: a kept call carries its id. +func (s State) TakeOnce(seat, ask string, bound time.Duration) (bool, error) { + if p := stdio.TakenProblem(seat, ask); p != "" { + return false, fmt.Errorf("taking an ask of %s: %s", seat, p) + } + _, err := s.CreateTTL(TakenKey(seat, ask), map[string]string{"seat": seat, "ask": ask}, max(bound, MinTTL)) + if errors.Is(err, stdio.ErrChanged) { + return false, nil + } + return err == nil, err +} diff --git a/go/process/state_integration_test.go b/go/process/state_integration_test.go index b84163d5..16fbf38b 100644 --- a/go/process/state_integration_test.go +++ b/go/process/state_integration_test.go @@ -4,7 +4,9 @@ package process import ( "errors" + "fmt" "strings" + "sync" "testing" "time" @@ -199,6 +201,31 @@ func TestStatePutAndKeysOnAGrantedBucket(t *testing.T) { if err != nil || strings.Join(keys, ",") != "a,b" { t.Fatalf("Keys: %v, %v, want a,b", keys, err) } + // Design 47 H2 §782: one connection's state calls are safe at once: gets, puts, keys and watches from many goroutines + // through the one handle, under the race detector. + var wg sync.WaitGroup + errs := make(chan error, 16*4) + for i := range 16 { + wg.Add(1) + go func() { + defer wg.Done() + key := fmt.Sprintf("c%d", i) + _, err := s.Put(key, i) + errs <- err + _, err = s.Get(key) + errs <- err + _, err = s.Keys() + errs <- err + errs <- s.Watch(key, func(stdio.StateChange) error { return nil }) + }() + } + wg.Wait() + close(errs) + for err := range errs { + if err != nil { + t.Fatalf("a concurrent state call: %v", err) + } + } } // 1840/S2: given a process connection granted its bucket, which already holds two keys and a deleted one, diff --git a/go/process/state_test.go b/go/process/state_test.go index 6950504c..32d197c9 100644 --- a/go/process/state_test.go +++ b/go/process/state_test.go @@ -3,6 +3,7 @@ package process import ( "context" "errors" + "fmt" "sort" "strings" "sync" @@ -227,6 +228,79 @@ func TestStatePutWritesAKeyWhole(t *testing.T) { if rev, err := dark.Put("k", 1); err != nil || rev == 0 || blackedOut != 0 { t.Fatalf("a Put through a short blackout: %d, %v", rev, err) } + // Design 47 H2 §775: a holder given the same ask id twice acts once; the id is kept under . with the + // ask's bound as its time to live; an ask with no id, or one that is not one key, is refused. + tkv := newFakeKV() + taken := StateOver("mesh-host", TakenState, tkv) // H-A-364: over a store the caller bound itself + acts := 0 + for range 2 { + if first, err := taken.TakeOnce("mesh-builds", "a-7", 10*time.Minute); err != nil { + t.Fatal(err) + } else if first { + acts++ + } + } + if acts != 1 || !tkv.withTTL[TakenKey("mesh-builds", "a-7")] || TakenKey("mesh-builds", "a-7") != "mesh-builds.a-7" { + t.Fatalf("acted %d times, keys with a TTL %v", acts, tkv.withTTL) + } + if first, err := taken.TakeOnce("mesh-wants", "a-7", time.Minute); !first || err != nil { + t.Fatalf("the same id on another seat is another ask: %v, %v", first, err) + } + for _, bad := range [][2]string{{"mesh-builds", ""}, {"mesh-builds", "a.7"}, {"", "a-8"}} { + if _, err := taken.TakeOnce(bad[0], bad[1], time.Minute); err == nil { + t.Errorf("seat %q, ask %q was taken", bad[0], bad[1]) + } + } + // Design 47 H2 §812 (H-A-380, H-A-386): a process's write that takes its state past prune-mark prunes it, oldest first + // by revision, to prune-target, in one journal line, reading the figures only at the bind and after the prune; a key + // kept under _keep/ survives, and no reader counts the keep. + t.Run("a write past its mark prunes to its target and a kept key survives", func(t *testing.T) { + var lines []string + sayPrune = func(line string) { lines = append(lines, line) } + t.Cleanup(func() { sayPrune = func(string) {} }) + kv := &pruneKV{fakeKV: newFakeKV(), cap: 10_000} + s, err := stateOn(kv).Pruned(stdio.Prune{Mark: 70, Target: 50}) + if err != nil { + t.Fatal(err) + } + value := strings.Repeat("x", 300) + if _, err := s.Put("k00", value); err != nil { + t.Fatal(err) + } + if err := s.Keep("k00", time.Now().Add(time.Hour)); err != nil { + t.Fatal(err) + } + if err := s.Keep("k01", time.Now().Add(-time.Hour)); err == nil { + t.Fatal("a keep until a time past was taken") + } + for i := 1; i < 18; i++ { + if _, err := s.Put(fmt.Sprintf("k%02d", i), value); err != nil { + t.Fatal(err) + } + if len(lines) == 0 && kv.held()*100 > 70*10_000 { + t.Fatalf("the state holds %d of 10000 bytes after k%02d and was not pruned", kv.held(), i) + } + } + if len(lines) != 1 || !strings.Contains(lines[0], "pruned the state mesh-mail's sends") { + t.Fatalf("the prunes said %q, want one line", lines) + } + if kv.statuss != 2 { + t.Fatalf("the figures were read %d times, want 2: at the bind and after the prune", kv.statuss) + } + if held := kv.held(); held*100 > 50*10_000 { + t.Fatalf("the prune left %d of 10000 bytes, above its target of 50 %%", held) + } + keys, err := s.Keys() + if err != nil { + t.Fatal(err) + } + if keys[0] != "k00" || strings.Contains(strings.Join(keys, " "), "_keep") || keys[1] == "k01" || keys[len(keys)-1] != "k17" { + t.Fatalf("after the prune the readers see %v: want k00 kept, the oldest others gone, k17 there, no keep", keys) + } + if e, _ := s.Get(stdio.KeepKey("k00")); e == nil { + t.Fatal("the keep itself was pruned") + } + }) } // 1840/S1: given a state, when its keys are listed, then they come sorted, and none is an empty list, not an error. @@ -298,25 +372,23 @@ func TestStateWatchHandsCurrentValuesThenChanges(t *testing.T) { t.Fatalf("no change %+v", want) } } -} - -// 1840/S2: a key narrowing reaches the bus as a subject pattern: `*` one name, `**` the rest, empty every key. -func TestStateWatchNarrowing(t *testing.T) { - for key, want := range map[string]string{"": ">", "a.b": "a.b", "a.*": "a.*", "a.**": "a.>", "**": ">"} { - if got := watchPattern(key); got != want { - t.Errorf("watchPattern(%q) = %q, want %q", key, got, want) + // 1840/S2: a key narrowing reaches the bus as a subject pattern: `*` one name, `**` the rest, empty every key. + t.Run("narrowing", func(t *testing.T) { + for key, want := range map[string]string{"": ">", "a.b": "a.b", "a.*": "a.*", "a.**": "a.>", "**": ">"} { + if got := watchPattern(key); got != want { + t.Errorf("watchPattern(%q) = %q, want %q", key, got, want) + } } - } -} - -// 1840/S2: given a handler that refuses a current value, then Watch returns its error. -func TestStateWatchReturnsAHandlersErrorOnCurrentValues(t *testing.T) { - kv := newFakeKV() - kv.initial = []jetstream.KeyValueEntry{watchEntry{fakeEntry{key: "a", value: []byte(`1`), rev: 1}, jetstream.KeyValuePut}} - boom := errors.New("boom") - if err := stateOn(kv).Watch("", func(stdio.StateChange) error { return boom }); !errors.Is(err, boom) { - t.Fatalf("Watch: %v, want the handler's error", err) - } + }) + // 1840/S2: given a handler that refuses a current value, then Watch returns its error. + t.Run("a handler's error on the current values is returned", func(t *testing.T) { + kv := newFakeKV() + kv.initial = []jetstream.KeyValueEntry{watchEntry{fakeEntry{key: "a", value: []byte(`1`), rev: 1}, jetstream.KeyValuePut}} + boom := errors.New("boom") + if err := stateOn(kv).Watch("", func(stdio.StateChange) error { return boom }); !errors.Is(err, boom) { + t.Fatalf("Watch: %v, want the handler's error", err) + } + }) } // 1840/S2: given a connection that closes before the current values arrive, then Watch returns an error. diff --git a/go/prune.go b/go/prune.go new file mode 100644 index 00000000..dbf818d3 --- /dev/null +++ b/go/prune.go @@ -0,0 +1,373 @@ +package stdio + +import ( + "encoding/json" + "errors" + "fmt" + "sort" + "strconv" + "strings" + "sync" + "time" + + "git.novox.be/novox/mesh-sdk/go/buscalls" +) + +// A kept store's owner prunes it toward its cap, judged at its own writes (design 47 H2 §812; H-A-380, H-A-386): when +// the fill passes prune-mark of the cap, the oldest keys by their current revision are removed until it is down to +// prune-target, one journal line per prune, with no extra read, no poll and no timer. A key kept by a person +// (`_state_keep`) is never pruned before its until: the keep lives in the pruned state itself under KeepPrefix. + +// The prune's settings and their defaults: prune-mark below the bus module's raise mark of 75 %. +const ( + PruneMarkSetting = "prune-mark" + PruneTargetSetting = "prune-target" + PruneMarkDefault = 70 + PruneTargetDefault = 50 + pruneSettingPercent = "a percentage of the cap between 1 and 100" +) + +// KeepPrefix is the reserved prefix a keep lives under in the state it keeps a key of (H-A-386): `_keep/`, with +// its until. A keep is never pruned and never counted as an entry by the readers (Keys, Page, Watch). +const KeepPrefix = "_keep/" + +// KeepKey is the key the keep of key lives under. +func KeepKey(key string) string { return KeepPrefix + key } + +// IsKeep is whether key is a keep rather than an entry of its state. +func IsKeep(key string) bool { return strings.HasPrefix(key, KeepPrefix) } + +// Keep is a keep's value: the key it keeps is never pruned before Until. +type Keep struct { + Until time.Time `json:"until"` +} + +// SettingPercent is the config value key read as a percentage ("70" or "70%"): def when none is composed or it is +// empty, an error naming the key for a value that is no whole number from 1 to 100. +func SettingPercent(key string, def int) (int, error) { + v, ok := Setting(key) + if v = strings.TrimSpace(v); !ok || v == "" { + return def, nil + } + n, err := strconv.Atoi(strings.TrimSpace(strings.TrimSuffix(v, "%"))) + if err != nil || n < 1 || n > 100 { + return 0, fmt.Errorf("the setting %s (%s) holds %q, which is not %s", key, SettingWord(key), v, pruneSettingPercent) + } + return n, nil +} + +// Prune is where a prune starts and where it ends, in percent of the cap. +type Prune struct { + Mark, Target int +} + +// PruneSettings is the module's prune-mark and prune-target, read through the settings reader with their defaults; a +// target not below the mark is refused, naming both. +func PruneSettings() (Prune, error) { + mark, err := SettingPercent(PruneMarkSetting, PruneMarkDefault) + if err != nil { + return Prune{}, err + } + target, err := SettingPercent(PruneTargetSetting, PruneTargetDefault) + if err != nil { + return Prune{}, err + } + if target >= mark { + return Prune{}, fmt.Errorf("the setting %s (%d %%) is not below %s (%d %%): a prune would never end below its mark", + PruneTargetSetting, target, PruneMarkSetting, mark) + } + return Prune{Mark: mark, Target: target}, nil +} + +// Passed is whether bytes of a cap of maxBytes passes the mark; an entry with no cap never does. +func (p Prune) Passed(bytes, maxBytes uint64) bool { + return maxBytes > 0 && bytes*100 > uint64(p.Mark)*maxBytes +} + +// PruneEntry is one current value of a pruned state as the prune reads it: its key, its current revision and the +// bytes it holds on the bus. +type PruneEntry struct { + Key string + Revision uint64 + Bytes uint64 + Value []byte +} + +// Choose is the keys a prune removes, oldest first by their current revision, until bytes is down to the target of +// maxBytes, and how many it skipped for a keep whose until is still to come. A keep itself is never chosen; a keep that +// cannot be read keeps its key (a prune never removes what a person may have kept). +func (p Prune) Choose(entries []PruneEntry, now time.Time, bytes, maxBytes uint64) (drop []string, kept int) { + keeps := map[string]bool{} + for _, e := range entries { + if !IsKeep(e.Key) { + continue + } + var k Keep + if err := json.Unmarshal(e.Value, &k); err != nil || k.Until.After(now) { + keeps[strings.TrimPrefix(e.Key, KeepPrefix)] = true + } + } + sorted := append([]PruneEntry(nil), entries...) + sort.Slice(sorted, func(i, j int) bool { return sorted[i].Revision < sorted[j].Revision }) + target := uint64(p.Target) * maxBytes / 100 + for _, e := range sorted { + if bytes <= target { + break + } + if IsKeep(e.Key) { + continue + } + if keeps[e.Key] { + kept++ + continue + } + drop = append(drop, e.Key) + bytes -= min(bytes, e.Bytes) + } + return drop, kept +} + +// Pruned is what one prune did: the state, how many keys it removed and skipped for a keep, and its fill before and +// after against the cap. +type Pruned struct { + State string `json:"state"` + Removed int `json:"removed"` + Kept int `json:"kept"` + From uint64 `json:"from-bytes"` + To uint64 `json:"to-bytes"` + MaxBytes uint64 `json:"max-bytes"` + Mark int `json:"mark"` + Target int `json:"target"` +} + +// Line is the prune's one journal line. +func (r Pruned) Line() string { + return fmt.Sprintf("pruned the state %s: removed %d keys oldest first (%d kept by a keep), from %d to %d bytes of its cap of %d (mark %d %%, target %d %%)", + r.State, r.Removed, r.Kept, r.From, r.To, r.MaxBytes, r.Mark, r.Target) +} + +// Prunable is a state its owner prunes: the generic tools `_state_prune_now` and `_state_keep` act on it. +type Prunable interface { + // PruneNow prunes the state to its target now, whatever its fill. + PruneNow() (Pruned, error) + // Keep keeps key from the prune until the time given. + Keep(key string, until time.Time) error +} + +// Pruner is the prune helper of one state, the same for a process and a bundle: Entries reads the state's current +// values (keeps included) once per prune, Remove removes one key, Figures reads the entry's bytes and cap. A process +// gives Wrote the bytes each write puts and Figures is read at Bind and after each prune; a bundle gives Filled the +// fill its write's answer carries. +type Pruner struct { + State string + Prune Prune + Entries func() ([]PruneEntry, error) + Remove func(key string) error + Figures func() (bytes, maxBytes uint64, err error) + // Say writes the prune's journal line. + Say func(line string) + Now func() time.Time + + mu sync.Mutex + bytes, limit uint64 +} + +// Bind reads the entry's figures once, the base of the running sum Wrote keeps. +func (p *Pruner) Bind() error { + bytes, limit, err := p.Figures() + if err != nil { + return fmt.Errorf("the state %s's fill could not be read for its prune: %w", p.State, err) + } + p.mu.Lock() + p.bytes, p.limit = bytes, limit + p.mu.Unlock() + return nil +} + +// Wrote adds the bytes a write put to the running sum and prunes when it passes the mark: the answer is the prune it +// ran, or nil. +func (p *Pruner) Wrote(n uint64) (*Pruned, error) { + p.mu.Lock() + defer p.mu.Unlock() + p.bytes += n + if !p.Prune.Passed(p.bytes, p.limit) { + return nil, nil + } + r, err := p.pruneLocked(p.bytes, p.limit, true) + return &r, err +} + +// Filled judges the fill a write's answer carries and prunes when it passes the mark: the answer is the prune it ran, +// or nil. +func (p *Pruner) Filled(f buscalls.Fill) (*Pruned, error) { + p.mu.Lock() + defer p.mu.Unlock() + p.bytes, p.limit = f.Bytes, f.MaxBytes + if !p.Prune.Passed(f.Bytes, f.MaxBytes) { + return nil, nil + } + r, err := p.pruneLocked(f.Bytes, f.MaxBytes, false) + return &r, err +} + +// PruneNow reads the figures and prunes to the target now. +func (p *Pruner) PruneNow() (Pruned, error) { + bytes, limit, err := p.Figures() + if err != nil { + return Pruned{State: p.State}, fmt.Errorf("the state %s's fill could not be read for its prune: %w", p.State, err) + } + p.mu.Lock() + defer p.mu.Unlock() + if limit == 0 { + return Pruned{State: p.State}, fmt.Errorf("the state %s names no cap, so it has no target to prune to", p.State) + } + return p.pruneLocked(bytes, limit, true) +} + +// pruneLocked removes the oldest keys from bytes down to the target of limit; reread reads the figures after it, the +// new base of the running sum. +func (p *Pruner) pruneLocked(bytes, limit uint64, reread bool) (Pruned, error) { + r := Pruned{State: p.State, From: bytes, MaxBytes: limit, Mark: p.Prune.Mark, Target: p.Prune.Target} + entries, err := p.Entries() + if err != nil { + return r, fmt.Errorf("the state %s could not be read for its prune: %w", p.State, err) + } + // The running sum is an upper bound (an overwrite's past value is the bucket's history to bound), so what the + // current values hold is the start when it is lower. + var held uint64 + for _, e := range entries { + held += e.Bytes + } + start := min(bytes, held) + now := time.Now + if p.Now != nil { + now = p.Now + } + drop, kept := p.Prune.Choose(entries, now(), start, limit) + r.Kept, r.To = kept, start + sizes := map[string]uint64{} + for _, e := range entries { + sizes[e.Key] = e.Bytes + } + var errs []error + for _, key := range drop { + if err := p.Remove(key); err != nil { + errs = append(errs, fmt.Errorf("%s: %w", key, err)) + continue + } + r.Removed++ + r.To -= min(r.To, sizes[key]) + } + if reread { + if b, l, err := p.Figures(); err == nil { + r.To, r.MaxBytes = b, l + } else { + errs = append(errs, fmt.Errorf("the fill after the prune: %w", err)) + } + } + p.bytes, p.limit = r.To, r.MaxBytes + if p.Say != nil { + p.Say(r.Line()) + } + if len(errs) > 0 { + return r, fmt.Errorf("the prune of the state %s: %w", p.State, errors.Join(errs...)) + } + return r, nil +} + +// PutOverhead is what a write puts on the bus beside its key and its value, an upper bound: the bucket's subject +// prefix, the message's headers and its record. +const PutOverhead = 96 + +// PutBytes is what a write of value under key puts on the bus, as the prune helper counts it. +func PutBytes(key string, value []byte) uint64 { return uint64(len(key) + len(value) + PutOverhead) } + +// PrunedState is a state a bundle prunes at its writes (§812): Put judges the fill the runtime's put answer carries +// (DMC-14, H-A-386) and prunes when it passes the mark. Its other calls are the state's own. +type PrunedState struct { + KeptState + pruner *Pruner + unsaid sync.Once +} + +// Pruned is the state pruned by p at its writes. +func (s KeptState) Pruned(p Prune) *PrunedState { + ps := &PrunedState{KeptState: s} + ps.pruner = &Pruner{State: s.name, Prune: p, Say: func(line string) { sayf("%s", line) }, + Remove: s.Delete, + Figures: func() (uint64, uint64, error) { + i, err := s.Info() + if err != nil { + return 0, 0, err + } + return i.Bytes, uint64(max(i.MaxBytes, 0)), nil + }, + Entries: func() ([]PruneEntry, error) { + var out []PruneEntry + cursor := "" + for { + page, err := s.page(cursor, 768<<10, stateTimeout/2) + if err != nil { + return nil, err + } + for _, e := range page.Entries { + out = append(out, PruneEntry{Key: e.Key, Revision: e.Revision, Bytes: PutBytes(e.Key, e.Value), Value: e.Value}) + } + if page.Next == "" || page.Next == cursor { + return out, nil + } + cursor = page.Next + } + }} + return ps +} + +// Put writes a key whole and returns the revision written; the fill its answer carries is judged, and past the mark +// the state is pruned to its target before Put returns. A runtime whose answer carries no fill is said once, as a +// state that is never pruned. +func (s *PrunedState) Put(key string, value any) (uint64, error) { + raw, err := s.ask("put", map[string]any{"key": key, "value": value}) + if err != nil { + return 0, err + } + var put struct { + Revision uint64 `json:"revision"` + Fill *buscalls.Fill `json:"fill"` + } + if err := json.Unmarshal(raw, &put); err != nil { + return 0, err + } + if put.Fill == nil { + s.unsaid.Do(func() { + sayf("the state %s is never pruned: the runtime's put answer carries no fill (design 47 H2 §812)", s.name) + }) + return put.Revision, nil + } + if _, err := s.pruner.Filled(*put.Fill); err != nil { + return put.Revision, err + } + return put.Revision, nil +} + +// PruneNow prunes the state to its target now. +func (s *PrunedState) PruneNow() (Pruned, error) { return s.pruner.PruneNow() } + +// Keep keeps key from the prune until until, written under KeepKey. +func (s *PrunedState) Keep(key string, until time.Time) error { + if err := KeepProblem(key, until); err != nil { + return err + } + _, err := s.Put(KeepKey(key), Keep{Until: until.UTC()}) + return err +} + +// KeepProblem is why key cannot be kept until until, or nil: a process and a bundle check a keep alike. +func KeepProblem(key string, until time.Time) error { + switch { + case key == "" || IsKeep(key): + return fmt.Errorf("%q is not a key a keep keeps", key) + case !until.After(time.Now()): + return fmt.Errorf("a keep until %s keeps nothing: until is a time still to come", until.Format(time.RFC3339)) + } + return nil +} diff --git a/go/seal/anonymous_test.go b/go/seal/anonymous_test.go new file mode 100644 index 00000000..3dc20386 --- /dev/null +++ b/go/seal/anonymous_test.go @@ -0,0 +1,61 @@ +package seal + +import ( + "bytes" + "crypto/sha256" + "encoding/base64" + "encoding/json" + "testing" + + "golang.org/x/crypto/nacl/box" +) + +// anonymousFixture holds the NaCl anonymous boxes golang.org/x/crypto/nacl/box seals (the box the vault's intake opens), +// with their one-time keys, for the SDK's TypeScript sealAnonymous to make byte for byte (design 47 H2, H-A-407). +type anonymousFixture struct { + About string `json:"about"` + Recipient struct { + Public string `json:"public"` + Private string `json:"private"` + } `json:"recipient"` + Vectors []struct { + Plaintext string `json:"plaintext"` + EphSecret string `json:"ephSecret"` + Sealed string `json:"sealed"` + } `json:"vectors"` +} + +const anonymousFixturePath = "testdata/sealed-anonymous-by-go.json" + +// anonymousVectors seals each plaintext with box.SealAnonymous to a recipient and one-time keys derived from fixed +// labels: test keys that guard nothing. +func anonymousVectors(t *testing.T) anonymousFixture { + t.Helper() + b64 := base64.StdEncoding.EncodeToString + seed := sha256.Sum256([]byte("mesh-sdk anonymous box test recipient")) + pub, priv, err := box.GenerateKey(bytes.NewReader(seed[:])) + if err != nil { + t.Fatal(err) + } + var f anonymousFixture + f.About = "Made by this package's anonymous_test.go with golang.org/x/crypto/nacl/box's SealAnonymous, its one-time keys fixed; the test proves the file is Go's bytes on every run. Test keys; they guard nothing." + f.Recipient.Public, f.Recipient.Private = b64(pub[:]), b64(priv[:]) + for i, plain := range []string{"", "a token minted for the forge", "ünïcode and a longer value: " + string(bytes.Repeat([]byte("x"), 300))} { + eph := sha256.Sum256([]byte{byte(i), 'e', 'p', 'h'}) + sealed, err := box.SealAnonymous(nil, []byte(plain), pub, bytes.NewReader(eph[:])) + if err != nil { + t.Fatal(err) + } + f.Vectors = append(f.Vectors, struct { + Plaintext string `json:"plaintext"` + EphSecret string `json:"ephSecret"` + Sealed string `json:"sealed"` + }{plain, b64(eph[:]), b64(sealed)}) + } + return f +} + +func mustJSON(v any) string { + raw, _ := json.Marshal(v) + return string(raw) +} diff --git a/go/seal/seal_test.go b/go/seal/seal_test.go index 2a2786e2..8c704617 100644 --- a/go/seal/seal_test.go +++ b/go/seal/seal_test.go @@ -6,6 +6,8 @@ import ( "encoding/json" "os" "testing" + + "golang.org/x/crypto/nacl/box" ) type fixture struct { @@ -75,6 +77,39 @@ func TestSealIsByteForByteTheLicenceManagersBox(t *testing.T) { t.Fatalf("reopened %q, %v", plain, err) } } + // The anonymous-box fixture the TypeScript tests read is what box.SealAnonymous makes, and each opens with the + // recipient's key as the vault's intake opens it (box.OpenAnonymous). MESH_SDK_WRITE_FIXTURES=1 writes it anew. + t.Run("the anonymous box fixture is Go's bytes", func(t *testing.T) { + want := anonymousVectors(t) + if os.Getenv("MESH_SDK_WRITE_FIXTURES") == "1" { + raw, _ := json.MarshalIndent(want, "", " ") + if err := os.WriteFile(anonymousFixturePath, append(raw, '\n'), 0o644); err != nil { + t.Fatal(err) + } + } + raw, err := os.ReadFile(anonymousFixturePath) + if err != nil { + t.Fatal(err) + } + var got anonymousFixture + if err := json.Unmarshal(raw, &got); err != nil { + t.Fatal(err) + } + if g, w := mustJSON(got), mustJSON(want); g != w { + t.Fatalf("the fixture is not Go's bytes:\n%s\nwant\n%s", g, w) + } + var pub, priv [32]byte + p, _ := base64.StdEncoding.DecodeString(got.Recipient.Public) + k, _ := base64.StdEncoding.DecodeString(got.Recipient.Private) + copy(pub[:], p) + copy(priv[:], k) + for _, v := range got.Vectors { + sealed, _ := base64.StdEncoding.DecodeString(v.Sealed) + if out, ok := box.OpenAnonymous(nil, sealed, &pub, &priv); !ok || string(out) != v.Plaintext { + t.Errorf("a box does not open to %q: %q, %v", v.Plaintext, out, ok) + } + } + }) } // A box the agent module's TypeScript sealed (the fixture the licence manager keeps) opens here too. @@ -117,40 +152,38 @@ func TestABoxOpensOnlyForItsRecipient(t *testing.T) { if again.Eph == box.Eph || again.IV == box.IV { t.Fatal("two boxes share a one-time key or iv") } -} - -// A box changed in any field, or of another version, does not open. -func TestATamperedBoxDoesNotOpen(t *testing.T) { - f := readFixture(t) - good := f.Sealed[1].Box - flip := func(s string) string { - b, _ := base64.StdEncoding.DecodeString(s) - b[len(b)-1] ^= 1 - return base64.StdEncoding.EncodeToString(b) - } - for name, box := range map[string]Box{ - "ct": {V: 1, Eph: good.Eph, IV: good.IV, Tag: good.Tag, Ct: flip(good.Ct)}, - "tag": {V: 1, Eph: good.Eph, IV: good.IV, Tag: flip(good.Tag), Ct: good.Ct}, - "iv": {V: 1, Eph: good.Eph, IV: flip(good.IV), Tag: good.Tag, Ct: good.Ct}, - "eph": {V: 1, Eph: flip(good.Eph), IV: good.IV, Tag: good.Tag, Ct: good.Ct}, - "version": {V: 2, Eph: good.Eph, IV: good.IV, Tag: good.Tag, Ct: good.Ct}, - } { - if _, err := Open(box, f.Recipient.PrivateKey); err == nil { - t.Fatalf("a box with a changed %s opened", name) + // A box changed in any field, or of another version, does not open. + t.Run("a tampered box does not open", func(t *testing.T) { + f := readFixture(t) + good := f.Sealed[1].Box + flip := func(s string) string { + b, _ := base64.StdEncoding.DecodeString(s) + b[len(b)-1] ^= 1 + return base64.StdEncoding.EncodeToString(b) } - } -} - -// A key that is not X25519 PEM is refused, not misread. -func TestAKeyOfAnotherKindIsRefused(t *testing.T) { - if _, err := Seal("x", "not a key"); err == nil { - t.Fatal("sealed to text") - } - f := readFixture(t) - if _, err := Open(f.Sealed[0].Box, f.Recipient.PublicKey); err == nil { - t.Fatal("opened with a public key") - } - if _, err := Seal("x", f.Recipient.PrivateKey); err == nil { - t.Fatal("sealed to a private key") - } + for name, box := range map[string]Box{ + "ct": {V: 1, Eph: good.Eph, IV: good.IV, Tag: good.Tag, Ct: flip(good.Ct)}, + "tag": {V: 1, Eph: good.Eph, IV: good.IV, Tag: flip(good.Tag), Ct: good.Ct}, + "iv": {V: 1, Eph: good.Eph, IV: flip(good.IV), Tag: good.Tag, Ct: good.Ct}, + "eph": {V: 1, Eph: flip(good.Eph), IV: good.IV, Tag: good.Tag, Ct: good.Ct}, + "version": {V: 2, Eph: good.Eph, IV: good.IV, Tag: good.Tag, Ct: good.Ct}, + } { + if _, err := Open(box, f.Recipient.PrivateKey); err == nil { + t.Fatalf("a box with a changed %s opened", name) + } + } + }) + // A key that is not X25519 PEM is refused, not misread. + t.Run("a key of another kind is refused", func(t *testing.T) { + if _, err := Seal("x", "not a key"); err == nil { + t.Fatal("sealed to text") + } + f := readFixture(t) + if _, err := Open(f.Sealed[0].Box, f.Recipient.PublicKey); err == nil { + t.Fatal("opened with a public key") + } + if _, err := Seal("x", f.Recipient.PrivateKey); err == nil { + t.Fatal("sealed to a private key") + } + }) } diff --git a/go/seal/testdata/sealed-anonymous-by-go.json b/go/seal/testdata/sealed-anonymous-by-go.json new file mode 100644 index 00000000..070ae684 --- /dev/null +++ b/go/seal/testdata/sealed-anonymous-by-go.json @@ -0,0 +1,24 @@ +{ + "about": "Made by this package's anonymous_test.go with golang.org/x/crypto/nacl/box's SealAnonymous, its one-time keys fixed; the test proves the file is Go's bytes on every run. Test keys; they guard nothing.", + "recipient": { + "public": "n2nWtTM35RrjXe1pPBzIHWRfSvoE5AMFxFWA92lYmj4=", + "private": "0zveHnrRJV0at3GITmix/NpjX1sJPgsu0tGZIXFVKos=" + }, + "vectors": [ + { + "plaintext": "", + "ephSecret": "CU2adSGwGEEx6/xxSvrQ8Vfq6Yc+kR+2yX7fFriYCbg=", + "sealed": "8qO+LLqjzgOtuE2KVM6bKDB3dMiMiAuEc0hgRwGe03wNBJ60VwAAIw4xPJLmmT1g" + }, + { + "plaintext": "a token minted for the forge", + "ephSecret": "CvmgsKRsGyTmAFRD4EB0mAPpBVN3I4yLJJkkcfbqX0A=", + "sealed": "erK5lMtu+k90Thig1rpUq2jJ9DzF/WZWvVvMPNO+hiPF5uL5jnqNB7h3hQro13ZiKwK4O/hm3MvZm9glM8KqSBoeApTrVWnvP/Ekzg==" + }, + { + "plaintext": "ünïcode and a longer value: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", + "ephSecret": "ypSqdS+hEEoCsEDT6K87sc/I/h9yi06j08c1bKEJAQw=", + "sealed": "QQ7wBpBtvQH4BvkicANk50VlTJ/puVD+BhRgD9ccAl6NriByxkFqxb3GQUy++HAhByqR9vd+NSSt59dVmjKIFdiiJKCvH6zgGWkHEcVBtn1F5PUzscO+yhEwxuuPK1pLJ24PBdcef9x111Yu4aKgG31mt5z0Bnxs9YZAobz8Nx2vkPYfz1Fzu6Tqqq9bElH9rbR/Ua+ChpGu0BgnKgAWRQlnXBOjrVUC4DFa4wFDfCROgFWC1rM3fQCd8Tm4OHpa6RMRhE9pafi8OMP0XIwUrL29JDkcyMtGPuDab3f6NTgGI+yTCHxNgbOupxeA18Ied7om9rt1oAsEfo2SrYEvzplFCgYpDuXmf693zNPFKuGHd18FZmQKwp6gODpO2t+UCEzJa5EB/9vaf44BWcRIDLWiLzx2Pg0QA3DxZziSL4eff9TOeN8qUHH5pYdU5Cdu7clbhImNoe9kqNSZb3NOwav+HZh4SjtfzcHrBTMWbKs8i0jZYLgxcEcr" + } + ] +} diff --git a/go/seat.go b/go/seat.go index 9bf4866b..f6ff2e11 100644 --- a/go/seat.go +++ b/go/seat.go @@ -2,6 +2,8 @@ package stdio import ( "encoding/json" + "errors" + "fmt" "strings" "time" ) @@ -201,6 +203,38 @@ func (s KeptState) CreateTTL(key string, value any, ttl time.Duration) (uint64, return s.revisionOf(s.ask("create", map[string]any{"key": key, "value": value, "ttl_seconds": ttlSeconds(ttl)})) } +// TakenState is the state a holder of a kept call keeps the ids of the asks it took in (design 47 H2 §775, §820): a +// state of 8 MiB whose keys live the ask's bound and are deleted by their time to live alone, never pruned. +const TakenState = "taken" + +// TakenKey is the key an ask taken from seat is kept under in TakenState: `.`. +func TakenKey(seat, ask string) string { return seat + "." + ask } + +// TakenProblem is why seat and ask name no taken key, or empty: a seat's name and an ask's id are each one key token. +func TakenProblem(seat, ask string) string { + switch { + case ask == "" || strings.ContainsAny(ask, ".*> "): + return fmt.Sprintf("%q is not an ask id: an ask id holds no dot and no wildcard, naming one key and no other", ask) + case seat == "" || strings.ContainsAny(seat, ".*> "): + return fmt.Sprintf("%q is not a seat's name", seat) + } + return "" +} + +// TakeOnce says whether this holder takes the ask ask of seat now (design 47 H2 §775; ADR 0668 §2; H-A-359): the +// first time it is given it is kept under TakenKey for the ask's bound and TakeOnce answers true; given again within +// the bound, false, and the holder acts once. s is the holder's TakenState, declared `key-ttl`. +func (s KeptState) TakeOnce(seat, ask string, bound time.Duration) (bool, error) { + if p := TakenProblem(seat, ask); p != "" { + return false, fmt.Errorf("taking an ask of %s: %s", seat, p) + } + _, err := s.CreateTTL(TakenKey(seat, ask), map[string]string{"seat": seat, "ask": ask}, max(bound, time.Second)) + if errors.Is(err, ErrChanged) { + return false, nil + } + return err == nil, err +} + // Update writes a key only when it is still at revision, and returns the new one; a key changed // meanwhile is refused, so of two writers that read one value only the first writes. func (s KeptState) Update(key string, value any, revision uint64) (uint64, error) { @@ -230,3 +264,39 @@ func subjectMatches(pattern, subject string) bool { } return len(p) == len(s) } + +// WaitingAsk is one ask waiting in a seat's work queue, as the runtime reads it without taking it (design 47 H2 §777, +// H-A-410): the ask's id, the module that asked it and its age in seconds. Again says it was handed out and handed back +// (a nak) and no worker holds it: waiting again (H-A-410's addendum). +type WaitingAsk struct { + ID string `json:"id"` + Module string `json:"module"` + AgeSeconds float64 `json:"age_seconds"` + Again bool `json:"again,omitempty"` +} + +// Waiting is the asks waiting in a seat's work queue, oldest first as the queue holds them; Capped says the runtime cut +// them at 768 KB. +type Waiting struct { + Asks []WaitingAsk `json:"asks"` + Capped bool `json:"capped"` +} + +// QueueWaiting is the asks waiting in the work queue of seat, this holder's own queue, which a holder's asks tool answers +// (design 47 H2 §777, H-A-410): asked of the runtime as mesh/queue.waiting, which reads the queue's waiting messages +// without taking them. A refusal is the runtime's error. +func QueueWaiting(seat string) (Waiting, error) { + c, err := live() + if err != nil { + return Waiting{}, err + } + raw, err := c.ask("mesh/queue.waiting", map[string]any{"seat": seat}, seatTimeout) + if err != nil { + return Waiting{}, err + } + var w Waiting + if err := json.Unmarshal(raw, &w); err != nil { + return Waiting{}, fmt.Errorf("the runtime answered the waiting asks of %s in no form this reads: %w", seat, err) + } + return w, nil +} diff --git a/go/seatdef/seatdef.go b/go/seatdef/seatdef.go index a9a6fa37..1ca8cb08 100644 --- a/go/seatdef/seatdef.go +++ b/go/seatdef/seatdef.go @@ -49,10 +49,13 @@ var Domains = []string{"Module", "Config", "Core", "Provisioning", "Identity and // Definition is one seat's contract. type Definition struct { - Format int `json:"format"` - Seat string `json:"seat"` - Scope string `json:"scope"` - Grain string `json:"grain"` + Format int `json:"format"` + // Form is the definition's form (design 47 H2 §780; H-A-352): raised at every field its values, events, accepts and + // answers add or remove. Absent reads 1; below 1 is refused. + Form int `json:"form,omitempty"` + Seat string `json:"seat"` + Scope string `json:"scope"` + Grain string `json:"grain"` // Decision is the record that made it a seat. Decision string `json:"decision"` // Domain is the glossary domain the seat belongs to, by its heading (ADR 0636 §5): the indexer takes a seat of the @@ -265,6 +268,40 @@ type Message struct { // `accept.`, the kind in the body alone (design 47 G §600, A-170). Its generated names and an invoke name it // `.`, so the same kind name may ride more than one stream (design 47 G §415). Stream string `json:"stream,omitempty"` + // The work queue's settings an accepted kind names (design 47 H2 §777; H-A-350 DMC-1, H-A-362), from which mesh-sdk's + // buscompose derives the seat's work-queue stream and its one shared worker: durations as Go writes them ("24h", + // "2m", "60s", "20s"), counts as integers. Each absent is buscompose's default. + MaxAge string `json:"max-age,omitempty"` + DuplicatesWindow string `json:"duplicates-window,omitempty"` + AckWait string `json:"ack-wait,omitempty"` + InProgress string `json:"in-progress,omitempty"` + MaxMsgs int `json:"max-msgs,omitempty"` + MaxAckPending int `json:"max-ack-pending,omitempty"` + MaxDeliveries int `json:"max-deliveries,omitempty"` +} + +// QueueProblems is what is wrong with an accepted kind's work-queue settings: a duration that is not one above zero, or +// a count below zero. +func (m Message) QueueProblems(where string) Problems { + var ps Problems + for _, d := range []struct{ name, value string }{{"max-age", m.MaxAge}, {"duplicates-window", m.DuplicatesWindow}, + {"ack-wait", m.AckWait}, {"in-progress", m.InProgress}} { + if d.value == "" { + continue + } + if v, err := time.ParseDuration(d.value); err != nil || v <= 0 { + ps = append(ps, Problem{where + "." + d.name, fmt.Sprintf("%q is not a duration above zero, as \"24h\" or \"60s\"", d.value)}) + } + } + for _, n := range []struct { + name string + value int + }{{"max-msgs", m.MaxMsgs}, {"max-ack-pending", m.MaxAckPending}, {"max-deliveries", m.MaxDeliveries}} { + if n.value < 0 { + ps = append(ps, Problem{where + "." + n.name, fmt.Sprintf("%d is below zero", n.value)}) + } + } + return ps } // What an event's keep says where it says less (design 47 G §396, §331). @@ -468,7 +505,9 @@ type Path struct { // Part is verbs, events and types several definitions include whole. type Part struct { - Format int `json:"format"` + Format int `json:"format"` + // Form is the part's own form, for what it adds (H-A-352): absent reads 1; below 1 is refused. + Form int `json:"form,omitempty"` Part string `json:"part"` Description string `json:"description,omitempty"` Verbs []Verb `json:"verbs,omitempty"` @@ -528,9 +567,21 @@ func Parse(raw []byte) (Definition, error) { if ps := d.check(); len(ps) > 0 { return Definition{}, ps } + d.Form = formRead(d.Form) return d, nil } +// formRead is a form as read: absent is 1 (H-A-352). +func formRead(n int) int { return max(n, 1) } + +// formProblems refuses a form below 1; absent reads 1. +func formProblems(n int) Problems { + if n < 0 { + return Problems{{"form", fmt.Sprintf("form %d: a form is 1 or more, and absent reads 1", n)}} + } + return nil +} + // ParsePart reads one part. func ParsePart(raw []byte) (Part, error) { var p Part @@ -544,11 +595,13 @@ func ParsePart(raw []byte) (Part, error) { if !wordName.MatchString(p.Part) { ps = append(ps, Problem{"part", "missing, or not a lower-case name"}) } + ps = append(ps, formProblems(p.Form)...) ps = append(ps, checkVerbs(p.Verbs, true)...) ps = append(ps, checkMessages("events", p.Events, true)...) if len(ps) > 0 { return Part{}, ps } + p.Form = formRead(p.Form) return p, nil } @@ -633,6 +686,7 @@ func (d Definition) check() Problems { if d.Format != Format { ps = append(ps, Problem{"format", formatProblem(d.Format)}) } + ps = append(ps, formProblems(d.Form)...) if d.Seat == "" { ps = append(ps, Problem{"seat", "missing"}) } else if !seatName.MatchString(d.Seat) { @@ -868,6 +922,7 @@ func checkMessages(field string, ms []Message, whole bool) Problems { ps = append(ps, Problem{where + ".keep", "an event of one subject kept last per subject is one message: give it a subject with parts"}) } } + ps = append(ps, m.QueueProblems(where)...) if m.Subject != "" && field != "events" { ps = append(ps, Problem{where + ".subject", "only an event carries a subject of its own"}) } else if why := checkSubject(m); why != "" { diff --git a/go/seatdef/seatdef_test.go b/go/seatdef/seatdef_test.go index cf5730d5..383b7d17 100644 --- a/go/seatdef/seatdef_test.go +++ b/go/seatdef/seatdef_test.go @@ -154,6 +154,10 @@ func TestADefinitionIsReadStrictlyNamingWhatIsWrong(t *testing.T) { `"accepts":[{"name":"step","body":{},"stream":"check"},{"name":"step","body":{},"stream":"check"}],"events":`, 1), `"step" twice`}, {"a stream that is none", strings.Replace(whole, `"events":`, `"accepts":[{"name":"step","body":{},"stream":"mail"}],"events":`, 1), `accepts[0].stream: "mail" is not check or delivery`}, + // Design 47 H2 §777, H-A-362: an accepted kind names its work queue's settings, durations as Go writes them. + {"a work queue's settings", strings.Replace(whole, `"events":`, `"accepts":[{"name":"step","body":{},"max-age":"24h","duplicates-window":"2m","ack-wait":"60s","in-progress":"20s","max-msgs":10000,"max-ack-pending":64,"max-deliveries":5}],"events":`, 1), ""}, + {"a max age that is no duration", strings.Replace(whole, `"events":`, `"accepts":[{"name":"step","body":{},"max-age":"a day"}],"events":`, 1), "accepts[0].max-age"}, + {"a negative max-msgs", strings.Replace(whole, `"events":`, `"accepts":[{"name":"step","body":{},"max-msgs":-1}],"events":`, 1), "accepts[0].max-msgs"}, {"an event on a stream", strings.Replace(whole, `"name":"created"`, `"name":"created","stream":"check"`, 1), "only an accepted kind rides a stream"}, {"over 256 KiB", listGrain[:len(listGrain)-1] + `,"description":"` + strings.Repeat("x", MaxSize) + `"}`, ErrTooLarge.Error()}, @@ -173,6 +177,35 @@ func TestADefinitionIsReadStrictlyNamingWhatIsWrong(t *testing.T) { `"}`)); !errors.Is(err, ErrTooLarge) { t.Errorf("over 256 KiB is ErrTooLarge, got %v", err) } + // Design 47 H2 §780, H-A-352: a definition and a part carry their form; absent reads 1, below 1 is refused. + for _, c := range []struct { + raw string + form int + bad bool + }{ + {whole, 1, false}, + {strings.Replace(whole, `"seat":`, `"form":3,"seat":`, 1), 3, false}, + {strings.Replace(whole, `"seat":`, `"form":-1,"seat":`, 1), 0, true}, + } { + d, err := Parse([]byte(c.raw)) + if c.bad != (err != nil) || (!c.bad && d.Form != c.form) || (c.bad && !strings.Contains(err.Error(), "form")) { + t.Errorf("a definition of form %d read as %d, %v", c.form, d.Form, err) + } + } + for _, c := range []struct { + raw string + form int + bad bool + }{ + {`{"format":1,"part":"registry"}`, 1, false}, + {`{"format":1,"form":2,"part":"registry"}`, 2, false}, + {`{"format":1,"form":-2,"part":"registry"}`, 0, true}, + } { + p, err := ParsePart([]byte(c.raw)) + if c.bad != (err != nil) || (!c.bad && p.Form != c.form) { + t.Errorf("a part %s read as form %d, %v", c.raw, p.Form, err) + } + } // A part is taken in whole, its verbs before the definition's own; a part not given, or a type defined twice, // is refused. diff --git a/go/seatlist/bootstrap/git.json b/go/seatlist/bootstrap/git.json index db42887f..b038f415 100644 --- a/go/seatlist/bootstrap/git.json +++ b/go/seatlist/bootstrap/git.json @@ -5,7 +5,7 @@ "grain": "whole", "domain": "Delivery", "decision": "novox/hq ADR 0111", - "description": "The forge: every read and act a delivery part makes of the mesh's git repositories, and what the forge says happened (design 47 *The seat git*, ADR 0596 \u00a71, \u00a72). A failed read answers an error, never an empty answer.", + "description": "The forge: every read and act a delivery part makes of the mesh's git repositories, and what the forge says happened (design 47 *The seat git*, ADR 0596 §1, §2). A failed read answers an error, never an empty answer.", "delivers": "git", "verbs": [ { @@ -48,6 +48,14 @@ }, "error-kind": { "$ref": "#/defs/ErrorKind" + }, + "part": { + "type": "string", + "description": "with the error kind away, what did not answer (design 47 H2 §776)" + }, + "words": { + "type": "string", + "description": "with the error kind away, the holder's words (design 47 H2 §776)" } }, "description": "what pull answers; error and error-kind alone when it fails" @@ -96,6 +104,14 @@ }, "error-kind": { "$ref": "#/defs/ErrorKind" + }, + "part": { + "type": "string", + "description": "with the error kind away, what did not answer (design 47 H2 §776)" + }, + "words": { + "type": "string", + "description": "with the error kind away, the holder's words (design 47 H2 §776)" } }, "description": "what branch answers; error and error-kind alone when it fails" @@ -126,6 +142,14 @@ }, "error-kind": { "$ref": "#/defs/ErrorKind" + }, + "part": { + "type": "string", + "description": "with the error kind away, what did not answer (design 47 H2 §776)" + }, + "words": { + "type": "string", + "description": "with the error kind away, the holder's words (design 47 H2 §776)" } }, "description": "what open-pulls answers; pulls always said, empty when none is open; error and error-kind beside an empty pulls when it fails", @@ -171,9 +195,9 @@ "tags": { "type": "array", "items": { - "type": "string" + "$ref": "#/defs/Tag" }, - "description": "the tags of this page" + "description": "the tags of this page, each with the commit it names (design 47 H2 §796)" }, "page": { "type": "integer", @@ -193,6 +217,14 @@ }, "error-kind": { "$ref": "#/defs/ErrorKind" + }, + "part": { + "type": "string", + "description": "with the error kind away, what did not answer (design 47 H2 §776)" + }, + "words": { + "type": "string", + "description": "with the error kind away, the holder's words (design 47 H2 §776)" } }, "description": "what tags answers; error and error-kind alone when it fails" @@ -201,7 +233,7 @@ }, { "name": "trees", - "description": "The trees of one commit: the root tree and the tree of every directory that holds a module.json, by their git hashes; with files, every blob under each named directory; with paths, the object of each named path. A listing the forge cut is refused, never answered in part.", + "description": "The trees of one commit: the root tree and the tree of every directory that holds a module.json, by their git hashes; with files, every blob under each named directory; with paths, the object of each named path. A listing the forge cut is refused, never answered in part. A directory named in files or paths that the commit does not hold answers the error kind not-found (design 47 H2 §796).", "input": { "type": "object", "properties": { @@ -284,6 +316,14 @@ }, "error-kind": { "$ref": "#/defs/ErrorKind" + }, + "part": { + "type": "string", + "description": "with the error kind away, what did not answer (design 47 H2 §776)" + }, + "words": { + "type": "string", + "description": "with the error kind away, the holder's words (design 47 H2 §776)" } }, "description": "what trees answers; error and error-kind alone when it fails" @@ -350,6 +390,14 @@ }, "error-kind": { "$ref": "#/defs/ErrorKind" + }, + "part": { + "type": "string", + "description": "with the error kind away, what did not answer (design 47 H2 §776)" + }, + "words": { + "type": "string", + "description": "with the error kind away, the holder's words (design 47 H2 §776)" } }, "description": "what file answers; error and error-kind alone when it fails" @@ -410,6 +458,14 @@ }, "error-kind": { "$ref": "#/defs/ErrorKind" + }, + "part": { + "type": "string", + "description": "with the error kind away, what did not answer (design 47 H2 §776)" + }, + "words": { + "type": "string", + "description": "with the error kind away, the holder's words (design 47 H2 §776)" } }, "description": "what contains answers; error and error-kind alone when it fails" @@ -464,6 +520,14 @@ }, "error-kind": { "$ref": "#/defs/ErrorKind" + }, + "part": { + "type": "string", + "description": "with the error kind away, what did not answer (design 47 H2 §776)" + }, + "words": { + "type": "string", + "description": "with the error kind away, the holder's words (design 47 H2 §776)" } }, "description": "what statuses answers; error and error-kind alone when it fails" @@ -546,6 +610,14 @@ }, "error-kind": { "$ref": "#/defs/ErrorKind" + }, + "part": { + "type": "string", + "description": "with the error kind away, what did not answer (design 47 H2 §776)" + }, + "words": { + "type": "string", + "description": "with the error kind away, the holder's words (design 47 H2 §776)" } }, "description": "what status answers; error and error-kind alone when it fails" @@ -607,6 +679,14 @@ }, "error-kind": { "$ref": "#/defs/ErrorKind" + }, + "part": { + "type": "string", + "description": "with the error kind away, what did not answer (design 47 H2 §776)" + }, + "words": { + "type": "string", + "description": "with the error kind away, the holder's words (design 47 H2 §776)" } }, "description": "what comment answers; error and error-kind alone when it fails" @@ -668,6 +748,14 @@ }, "error-kind": { "$ref": "#/defs/ErrorKind" + }, + "part": { + "type": "string", + "description": "with the error kind away, what did not answer (design 47 H2 §776)" + }, + "words": { + "type": "string", + "description": "with the error kind away, the holder's words (design 47 H2 §776)" } }, "description": "what view answers; error and error-kind alone when it fails" @@ -730,6 +818,14 @@ }, "error-kind": { "$ref": "#/defs/ErrorKind" + }, + "part": { + "type": "string", + "description": "with the error kind away, what did not answer (design 47 H2 §776)" + }, + "words": { + "type": "string", + "description": "with the error kind away, the holder's words (design 47 H2 §776)" } }, "description": "what note answers; error and error-kind alone when it fails" @@ -738,7 +834,7 @@ }, { "name": "merge-pull", - "description": "Merge one or more pull requests with the forge tools' account, each at the head the caller names: one merge call, a delivery's start and, with two or more members, its group (design 47 H \u00a7734, \u00a7735, \u00a7766; ADR 0682 \u00a71, \u00a72). Every member is judged before any is merged: still the pull request's head, its base a watched branch, no run at the head's build-input fingerprint failed a required step (judged from its own state failed-steps), registrations heard; no branch protection is read. When one member is refused, none is merged. Then it merges them in the order named, writing the call in its state calls before the first merge, and says merge-called once. It answers once, at the call's end, naming the call and each member's outcome; a member already merged is refused already-merged, naming its commit.", + "description": "Merge one or more pull requests with the forge tools' account, each at the head the caller names: one merge call, a delivery's start and, with two or more members, its group (design 47 H §734, §735, §766; ADR 0682 §1, §2). Every member is judged before any is merged: still the pull request's head, its base a watched branch, no run at the head's build-input fingerprint failed a required step (judged from its own state failed-steps), registrations heard; no branch protection is read. When one member is refused, none is merged. Then it merges them in the order named, writing the call in its state calls before the first merge, and says merge-called once. It answers once, at the call's end, naming the call and each member's outcome; a member already merged is refused already-merged, naming its commit.", "input": { "type": "object", "properties": { @@ -805,6 +901,14 @@ }, "error-kind": { "$ref": "#/defs/ErrorKind" + }, + "part": { + "type": "string", + "description": "with the error kind away, what did not answer (design 47 H2 §776)" + }, + "words": { + "type": "string", + "description": "with the error kind away, the holder's words (design 47 H2 §776)" } }, "description": "what merge-pull answers at the call's end; error and error-kind alone when it fails" @@ -815,7 +919,7 @@ { "name": "tag", "class": "act", - "description": "Write the git tag of a released version, go/vMAJOR.MINOR.PATCH, at the commit its build was made from; the builder writes it only on the catalog's version-released (design 47 G section 370). A tag already at that commit is answered performed. An error answer carries error and error-kind.", + "description": "Write the git tag of a released version, /vMAJOR.MINOR.PATCH for a Go package (design 47 H2 §796), at the commit its build was made from; the builder writes it only on the catalog's version-released (design 47 G section 370). A tag already at that commit is answered performed. An error answer carries error and error-kind.", "input": { "type": "object", "properties": { @@ -856,6 +960,14 @@ }, "error-kind": { "$ref": "#/defs/ErrorKind" + }, + "part": { + "type": "string", + "description": "with the error kind away, what did not answer (design 47 H2 §776)" + }, + "words": { + "type": "string", + "description": "with the error kind away, the holder's words (design 47 H2 §776)" } }, "description": "what tag answers; error and error-kind alone when it fails" @@ -1259,7 +1371,7 @@ "name": "calls", "template": "state", "history": 1, - "description": "every merge call, written before its first merge; one whose merge-called is not said is said at the holder's start; deleted past the stream merge-called's age (design 47 H \u00a7766, H-A-47)", + "description": "every merge call, written before its first merge; one whose merge-called is not said is said at the holder's start; deleted past the stream merge-called's age (design 47 H §766, H-A-47)", "key-forms": [ { "key": "call.", @@ -1328,7 +1440,7 @@ "description": "its page on the forge" } }, - "description": "one pull request as the forge holds it; never its description (ADR 0659 \u00a71)", + "description": "one pull request as the forge holds it; never its description (ADR 0659 §1)", "required": [ "repository", "number", @@ -1361,7 +1473,7 @@ "description": "whether the forge's merging account may write the repository: false, its statuses and its merge are refused" } }, - "description": "what the forge would merge now, for an open pull request (issues 1275, 1945; ADR 0441 \u00a71): answered whole or not at all, and a read the forge could not make answers the verb's error, never a guess; absent for a merged or closed one", + "description": "what the forge would merge now, for an open pull request (issues 1275, 1945; ADR 0441 §1): answered whole or not at all, and a read the forge could not make answers the verb's error, never a guess; absent for a merged or closed one", "required": [ "mergeable", "protected", @@ -1471,9 +1583,10 @@ "enum": [ "refused", "not-found", - "failed" + "failed", + "away" ], - "description": "which kind of error, set with error alone: refused, the forge or the holder refused and error says why in the forge's own words; not-found, the forge said that what was asked about (a repository, a commit, a pull request, a branch or a tag) does not exist, so a caller never reads it out of error's words; failed, the forge failed or could not be reached, so the act or read could not be performed" + "description": "which kind of error, set with error alone: refused, the forge or the holder refused and error says why in the forge's own words; not-found, the forge said that what was asked about (a repository, a commit, a pull request, a branch or a tag) does not exist, so a caller never reads it out of error's words; failed, the forge failed or could not be reached, so the act or read could not be performed; away, a part the holder depends on (the forge, its store) is away, with part naming what did not answer and words: a caller reads it as a transport fault and calls again (design 47 H2 §776)" }, "FailedStep": { "type": "object", @@ -1529,7 +1642,7 @@ }, "MemberOutcome": { "type": "object", - "description": "one member of a merge call and where it stands (design 47 H \u00a7766, H-A-22)", + "description": "one member of a merge call and where it stands (design 47 H §766, H-A-22)", "properties": { "repository": { "type": "string" @@ -1564,7 +1677,7 @@ }, "Call": { "type": "object", - "description": "one merge call, written before its first merge and updated as each member merges (design 47 H \u00a7766)", + "description": "one merge call, written before its first merge and updated as each member merges (design 47 H §766)", "properties": { "by": { "type": "string", @@ -1640,12 +1753,30 @@ "head", "state" ] + }, + "Tag": { + "type": "object", + "description": "a tag and the commit it names (design 47 H2 §796)", + "properties": { + "tag": { + "type": "string" + }, + "commit": { + "type": "string", + "description": "the commit it names, its full 40-hex sha", + "pattern": "^[0-9a-f]{40}$" + } + }, + "required": [ + "tag", + "commit" + ] } }, "bodies": { "MergeCalled": { "type": "object", - "description": "a merge call's members merged or refused, said once by the forge module (design 47 H \u00a7736, \u00a7767)", + "description": "a merge call's members merged or refused, said once by the forge module (design 47 H §736, §767)", "properties": { "call": { "type": "string", @@ -1702,7 +1833,7 @@ }, "refused": { "type": "array", - "description": "one per member not merged after every member was judged (design 47 H \u00a7767)", + "description": "one per member not merged after every member was judged (design 47 H §767)", "items": { "type": "object", "properties": { @@ -1732,5 +1863,6 @@ "members" ] } - } + }, + "form": 2 } diff --git a/go/seatlist/bootstrap/mesh-catalog.json b/go/seatlist/bootstrap/mesh-catalog.json index a6d62b1a..f40e9799 100644 --- a/go/seatlist/bootstrap/mesh-catalog.json +++ b/go/seatlist/bootstrap/mesh-catalog.json @@ -253,6 +253,49 @@ }, "class": "read" }, + { + "name": "seat-list", + "class": "read", + "description": "The trunk's seat list as the mesh holds it, the object of the holder's objects seat-lists of that digest (design 47 H2 §814, §820; H-A-441, H-A-443): its digest and size, then its bytes (the definitions resolved, a JSON array in the form of a composer request's seats, sorted by seat) in chunks of at most 512 KiB on the chunk subject of the call, each acknowledged by the caller on the ack subject within 30 seconds. A validate run's asker fills the request's seats from these bytes. Reads only.", + "input": { + "type": "object", + "properties": { + "call": { + "type": "string", + "description": "128 random bits the caller chooses: the subject token of this send's chunks and acknowledgements", + "minLength": 1 + } + }, + "required": [ + "call" + ] + }, + "answer": { + "type": "object", + "properties": { + "digest": { + "type": "string", + "description": "the seat list's sha256, the object's name in seat-lists" + }, + "size": { + "type": "integer", + "minimum": 0, + "description": "the seat list's size in bytes" + }, + "chunks": { + "type": "integer", + "minimum": 0, + "description": "how many chunks are sent" + } + }, + "required": [ + "digest", + "size", + "chunks" + ] + }, + "chunks": true + }, { "name": "revision", "description": "The catalog half of the revision view (ADR 0656 §6): one catalog graph revision with its entries whole, the trunk heads, the seat definitions by digest, and each manifest's verdict and fingerprint; the trunk's newest when no revision and no candidate is named, or a candidate's for its group and heads. Answered by the holder from what it holds, in its own runtime (ADR 0663), in pages of entries. Reads only. A watched branch with no revision yet answers {none-yet: true} alone (design 47 G section 530).", @@ -312,7 +355,7 @@ }, { "name": "registrations", - "description": "Every module's registration, or one with every version and who changed it on whose warrant (design 47 *A module's registration*). Its readers take a first view here, then hear registration-changed.", + "description": "Every module's registration, or one with every version and who changed it on whose warrant (design 47 *A module's registration*). Its readers take a first view here, then hear registration-changed. Filtered by module or repository while it reads and paged by the cursor next, at most 768 KB a page (design 47 H2 §806, §811).", "input": { "type": "object", "properties": { @@ -323,6 +366,14 @@ "history": { "type": "boolean", "description": "with every version of the one module" + }, + "repository": { + "type": "string", + "description": "only the registrations of this repository, /, applied while reading (design 47 H2 §806)" + }, + "next": { + "type": "string", + "description": "the cursor the previous page answered, opaque; absent for the first page (design 47 H2 §779)" } } }, @@ -344,6 +395,18 @@ }, "fill": { "$ref": "#/defs/Fill" + }, + "capped": { + "type": "boolean", + "description": "yes when the answer stopped at 768 KB; the rest is read from next (design 47 H2 §806)" + }, + "total": { + "type": "integer", + "description": "how many registrations the filters hold in all (design 47 H2 §811)" + }, + "next": { + "type": "string", + "description": "the cursor of the next page, opaque; empty at the end (design 47 H2 §779)" } }, "required": [ @@ -657,6 +720,32 @@ "source": { "type": "integer" }, + "call": { + "type": "string", + "description": "what made the revision, as the revision's call names it (design 47 H-A-212, H-A-213); absent for a candidate's" + }, + "call-kind": { + "type": "string", + "enum": [ + "take-in", + "release", + "take-in-again", + "recompute" + ], + "description": "the kind of what made the revision, which a reader reads instead of call's words (design 47 H-A-213 (3), H-A-214, H-A-215, H-A-217); absent for a candidate's" + }, + "module": { + "type": "string", + "description": "for call-kind release, the module released (H-A-215)" + }, + "version": { + "type": "string", + "description": "for call-kind release, the version released (H-A-215)" + }, + "repository": { + "type": "string", + "description": "for call-kind take-in-again, the repository taken in again (H-A-215)" + }, "group": { "type": "string" }, @@ -1448,6 +1537,13 @@ "$ref": "#/defs/Registration" }, "description": "a module's registration" + }, + { + "key": "source..", + "value": { + "$ref": "#/defs/Registration" + }, + "description": "a source-only registration of a repository with no module (design 47 H2 §815)" } ] }, @@ -1462,6 +1558,13 @@ "$ref": "#/defs/Registration" }, "description": "one version of a module's registration" + }, + { + "key": "source...", + "value": { + "$ref": "#/defs/Registration" + }, + "description": "one version of a source-only registration (design 47 H2 §815)" } ] }, @@ -1594,6 +1697,10 @@ "type": "string", "description": "its manifest's verdict: accepted, refused or held" }, + "class": { + "type": "string", + "description": "core or leaf: core when its manifest claims a seat, leaf when it claims none; an entry without it reads as leaf (design 47 H2 H-A-394, H-A-431)" + }, "fp": { "type": "string", "description": "its fingerprint (src2: and 64 hex digits); absent when it has none" @@ -1871,6 +1978,32 @@ "type": "integer", "description": "the source revision it was taken in from; for a candidate, the source revision its heads are merged onto" }, + "call": { + "type": "string", + "description": "what made the revision (design 47 H-A-212, H-A-213): a take-in's the gate's call its source revision names, a re-pin's release of , a take-in again's take-in again of ; absent for a candidate's" + }, + "call-kind": { + "type": "string", + "enum": [ + "take-in", + "release", + "take-in-again", + "recompute" + ], + "description": "the kind of what made the revision, which a reader reads instead of call's words (design 47 H-A-213 (3), H-A-214, H-A-215, H-A-217): a take-in's, a re-pin's on a module's release, a take-in again's, a recompute's that moved a fingerprint (its call naming the reason); absent for a candidate's" + }, + "module": { + "type": "string", + "description": "for call-kind release, the module released (H-A-215)" + }, + "version": { + "type": "string", + "description": "for call-kind release, the version released (H-A-215)" + }, + "repository": { + "type": "string", + "description": "for call-kind take-in-again, the repository taken in again (H-A-215)" + }, "group": { "type": "string", "description": "a candidate's group" @@ -1927,6 +2060,10 @@ "$ref": "#/defs/TakeInEntry" }, "description": "the movers of the take-in that wrote it, so a redelivered source-moved whose revision is kept says take-in.taken again from them, recomputing nothing (design 47 H §743, H-A-92); absent for one with no mover" + }, + "base": { + "type": "integer", + "description": "the newest kept revision of its branch below it, which the take-in that wrote it pinned for its pipeline: the base of the validate step's case before (design 47 H-A-147, H-A-151, H-A-154); absent on the branch's first take-in" } }, "required": [ @@ -2078,14 +2215,15 @@ }, "Registration": { "type": "object", - "description": "a module's registration (ADR 0666 section 2; design 47 *A module's registration*): its git origin, its manifest's path, its watched branches and its policy, every version with who changed it and the warrant of the operator's Approve", + "description": "a module's registration (ADR 0666 section 2; design 47 *A module's registration*): its git origin, its manifest's path, its watched branches and its policy, every version with who changed it and the warrant of the operator's Approve; or a source-only registration of a repository with no module (design 47 H2 §815): its origin, one watched branch, its policy and its check, naming no module and no path, kept under source..", "properties": { "v": { "type": "integer", "description": "the layout version" }, "module": { - "type": "string" + "type": "string", + "description": "the module; absent for a source-only registration" }, "origin": { "type": "string", @@ -2093,7 +2231,7 @@ }, "path": { "type": "string", - "description": "the path of its manifest in the repository" + "description": "the path of its manifest in the repository; absent for a source-only registration" }, "branches": { "type": "array", @@ -2121,12 +2259,14 @@ "at": { "type": "string", "format": "date-time" + }, + "check": { + "$ref": "#/defs/SourceCheck", + "description": "for a source-only registration, the one step its heads run (design 47 H2 §815)" } }, "required": [ - "module", "origin", - "path", "branches", "policy" ] @@ -2157,9 +2297,10 @@ "placed", "review required", "unknown module", - "nothing moves" + "nothing moves", + "holds-manifest" ], - "description": "why no ask was raised" + "description": "why no ask was raised; holds-manifest: a source-only registration whose repository holds a module.json at its root or under modules/ at the branch's head (design 47 H2 §815; H-A-450)" }, "words": { "type": "string", @@ -2469,6 +2610,34 @@ "ref", "language" ] + }, + "SourceCheck": { + "type": "object", + "description": "the check of a source-only registration: a build seat and a step kind, run at the repository's root, never a free command (design 47 H2 §815)", + "properties": { + "seat": { + "type": "string", + "enum": [ + "node-build-go", + "node-build-node", + "node-build-image" + ], + "description": "the build seat that runs it" + }, + "step": { + "type": "string", + "enum": [ + "lint", + "unit", + "integration" + ], + "description": "the step kind" + } + }, + "required": [ + "seat", + "step" + ] } }, "bodies": { @@ -2567,5 +2736,6 @@ "call" ] } - } + }, + "form": 5 } diff --git a/go/seatlist/bootstrap/mesh-delivery.json b/go/seatlist/bootstrap/mesh-delivery.json index 015f9657..08e74ab4 100644 --- a/go/seatlist/bootstrap/mesh-delivery.json +++ b/go/seatlist/bootstrap/mesh-delivery.json @@ -9,7 +9,7 @@ "verbs": [ { "name": "deliveries", - "description": "Every delivery not final, and those that ended in the last day, one line each: its id (owner/repository@commit), its state, its group, what it waits for and since when. Narrowed by state, repository or group.", + "description": "Every delivery not final, and those that ended in the last day, one line each: its id (owner/repository@commit), its state, its group, what it waits for and since when. Narrowed by state, repository or group. Filtered by repository, module, outcome and since while it reads, at most 768 KB a page, capped saying the rest is read from the cursor next (design 47 H2 §811).", "input": { "properties": { "all": { @@ -31,13 +31,55 @@ "state": { "description": "one state, e.g. delivering or held", "type": "string" + }, + "module": { + "type": "string", + "description": "only the deliveries that move this module, applied while reading (design 47 H2 §811)" + }, + "outcome": { + "type": "string", + "description": "only the deliveries that ended with this outcome, applied while reading (design 47 H2 §811)" + }, + "since": { + "type": "string", + "format": "date-time", + "description": "only the deliveries opened at or after this time, applied while reading (design 47 H2 §811)" + }, + "next": { + "type": "string", + "description": "the cursor the previous page answered, opaque; absent for the first page (design 47 H2 §779)" } }, "type": "object" }, "answer": { "type": "object", - "description": "what deliveries answers, as its description says" + "description": "what deliveries answers, as its description says", + "properties": { + "deliveries": { + "type": "array", + "items": { + "type": "object", + "description": "one delivery's record, as the gate keeps it" + }, + "description": "this page's deliveries, the open ones first, then the newest" + }, + "capped": { + "type": "boolean", + "description": "yes when the answer stopped at 768 KB; the rest is read from next (design 47 H2 §811)" + }, + "total": { + "type": "integer", + "description": "how many deliveries the filters hold in all (design 47 H2 §811)" + }, + "next": { + "type": "string", + "description": "the cursor of the next page, opaque; empty at the end (design 47 H2 §779)" + } + }, + "required": [ + "deliveries" + ] }, "class": "read" }, @@ -196,7 +238,7 @@ "name": "record-opened", "body": { "type": "object", - "description": "a delivery record opened for one source revision of a merge call (design 47 H \u00a7737)", + "description": "a delivery record opened for one source revision of a merge call (design 47 H §737)", "properties": { "record": { "type": "string", @@ -319,7 +361,7 @@ "subject": "source-moved..", "body": { "type": "object", - "description": "a source revision of a watched branch, named once per merge call (design 47 H \u00a7737)", + "description": "a source revision of a watched branch, named once per merge call (design 47 H §737)", "properties": { "v": { "type": "integer" @@ -337,7 +379,7 @@ }, "start": { "type": "boolean", - "description": "true only for a branch's first revision (design 47 H \u00a7768)" + "description": "true only for a branch's first revision (design 47 H §768)" }, "heads": { "type": "array", @@ -448,7 +490,7 @@ }, "call": { "type": "string", - "description": "the merge call that moved it; a call the newest revision already carries writes nothing (design 47 H \u00a7737, H-A-33)" + "description": "the merge call that moved it; a call the newest revision already carries writes nothing (design 47 H §737, H-A-33)" } }, "required": [ @@ -512,7 +554,7 @@ "name": "heads", "template": "state", "history": 1, - "description": "each head the forge bridge's head-moved said that source. does not name for that repository; deleted when source. names it (design 47 H \u00a7740)", + "description": "each head the forge bridge's head-moved said that source. does not name for that repository; deleted when source. names it (design 47 H §740)", "key-forms": [ { "key": ".", @@ -607,6 +649,10 @@ "type": "string", "format": "date-time", "description": "when the gate first said it" + }, + "record": { + "type": "string", + "description": "the delivery record the act is for, by its source revision, .; absent on an act said before it was named, which bridge_owed_again by record never finds (design 47 H2 §807, H-A-432)" } }, "required": [ @@ -672,5 +718,6 @@ "head" ] } - } + }, + "form": 2 } diff --git a/go/seatlist/bootstrap/node-engine.json b/go/seatlist/bootstrap/node-engine.json index f33b6cbc..6214da6a 100644 --- a/go/seatlist/bootstrap/node-engine.json +++ b/go/seatlist/bootstrap/node-engine.json @@ -1,11 +1,12 @@ { "format": 1, + "form": 4, "seat": "node-engine", "scope": "node", "grain": "whole", "domain": "Core", "decision": "novox/hq ADR 0563", - "description": "The node seat every node's engine holds (novox/hq ADR 0563, ADR 0618 §3, §4): its hand acts on the operator's warrant, its read-only judgements, its compositions and what it reads of a manifest. Each verb is answered on this node's subject, mesh.seat.node-engine.tool.., and each event is published on mesh.seat.node-engine.event.., the node's name its last token, or, for an event whose subject names , there (composed., running..): the grants part grants a node's account only its own node's subjects (ADR 0363 §2).", + "description": "The node seat every node's engine holds (novox/hq ADR 0563, ADR 0618 \u00a73, \u00a74): its hand acts on the operator's warrant, its read-only judgements, its compositions and what it reads of a manifest. Each verb is answered on this node's subject, mesh.seat.node-engine.tool.., and each event is published on mesh.seat.node-engine.event.., the node's name its last token, or, for an event whose subject names , there (composed., running..): the grants part grants a node's account only its own node's subjects (ADR 0363 \u00a72).", "acts": [ { "step": "compose", @@ -15,7 +16,7 @@ "verbs": [ { "name": "hand-over", - "description": "Hand a directory this machine uses as found to the mesh (novox/hq ADR 0563 §3, §5): the engine judges the path as it does every hand-over (absolute, as it states it, used as found, reached through no link) and refuses in words, or asks the operator on their channel at the level approve, binding the owner, group and mode it will set. Nothing changes on the call: only the operator's yes hands it over. One ask is open per path; acts shows where it stands.", + "description": "Hand a directory this machine uses as found to the mesh (novox/hq ADR 0563 \u00a73, \u00a75): the engine judges the path as it does every hand-over (absolute, as it states it, used as found, reached through no link) and refuses in words, or asks the operator on their channel at the level approve, binding the owner, group and mode it will set. Nothing changes on the call: only the operator's yes hands it over. One ask is open per path; acts shows where it stands.", "input": { "properties": { "path": { @@ -39,7 +40,7 @@ }, { "name": "access-act", - "description": "Ask the operator for the take act on a library whose root holds data outside its group (novox/hq ADR 0328 §7, ADR 0563 §3, §4), on the library's home engine: the access as declared and its root as last declared, judged and refused in words or asked at the level approve. Only the operator's yes sets the library's group, group write and setgid under its root. One ask is open per access.", + "description": "Ask the operator for the take act on a library whose root holds data outside its group (novox/hq ADR 0328 \u00a77, ADR 0563 \u00a73, \u00a74), on the library's home engine: the access as declared and its root as last declared, judged and refused in words or asked at the level approve. Only the operator's yes sets the library's group, group write and setgid under its root. One ask is open per access.", "input": { "properties": { "access": { @@ -68,7 +69,7 @@ }, { "name": "judge-directories", - "description": "Which of this machine's declared directories the engine would refuse to make or keep, one line each (novox/hq ADR 0563 §2). Read-only: changes no owner or mode.", + "description": "Which of this machine's declared directories the engine would refuse to make or keep, one line each (novox/hq ADR 0563 \u00a72). Read-only: changes no owner or mode.", "input": { "properties": {}, "type": "object" @@ -80,7 +81,7 @@ }, { "name": "setuid-search", - "description": "Throw away the engine's last search for setuid programs and start a full one (novox/hq ADR 0563 §2); until it completes root-free answers that the machine is not judged yet. Refused while one runs, and when one was started within the hour, naming when the next may start.", + "description": "Throw away the engine's last search for setuid programs and start a full one (novox/hq ADR 0563 \u00a72); until it completes root-free answers that the machine is not judged yet. Refused while one runs, and when one was started within the hour, naming when the next may start.", "input": { "properties": { "why": { @@ -97,7 +98,7 @@ }, { "name": "root-free", - "description": "Whether this machine is root-free (novox/hq ADR 0259 §8, ADR 0563 §6): free only on the engine's own verdict, at most fifteen minutes old, that the account agents run as cannot become root without a person, and the login shell's execute not served here. Anything else is no pass, and says why. Answered by this machine's engine alone.", + "description": "Whether this machine is root-free (novox/hq ADR 0259 \u00a78, ADR 0563 \u00a76): free only on the engine's own verdict, at most fifteen minutes old, that the account agents run as cannot become root without a person, and the login shell's execute not served here. Anything else is no pass, and says why. Answered by this machine's engine alone.", "input": { "properties": {}, "type": "object" @@ -121,7 +122,7 @@ }, { "name": "acts", - "description": "Every act this machine's engine was asked through this seat (novox/hq ADR 0563 §9): each hand-over, access act and setuid search, open, done or refused, with who asked, the ask and the operator's warrant. Read-only.", + "description": "Every act this machine's engine was asked through this seat (novox/hq ADR 0563 \u00a79): each hand-over, access act and setuid search, open, done or refused, with who asked, the ask and the operator's warrant. Read-only.", "input": { "properties": { "id": { @@ -242,7 +243,7 @@ "type": "string" }, "digest": { - "description": "a composition's digest (sha256:…), to read that revision rather than the latest", + "description": "a composition's digest (sha256:\u2026), to read that revision rather than the latest", "type": "string" }, "module": { @@ -287,7 +288,7 @@ }, "self": { "type": "boolean", - "description": "the node composes the module itself (ADR 0547 §2)" + "description": "the node composes the module itself (ADR 0547 \u00a72)" }, "backup": { "type": "string", @@ -455,6 +456,32 @@ }, "description": "every resource that differs from what runs, when asked" }, + "drift": { + "type": "array", + "description": "when the diff is asked, each directory entry of what runs that the mesh made, as its record holds, whose owner or mode on disk differ from it (H-A-392 (b), H-A-458 (2)): read-only; the engine sets it right at its next start or apply of the module", + "items": { + "type": "object", + "required": [ + "id", + "path", + "drift" + ], + "properties": { + "id": { + "type": "string", + "description": "the directory entry's id in the composition" + }, + "path": { + "type": "string", + "description": "its path on the machine" + }, + "drift": { + "type": "string", + "description": "how its owner or mode on disk differ, in words" + } + } + } + }, "own": { "type": "object", "properties": { @@ -517,6 +544,27 @@ "withheld-unknown": { "type": "string", "description": "withheld unknown: the grants part does not answer, when its verb withheld did not answer" + }, + "withheld-parts": { + "type": "array", + "description": "each part a compose kept because its read did not answer, with why (design 08 control-verbs \u00a74, H-A-321): for the packet filter's holder the part node-packet-filter.allowed, its filter file kept as it runs", + "items": { + "type": "object", + "required": [ + "part", + "why" + ], + "properties": { + "part": { + "type": "string", + "description": "the read asked, as ." + }, + "why": { + "type": "string", + "description": "why it did not answer, naming the holder asked" + } + } + } } }, "required": [ @@ -529,6 +577,63 @@ }, "class": "read" }, + { + "name": "composition-diff", + "description": "One module's composed values read by resource (design 47 H2 \u00a7813 (3), H-A-349 DQ-12b): given a module and a revision or a digest of its composition, the entries that differ between the composition that runs and that one, per resource, environment entries by name, a secret's value masked. Read-only. Answered by this machine's engine alone.", + "class": "read", + "input": { + "type": "object", + "required": [ + "module" + ], + "properties": { + "module": { + "type": "string", + "description": "the module, as placed on this machine; node for the node's own" + }, + "revision": { + "type": "integer", + "description": "the revision of the module's composition key to compare with, from the revisions the bucket keeps; give it or digest" + }, + "digest": { + "type": "string", + "description": "the digest (sha256:\u2026) of the composition to compare with; give it or revision" + } + } + }, + "answer": { + "type": "object", + "required": [ + "node", + "module", + "resources" + ], + "properties": { + "node": { + "type": "string", + "description": "the node" + }, + "module": { + "type": "string", + "description": "the module" + }, + "running": { + "type": "string", + "description": "the digest of the composition that runs; absent when nothing runs" + }, + "other": { + "$ref": "#/defs/Revision" + }, + "resources": { + "type": "array", + "description": "each resource with an entry that differs; empty when none differs", + "items": { + "$ref": "#/defs/CompositionDiffResource" + } + } + } + } + }, { "name": "compose", "description": "Compose this machine's modules now and say how it went, so a composition that failed, or a module held, recovers without a restart: what the pass refused or held, each with why, and when the engine composes again on its own after a seat that did not answer (a deadline, no responders: retried with a growing wait; a refusal never). With module, that module's apply is also tried again. Changes only what a pass the engine makes on its own would change.", @@ -595,7 +700,7 @@ }, { "name": "compose-candidate", - "description": "Compose a check's candidate on this machine in shadow, with its own sealed values and the indexer's manifest verdicts, and say per module placed here whether it composes (with its digest and whether the candidate changes it), is refused by the candidate or is held without it, each with why, and the verdict of the machine's own composition (novox/hq ADR 0593 §7). Read-only: applies nothing and writes nothing. Answered by this machine's engine alone, within the asker's bound.", + "description": "Compose a check's candidate on this machine in shadow, with its own sealed values and the indexer's manifest verdicts, and say per module placed here whether it composes (with its digest and whether the candidate changes it), is refused by the candidate or is held without it, each with why, and the verdict of the machine's own composition (novox/hq ADR 0593 \u00a77). Read-only: applies nothing and writes nothing. Answered by this machine's engine alone, within the asker's bound.", "optional": true, "input": { "properties": { @@ -640,7 +745,7 @@ "unready", "not-judged" ], - "description": "the node's verdict (ADR 0593 §7, §9)" + "description": "the node's verdict (ADR 0593 \u00a77, \u00a79)" }, "modules": { "type": "array", @@ -686,7 +791,7 @@ { "name": "manifest-reads", "optional": true, - "description": "The reader capabilities this machine's engine states (manifest words, forms and seat names it reads, novox/hq ADR 0618 §3) and when it last stated them. An engine built before it said them states none, which a reader takes as the baseline. Read-only. Answered by this machine's engine alone.", + "description": "The reader capabilities this machine's engine states (manifest words, forms and seat names it reads, novox/hq ADR 0618 \u00a73) and when it last stated them. An engine built before it said them states none, which a reader takes as the baseline. Read-only. Answered by this machine's engine alone.", "input": { "type": "object", "properties": {} @@ -720,7 +825,7 @@ "node", "stated-at" ], - "description": "what this machine's engine reads of a manifest (ADR 0618 §3)" + "description": "what this machine's engine reads of a manifest (ADR 0618 \u00a73)" }, "class": "read" }, @@ -787,7 +892,7 @@ }, { "name": "seat-list", - "description": "The seat list this engine holds and what it waits for (novox/hq ADR 0636 §7): each seat's source (trunk or bootstrap) and digest, the seats held from the bootstrap alone, the definitions refused, the last error and whether the seat mesh-catalog has answered. Read-only.", + "description": "The seat list this engine holds and what it waits for (novox/hq ADR 0636 \u00a77): each seat's source (trunk or bootstrap) and digest, the seats held from the bootstrap alone, the definitions refused, the last error and whether the seat mesh-catalog has answered. Read-only.", "input": { "type": "object", "properties": {} @@ -798,7 +903,7 @@ }, { "name": "seat-list-again", - "description": "Make this engine read the seat list from the seat mesh-catalog now (novox/hq ADR 0636 §7), the recovery of a list unread; it answers what it holds afterwards, and the error when the read failed. A read that changed nothing still ends the engine's wait.", + "description": "Make this engine read the seat list from the seat mesh-catalog now (novox/hq ADR 0636 \u00a77), the recovery of a list unread; it answers what it holds afterwards, and the error when the read failed. A read that changed nothing still ends the engine's wait.", "input": { "type": "object", "properties": {} @@ -1057,6 +1162,52 @@ "$ref": "#/defs/ProcessBuilds" }, "class": "read" + }, + { + "name": "converged", + "description": "The node's convergence state, which the node-engine keeps (ADR 0168, ADR 0683; design 33 control-verbs \u00a710): adopted or converged, since when it became so, for an adopted node the kind of the firewall found, and the node's links as the engine classifies them: the tunnel's interface and the outward links (design 47 H-A-325). Read-only: the holder of node-packet-filter asks it at its start and at every allow and disallow, and refuses when it does not answer.", + "class": "read", + "input": { + "type": "object", + "properties": {} + }, + "answer": { + "type": "object", + "required": [ + "state", + "tunnel", + "outward" + ], + "properties": { + "state": { + "type": "string", + "enum": [ + "adopted", + "converged" + ], + "description": "the node's mode: adopted (the found firewall's terms) or converged (the mesh's own filter)" + }, + "since": { + "type": "string", + "description": "when the node became so, RFC 3339; absent where the engine recorded no time for it" + }, + "found": { + "type": "string", + "description": "the kind of the firewall found, for an adopted node" + }, + "tunnel": { + "type": "string", + "description": "the private network's tunnel interface on this node, as the engine's filter render names it" + }, + "outward": { + "type": "array", + "items": { + "type": "string" + }, + "description": "the links of this node that face outside, as the engine classifies them for its filter render (ADR 0140); an allowance with no interface never opens one" + } + } + } } ], "events": [ @@ -1205,6 +1356,14 @@ "type": "boolean", "description": "the move changes the build alone" }, + "run": { + "type": "string", + "description": "the composition run's pipeline sequence, in decimal, the module was applied from (design 47 H2 §813 (1), H-A-388): a restarted engine applies no older run again" + }, + "pipeline": { + "type": "string", + "description": "the pipeline of the want the module was applied from, its key ., beside its run (H-A-429 (1), H-A-458 (1)): a want's run is compared with this run only when the want's branch is the same, the key without its last .; absent for a want in none" + }, "kept": { "type": "array", "items": { @@ -1696,7 +1855,7 @@ "description": "one finding of the observer that wrote it" } ], - "description": "the findings this node's engine raises about itself, one key per finding, in the layout of the seat mesh-conditions (its findings state): a key written raises the condition of the same key, a key deleted clears it (novox/hq ADR 0564 §2, ADR 0636 §1); today `machine..seat-list-unread`" + "description": "the findings this node's engine raises about itself, one key per finding, in the layout of the seat mesh-conditions (its findings state): a key written raises the condition of the same key, a key deleted clears it (novox/hq ADR 0564 \u00a72, ADR 0636 \u00a71); today `machine..seat-list-unread`" }, { "name": "bounds", @@ -1708,7 +1867,7 @@ "key": ".", "value": { "type": "object", - "description": "one bound a wait of the node-engine runs against, kept with its own time to live: its expiry is heard as the bus's delete marker and ends the wait (design 47 G §560, A-77)", + "description": "one bound a wait of the node-engine runs against, kept with its own time to live: its expiry is heard as the bus's delete marker and ends the wait (design 47 G \u00a7560, A-77)", "properties": { "v": { "type": "integer", @@ -1815,7 +1974,7 @@ }, "auto": { "type": "boolean", - "description": "the engine asked it itself, on finding a library's root holding data outside its group (ADR 0563 §4)" + "description": "the engine asked it itself, on finding a library's root holding data outside its group (ADR 0563 \u00a74)" } }, "required": [ @@ -1825,7 +1984,7 @@ "at", "state" ], - "description": "one act asked through the seat, as acts reads it (ADR 0563 §9)" + "description": "one act asked through the seat, as acts reads it (ADR 0563 \u00a79)" }, "ActSaid": { "type": "object", @@ -1870,7 +2029,7 @@ "state", "node" ], - "description": "what the seat says of an act: the act, the path or access, the caller as the runtime proved it and the ask (ADR 0563 §1)" + "description": "what the seat says of an act: the act, the path or access, the caller as the runtime proved it and the ask (ADR 0563 \u00a71)" }, "Acts": { "type": "object", @@ -1961,7 +2120,7 @@ "at", "why" ], - "description": "what a node-engine says when what it reads changed or its node returned (ADR 0618 §4)" + "description": "what a node-engine says when what it reads changed or its node returned (ADR 0618 \u00a74)" }, "Module": { "type": "object", @@ -2061,7 +2220,7 @@ "why", "judged" ], - "description": "the engine's root-free verdict (ADR 0563 §6)" + "description": "the engine's root-free verdict (ADR 0563 \u00a76)" }, "Said": { "type": "object", @@ -2106,7 +2265,7 @@ }, "SeatList": { "type": "object", - "description": "what an engine holds of the seat list (novox/hq ADR 0636 §7)", + "description": "what an engine holds of the seat list (novox/hq ADR 0636 \u00a77)", "properties": { "status": { "type": "object", @@ -2451,6 +2610,10 @@ "type": "string" } }, + "declaration-reads": { + "type": "integer", + "description": "the declaration Version its engine reads (design 47 H2 §813 (12), H-A-416 (2), H-A-419): the composer writes this node's declaration at the lower of its own Version and this one, 1 when none is stated" + }, "runner-capabilities": { "type": "array", "items": { @@ -3160,7 +3323,7 @@ }, "Running": { "type": "object", - "description": "a module's build running on a node, and its health: what the node-engines of other nodes and the grants part decide a seat's pass from (ADR 0660 §2), and the assigner reads a provider's and a canary's health and a put's end from (design 47 G 520); said again on the same subject each time healthy changes and for every module the have holds at the engine's start, kept last per subject with no age limit", + "description": "a module's build running on a node, and its health: what the node-engines of other nodes and the grants part decide a seat's pass from (ADR 0660 \u00a72), and the assigner reads a provider's and a canary's health and a put's end from (design 47 G 520); said again on the same subject each time healthy changes and for every module the have holds at the engine's start, kept last per subject with no age limit", "properties": { "v": { "type": "integer", @@ -3331,6 +3494,52 @@ "step", "since" ] + }, + "CompositionDiffEntry": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "the entry: an environment entry by its name, else the part of the resource that differs" + }, + "running": { + "type": "string", + "description": "its value in the running composition; absent where the running one lacks it or the value is a secret" + }, + "other": { + "type": "string", + "description": "its value in the other composition; absent where the other lacks it or the value is a secret" + }, + "masked": { + "type": "boolean", + "description": "the value is a secret, so neither value is given" + } + }, + "description": "one entry that differs between the running composition and the other" + }, + "CompositionDiffResource": { + "type": "object", + "required": [ + "resource", + "entries" + ], + "properties": { + "resource": { + "type": "string", + "description": "the resource, as the composition names it" + }, + "entries": { + "type": "array", + "items": { + "$ref": "#/defs/CompositionDiffEntry" + }, + "description": "the entries of the resource that differ" + } + }, + "description": "one resource with the entries of it that differ" } } } diff --git a/go/seatlist/seatlist_test.go b/go/seatlist/seatlist_test.go index adce5239..a9ed1bb1 100644 --- a/go/seatlist/seatlist_test.go +++ b/go/seatlist/seatlist_test.go @@ -370,6 +370,13 @@ func TestTheBootstrapEqualsTheTestsSeatList(t *testing.T) { t.Errorf("%s: bootstrap/%s.json differs from seatlisttest/seats/%s.json", b.Seat, b.Seat, b.Seat) } } + // Design 47 H2, H-A-452: the tests' seat list holds the seats the live list holds, the recovery seat among them, so a + // test judges a manifest against the seats the running validator judges it against. + for _, seat := range []string{"mesh-builds", "mesh-delivery-view", "mesh-grants", "mesh-pipeline-runner", "mesh-recovery"} { + if _, ok := tests[seat]; !ok { + t.Errorf("seatlisttest holds no definition of %s", seat) + } + } } // A host whose path for the seat's events became live after the seat spoke reads the list at once, once: a read diff --git a/go/seatlist/seatlisttest/parts/build-step.json b/go/seatlist/seatlisttest/parts/build-step.json index 55920376..49f33cfc 100644 --- a/go/seatlist/seatlisttest/parts/build-step.json +++ b/go/seatlist/seatlisttest/parts/build-step.json @@ -5,15 +5,57 @@ "verbs": [ { "name": "current", - "description": "The asks this node runs now (each ask's id, kind, module and when it started), its build capacity, and whether it is paused, taking no new ask.", + "description": "What this node's holder takes and runs (design 47 H2 §801, H-A-367 (2)): whether it takes asks and why not (paused, capacity 0, unhealthy) since when, its capacity, and every ask it runs, whoever asked it (the builder's delivery asks included).", "input": { "type": "object", "properties": {} }, "answer": { "type": "object", - "description": "the steps running, the capacity and whether paused" - } + "properties": { + "node": { + "type": "string", + "description": "the node this holder runs on" + }, + "takes": { + "type": "boolean", + "description": "whether it takes asks now" + }, + "why-not": { + "type": "string", + "enum": [ + "paused", + "capacity 0", + "unhealthy" + ], + "description": "why it takes none; absent while it takes asks" + }, + "since": { + "type": "string", + "format": "date-time", + "description": "since when it takes asks or not, as takes says" + }, + "capacity": { + "type": "integer", + "minimum": 0, + "description": "how many steps it runs at once" + }, + "asks": { + "type": "array", + "items": { + "$ref": "#/defs/CurrentAsk" + }, + "description": "every ask it runs, whoever asked it" + } + }, + "required": [ + "node", + "takes", + "capacity", + "asks" + ] + }, + "class": "read" }, { "name": "pause", @@ -38,6 +80,280 @@ "type": "object", "description": "whether the node is paused" } + }, + { + "name": "drop", + "description": "End an ask this node holds for a caller that no longer waits for it: a step not yet running is ended when the bus gives it, one running is cancelled; both end the step could not run, its why naming the drop, who asked it and why, never an outcome of its own (one way for an ask that never ran; design 47 H2 H-A-401). Writes dropped. in the pauses state; the build module removes the key when it has acted.", + "input": { + "type": "object", + "properties": { + "id": { + "type": "string", + "description": "the ask's id" + }, + "by": { + "type": "string", + "description": "who drops it" + }, + "why": { + "type": "string", + "description": "why the caller no longer waits" + } + }, + "required": [ + "id", + "why" + ] + }, + "answer": { + "type": "object", + "description": "whether the ask was held here" + }, + "class": "act" + }, + { + "name": "kept-artifact", + "description": "The one build this node keeps instead of storing it through the seat oci-artifact-registry: the new build of that seat's holder, when the running holder refused or did not answer the store (design 47 *The artifact store's holder takes its own new build even when its seat is the one broken*). Answers where the bytes are, as an object by their digest on the bus (the old objects form, ADR 0619 §2), never in the answer; the node-engine checks what it fetched against the digest and applies nothing that differs. A module that claims no oci-artifact-registry is refused by name, so a store outage never becomes a delivery from build nodes; so is a build this node no longer keeps, or keeps under another digest.", + "input": { + "type": "object", + "properties": { + "module": { + "type": "string", + "description": "the module, the holder of oci-artifact-registry", + "minLength": 1 + }, + "fp": { + "type": "string", + "description": "its build's fingerprint", + "minLength": 1 + }, + "digest": { + "type": "string", + "description": "the artifact's sha256, 64 lowercase hex digits, as the builder's done key names it", + "pattern": "^[0-9a-f]{64}$" + } + }, + "required": [ + "module", + "fp", + "digest" + ] + }, + "answer": { + "type": "object", + "properties": { + "module": { + "type": "string", + "description": "the module asked" + }, + "fp": { + "type": "string", + "description": "the fingerprint asked" + }, + "digest": { + "type": "string", + "description": "the digest of the bytes kept", + "pattern": "^[0-9a-f]{64}$" + }, + "size": { + "type": "integer", + "description": "bytes" + }, + "store": { + "type": "string", + "description": "the object store the bytes are in, the build module's kept builds, as its manifest's objects names it on the bus" + }, + "object": { + "type": "string", + "description": "the object's name in it: the digest" + }, + "refused": { + "type": "string", + "enum": [ + "claims no oci-artifact-registry", + "not kept", + "digest differs" + ], + "description": "why nothing is answered: the module claims no oci-artifact-registry, this node keeps no build of it at that fingerprint (never kept, or swept), or keeps it under another digest" + }, + "words": { + "type": "string", + "description": "the refusal in words, naming the module" + }, + "error": { + "type": "string", + "description": "why it could not answer" + } + }, + "required": [ + "module", + "fp" + ] + }, + "class": "read" + }, + { + "name": "steps", + "description": "Which pipeline steps an artifact can run, found by its language's convention (design 47 *Pipeline steps and the policy*): lint always, unit and integration tests where its packages hold them. Kept by the artifact's fingerprint; again finds them anew. With drop and why it deletes the artifact's kept key instead, writes the act in the hand-act log of the seat mesh-hand-acts first (a log that refuses deletes nothing and answers error), and answers dropped (design 47 G section 463, section 480).", + "input": { + "type": "object", + "properties": { + "module": { + "type": "string" + }, + "artifact": { + "type": "string", + "description": "the key of detected-steps, ." + }, + "fingerprint": { + "type": "string" + }, + "fetch": { + "$ref": "#/defs/Fetch" + }, + "again": { + "type": "boolean", + "description": "find them anew, not from the kept answer" + }, + "drop": { + "type": "boolean", + "description": "delete the artifact's kept key instead of detecting; needs why" + }, + "why": { + "type": "string", + "description": "with drop, why: a non-empty text, written in the hand-act log with the caller the bus proved" + } + }, + "required": [ + "module", + "artifact" + ], + "description": "fetch is required unless drop is given, and why with drop" + }, + "answer": { + "type": "object", + "properties": { + "artifact": { + "type": "string" + }, + "fingerprint": { + "type": "string" + }, + "steps": { + "type": "array", + "items": { + "$ref": "#/defs/DetectedStep" + } + }, + "kept": { + "type": "boolean", + "description": "answered from the kept answer" + }, + "dropped": { + "type": "string", + "description": "the artifact whose key a drop deleted" + }, + "error": { + "type": "string", + "description": "why a drop deleted nothing, naming the hand-act log when it refused" + } + }, + "required": [] + }, + "class": "act" + }, + { + "name": "answer", + "class": "read", + "description": "A step's answer by its ask, when the step's key or a run's step-ended names it as `answer-object` because the answer is over what the steps state's value or the event carries (design 47 H2 H-A-412): the kept end's bytes in chunks of at most 512 KiB on the chunk subject of the call, each acknowledged by the caller on the ack subject within 30 seconds; the answer is of any size. An answer inline in the key or the event is never asked here; a run's test output is the verb output's.", + "input": { + "type": "object", + "properties": { + "ask": { + "type": "string", + "description": "the ask's id, the `answer-object` of its step key or its step-ended naming it", + "minLength": 1 + }, + "call": { + "type": "string", + "description": "128 random bits the caller chooses: the subject token of this send's chunks and acknowledgements", + "minLength": 1 + } + }, + "required": [ + "ask", + "call" + ] + }, + "answer": { + "type": "object", + "properties": { + "ask": { + "type": "string", + "description": "the ask's id" + }, + "chunks": { + "type": "integer", + "minimum": 0, + "description": "how many chunks are sent; 0 when this node keeps no answer of that ask" + } + }, + "required": [ + "ask", + "chunks" + ] + }, + "chunks": true + }, + { + "name": "output", + "class": "read", + "description": "A step's output by its ask (design 47 H2 §791, H-A-367 (1)): the output's last 64 KB, its size, the failing tests with their words, or no longer kept once past the outputs objects' max-age. Asked by the runner's runner_step_output of the holder on the node the step key names.", + "input": { + "type": "object", + "properties": { + "ask": { + "type": "string", + "description": "the ask's id", + "minLength": 1 + } + }, + "required": [ + "ask" + ] + }, + "answer": { + "type": "object", + "properties": { + "ask": { + "type": "string", + "description": "the ask's id" + }, + "output": { + "type": "string", + "description": "the output's last 64 KB" + }, + "size": { + "type": "integer", + "minimum": 0, + "description": "the whole output's size in bytes" + }, + "failing": { + "type": "array", + "items": { + "$ref": "#/defs/FailingTest" + }, + "description": "the failing tests with their words" + }, + "no-longer-kept": { + "type": "boolean", + "description": "the output is past the outputs objects' max-age, kept no longer" + } + }, + "required": [ + "ask" + ] + } } ], "defs": { @@ -53,7 +369,7 @@ }, "kind": { "type": "string", - "description": "build, check or compose" + "description": "build, check, compose, step or install" }, "language": { "type": "string", @@ -67,6 +383,10 @@ "description": "when (RFC 3339 time)", "format": "date-time" }, + "version": { + "type": "string", + "description": "the version MAJOR.MINOR.PATCH the builder reserved for the build, passed to it as MESH_VERSION; empty for a build that reserves none, which stamps none (design 47 G section 573)" + }, "build": { "type": "object", "properties": { @@ -130,6 +450,17 @@ "items": { "type": "string" } + }, + "toolchain": { + "$ref": "#/defs/ToolchainUsed", + "description": "the toolchain the checker agreed for the language (ADR 0584 clause 2): a node on another declines the check" + }, + "node-names": { + "type": "array", + "items": { + "type": "string" + }, + "description": "the node names the check's tests run against" } }, "required": [ @@ -150,7 +481,7 @@ "type": "string" }, "request": { - "description": "the compose-candidate request" + "description": "the compose-candidate request, in one of two forms: the request as a JSON object, or, for one near the bus's message limit, a JSON string \"gzip+base64:\" followed by the base64 of the request object gzip-compressed; a holder that does not know the packed form refuses it, naming that it holds no request object" }, "reader": { "type": "string" @@ -163,11 +494,23 @@ "request" ] }, + "run": { + "$ref": "#/defs/RunJob", + "description": "a run: a program built from a change and run in the build seat's sandbox over the heads, its input asked of a verb of the asker's seat; the Go seat's check stream alone carries it, as a compose" + }, "needs": { "type": "array", "items": { "type": "string" - } + }, + "description": "what the ask needs of the holder's node, by seat name: a check's packages' needs, and a pipeline step's, read from its artifact's check-needs in the module's manifest; a holder whose node lacks one declines the ask (ADR 0552 sections 2, 4; design 47 H2 H-A-375)" + }, + "not-on": { + "type": "array", + "items": { + "type": "string" + }, + "description": "the nodes that declined the step before, to which the runner puts it again under a new id: a holder on a listed node hands it back at once, unacknowledged, for another node to take (design 47 H2 H-A-445)" }, "packages": { "type": "object", @@ -175,6 +518,14 @@ "type": "string" }, "description": "package to the tagged version a build pins" + }, + "step": { + "$ref": "#/defs/StepRun", + "description": "a pipeline step: lint, unit tests or integration tests of one artifact at one commit, sent by the runner as the accepted kind step (design 47 *The steps and who serves them*)" + }, + "install": { + "$ref": "#/defs/InstallAsk", + "description": "the install job of the accepted kind install of node-build-node alone (design 47 H2 §800, H-A-403): the module's npm tree its lock names, published as the module's archive :node-modules under the ask's version" } }, "required": [ @@ -208,6 +559,10 @@ "items": { "type": "string" } + }, + "kept": { + "type": "string", + "description": "the digest of the artifact this node kept for the artifact store's holder instead of storing it through the seat oci-artifact-registry (the verb kept-artifact serves it); the keeping node is the step's own; absent when every artifact was stored" } }, "required": [ @@ -218,7 +573,7 @@ "type": "object", "properties": { "fetch": { - "$ref": "#/defs/Fetch" + "$ref": "#/defs/FetchAnswer" }, "levels": { "type": "array", @@ -248,6 +603,129 @@ "built" ] }, + "RunJob": { + "type": "object", + "description": "a run ask's job: where its program is built, the heads it reads (each laid read-only at its repository's path), its input by reference, its arguments and what it is for", + "properties": { + "program": { + "$ref": "#/defs/RunProgram" + }, + "heads": { + "type": "array", + "items": { + "$ref": "#/defs/RunHead" + }, + "description": "the commits the program reads, a repository at most once" + }, + "input": { + "$ref": "#/defs/RunInput" + }, + "args": { + "type": "array", + "items": { + "type": "string" + }, + "description": "the program's arguments, each a word or a flag" + }, + "reader": { + "type": "string", + "description": "what the run is for, for a reader of the ask" + } + }, + "required": [ + "program", + "heads", + "input" + ] + }, + "RunProgram": { + "type": "object", + "description": "where a run's program is built: the repository and full commit fetched, the Go module's directory in it and the program's import path", + "properties": { + "repository": { + "type": "string" + }, + "commit": { + "type": "string" + }, + "dir": { + "type": "string" + }, + "package": { + "type": "string" + } + }, + "required": [ + "repository", + "commit", + "dir", + "package" + ] + }, + "RunHead": { + "type": "object", + "description": "one commit a run's program reads", + "properties": { + "repository": { + "type": "string" + }, + "commit": { + "type": "string" + } + }, + "required": [ + "repository", + "commit" + ] + }, + "RunInput": { + "type": "object", + "description": "a run's input by reference: the seat verb the asker serves it on (seat:.), the sha256 of its bytes and its size, at most 256 KiB", + "properties": { + "verb": { + "type": "string" + }, + "digest": { + "type": "string", + "description": "64 lowercase hex digits" + }, + "size": { + "type": "integer", + "description": "bytes" + } + }, + "required": [ + "verb", + "digest", + "size" + ] + }, + "RunAnswer": { + "type": "object", + "description": "what a run ask's step answers: whether this run built the program or took the one kept under its key, the key, the program's exit, the last line of its standard output and the first of its standard error", + "properties": { + "built": { + "type": "boolean" + }, + "key": { + "type": "string" + }, + "exit": { + "type": "integer" + }, + "output": { + "description": "raw when it is JSON, else a JSON string" + }, + "said": { + "type": "string" + } + }, + "required": [ + "built", + "key", + "exit" + ] + }, "Step": { "type": "object", "properties": { @@ -266,7 +744,7 @@ }, "state": { "type": "string", - "description": "running, passed, failed or refused" + "description": "running, again (ended on a transport fault, to be given again by the bus after again-at), passed, failed, refused or dropped (ended by the verb drop, its cause the reason)" }, "started": { "type": "string", @@ -284,6 +762,10 @@ "result": { "$ref": "#/defs/StepResult" }, + "answer-object": { + "type": "string", + "description": "the name of the object in the build module's steps objects that holds the step's answer when the step's value would be over what the steps state holds (design 47 G A-290, H2 H-A-412): a check's whole answer, its result then empty; a run's output, its result the run's answer without it. Read through the verb answer" + }, "what": { "type": "string" }, @@ -297,6 +779,19 @@ }, "owner": { "type": "string" + }, + "run": { + "type": "integer", + "description": "which run of the ask this is, counted by the bus's delivery count (1 for the first)" + }, + "again-at": { + "type": "string", + "description": "when the bus gives the ask again, for the state again (RFC 3339 time)", + "format": "date-time" + }, + "passing": { + "type": "boolean", + "description": "true when the step ended on a fault that passes (a transport fault), so a failed end after the last run says so" } }, "required": [ @@ -329,6 +824,10 @@ "type": "string", "description": "when (RFC 3339 time)", "format": "date-time" + }, + "digest": { + "type": "string", + "description": "the sha256: digest of the language's toolchain program as the build module runs it: the program of the module that provides that language's toolchain, never the build module's own (design 47 G section 472)" } }, "required": [ @@ -338,6 +837,30 @@ ], "description": "the toolchain a node reports" }, + "Drop": { + "type": "object", + "properties": { + "v": { + "type": "integer", + "description": "the layout version" + }, + "by": { + "type": "string" + }, + "why": { + "type": "string" + }, + "at": { + "type": "string", + "description": "when (RFC 3339 time)", + "format": "date-time" + } + }, + "required": [ + "why" + ], + "description": "an ask a caller no longer waits for" + }, "Pause": { "type": "object", "properties": { @@ -420,7 +943,7 @@ }, "Fetch": { "type": "object", - "description": "what a build module's fetch tool is asked: the repository and the full commit, the pull request's ref when only its head carries the commit, the trunk commit to merge onto for a merge preview, the module's directory and named paths, the mesh packages to build against, and the sibling repositories placed beside", + "description": "what a build module's fetch tool is asked: the repository and the full commit, the pull request's ref when only its head carries the commit, the trunk commit to merge onto for a merge preview, the module's directory and named paths, the mesh packages to build against, the group's TypeScript packages, and the sibling repositories placed beside", "properties": { "repository": { "type": "string", @@ -456,6 +979,13 @@ }, "description": "the version (a full commit) of each mesh package to build against, by its module path" }, + "group-packages": { + "type": "array", + "description": "a group's check: the TypeScript packages the group's members change, each a package of a member at its commit with its directory there; the build module packs each and places it by its package.json name in every module of the tree that names it, in place of the registry's (design 47 G A-284, A-285)", + "items": { + "$ref": "#/defs/FetchPackage" + } + }, "beside": { "type": "array", "items": { @@ -474,6 +1004,61 @@ ] }, "description": "the sibling repositories placed beside the fetched tree" + }, + "builds": { + "type": "array", + "description": "a group run's package members, each by its build's digest: the build module reads each from the artifact store at artifact-store:////blobs/, checks the bytes against the digest and lays them as the group package; a package named here is named in neither packages nor group-packages", + "items": { + "type": "object", + "properties": { + "package": { + "type": "string", + "description": "the name in its ecosystem (the Go module path, the npm name)" + }, + "module": { + "type": "string" + }, + "artifact": { + "type": "string" + }, + "digest": { + "type": "string", + "description": "the member's build digest, sha256:" + } + }, + "required": [ + "package", + "module", + "artifact", + "digest" + ] + } + } + }, + "required": [ + "repository", + "commit" + ] + }, + "FetchPackage": { + "type": "object", + "description": "one TypeScript package of a group's member: the member's repository, the commit packed (the member's head), the pull request's ref when only its head carries it, and the package's directory in the member (empty for the member's root)", + "properties": { + "repository": { + "type": "string", + "description": "/" + }, + "commit": { + "type": "string", + "description": "the full commit" + }, + "ref": { + "type": "string", + "description": "the pull request's ref, when only its head carries the commit" + }, + "path": { + "type": "string", + "description": "the package's directory in the member, empty for the member's root" } }, "required": [ @@ -492,6 +1077,10 @@ }, "recipe": { "type": "string" + }, + "digest": { + "type": "string", + "description": "the building node's digest of its toolchain program (kind 3's entry), never the build module's own program" } }, "required": [ @@ -533,6 +1122,30 @@ }, "reference": { "type": "string" + }, + "resolved": { + "type": "array", + "description": "one per mesh package the build resolved (the catalog graph entry's packages, the edges of kind 1)", + "items": { + "type": "object", + "properties": { + "package": { + "type": "string" + }, + "version": { + "type": "string" + }, + "digest": { + "type": "string", + "description": "sha256: of the package's bytes as the build fetched them" + } + }, + "required": [ + "package", + "version", + "digest" + ] + } } }, "required": [ @@ -656,7 +1269,7 @@ }, "StepResult": { "type": "object", - "description": "what a step answered, by the kind of ask: a build's artifacts, toolchain, imports and out-of-step modules; a check's fetch and levels; a compose's built, key and verdict; a declined ask's declined, needs and lacks. The fields of the kinds do not meet, so one layout holds the four", + "description": "what a step answered, by the kind of ask: a build's artifacts, toolchain, imports, out-of-step modules and the artifact it kept; a check's fetch and levels; a compose's built, key and verdict; a run's built, key, exit, output and said; a declined ask's declined, needs and lacks. The fields of the kinds do not meet, so one layout holds the five", "properties": { "artifacts": { "type": "array", @@ -679,8 +1292,12 @@ "type": "string" } }, + "kept": { + "type": "string", + "description": "the digest of the artifact this node kept for the artifact store's holder instead of storing it through the seat oci-artifact-registry (the verb kept-artifact serves it); the keeping node is the step's own; absent when every artifact was stored" + }, "fetch": { - "$ref": "#/defs/Fetch" + "$ref": "#/defs/FetchAnswer" }, "levels": { "type": "array", @@ -714,8 +1331,773 @@ "type": "string" }, "description": "what the holder's node lacks of it" + }, + "exit": { + "type": "integer", + "description": "a run's program's exit status" + }, + "output": { + "description": "a run's program's last line of standard output: raw when it is JSON, else a JSON string" + }, + "said": { + "type": "string", + "description": "a run's program's first line of standard error, or why the step could not run it" } } + }, + "ModuleTree": { + "type": "object", + "description": "the tree of one module directory in a merge preview: its git tree, the digest of its manifest, the lock files' digests and the digests of the named paths", + "properties": { + "tree": { + "type": "string" + }, + "manifest": { + "type": "string" + }, + "locks": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "paths": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "packages": { + "type": "array", + "items": { + "type": "string" + }, + "description": "no longer filled (ADR 0593 section 2); kept so a preview written before still reads" + } + }, + "required": [ + "tree", + "manifest" + ] + }, + "Program": { + "type": "object", + "description": "a changed file that is a compiled program (issue 1577): its path, its executable format, and the commit of the branch that added it when the merge's tree no longer holds it", + "properties": { + "path": { + "type": "string" + }, + "kind": { + "type": "string", + "description": "ELF, Mach-O or PE" + }, + "commit": { + "type": "string" + } + }, + "required": [ + "path", + "kind" + ] + }, + "PlacedBeside": { + "type": "object", + "description": "a sibling repository placed as a plain directory under the workspace's beside directory", + "properties": { + "repository": { + "type": "string" + }, + "commit": { + "type": "string" + }, + "dir": { + "type": "string" + } + }, + "required": [ + "repository", + "commit", + "dir" + ] + }, + "TakenVersion": { + "type": "object", + "description": "of a directory whose go.mod requires an asked mesh package, the version it is built against: the asked one, or go.mod's when it names a newer one, said", + "properties": { + "dir": { + "type": "string" + }, + "package": { + "type": "string" + }, + "asked": { + "type": "string" + }, + "named": { + "type": "string" + }, + "built": { + "type": "string" + }, + "sum": { + "type": "string" + }, + "said": { + "type": "string" + } + }, + "required": [ + "dir", + "package", + "asked", + "named", + "built" + ] + }, + "FetchRefusal": { + "type": "object", + "description": "a TypeScript module whose packages the fetch did not install, with why", + "properties": { + "module": { + "type": "string" + }, + "reason": { + "type": "string" + } + }, + "required": [ + "module", + "reason" + ] + }, + "FetchAnswer": { + "type": "object", + "description": "what a build module's fetch tool answers, whole: the workspace it made, the commit in it, the module directories found; and, when the call gave a base, the merge preview (the trunk commit, the merge commit, the files that conflict, the paths the merge changes, the compiled programs among them, the tree of each module directory). A head that does not merge cleanly is an answer, not an error: merge is empty and conflicts names the files. The Go module adds the sibling repositories placed and the package versions taken; the TypeScript module the directories installed and refused and the group's package", + "properties": { + "workspace": { + "type": "string" + }, + "commit": { + "type": "string" + }, + "modules": { + "type": "array", + "items": { + "type": "string" + } + }, + "base": { + "type": "string" + }, + "merge": { + "type": "string" + }, + "conflicts": { + "type": "array", + "items": { + "type": "string" + } + }, + "changed": { + "type": "array", + "items": { + "type": "string" + } + }, + "programs": { + "type": "array", + "items": { + "$ref": "#/defs/Program" + } + }, + "trees": { + "type": "object", + "additionalProperties": { + "$ref": "#/defs/ModuleTree" + } + }, + "beside": { + "type": "array", + "items": { + "$ref": "#/defs/PlacedBeside" + } + }, + "packages": { + "type": "array", + "items": { + "$ref": "#/defs/TakenVersion" + } + }, + "installed": { + "type": "array", + "items": { + "type": "string" + } + }, + "refused": { + "type": "array", + "items": { + "$ref": "#/defs/FetchRefusal" + } + }, + "group": { + "type": "string" + } + }, + "required": [ + "workspace", + "commit", + "modules" + ] + }, + "KeptResult": { + "type": "object", + "description": "a package's passing result the runner keeps, by the digest of what its tests read (ADR 0666 section 8): the step runs no package whose digest and level it names", + "properties": { + "package": { + "type": "string" + }, + "digest": { + "type": "string", + "description": "sha256: of what the package's tests read" + }, + "level": { + "type": "string", + "description": "unit or integration" + }, + "toolchain": { + "$ref": "#/defs/ToolchainUsed" + }, + "pipeline": { + "type": "string", + "description": "the pipeline whose step kept it" + } + }, + "required": [ + "package", + "digest", + "level" + ] + }, + "TestCounts": { + "type": "object", + "description": "how many tests a step ran and carried, per level (ADR 0666 section 8: the pipeline's end names any repository or catalog module over the test budget)", + "properties": { + "unit": { + "type": "integer" + }, + "integration": { + "type": "integer" + } + } + }, + "StepRun": { + "type": "object", + "description": "one pipeline step of one artifact at one commit (ADR 0666 sections 7, 8, 17): the step a build seat runs in its sandbox from the change's own heads, the names the lint step needs and the kept results it may skip", + "properties": { + "pipeline": { + "type": "string", + "description": "the runner's pipeline key, .., a branch escaped as a key's name is" + }, + "step": { + "type": "string", + "enum": [ + "lint", + "unit", + "integration" + ], + "description": "the pipeline step" + }, + "module": { + "type": "string" + }, + "artifact": { + "type": "string", + "description": "the artifact reference, :" + }, + "branch": { + "type": "string", + "description": "the watched branch, or the branch of a run on request" + }, + "commit": { + "type": "string", + "description": "the commit the step runs at" + }, + "revision": { + "type": "integer", + "description": "the catalog revision number of the watched branch the pipeline runs on, the indexer's count, which a build module reads a revision by (design 47 H-A-219); absent for a run on request" + }, + "source": { + "type": "integer", + "description": "the gate's source revision the pipeline's catalog revision was taken in from (design 47 H-A-219); absent for a run on request" + }, + "fingerprint": { + "type": "string", + "description": "the artifact's build-input fingerprint at the commit" + }, + "fetch": { + "$ref": "#/defs/Fetch", + "description": "where the step reads the change's heads" + }, + "toolchain": { + "$ref": "#/defs/ToolchainUsed" + }, + "kept": { + "type": "array", + "items": { + "$ref": "#/defs/KeptResult" + }, + "description": "the kept results of the artifact's packages, by digest and level (design 47 *Carried, not run*)" + }, + "rerun": { + "type": "boolean", + "description": "the step is asked again by the runner's rerun" + }, + "by": { + "type": "string", + "description": "for a rerun, the caller the runtime proved" + }, + "archive": { + "type": "string", + "description": "the reference of the install archive in the artifact store, which the Go build module fetches and unpacks into the workspace before the step (design 47 H2 §800, H-A-399 (2))" + } + }, + "required": [ + "pipeline", + "step", + "module", + "artifact", + "commit", + "fingerprint", + "fetch" + ] + }, + "StepEnded": { + "type": "object", + "description": "an ask's end, said by the build seat's holder on the node that ran it, whatever its asker (design 47 G section 602): a pipeline step's end names the pipeline, the commit and the fingerprint it was asked at, which the runner takes only from that holder's account (ADR 0666 sections 7, 21); the end of any other ask names its asker, the ask, the node, the outcome and when, and neither commit nor fingerprint (design 47 G section 548)", + "properties": { + "v": { + "type": "integer", + "description": "the layout version" + }, + "id": { + "type": "string", + "description": "the ask's id" + }, + "asker": { + "type": "string", + "description": "who asked it" + }, + "ask": { + "type": "string", + "description": "the ask's id, by which a reader takes only its own asks" + }, + "pipeline": { + "type": "string" + }, + "step": { + "type": "string", + "enum": [ + "lint", + "unit", + "install", + "integration" + ] + }, + "module": { + "type": "string" + }, + "artifact": { + "type": "string" + }, + "commit": { + "type": "string" + }, + "fingerprint": { + "type": "string" + }, + "node": { + "type": "string", + "description": "the node that ran it, as in the subject" + }, + "outcome": { + "type": "string", + "enum": [ + "passed", + "failed", + "nothing to run", + "could not run", + "declined" + ], + "description": "never passed for a step that ran nothing; declined for a step the holder's node lacks a need of, with needs and lacks, which the runner puts again on a node that has it (design 47 H2 H-A-375, H-A-445)" + }, + "tests": { + "$ref": "#/defs/TestCounts" + }, + "carried": { + "type": "array", + "items": { + "$ref": "#/defs/KeptResult" + }, + "description": "the packages not run because the ask's kept results held them" + }, + "kept": { + "type": "array", + "items": { + "$ref": "#/defs/KeptResult" + }, + "description": "the packages that passed at their digest, for the runner to keep" + }, + "said": { + "type": "string", + "description": "why it failed or could not run, naming the part not heard" + }, + "needs": { + "type": "array", + "items": { + "type": "string" + }, + "description": "for a declined step, the seats the package needs the node to hold (design 47 H2 H-A-445)" + }, + "lacks": { + "type": "array", + "items": { + "type": "string" + }, + "description": "for a declined step, what the node lacked of them (design 47 H2 H-A-445)" + }, + "output": { + "type": "string", + "description": "the run's answer (its program's standard output's last line) when it fits the event (design 47 G A-254)" + }, + "answer-object": { + "type": "string", + "description": "the name of the object that holds the run's answer, kept by the build seat's holder in its steps objects, when the answer is over what the event carries (design 47 G A-254, H2 H-A-412); read through the verb answer" + }, + "rerun": { + "type": "boolean" + }, + "by": { + "type": "string", + "description": "for a rerun, the caller the runtime proved" + }, + "started": { + "type": "string", + "format": "date-time" + }, + "ended": { + "type": "string", + "format": "date-time" + }, + "at": { + "type": "string", + "description": "when it ended (RFC 3339 time)", + "format": "date-time" + }, + "not-run": { + "type": "array", + "items": { + "$ref": "#/defs/NotRun" + }, + "description": "the tests that did not run, each with its reason, counted apart from passed and failed (design 47 H2 §800)" + }, + "archive": { + "type": "string", + "description": "for an install, the reference the installed tree's archive was published under in the artifact store (the seat oci-artifact-registry; design 47 H2 §800, H-A-399 (2))" + } + }, + "required": [ + "id", + "asker", + "ask", + "node", + "outcome", + "at" + ] + }, + "DetectedStep": { + "type": "object", + "properties": { + "step": { + "type": "string", + "enum": [ + "lint", + "unit", + "integration" + ] + }, + "packages": { + "type": "array", + "items": { + "type": "string" + }, + "description": "the packages the step runs, by the language's convention" + } + }, + "required": [ + "step" + ] + }, + "DetectedSteps": { + "type": "object", + "description": "the steps detected for one artifact at its newest build-input fingerprint, replaced when a newer fingerprint is detected and by steps with again (design 47 G section 326)", + "properties": { + "v": { + "type": "integer" + }, + "fingerprint": { + "type": "string" + }, + "steps": { + "type": "array", + "items": { + "$ref": "#/defs/DetectedStep" + } + }, + "at": { + "type": "string", + "format": "date-time" + } + }, + "required": [ + "v", + "fingerprint", + "steps" + ] + }, + "ToolchainReported": { + "type": "object", + "description": "a node's toolchain of a language, said at the build module's start and each time its toolchains key of the language changes (design 47 G section 492)", + "properties": { + "node": { + "type": "string" + }, + "language": { + "type": "string" + }, + "version": { + "type": "string" + }, + "recipe": { + "type": "string" + }, + "digest": { + "type": "string", + "description": "the digest of the program of the provision the build module requires for that language's toolchain, never the build module's own" + } + }, + "required": [ + "node", + "language", + "version", + "recipe", + "digest" + ] + }, + "StepProgress": { + "type": "object", + "description": "a running ask's progress, said once a minute while the holder runs it (design 47 G section 496)", + "properties": { + "asker": { + "type": "string" + }, + "ask": { + "type": "string" + }, + "phase": { + "type": "string", + "enum": [ + "fetch", + "build", + "run", + "report" + ] + }, + "at": { + "type": "string", + "format": "date-time" + } + }, + "required": [ + "asker", + "ask", + "phase", + "at" + ] + }, + "AskTaken": { + "type": "object", + "description": "an ask a build seat's holder took, said when it takes any ask, whatever its asker (design 47 G section 602)", + "properties": { + "asker": { + "type": "string" + }, + "ask": { + "type": "string", + "description": "the ask's id" + }, + "node": { + "type": "string", + "description": "the node that took it, as in the subject" + }, + "at": { + "type": "string", + "format": "date-time" + } + }, + "required": [ + "asker", + "ask", + "node", + "at" + ] + }, + "CurrentAsk": { + "type": "object", + "description": "an ask a holder runs now (design 47 H2 §801)", + "properties": { + "id": { + "type": "string", + "description": "the ask's id" + }, + "kind": { + "type": "string", + "description": "the ask's kind" + }, + "module": { + "type": "string" + }, + "started": { + "type": "string", + "format": "date-time" + }, + "asker": { + "type": "string", + "description": "who asked it" + } + }, + "required": [ + "id", + "kind", + "started", + "asker" + ] + }, + "FailingTest": { + "type": "object", + "description": "a failing test and its words (design 47 H2 §791)", + "properties": { + "test": { + "type": "string", + "description": "the test's name" + }, + "words": { + "type": "string", + "description": "what the test said" + } + }, + "required": [ + "test" + ] + }, + "NotRun": { + "type": "object", + "description": "a test that did not run and why (design 47 H2 §800, H-A-367 (3))", + "properties": { + "test": { + "type": "string", + "description": "the test's name" + }, + "reason": { + "type": "string", + "description": "why it did not run" + } + }, + "required": [ + "test", + "reason" + ] + }, + "InstallAsk": { + "type": "object", + "description": "the job of an ask of the kind install (design 47 H2 §800, H-A-367 (4), H-A-403): the npm tree a module's package lock names, installed with an integrity-checked install and no install scripts, and published to the artifact store (the seat oci-artifact-registry) as the module's archive :node-modules under the ask's version, which the builder's reservation covers", + "properties": { + "id": { + "type": "string", + "description": "the ask's id" + }, + "asker": { + "type": "string", + "description": "who asked it" + }, + "pipeline": { + "type": "string", + "description": "the runner's pipeline key" + }, + "module": { + "type": "string" + }, + "commit": { + "type": "string", + "description": "the commit the lock is read at" + }, + "fingerprint": { + "type": "string", + "description": "the module's build-input fingerprint at the commit" + }, + "lock": { + "type": "string", + "description": "the package lock's text at the commit" + } + }, + "required": [ + "id", + "asker", + "pipeline", + "module", + "commit", + "fingerprint", + "lock" + ] } - } + }, + "events": [ + { + "name": "step-ended", + "subject": "step-ended.", + "body": { + "$ref": "#/defs/StepEnded" + }, + "keep": { + "max-age-days": 90, + "max-mib": 128 + } + }, + { + "name": "toolchain-reported", + "subject": "toolchain-reported.", + "body": { + "$ref": "#/defs/ToolchainReported" + }, + "keep": { + "max-age-days": 90, + "max-mib": 16 + } + }, + { + "name": "step-progress", + "subject": "step-progress.", + "body": { + "$ref": "#/defs/StepProgress" + } + }, + { + "name": "ask-taken", + "subject": "ask-taken.", + "body": { + "$ref": "#/defs/AskTaken" + } + } + ], + "form": 3 } diff --git a/go/seatlist/seatlisttest/parts/registry.json b/go/seatlist/seatlisttest/parts/registry.json index 1a02c7e0..671d4b4d 100644 --- a/go/seatlist/seatlisttest/parts/registry.json +++ b/go/seatlist/seatlisttest/parts/registry.json @@ -193,7 +193,7 @@ }, { "name": "remove", - "description": "Remove one tagged version; the holder refuses an untagged one but for the retention removal of a bundle, archive or copied base by the artifact sweep's holder. Every removal emits removed with its reason. A tagged version the store cannot remove alone, because another tag names its digest, stays whole, its tag and bytes kept: the answer says kept with the reason shared-digest and the tags still naming the digest, removed false, and kept is emitted, never removed; the caller ends there without a retry.", + "description": "Remove one tagged version; the holder refuses an untagged one but for the retention removal of a bundle, archive or copied base, which the artifact sweep's holder asks; the caller is not judged (design 47 H2 §813 (7)). Every removal emits removed with its reason. A tagged version the store cannot remove alone, because another tag names its digest, stays whole, its tag and bytes kept: the answer says kept with the reason shared-digest and the tags still naming the digest, removed false, and kept is emitted, never removed; the caller ends there without a retry.", "input": { "type": "object", "properties": { diff --git a/go/seatlist/seatlisttest/seats/git.json b/go/seatlist/seatlisttest/seats/git.json index db42887f..b038f415 100644 --- a/go/seatlist/seatlisttest/seats/git.json +++ b/go/seatlist/seatlisttest/seats/git.json @@ -5,7 +5,7 @@ "grain": "whole", "domain": "Delivery", "decision": "novox/hq ADR 0111", - "description": "The forge: every read and act a delivery part makes of the mesh's git repositories, and what the forge says happened (design 47 *The seat git*, ADR 0596 \u00a71, \u00a72). A failed read answers an error, never an empty answer.", + "description": "The forge: every read and act a delivery part makes of the mesh's git repositories, and what the forge says happened (design 47 *The seat git*, ADR 0596 §1, §2). A failed read answers an error, never an empty answer.", "delivers": "git", "verbs": [ { @@ -48,6 +48,14 @@ }, "error-kind": { "$ref": "#/defs/ErrorKind" + }, + "part": { + "type": "string", + "description": "with the error kind away, what did not answer (design 47 H2 §776)" + }, + "words": { + "type": "string", + "description": "with the error kind away, the holder's words (design 47 H2 §776)" } }, "description": "what pull answers; error and error-kind alone when it fails" @@ -96,6 +104,14 @@ }, "error-kind": { "$ref": "#/defs/ErrorKind" + }, + "part": { + "type": "string", + "description": "with the error kind away, what did not answer (design 47 H2 §776)" + }, + "words": { + "type": "string", + "description": "with the error kind away, the holder's words (design 47 H2 §776)" } }, "description": "what branch answers; error and error-kind alone when it fails" @@ -126,6 +142,14 @@ }, "error-kind": { "$ref": "#/defs/ErrorKind" + }, + "part": { + "type": "string", + "description": "with the error kind away, what did not answer (design 47 H2 §776)" + }, + "words": { + "type": "string", + "description": "with the error kind away, the holder's words (design 47 H2 §776)" } }, "description": "what open-pulls answers; pulls always said, empty when none is open; error and error-kind beside an empty pulls when it fails", @@ -171,9 +195,9 @@ "tags": { "type": "array", "items": { - "type": "string" + "$ref": "#/defs/Tag" }, - "description": "the tags of this page" + "description": "the tags of this page, each with the commit it names (design 47 H2 §796)" }, "page": { "type": "integer", @@ -193,6 +217,14 @@ }, "error-kind": { "$ref": "#/defs/ErrorKind" + }, + "part": { + "type": "string", + "description": "with the error kind away, what did not answer (design 47 H2 §776)" + }, + "words": { + "type": "string", + "description": "with the error kind away, the holder's words (design 47 H2 §776)" } }, "description": "what tags answers; error and error-kind alone when it fails" @@ -201,7 +233,7 @@ }, { "name": "trees", - "description": "The trees of one commit: the root tree and the tree of every directory that holds a module.json, by their git hashes; with files, every blob under each named directory; with paths, the object of each named path. A listing the forge cut is refused, never answered in part.", + "description": "The trees of one commit: the root tree and the tree of every directory that holds a module.json, by their git hashes; with files, every blob under each named directory; with paths, the object of each named path. A listing the forge cut is refused, never answered in part. A directory named in files or paths that the commit does not hold answers the error kind not-found (design 47 H2 §796).", "input": { "type": "object", "properties": { @@ -284,6 +316,14 @@ }, "error-kind": { "$ref": "#/defs/ErrorKind" + }, + "part": { + "type": "string", + "description": "with the error kind away, what did not answer (design 47 H2 §776)" + }, + "words": { + "type": "string", + "description": "with the error kind away, the holder's words (design 47 H2 §776)" } }, "description": "what trees answers; error and error-kind alone when it fails" @@ -350,6 +390,14 @@ }, "error-kind": { "$ref": "#/defs/ErrorKind" + }, + "part": { + "type": "string", + "description": "with the error kind away, what did not answer (design 47 H2 §776)" + }, + "words": { + "type": "string", + "description": "with the error kind away, the holder's words (design 47 H2 §776)" } }, "description": "what file answers; error and error-kind alone when it fails" @@ -410,6 +458,14 @@ }, "error-kind": { "$ref": "#/defs/ErrorKind" + }, + "part": { + "type": "string", + "description": "with the error kind away, what did not answer (design 47 H2 §776)" + }, + "words": { + "type": "string", + "description": "with the error kind away, the holder's words (design 47 H2 §776)" } }, "description": "what contains answers; error and error-kind alone when it fails" @@ -464,6 +520,14 @@ }, "error-kind": { "$ref": "#/defs/ErrorKind" + }, + "part": { + "type": "string", + "description": "with the error kind away, what did not answer (design 47 H2 §776)" + }, + "words": { + "type": "string", + "description": "with the error kind away, the holder's words (design 47 H2 §776)" } }, "description": "what statuses answers; error and error-kind alone when it fails" @@ -546,6 +610,14 @@ }, "error-kind": { "$ref": "#/defs/ErrorKind" + }, + "part": { + "type": "string", + "description": "with the error kind away, what did not answer (design 47 H2 §776)" + }, + "words": { + "type": "string", + "description": "with the error kind away, the holder's words (design 47 H2 §776)" } }, "description": "what status answers; error and error-kind alone when it fails" @@ -607,6 +679,14 @@ }, "error-kind": { "$ref": "#/defs/ErrorKind" + }, + "part": { + "type": "string", + "description": "with the error kind away, what did not answer (design 47 H2 §776)" + }, + "words": { + "type": "string", + "description": "with the error kind away, the holder's words (design 47 H2 §776)" } }, "description": "what comment answers; error and error-kind alone when it fails" @@ -668,6 +748,14 @@ }, "error-kind": { "$ref": "#/defs/ErrorKind" + }, + "part": { + "type": "string", + "description": "with the error kind away, what did not answer (design 47 H2 §776)" + }, + "words": { + "type": "string", + "description": "with the error kind away, the holder's words (design 47 H2 §776)" } }, "description": "what view answers; error and error-kind alone when it fails" @@ -730,6 +818,14 @@ }, "error-kind": { "$ref": "#/defs/ErrorKind" + }, + "part": { + "type": "string", + "description": "with the error kind away, what did not answer (design 47 H2 §776)" + }, + "words": { + "type": "string", + "description": "with the error kind away, the holder's words (design 47 H2 §776)" } }, "description": "what note answers; error and error-kind alone when it fails" @@ -738,7 +834,7 @@ }, { "name": "merge-pull", - "description": "Merge one or more pull requests with the forge tools' account, each at the head the caller names: one merge call, a delivery's start and, with two or more members, its group (design 47 H \u00a7734, \u00a7735, \u00a7766; ADR 0682 \u00a71, \u00a72). Every member is judged before any is merged: still the pull request's head, its base a watched branch, no run at the head's build-input fingerprint failed a required step (judged from its own state failed-steps), registrations heard; no branch protection is read. When one member is refused, none is merged. Then it merges them in the order named, writing the call in its state calls before the first merge, and says merge-called once. It answers once, at the call's end, naming the call and each member's outcome; a member already merged is refused already-merged, naming its commit.", + "description": "Merge one or more pull requests with the forge tools' account, each at the head the caller names: one merge call, a delivery's start and, with two or more members, its group (design 47 H §734, §735, §766; ADR 0682 §1, §2). Every member is judged before any is merged: still the pull request's head, its base a watched branch, no run at the head's build-input fingerprint failed a required step (judged from its own state failed-steps), registrations heard; no branch protection is read. When one member is refused, none is merged. Then it merges them in the order named, writing the call in its state calls before the first merge, and says merge-called once. It answers once, at the call's end, naming the call and each member's outcome; a member already merged is refused already-merged, naming its commit.", "input": { "type": "object", "properties": { @@ -805,6 +901,14 @@ }, "error-kind": { "$ref": "#/defs/ErrorKind" + }, + "part": { + "type": "string", + "description": "with the error kind away, what did not answer (design 47 H2 §776)" + }, + "words": { + "type": "string", + "description": "with the error kind away, the holder's words (design 47 H2 §776)" } }, "description": "what merge-pull answers at the call's end; error and error-kind alone when it fails" @@ -815,7 +919,7 @@ { "name": "tag", "class": "act", - "description": "Write the git tag of a released version, go/vMAJOR.MINOR.PATCH, at the commit its build was made from; the builder writes it only on the catalog's version-released (design 47 G section 370). A tag already at that commit is answered performed. An error answer carries error and error-kind.", + "description": "Write the git tag of a released version, /vMAJOR.MINOR.PATCH for a Go package (design 47 H2 §796), at the commit its build was made from; the builder writes it only on the catalog's version-released (design 47 G section 370). A tag already at that commit is answered performed. An error answer carries error and error-kind.", "input": { "type": "object", "properties": { @@ -856,6 +960,14 @@ }, "error-kind": { "$ref": "#/defs/ErrorKind" + }, + "part": { + "type": "string", + "description": "with the error kind away, what did not answer (design 47 H2 §776)" + }, + "words": { + "type": "string", + "description": "with the error kind away, the holder's words (design 47 H2 §776)" } }, "description": "what tag answers; error and error-kind alone when it fails" @@ -1259,7 +1371,7 @@ "name": "calls", "template": "state", "history": 1, - "description": "every merge call, written before its first merge; one whose merge-called is not said is said at the holder's start; deleted past the stream merge-called's age (design 47 H \u00a7766, H-A-47)", + "description": "every merge call, written before its first merge; one whose merge-called is not said is said at the holder's start; deleted past the stream merge-called's age (design 47 H §766, H-A-47)", "key-forms": [ { "key": "call.", @@ -1328,7 +1440,7 @@ "description": "its page on the forge" } }, - "description": "one pull request as the forge holds it; never its description (ADR 0659 \u00a71)", + "description": "one pull request as the forge holds it; never its description (ADR 0659 §1)", "required": [ "repository", "number", @@ -1361,7 +1473,7 @@ "description": "whether the forge's merging account may write the repository: false, its statuses and its merge are refused" } }, - "description": "what the forge would merge now, for an open pull request (issues 1275, 1945; ADR 0441 \u00a71): answered whole or not at all, and a read the forge could not make answers the verb's error, never a guess; absent for a merged or closed one", + "description": "what the forge would merge now, for an open pull request (issues 1275, 1945; ADR 0441 §1): answered whole or not at all, and a read the forge could not make answers the verb's error, never a guess; absent for a merged or closed one", "required": [ "mergeable", "protected", @@ -1471,9 +1583,10 @@ "enum": [ "refused", "not-found", - "failed" + "failed", + "away" ], - "description": "which kind of error, set with error alone: refused, the forge or the holder refused and error says why in the forge's own words; not-found, the forge said that what was asked about (a repository, a commit, a pull request, a branch or a tag) does not exist, so a caller never reads it out of error's words; failed, the forge failed or could not be reached, so the act or read could not be performed" + "description": "which kind of error, set with error alone: refused, the forge or the holder refused and error says why in the forge's own words; not-found, the forge said that what was asked about (a repository, a commit, a pull request, a branch or a tag) does not exist, so a caller never reads it out of error's words; failed, the forge failed or could not be reached, so the act or read could not be performed; away, a part the holder depends on (the forge, its store) is away, with part naming what did not answer and words: a caller reads it as a transport fault and calls again (design 47 H2 §776)" }, "FailedStep": { "type": "object", @@ -1529,7 +1642,7 @@ }, "MemberOutcome": { "type": "object", - "description": "one member of a merge call and where it stands (design 47 H \u00a7766, H-A-22)", + "description": "one member of a merge call and where it stands (design 47 H §766, H-A-22)", "properties": { "repository": { "type": "string" @@ -1564,7 +1677,7 @@ }, "Call": { "type": "object", - "description": "one merge call, written before its first merge and updated as each member merges (design 47 H \u00a7766)", + "description": "one merge call, written before its first merge and updated as each member merges (design 47 H §766)", "properties": { "by": { "type": "string", @@ -1640,12 +1753,30 @@ "head", "state" ] + }, + "Tag": { + "type": "object", + "description": "a tag and the commit it names (design 47 H2 §796)", + "properties": { + "tag": { + "type": "string" + }, + "commit": { + "type": "string", + "description": "the commit it names, its full 40-hex sha", + "pattern": "^[0-9a-f]{40}$" + } + }, + "required": [ + "tag", + "commit" + ] } }, "bodies": { "MergeCalled": { "type": "object", - "description": "a merge call's members merged or refused, said once by the forge module (design 47 H \u00a7736, \u00a7767)", + "description": "a merge call's members merged or refused, said once by the forge module (design 47 H §736, §767)", "properties": { "call": { "type": "string", @@ -1702,7 +1833,7 @@ }, "refused": { "type": "array", - "description": "one per member not merged after every member was judged (design 47 H \u00a7767)", + "description": "one per member not merged after every member was judged (design 47 H §767)", "items": { "type": "object", "properties": { @@ -1732,5 +1863,6 @@ "members" ] } - } + }, + "form": 2 } diff --git a/go/seatlist/seatlisttest/seats/mesh-builds.json b/go/seatlist/seatlisttest/seats/mesh-builds.json new file mode 100644 index 00000000..b19a7f79 --- /dev/null +++ b/go/seatlist/seatlisttest/seats/mesh-builds.json @@ -0,0 +1,707 @@ +{ + "format": 1, + "seat": "mesh-builds", + "scope": "mesh", + "grain": "whole", + "domain": "Delivery", + "decision": "novox/hq ADR 0666", + "description": "The builder's seat: it says each artifact it published and each agreed toolchain, and answers a module's reserved version. Held once for the mesh (design 47 G section 52, section 307).", + "verbs": [ + { + "name": "reserved", + "class": "read", + "description": "A module's reserved version with each artifact's digest, for a registry holder deciding whether to publish; asked by the module or by a package's name in its ecosystem (design 47 G section 369, section 478; design 51, Publishing).", + "input": { + "type": "object", + "properties": { + "module": { + "type": "string", + "description": "the module; give this or name" + }, + "name": { + "type": "string", + "description": "a package's or an image's name in its ecosystem (the Go module path, the npm name, the image name), in place of module; a name no entry carries, or more than one entry carries, is answered error naming it (design 47 G section 478, section 491)" + }, + "version": { + "type": "string", + "description": "the version, when asked for a particular one" + } + }, + "required": [] + }, + "answer": { + "type": "object", + "properties": { + "module": { + "type": "string", + "description": "the module" + }, + "version": { + "type": "string", + "description": "the reserved version, MAJOR.MINOR.PATCH" + }, + "artifacts": { + "type": "array", + "items": { + "type": "object", + "properties": { + "artifact": { + "type": "string", + "description": "the artifact reference, :" + }, + "digest": { + "type": "string", + "description": "its digest once built; empty before" + } + }, + "required": [ + "artifact" + ] + } + }, + "released": { + "type": "boolean", + "description": "true when the version is released: the catalog holds it, and artifacts are the ones its release recorded with their digests; false when it is reserved (design 47 G 582, 679)" + }, + "error": { + "type": "string", + "description": "a name no entry carries, or a version neither reserved nor released, naming which; set alone (design 47 G 582, 679)" + } + }, + "description": "the reservation or release, or the error; module, version and artifacts are present whenever error is not (design 47 G 679)" + } + }, + { + "name": "published", + "class": "read", + "description": "Whether an artifact built at a build fingerprint is published, from the builder's reservations and the registries' record, for the pipeline runner writing a mover's wait on a package: a package published before the wait was written is read, not waited for (design 47 H-A-326, H-A-328, H-A-274).", + "input": { + "type": "object", + "properties": { + "module": { + "type": "string", + "description": "the module that builds the package" + }, + "artifact": { + "type": "string", + "description": "the package's artifact reference, :" + }, + "fingerprint": { + "type": "string", + "description": "the build fingerprint the package is built at, which the answer is matched by" + } + }, + "required": [ + "module", + "artifact", + "fingerprint" + ] + }, + "answer": { + "type": "object", + "properties": { + "published": { + "type": "boolean", + "description": "true when the artifact built at the fingerprint is published; a number a take-over withdrew is never published (design 47 H-A-331)" + }, + "version": { + "type": "string", + "description": "the version it is published at, MAJOR.MINOR.PATCH, when published" + }, + "digest": { + "type": "string", + "description": "the published artifact's digest, when published; absent for a release kept before the digest was kept (design 47 H-A-331)" + }, + "at": { + "type": "string", + "format": "date-time", + "description": "when the builder heard or confirmed the publish, when published; absent for a mark written before the publish time was kept (design 47 H-A-331)" + } + }, + "required": [ + "published" + ] + } + }, + { + "name": "drop", + "description": "End an ask on the seat's work queue for a caller that no longer waits for it (its step ended by its bound): one waiting is ended dropped when it is pulled, one taken is cancelled; both end the step could-not-run, its why naming the drop and who asked it, so no holder works on it after (design 47 H2 §778; H-A-401; ADR 0664 §4).", + "input": { + "type": "object", + "properties": { + "id": { + "type": "string", + "description": "the ask's id" + }, + "by": { + "type": "string", + "description": "who drops it" + }, + "why": { + "type": "string", + "description": "why the caller no longer waits" + } + }, + "required": [ + "id", + "why" + ] + }, + "answer": { + "type": "object", + "description": "whether the ask was held here" + }, + "class": "act" + }, + { + "name": "publish-queue", + "class": "read", + "description": "What the builder's work queue publishes holds, read by the builder from its own stream (design 47 H2 §793; H-A-454): the items' count per registry seat holder and class with the oldest and newest queue time, and the first 50 in the order they are pulled (the live class first, each class oldest first); module, and version, list only that module's items. A queue read past 4,000 items is answered capped, naming the items read and the queue's size.", + "input": { + "type": "object", + "properties": { + "module": { + "type": "string", + "description": "list only this module's items" + }, + "version": { + "type": "string", + "description": "with module: only this version's, MAJOR.MINOR.PATCH" + } + }, + "required": [] + }, + "answer": { + "type": "object", + "properties": { + "total": { + "type": "integer", + "description": "how many items were read" + }, + "counts": { + "type": "array", + "items": { + "type": "object", + "properties": { + "holder": { + "type": "string", + "description": "the registry seat's holder" + }, + "class": { + "type": "string", + "enum": [ + "live", + "again" + ], + "description": "the items' class" + }, + "count": { + "type": "integer", + "description": "how many items of the class the holder's queue holds" + }, + "oldest": { + "type": "string", + "format": "date-time", + "description": "when the oldest of them was queued" + }, + "newest": { + "type": "string", + "format": "date-time", + "description": "when the newest of them was queued" + } + }, + "required": [ + "holder", + "class", + "count", + "oldest", + "newest" + ] + }, + "description": "per holder and class, sorted by holder, then class" + }, + "queue": { + "type": "array", + "items": { + "type": "object", + "properties": { + "holder": { + "type": "string", + "description": "the registry seat's holder the item is sent to, a module" + }, + "module": { + "type": "string", + "description": "the module whose version is published" + }, + "version": { + "type": "string", + "description": "the reserved version, MAJOR.MINOR.PATCH" + }, + "class": { + "type": "string", + "enum": [ + "live", + "again" + ], + "description": "live, a build a pipeline's step waits on, pulled first; again, asked again at a start" + }, + "artifacts": { + "type": "object", + "additionalProperties": { + "type": "string" + }, + "description": "each artifact reference, :, with the digest to publish" + }, + "queued": { + "type": "string", + "format": "date-time", + "description": "when the item was put on the work queue" + } + }, + "required": [ + "holder", + "module", + "version", + "class", + "queued" + ] + }, + "description": "the first 50 items listed, per holder in the order they are pulled" + }, + "listed": { + "type": "integer", + "description": "how many of total the answer lists" + }, + "capped": { + "type": "boolean", + "description": "true when the queue held more items than were read; every count is then of the items read" + }, + "items-read": { + "type": "integer", + "description": "with capped: how many items were read" + }, + "queue-items": { + "type": "integer", + "description": "with capped: how many items the queue holds, from its stream's state" + }, + "queue-bytes": { + "type": "integer", + "description": "with capped: the queue's size in bytes" + } + }, + "required": [ + "total", + "counts", + "queue", + "listed" + ] + } + } + ], + "accepts": [ + { + "name": "step", + "body": { + "$ref": "#/defs/StepRun" + }, + "max-age": "24h", + "max-msgs": 10000, + "ack-wait": "60s", + "in-progress": "20s", + "max-ack-pending": 1000, + "max-deliveries": 5, + "duplicates-window": "2m" + } + ], + "events": [ + { + "name": "package-published", + "keep": { + "max-age-days": 90, + "max-mib": 128 + }, + "body": { + "$ref": "#/defs/PackagePublished" + } + }, + { + "name": "toolchain-agreed", + "keep": { + "max-age-days": 90, + "max-mib": 16 + }, + "body": { + "$ref": "#/defs/ToolchainAgreed" + } + }, + { + "name": "step-ended", + "subject": "step-ended.", + "body": { + "$ref": "#/defs/StepEnded" + }, + "keep": { + "max-age-days": 90, + "max-mib": 32 + } + }, + { + "name": "ask-taken", + "subject": "ask-taken.", + "body": { + "type": "object", + "properties": { + "ask": { + "type": "string", + "description": "the ask's id" + }, + "node": { + "type": "string", + "description": "the node whose holder took it" + }, + "at": { + "type": "string", + "format": "date-time", + "description": "when it was taken" + } + }, + "required": [ + "ask", + "node", + "at" + ], + "description": "the seat's holder took the ask from the seat's work queue, at the pull: only the holder says it; the runner arms the ask's bound from it (design 47 H2 §777, §778)" + } + }, + { + "name": "version-reserved", + "body": { + "type": "object", + "properties": { + "module": { + "type": "string" + }, + "artifact": { + "type": "string" + }, + "path": { + "type": "string", + "description": "the package's path" + }, + "version": { + "type": "string" + }, + "fingerprint": { + "type": "string" + }, + "revision": { + "type": "string", + "description": "the catalog revision the reservation was made at" + } + }, + "required": [ + "module", + "artifact", + "path", + "version", + "fingerprint", + "revision" + ], + "description": "the builder reserved a version for a module's artifact at a fingerprint: the indexer re-pins once, at the reservation (design 47 H2 §805)" + } + } + ], + "family": { + "name": "mesh-builds.events", + "max-age-days": 7 + }, + "defs": { + "Dependency": { + "type": "object", + "properties": { + "part": { + "type": "string", + "description": "the dependency's part: for kind 1 the package's artifact reference, :; for kind 3 the agreed toolchain's part (design 47 G 466, 673)" + }, + "kind": { + "type": "integer", + "enum": [ + 1, + 3 + ], + "description": "1 an imported package, 3 the build toolchain (design 47 G section 374)" + }, + "version": { + "type": "string", + "description": "the version of the dependency the build resolved" + }, + "digest": { + "type": "string", + "description": "the digest of the part, as the build used it" + } + }, + "required": [ + "part", + "kind", + "version", + "digest" + ], + "description": "a dependency the build resolved: one item of kind 1 for each resolved package and one of kind 3 for the toolchain, named by the build module in its step key and carried by the builder (design 47 G 374, 673)" + }, + "PackagePublished": { + "type": "object", + "properties": { + "v": { + "type": "integer" + }, + "artifact": { + "type": "string", + "description": "the package's artifact reference, :, from the catalog entry" + }, + "module": { + "type": "string", + "description": "the module" + }, + "version": { + "type": "string", + "description": "the version the build reserved, MAJOR.MINOR.PATCH" + }, + "digest": { + "type": "string", + "description": "the published artifact's digest" + }, + "branch": { + "type": "string", + "description": "the watched branch the build was made from" + }, + "source": { + "type": "integer", + "description": "the source revision of that branch the build was made from" + }, + "revision": { + "type": "integer", + "description": "the catalog revision number of that branch the build was made from, the indexer's count (design 47 H-A-218, H-A-220); absent from a reservation made before the field" + }, + "dependencies": { + "type": "array", + "items": { + "$ref": "#/defs/Dependency" + }, + "description": "each dependency the build resolved, with its version and digest" + }, + "at": { + "type": "string", + "format": "date-time" + } + }, + "required": [ + "v", + "artifact", + "module", + "version", + "digest", + "branch", + "source" + ], + "description": "one artifact published at the module's reserved version (design 47 G section 52, section 334)" + }, + "ToolchainAgreed": { + "type": "object", + "properties": { + "v": { + "type": "integer" + }, + "language": { + "type": "string", + "description": "the language whose agreed toolchain changed" + }, + "version": { + "type": "string", + "description": "the agreed toolchain's version" + }, + "digests": { + "type": "array", + "description": "each digest the nodes reporting the agreed version report, as it stood when the version was agreed, with those nodes (design 47 G section 472, section 495)", + "items": { + "type": "object", + "properties": { + "digest": { + "type": "string", + "description": "the toolchain part's digest" + }, + "nodes": { + "type": "array", + "items": { + "type": "string" + }, + "description": "the nodes reporting that digest" + } + }, + "required": [ + "digest", + "nodes" + ] + } + }, + "at": { + "type": "string", + "format": "date-time" + } + }, + "required": [ + "v", + "language", + "version", + "digests" + ], + "description": "the agreed toolchain of a language changed (design 47 G section 376, ADR 0584 section 1)" + }, + "StepRun": { + "type": "object", + "description": "the build step of one module's pipeline (design 47 G section 230, A-97): the runner asks the seat mesh-builds to build a candidate's entries, each named by its fingerprint, and the builder, the one entry point for a candidate, takes the ask and answers the step's end", + "properties": { + "pipeline": { + "type": "string", + "description": "the runner's pipeline key, .., a branch escaped as a key's name is" + }, + "step": { + "type": "string", + "enum": [ + "build" + ] + }, + "branch": { + "type": "string", + "description": "the watched branch, or the branch of a run on request" + }, + "revision": { + "type": "integer", + "description": "the catalog graph revision of the branch the pipeline runs for" + }, + "source": { + "type": "integer", + "description": "the source revision of the watched branch; absent for a run on request" + }, + "rerun": { + "type": "boolean", + "description": "the step is asked again by the runner's rerun" + }, + "by": { + "type": "string", + "description": "for a rerun, the caller the runtime proved" + }, + "entries": { + "type": "array", + "items": { + "$ref": "#/defs/BuildEntry" + }, + "description": "each entry the candidate's build builds, by module and fingerprint" + } + }, + "required": [ + "pipeline", + "step", + "branch", + "entries" + ] + }, + "StepEnded": { + "type": "object", + "description": "the build step's end, said by the builder on the node it runs on (design 47 G A-97; ADR 0666 section 21): the runner takes it only from that holder's account, naming the ask and each entry's fingerprint it was asked at", + "properties": { + "v": { + "type": "integer", + "description": "the layout version" + }, + "id": { + "type": "string", + "description": "the ask's id" + }, + "pipeline": { + "type": "string" + }, + "step": { + "type": "string", + "enum": [ + "build" + ] + }, + "node": { + "type": "string", + "description": "the node the builder ran it on, as in the subject" + }, + "outcome": { + "type": "string", + "enum": [ + "passed", + "failed", + "nothing to run", + "could not run" + ], + "description": "never passed for a build that built nothing" + }, + "said": { + "type": "string", + "description": "why it failed or could not run: the entry not built, the part not heard" + }, + "rerun": { + "type": "boolean" + }, + "by": { + "type": "string", + "description": "for a rerun, the caller the runtime proved" + }, + "started": { + "type": "string", + "format": "date-time" + }, + "ended": { + "type": "string", + "format": "date-time" + }, + "entries": { + "type": "array", + "items": { + "$ref": "#/defs/BuildEntry" + }, + "description": "each entry the step built, by module and fingerprint" + }, + "packages": { + "type": "object", + "additionalProperties": { + "type": "string" + }, + "description": "the pins the step's entries were built with, each mesh package's version by its module path, the ones the builder took from the registry for a package the entry named with none included; the runner asks the entry's held test steps with them (H-A-207)" + } + }, + "required": [ + "id", + "pipeline", + "step", + "node", + "outcome", + "entries" + ] + }, + "BuildEntry": { + "type": "object", + "description": "one entry a candidate's build builds: a module at the fingerprint of its build inputs, and the commit that fingerprint was read at", + "properties": { + "module": { + "type": "string" + }, + "commit": { + "type": "string" + }, + "fingerprint": { + "type": "string", + "description": "the module's build-input fingerprint at the commit" + } + }, + "required": [ + "module", + "fingerprint" + ] + } + } +} diff --git a/go/seatlist/seatlisttest/seats/mesh-catalog.json b/go/seatlist/seatlisttest/seats/mesh-catalog.json index a6d62b1a..f40e9799 100644 --- a/go/seatlist/seatlisttest/seats/mesh-catalog.json +++ b/go/seatlist/seatlisttest/seats/mesh-catalog.json @@ -253,6 +253,49 @@ }, "class": "read" }, + { + "name": "seat-list", + "class": "read", + "description": "The trunk's seat list as the mesh holds it, the object of the holder's objects seat-lists of that digest (design 47 H2 §814, §820; H-A-441, H-A-443): its digest and size, then its bytes (the definitions resolved, a JSON array in the form of a composer request's seats, sorted by seat) in chunks of at most 512 KiB on the chunk subject of the call, each acknowledged by the caller on the ack subject within 30 seconds. A validate run's asker fills the request's seats from these bytes. Reads only.", + "input": { + "type": "object", + "properties": { + "call": { + "type": "string", + "description": "128 random bits the caller chooses: the subject token of this send's chunks and acknowledgements", + "minLength": 1 + } + }, + "required": [ + "call" + ] + }, + "answer": { + "type": "object", + "properties": { + "digest": { + "type": "string", + "description": "the seat list's sha256, the object's name in seat-lists" + }, + "size": { + "type": "integer", + "minimum": 0, + "description": "the seat list's size in bytes" + }, + "chunks": { + "type": "integer", + "minimum": 0, + "description": "how many chunks are sent" + } + }, + "required": [ + "digest", + "size", + "chunks" + ] + }, + "chunks": true + }, { "name": "revision", "description": "The catalog half of the revision view (ADR 0656 §6): one catalog graph revision with its entries whole, the trunk heads, the seat definitions by digest, and each manifest's verdict and fingerprint; the trunk's newest when no revision and no candidate is named, or a candidate's for its group and heads. Answered by the holder from what it holds, in its own runtime (ADR 0663), in pages of entries. Reads only. A watched branch with no revision yet answers {none-yet: true} alone (design 47 G section 530).", @@ -312,7 +355,7 @@ }, { "name": "registrations", - "description": "Every module's registration, or one with every version and who changed it on whose warrant (design 47 *A module's registration*). Its readers take a first view here, then hear registration-changed.", + "description": "Every module's registration, or one with every version and who changed it on whose warrant (design 47 *A module's registration*). Its readers take a first view here, then hear registration-changed. Filtered by module or repository while it reads and paged by the cursor next, at most 768 KB a page (design 47 H2 §806, §811).", "input": { "type": "object", "properties": { @@ -323,6 +366,14 @@ "history": { "type": "boolean", "description": "with every version of the one module" + }, + "repository": { + "type": "string", + "description": "only the registrations of this repository, /, applied while reading (design 47 H2 §806)" + }, + "next": { + "type": "string", + "description": "the cursor the previous page answered, opaque; absent for the first page (design 47 H2 §779)" } } }, @@ -344,6 +395,18 @@ }, "fill": { "$ref": "#/defs/Fill" + }, + "capped": { + "type": "boolean", + "description": "yes when the answer stopped at 768 KB; the rest is read from next (design 47 H2 §806)" + }, + "total": { + "type": "integer", + "description": "how many registrations the filters hold in all (design 47 H2 §811)" + }, + "next": { + "type": "string", + "description": "the cursor of the next page, opaque; empty at the end (design 47 H2 §779)" } }, "required": [ @@ -657,6 +720,32 @@ "source": { "type": "integer" }, + "call": { + "type": "string", + "description": "what made the revision, as the revision's call names it (design 47 H-A-212, H-A-213); absent for a candidate's" + }, + "call-kind": { + "type": "string", + "enum": [ + "take-in", + "release", + "take-in-again", + "recompute" + ], + "description": "the kind of what made the revision, which a reader reads instead of call's words (design 47 H-A-213 (3), H-A-214, H-A-215, H-A-217); absent for a candidate's" + }, + "module": { + "type": "string", + "description": "for call-kind release, the module released (H-A-215)" + }, + "version": { + "type": "string", + "description": "for call-kind release, the version released (H-A-215)" + }, + "repository": { + "type": "string", + "description": "for call-kind take-in-again, the repository taken in again (H-A-215)" + }, "group": { "type": "string" }, @@ -1448,6 +1537,13 @@ "$ref": "#/defs/Registration" }, "description": "a module's registration" + }, + { + "key": "source..", + "value": { + "$ref": "#/defs/Registration" + }, + "description": "a source-only registration of a repository with no module (design 47 H2 §815)" } ] }, @@ -1462,6 +1558,13 @@ "$ref": "#/defs/Registration" }, "description": "one version of a module's registration" + }, + { + "key": "source...", + "value": { + "$ref": "#/defs/Registration" + }, + "description": "one version of a source-only registration (design 47 H2 §815)" } ] }, @@ -1594,6 +1697,10 @@ "type": "string", "description": "its manifest's verdict: accepted, refused or held" }, + "class": { + "type": "string", + "description": "core or leaf: core when its manifest claims a seat, leaf when it claims none; an entry without it reads as leaf (design 47 H2 H-A-394, H-A-431)" + }, "fp": { "type": "string", "description": "its fingerprint (src2: and 64 hex digits); absent when it has none" @@ -1871,6 +1978,32 @@ "type": "integer", "description": "the source revision it was taken in from; for a candidate, the source revision its heads are merged onto" }, + "call": { + "type": "string", + "description": "what made the revision (design 47 H-A-212, H-A-213): a take-in's the gate's call its source revision names, a re-pin's release of , a take-in again's take-in again of ; absent for a candidate's" + }, + "call-kind": { + "type": "string", + "enum": [ + "take-in", + "release", + "take-in-again", + "recompute" + ], + "description": "the kind of what made the revision, which a reader reads instead of call's words (design 47 H-A-213 (3), H-A-214, H-A-215, H-A-217): a take-in's, a re-pin's on a module's release, a take-in again's, a recompute's that moved a fingerprint (its call naming the reason); absent for a candidate's" + }, + "module": { + "type": "string", + "description": "for call-kind release, the module released (H-A-215)" + }, + "version": { + "type": "string", + "description": "for call-kind release, the version released (H-A-215)" + }, + "repository": { + "type": "string", + "description": "for call-kind take-in-again, the repository taken in again (H-A-215)" + }, "group": { "type": "string", "description": "a candidate's group" @@ -1927,6 +2060,10 @@ "$ref": "#/defs/TakeInEntry" }, "description": "the movers of the take-in that wrote it, so a redelivered source-moved whose revision is kept says take-in.taken again from them, recomputing nothing (design 47 H §743, H-A-92); absent for one with no mover" + }, + "base": { + "type": "integer", + "description": "the newest kept revision of its branch below it, which the take-in that wrote it pinned for its pipeline: the base of the validate step's case before (design 47 H-A-147, H-A-151, H-A-154); absent on the branch's first take-in" } }, "required": [ @@ -2078,14 +2215,15 @@ }, "Registration": { "type": "object", - "description": "a module's registration (ADR 0666 section 2; design 47 *A module's registration*): its git origin, its manifest's path, its watched branches and its policy, every version with who changed it and the warrant of the operator's Approve", + "description": "a module's registration (ADR 0666 section 2; design 47 *A module's registration*): its git origin, its manifest's path, its watched branches and its policy, every version with who changed it and the warrant of the operator's Approve; or a source-only registration of a repository with no module (design 47 H2 §815): its origin, one watched branch, its policy and its check, naming no module and no path, kept under source..", "properties": { "v": { "type": "integer", "description": "the layout version" }, "module": { - "type": "string" + "type": "string", + "description": "the module; absent for a source-only registration" }, "origin": { "type": "string", @@ -2093,7 +2231,7 @@ }, "path": { "type": "string", - "description": "the path of its manifest in the repository" + "description": "the path of its manifest in the repository; absent for a source-only registration" }, "branches": { "type": "array", @@ -2121,12 +2259,14 @@ "at": { "type": "string", "format": "date-time" + }, + "check": { + "$ref": "#/defs/SourceCheck", + "description": "for a source-only registration, the one step its heads run (design 47 H2 §815)" } }, "required": [ - "module", "origin", - "path", "branches", "policy" ] @@ -2157,9 +2297,10 @@ "placed", "review required", "unknown module", - "nothing moves" + "nothing moves", + "holds-manifest" ], - "description": "why no ask was raised" + "description": "why no ask was raised; holds-manifest: a source-only registration whose repository holds a module.json at its root or under modules/ at the branch's head (design 47 H2 §815; H-A-450)" }, "words": { "type": "string", @@ -2469,6 +2610,34 @@ "ref", "language" ] + }, + "SourceCheck": { + "type": "object", + "description": "the check of a source-only registration: a build seat and a step kind, run at the repository's root, never a free command (design 47 H2 §815)", + "properties": { + "seat": { + "type": "string", + "enum": [ + "node-build-go", + "node-build-node", + "node-build-image" + ], + "description": "the build seat that runs it" + }, + "step": { + "type": "string", + "enum": [ + "lint", + "unit", + "integration" + ], + "description": "the step kind" + } + }, + "required": [ + "seat", + "step" + ] } }, "bodies": { @@ -2567,5 +2736,6 @@ "call" ] } - } + }, + "form": 5 } diff --git a/go/seatlist/seatlisttest/seats/mesh-delivery-view.json b/go/seatlist/seatlisttest/seats/mesh-delivery-view.json new file mode 100644 index 00000000..9b78f1f0 --- /dev/null +++ b/go/seatlist/seatlisttest/seats/mesh-delivery-view.json @@ -0,0 +1,96 @@ +{ + "format": 1, + "seat": "mesh-delivery-view", + "scope": "mesh", + "grain": "whole", + "domain": "Delivery", + "decision": "novox/hq ADR 0540", + "description": "The status view of deliveries (ADR 0540): it serves no verb and says no seat event; its tools and its events delivered, moving, settled and stuck stay its holder's own, and it keeps the person's answers given in the web view.", + "states": [ + { + "name": "answers", + "template": "state", + "view": true, + "history": 5, + "description": "the person's answer in the web view to an ask of the seat mesh-wants' state asks of a kind that needs no proof of who answered; only the holder writes a key, and only the holder deletes it, once the ask's key has left asks (design 47 G section 717)", + "key-forms": [ + { + "key": "hold...view", + "value": { + "$ref": "#/defs/Answer" + }, + "description": "the status view's answer to the ask of the seat mesh-wants' state asks under hold.. (design 47 G A-299)" + }, + { + "key": "restore....view", + "value": { + "$ref": "#/defs/Answer" + }, + "description": "the status view's answer to the ask of the seat mesh-wants' state asks under restore... (design 47 G A-299)" + }, + { + "key": "backup....view", + "value": { + "$ref": "#/defs/Answer" + }, + "description": "the status view's answer to the ask of the seat mesh-wants' state asks under backup... (design 47 G A-299)" + }, + { + "key": "record....view", + "value": { + "$ref": "#/defs/Answer" + }, + "description": "the status view's answer to the ask of the seat mesh-wants' state asks under record... (design 47 G A-299)" + }, + { + "key": "outside....view", + "value": { + "$ref": "#/defs/Answer" + }, + "description": "the status view's answer to the ask of the seat mesh-wants' state asks under outside... (design 47 G A-299)" + }, + { + "key": "revert....view", + "value": { + "$ref": "#/defs/Answer" + }, + "description": "the status view's answer to the ask of the seat mesh-wants' state asks under revert... (design 47 G A-299)" + }, + { + "key": "assignments..view", + "value": { + "$ref": "#/defs/Answer" + }, + "description": "the status view's answer to the ask of the seat mesh-wants' state asks under assignments. (design 47 G A-299)" + } + ] + } + ], + "defs": { + "Answer": { + "type": "object", + "description": "the status view's answer to an ask", + "properties": { + "v": { + "type": "integer" + }, + "asking": { + "type": "string", + "description": "the asking it answers" + }, + "option": { + "type": "string", + "description": "the option the person chose" + }, + "words": { + "type": "string", + "description": "the person's words with it" + } + }, + "required": [ + "asking", + "option" + ] + } + } +} diff --git a/go/seatlist/seatlisttest/seats/mesh-delivery.json b/go/seatlist/seatlisttest/seats/mesh-delivery.json index 015f9657..08e74ab4 100644 --- a/go/seatlist/seatlisttest/seats/mesh-delivery.json +++ b/go/seatlist/seatlisttest/seats/mesh-delivery.json @@ -9,7 +9,7 @@ "verbs": [ { "name": "deliveries", - "description": "Every delivery not final, and those that ended in the last day, one line each: its id (owner/repository@commit), its state, its group, what it waits for and since when. Narrowed by state, repository or group.", + "description": "Every delivery not final, and those that ended in the last day, one line each: its id (owner/repository@commit), its state, its group, what it waits for and since when. Narrowed by state, repository or group. Filtered by repository, module, outcome and since while it reads, at most 768 KB a page, capped saying the rest is read from the cursor next (design 47 H2 §811).", "input": { "properties": { "all": { @@ -31,13 +31,55 @@ "state": { "description": "one state, e.g. delivering or held", "type": "string" + }, + "module": { + "type": "string", + "description": "only the deliveries that move this module, applied while reading (design 47 H2 §811)" + }, + "outcome": { + "type": "string", + "description": "only the deliveries that ended with this outcome, applied while reading (design 47 H2 §811)" + }, + "since": { + "type": "string", + "format": "date-time", + "description": "only the deliveries opened at or after this time, applied while reading (design 47 H2 §811)" + }, + "next": { + "type": "string", + "description": "the cursor the previous page answered, opaque; absent for the first page (design 47 H2 §779)" } }, "type": "object" }, "answer": { "type": "object", - "description": "what deliveries answers, as its description says" + "description": "what deliveries answers, as its description says", + "properties": { + "deliveries": { + "type": "array", + "items": { + "type": "object", + "description": "one delivery's record, as the gate keeps it" + }, + "description": "this page's deliveries, the open ones first, then the newest" + }, + "capped": { + "type": "boolean", + "description": "yes when the answer stopped at 768 KB; the rest is read from next (design 47 H2 §811)" + }, + "total": { + "type": "integer", + "description": "how many deliveries the filters hold in all (design 47 H2 §811)" + }, + "next": { + "type": "string", + "description": "the cursor of the next page, opaque; empty at the end (design 47 H2 §779)" + } + }, + "required": [ + "deliveries" + ] }, "class": "read" }, @@ -196,7 +238,7 @@ "name": "record-opened", "body": { "type": "object", - "description": "a delivery record opened for one source revision of a merge call (design 47 H \u00a7737)", + "description": "a delivery record opened for one source revision of a merge call (design 47 H §737)", "properties": { "record": { "type": "string", @@ -319,7 +361,7 @@ "subject": "source-moved..", "body": { "type": "object", - "description": "a source revision of a watched branch, named once per merge call (design 47 H \u00a7737)", + "description": "a source revision of a watched branch, named once per merge call (design 47 H §737)", "properties": { "v": { "type": "integer" @@ -337,7 +379,7 @@ }, "start": { "type": "boolean", - "description": "true only for a branch's first revision (design 47 H \u00a7768)" + "description": "true only for a branch's first revision (design 47 H §768)" }, "heads": { "type": "array", @@ -448,7 +490,7 @@ }, "call": { "type": "string", - "description": "the merge call that moved it; a call the newest revision already carries writes nothing (design 47 H \u00a7737, H-A-33)" + "description": "the merge call that moved it; a call the newest revision already carries writes nothing (design 47 H §737, H-A-33)" } }, "required": [ @@ -512,7 +554,7 @@ "name": "heads", "template": "state", "history": 1, - "description": "each head the forge bridge's head-moved said that source. does not name for that repository; deleted when source. names it (design 47 H \u00a7740)", + "description": "each head the forge bridge's head-moved said that source. does not name for that repository; deleted when source. names it (design 47 H §740)", "key-forms": [ { "key": ".", @@ -607,6 +649,10 @@ "type": "string", "format": "date-time", "description": "when the gate first said it" + }, + "record": { + "type": "string", + "description": "the delivery record the act is for, by its source revision, .; absent on an act said before it was named, which bridge_owed_again by record never finds (design 47 H2 §807, H-A-432)" } }, "required": [ @@ -672,5 +718,6 @@ "head" ] } - } + }, + "form": 2 } diff --git a/go/seatlist/seatlisttest/seats/mesh-grants.json b/go/seatlist/seatlisttest/seats/mesh-grants.json new file mode 100644 index 00000000..779bb1a4 --- /dev/null +++ b/go/seatlist/seatlisttest/seats/mesh-grants.json @@ -0,0 +1,293 @@ +{ + "format": 1, + "seat": "mesh-grants", + "scope": "mesh", + "grain": "whole", + "domain": "Identity and access", + "decision": "novox/hq ADR 0656, ADR 0547, ADR 0636", + "description": "Who may say what on the bus: the grants part composes each module's membership, its buckets, logs and streams, the bus's user list and grants (design 47 'The grants part'). An acting part of the step grant (ADR 0656 1): an entry whose grant waits for its new version is held on its running build within the bound, said by wait-held and wait-ended; a store passing 80 % of its cap is said by filling, and back below 70 % by room; grants-for answers the grants a candidate would compose (design 47 G 331, 441).", + "acts": [ + { + "step": "grant", + "on": "every" + } + ], + "verbs": [ + { + "name": "grants-for", + "description": "The grants a candidate would compose, per module: the grants it would add and remove against the branch's newest, each named by its subject and verb. The candidate's catalog half is taken by mesh-catalog's revision (design 47 G 441).", + "class": "read", + "input": { + "type": "object", + "properties": { + "source": { + "type": "string", + "description": "the candidate's source revision" + }, + "run": { + "type": "string", + "description": "or the id of a run" + }, + "branch": { + "type": "string", + "description": "its watched branch" + } + }, + "required": [ + "branch" + ], + "additionalProperties": false + }, + "answer": { + "$ref": "#/defs/GrantsFor" + } + }, + { + "name": "withheld", + "description": "The marked subjects withheld from each module a node's runtime carries, and why: where its tool runner does not state approval-mark/1, or its reading is not the running runner's (design 46 approval mark 6).", + "class": "read", + "input": { + "type": "object", + "properties": { + "node": { + "type": "string", + "description": "the node whose runtime's withheld subjects are asked" + } + }, + "required": [ + "node" + ], + "additionalProperties": false + }, + "answer": { + "$ref": "#/defs/Withheld" + } + } + ], + "events": [ + { + "name": "wait-held", + "body": { + "$ref": "#/defs/Wait" + } + }, + { + "name": "wait-ended", + "body": { + "$ref": "#/defs/Wait" + } + }, + { + "name": "filling", + "body": { + "$ref": "#/defs/Fill" + } + }, + { + "name": "room", + "body": { + "$ref": "#/defs/Fill" + } + } + ], + "family": { + "name": "mesh-grants.events", + "max-age-days": 7 + }, + "defs": { + "Wait": { + "type": "object", + "description": "a granting wait the grants part holds: an entry whose grant waits until the grants part's new version runs (design 47 *Each step's wait is its own part's*)", + "properties": { + "step": { + "type": "string", + "description": "the step, grant", + "enum": [ + "grant" + ] + }, + "part": { + "type": "string", + "description": "the acting part waited for" + }, + "version": { + "type": "string", + "description": "its new version's fingerprint" + }, + "module": { + "type": "string", + "description": "the entry that waits" + }, + "fp": { + "type": "string", + "description": "the entry's fingerprint, whose grant waits" + }, + "since": { + "type": "string", + "format": "date-time" + }, + "bound": { + "type": "string", + "description": "a duration such as 10m" + }, + "how": { + "type": "string", + "description": "on wait-ended: ran or released", + "enum": [ + "ran", + "released" + ] + }, + "by": { + "type": "string", + "description": "on a release, who released it and why" + }, + "clock-from": { + "type": "string", + "format": "date-time", + "description": "when the new version's build was done, as the builder's build.ended says; the bound's clock starts there (design 47 G 266)" + } + }, + "required": [ + "step", + "part", + "version", + "module", + "since", + "bound", + "clock-from" + ] + }, + "Fill": { + "type": "object", + "description": "a store's fill against its cap", + "properties": { + "store": { + "type": "string", + "description": "the store" + }, + "bytes": { + "type": "integer" + }, + "cap": { + "type": "integer", + "description": "bytes" + }, + "at": { + "type": "string", + "format": "date-time" + } + }, + "required": [ + "store", + "bytes", + "cap" + ] + }, + "Grant": { + "type": "object", + "properties": { + "subject": { + "type": "string", + "description": "the bus subject" + }, + "verb": { + "type": "string", + "description": "the verb or event the grant is for" + } + }, + "required": [ + "subject", + "verb" + ] + }, + "GrantsFor": { + "type": "object", + "description": "the grants a candidate would compose, per module, added and removed against the branch's newest", + "properties": { + "modules": { + "type": "array", + "items": { + "type": "object", + "properties": { + "module": { + "type": "string" + }, + "added": { + "type": "array", + "items": { + "$ref": "#/defs/Grant" + } + }, + "removed": { + "type": "array", + "items": { + "$ref": "#/defs/Grant" + } + } + }, + "required": [ + "module", + "added", + "removed" + ] + } + }, + "error": { + "type": "string", + "description": "why it could not be answered: the catalog half of the candidate is not held" + } + } + }, + "Withheld": { + "type": "object", + "description": "a node's withheld subjects, by module", + "properties": { + "node": { + "type": "string" + }, + "modules": { + "type": "array", + "items": { + "type": "object", + "properties": { + "module": { + "type": "string" + }, + "subjects": { + "type": "array", + "items": { + "$ref": "#/defs/WithheldSubject" + } + } + }, + "required": [ + "module", + "subjects" + ] + } + } + }, + "required": [ + "node", + "modules" + ] + }, + "WithheldSubject": { + "type": "object", + "description": "one subject withheld, and why", + "properties": { + "subject": { + "type": "string" + }, + "why": { + "type": "string" + } + }, + "required": [ + "subject", + "why" + ] + } + } +} diff --git a/go/seatlist/seatlisttest/seats/mesh-healers.json b/go/seatlist/seatlisttest/seats/mesh-healers.json index 23772b08..38b0bb48 100644 --- a/go/seatlist/seatlisttest/seats/mesh-healers.json +++ b/go/seatlist/seatlisttest/seats/mesh-healers.json @@ -135,6 +135,66 @@ "outcome" ] } + }, + { + "name": "attempt-changed", + "subject": "attempt-changed..", + "keep": { + "max-age-days": 7, + "max-mib": 16, + "last-per-subject": true + }, + "body": { + "type": "object", + "description": "an attempt as the attempts state holds it, whole, with until, or removed: true when the healer ends it; said again for every open attempt at the healers' start (design 47 G 541, 665)", + "properties": { + "healer": { + "type": "string" + }, + "condition": { + "type": "string" + }, + "kind": { + "type": "string" + }, + "budget-key": { + "type": "string", + "description": "what the budget counts by: the condition's key unless the healer says another" + }, + "act": { + "type": "string" + }, + "outcome": { + "type": "string", + "enum": [ + "acting", + "acted", + "failed", + "escalated" + ] + }, + "said": { + "type": "string" + }, + "begun": { + "type": "string", + "format": "date-time" + }, + "finished": { + "type": "string", + "format": "date-time" + }, + "until": { + "type": "string", + "format": "date-time", + "description": "the end of that key's lifetime, which the keeper drops the attempt at on its own clock" + }, + "removed": { + "type": "boolean", + "description": "true when the healer ends the attempt" + } + } + } } ], "family": { diff --git a/go/seatlist/seatlisttest/seats/mesh-module-assigner.json b/go/seatlist/seatlisttest/seats/mesh-module-assigner.json index cd483f51..cd98e02c 100644 --- a/go/seatlist/seatlisttest/seats/mesh-module-assigner.json +++ b/go/seatlist/seatlisttest/seats/mesh-module-assigner.json @@ -154,6 +154,39 @@ "type": "object", "description": "what was judged or asked again" } + }, + { + "name": "placement", + "description": "Read the placements of one module as the assignments rows hold them: for each watched branch a node runs it from, the nodes that run it from there.", + "optional": true, + "input": { + "properties": { + "module": { + "description": "the module", + "type": "string" + } + }, + "required": [ + "module" + ], + "type": "object" + }, + "answer": { + "properties": { + "placements": { + "type": "array", + "items": { + "$ref": "#/defs/ModulePlacement" + }, + "description": "one per branch and build, the nodes that run the module from it" + } + }, + "required": [ + "placements" + ], + "type": "object" + }, + "class": "read" } ], "events": [ @@ -190,6 +223,7 @@ { "name": "assignments", "template": "state", + "view": true, "key-forms": [ { "key": ".node", @@ -316,6 +350,13 @@ "assignments": { "$ref": "#/defs/Assignments", "description": "the node's assignments with the proposal laid over it; absent when the proposal takes its key away" + }, + "added": { + "type": "object", + "additionalProperties": { + "$ref": "#/defs/AddedModule" + }, + "description": "by module name, each module the candidate adds to this node without a want there: the `version` and `parts` the want it would write carries (design 47 G A-316); the node-engine builds such a module from this entry" } }, "required": [ @@ -364,6 +405,7 @@ { "name": "seat-holders", "template": "state", + "view": true, "key-forms": [ { "key": "", @@ -1007,6 +1049,46 @@ "scope", "holders" ] + }, + "ModulePlacement": { + "type": "object", + "description": "the nodes that run a module from one watched branch", + "properties": { + "branch": { + "type": "string" + }, + "build": { + "type": "string", + "description": "the fingerprint of the build these nodes run from the branch; absent, the newest delivered of the branch (design 47 H 750, H-A-39): one row per branch and build" + }, + "nodes": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "branch", + "nodes" + ] + }, + "AddedModule": { + "type": "object", + "description": "what the want a candidate would write for a module carries of its build, in the want's own forms (the seat mesh-wants' state wants: `version` and `parts`; copied here because a seat definition cannot reuse another repository's seat's forms, design 47 G A-316)", + "properties": { + "version": { + "type": "string", + "description": "the module's released version, for people; none for a build of another watched branch (design 47 G section 379)" + }, + "parts": { + "type": "object", + "additionalProperties": { + "type": "string" + }, + "description": "the digest of each of the module's parts, by `:` (design 47 G section 379)" + } + } } } } diff --git a/go/seatlist/seatlisttest/seats/mesh-pipeline-runner.json b/go/seatlist/seatlisttest/seats/mesh-pipeline-runner.json new file mode 100644 index 00000000..7c49d7d1 --- /dev/null +++ b/go/seatlist/seatlisttest/seats/mesh-pipeline-runner.json @@ -0,0 +1,2342 @@ +{ + "format": 1, + "seat": "mesh-pipeline-runner", + "scope": "mesh", + "grain": "whole", + "domain": "Delivery", + "decision": "novox/hq ADR 0666", + "description": "The pipeline runner: it runs each module's pipeline on each of its watched branches, asking each step of the seat that serves it and hearing its end, and says each pipeline's and each step's start and end (design 47 *Pipelines per module*). Held once for the mesh; a foundation part.", + "verbs": [ + { + "name": "run", + "description": "The run on request of one or more members, each a module at a named commit, through their pipelines; delivers nothing, replacing the checker's try (ADR 0666 section 18). Says run-started.", + "input": { + "type": "object", + "properties": { + "members": { + "type": "array", + "items": { + "$ref": "#/defs/Member" + }, + "minItems": 1 + }, + "why": { + "type": "string" + } + }, + "required": [ + "members" + ] + }, + "answer": { + "type": "object", + "properties": { + "run": { + "type": "string" + }, + "pipelines": { + "type": "array", + "items": { + "type": "string" + }, + "description": "the pipelines' keys" + } + }, + "required": [ + "run" + ] + }, + "class": "act" + }, + { + "name": "start", + "description": "A delivering pipeline for a watched branch's head, by hand; written in the hand-act log.", + "input": { + "type": "object", + "properties": { + "module": { + "type": "string" + }, + "branch": { + "type": "string" + }, + "why": { + "type": "string", + "description": "why, written with the caller the runtime proves in the hand-act log" + } + }, + "required": [ + "module", + "branch", + "why" + ] + }, + "answer": { + "type": "object", + "properties": { + "pipeline": { + "type": "string" + } + }, + "required": [ + "pipeline" + ] + }, + "class": "act" + }, + { + "name": "show", + "description": "One pipeline: naming a mover, its steps alone; with no mover, each mover's row (its module, outcome and failed step), never every step of every mover, or the steps of any mover the step and state filters hold; and its step log, with the stores' fill (design 47 H2 §789, H-A-376). Reads only.", + "input": { + "type": "object", + "properties": { + "pipeline": { + "type": "string", + "description": "the pipeline's key" + }, + "mover": { + "type": "string", + "description": "a mover's module: the answer's pipeline then holds that mover's steps alone (H-A-187); with neither mover, step nor state the answer holds each mover's row (design 47 H2 H-A-376)" + }, + "step": { + "type": "string", + "enum": [ + "validate", + "build", + "lint", + "unit", + "integration", + "composition", + "review", + "approval" + ], + "description": "only the steps of this kind, applied while reading (design 47 H2 §789)" + }, + "state": { + "type": "string", + "description": "only the steps whose outcome is this, applied while reading (design 47 H2 §789)" + }, + "next": { + "type": "string", + "description": "the cursor the previous page answered, opaque; absent for the first page (design 47 H2 §779, §789)" + } + }, + "required": [ + "pipeline" + ] + }, + "answer": { + "type": "object", + "properties": { + "pipeline": { + "$ref": "#/defs/Pipeline" + }, + "log": { + "type": "array", + "items": { + "$ref": "#/defs/LogEntry" + } + }, + "fill": { + "type": "array", + "items": { + "$ref": "#/defs/Fill" + } + }, + "capped": { + "type": "boolean", + "description": "yes when the answer stopped at 768 KB; the rest is read from next (design 47 H2 §789)" + }, + "total": { + "type": "integer", + "description": "how many steps the filters hold in all, or how many movers for the movers' rows (design 47 H2 §789, H-A-376)" + }, + "next": { + "type": "string", + "description": "the cursor of the next page, opaque; empty at the end (design 47 H2 §779)" + }, + "movers": { + "type": "array", + "items": { + "$ref": "#/defs/MoverRow" + }, + "description": "each mover's row, when show names no mover, step or state (design 47 H2 H-A-376)" + } + }, + "required": [ + "pipeline" + ] + }, + "class": "read" + }, + { + "name": "list", + "description": "The pipelines and runs of a module, a branch or a commit, newest first: by default one short row per pipeline, the newest 20 (its key, call, start, end, outcome, movers' count and first failed or could-not-run step with its module and why); the whole pipelines, each record with its movers and steps counted, never its steps (show gives those), only with full and a limit (design 47 H2 H-A-376). Reads only.", + "input": { + "type": "object", + "properties": { + "module": { + "type": "string" + }, + "branch": { + "type": "string" + }, + "commit": { + "type": "string" + }, + "run": { + "type": "string" + }, + "limit": { + "type": "integer", + "description": "at most this many, newest first; the rows default to 20, and full needs one (design 47 H2 H-A-376)" + }, + "full": { + "type": "boolean", + "description": "answer the whole pipelines in place of the rows, with a limit (design 47 H2 H-A-376)" + } + } + }, + "answer": { + "type": "object", + "properties": { + "rows": { + "type": "array", + "items": { + "$ref": "#/defs/PipelineRow" + }, + "description": "one short row per pipeline, without full (design 47 H2 H-A-376)" + }, + "pipelines": { + "type": "array", + "items": { + "$ref": "#/defs/Pipeline" + }, + "description": "the whole pipelines, only with full (design 47 H2 H-A-376)" + }, + "fill": { + "$ref": "#/defs/Fill" + } + }, + "required": [] + }, + "class": "read" + }, + { + "name": "stands", + "description": "What stands still, and since when: each step not ended and each wait held. Reads only.", + "input": { + "type": "object", + "properties": {} + }, + "answer": { + "type": "object", + "properties": { + "stands": { + "type": "array", + "items": { + "$ref": "#/defs/Wait" + } + } + }, + "required": [ + "stands" + ] + }, + "class": "read" + }, + { + "name": "why-carried", + "description": "Where a step's result was carried from, with the kept results it rests on. Reads only.", + "input": { + "type": "object", + "properties": { + "pipeline": { + "type": "string", + "description": "the pipeline's key" + }, + "step": { + "type": "string", + "enum": [ + "validate", + "build", + "lint", + "unit", + "integration", + "composition", + "review", + "approval" + ] + }, + "artifact": { + "type": "string" + } + }, + "required": [ + "pipeline", + "step" + ] + }, + "answer": { + "type": "object", + "properties": { + "carried-from": { + "type": "string" + }, + "kept": { + "type": "array", + "items": { + "$ref": "#/defs/KeptResult" + } + } + } + }, + "class": "read" + }, + { + "name": "kept", + "description": "The kept results, by digest, package or level, with the store's fill. Reads only.", + "input": { + "type": "object", + "properties": { + "digest": { + "type": "string" + }, + "package": { + "type": "string" + }, + "level": { + "type": "string" + } + } + }, + "answer": { + "type": "object", + "properties": { + "kept": { + "type": "array", + "items": { + "$ref": "#/defs/KeptResult" + } + }, + "fill": { + "$ref": "#/defs/Fill" + } + }, + "required": [ + "kept" + ] + }, + "class": "read" + }, + { + "name": "review", + "description": "A code review's result for a commit and fingerprint, taken from the reviewer's account and refused from the commit's author (ADR 0666 section 22).", + "input": { + "type": "object", + "properties": { + "module": { + "type": "string" + }, + "commit": { + "type": "string" + }, + "fingerprint": { + "type": "string" + }, + "outcome": { + "type": "string", + "enum": [ + "passed", + "failed" + ] + }, + "said": { + "type": "string", + "description": "what the review found" + } + }, + "required": [ + "module", + "commit", + "fingerprint", + "outcome" + ] + }, + "answer": { + "type": "object", + "properties": { + "pipeline": { + "type": "string" + }, + "result": { + "$ref": "#/defs/StepResult" + } + }, + "required": [ + "result" + ] + }, + "class": "act" + }, + { + "name": "rerun", + "description": "One step again; its step-ended says it is a rerun and the caller the runtime proved. Written in the hand-act log.", + "input": { + "type": "object", + "properties": { + "pipeline": { + "type": "string", + "description": "the pipeline's key" + }, + "module": { + "type": "string", + "description": "the mover's module, so a pipeline of several movers is never ambiguous; a call naming none on a pipeline of one mover is taken as that mover (design 47 H-A-101)" + }, + "step": { + "type": "string", + "enum": [ + "validate", + "build", + "lint", + "unit", + "integration", + "composition", + "review", + "approval" + ] + }, + "artifact": { + "type": "string" + }, + "why": { + "type": "string", + "description": "why, written with the caller the runtime proves in the hand-act log" + } + }, + "required": [ + "pipeline", + "step", + "why" + ] + }, + "answer": { + "type": "object", + "properties": { + "pipeline": { + "type": "string" + }, + "result": { + "$ref": "#/defs/StepResult" + } + }, + "required": [ + "pipeline" + ] + }, + "class": "act" + }, + { + "name": "accept", + "description": "A failed optional step accepted, with who and why, in the hand-act log.", + "input": { + "type": "object", + "properties": { + "pipeline": { + "type": "string", + "description": "the pipeline's key" + }, + "module": { + "type": "string", + "description": "the mover's module, so a pipeline of several movers is never ambiguous; a call naming none on a pipeline of one mover is taken as that mover (design 47 H-A-101)" + }, + "step": { + "type": "string", + "enum": [ + "validate", + "build", + "lint", + "unit", + "integration", + "composition", + "review", + "approval" + ] + }, + "artifact": { + "type": "string" + }, + "why": { + "type": "string", + "description": "why, written with the caller the runtime proves in the hand-act log" + } + }, + "required": [ + "pipeline", + "step", + "why" + ] + }, + "answer": { + "type": "object", + "properties": { + "result": { + "$ref": "#/defs/StepResult" + } + }, + "required": [ + "result" + ] + }, + "class": "act" + }, + { + "name": "cancel", + "description": "A pipeline ended cancelled, its waiting asks dropped by each build seat's drop. Written in the hand-act log.", + "input": { + "type": "object", + "properties": { + "pipeline": { + "type": "string", + "description": "the pipeline's key" + }, + "why": { + "type": "string", + "description": "why, written with the caller the runtime proves in the hand-act log" + } + }, + "required": [ + "pipeline", + "why" + ] + }, + "answer": { + "type": "object", + "properties": { + "pipeline": { + "$ref": "#/defs/Pipeline" + } + }, + "required": [ + "pipeline" + ] + }, + "class": "act" + }, + { + "name": "end-again", + "description": "A running pipeline's end judged once more, as at a step's end: it ends when it can, failed as soon as it can no longer pass (design 47 H §746, H-A-163, H-A-164). Written in the hand-act log.", + "input": { + "type": "object", + "properties": { + "pipeline": { + "type": "string", + "description": "the pipeline's key" + }, + "why": { + "type": "string", + "description": "why, written with the caller the runtime proves in the hand-act log" + } + }, + "required": [ + "pipeline", + "why" + ] + }, + "answer": { + "type": "object", + "properties": { + "pipeline": { + "$ref": "#/defs/Pipeline" + } + }, + "required": [ + "pipeline" + ] + }, + "class": "act" + }, + { + "name": "place", + "description": "A waiting pipeline placed ahead of its turn, its steps asked now; written in the hand-act log (design 47 H §748, H-A-53).", + "input": { + "type": "object", + "properties": { + "pipeline": { + "type": "string", + "description": "the pipeline's key" + }, + "why": { + "type": "string", + "description": "why, written with the caller the runtime proves in the hand-act log" + } + }, + "required": [ + "pipeline", + "why" + ] + }, + "answer": { + "type": "object", + "properties": { + "pipeline": { + "$ref": "#/defs/Pipeline" + } + }, + "required": [ + "pipeline" + ] + }, + "class": "act" + }, + { + "name": "place-unfinished", + "description": "A mover kept in the state unfinished put in a pipeline of its own at once, keyed ..unfinished.; written in the hand-act log (design 47 H2 §786, H-A-398).", + "input": { + "type": "object", + "properties": { + "key": { + "type": "string", + "description": "the unfinished entry's key, ." + }, + "why": { + "type": "string", + "description": "why, written with the caller the runtime proves in the hand-act log" + } + }, + "required": [ + "key", + "why" + ] + }, + "answer": { + "type": "object", + "properties": { + "pipeline": { + "type": "string", + "description": "the key of the pipeline the mover is put in" + } + }, + "required": [ + "pipeline" + ] + }, + "class": "act" + }, + { + "name": "ask-again", + "description": "A declined approval step asked again (ADR 0666 section 10). Written in the hand-act log.", + "input": { + "type": "object", + "properties": { + "pipeline": { + "type": "string", + "description": "the pipeline's key" + }, + "module": { + "type": "string", + "description": "the mover's module, so a pipeline of several movers is never ambiguous; a call naming none on a pipeline of one mover is taken as that mover (design 47 H-A-101)" + }, + "why": { + "type": "string", + "description": "why, written with the caller the runtime proves in the hand-act log" + } + }, + "required": [ + "pipeline", + "why" + ] + }, + "answer": { + "type": "object", + "properties": { + "result": { + "$ref": "#/defs/StepResult" + } + }, + "required": [ + "result" + ] + }, + "class": "act" + }, + { + "name": "test-runs", + "class": "read", + "description": "Each test run the runner keeps (design 47 G §728): its packages read from its output object, ask, node, readers, outcome and counts, and the state's fill against its cap; with candidate, level, digest or module, only those.", + "input": { + "type": "object", + "properties": { + "candidate": { + "type": "string", + "description": "the candidate's key" + }, + "level": { + "type": "string", + "enum": [ + "lint", + "unit", + "integration" + ] + }, + "digest": { + "type": "string" + }, + "module": { + "type": "string" + } + } + }, + "answer": { + "type": "object", + "properties": { + "runs": { + "type": "array", + "items": { + "$ref": "#/defs/TestRun" + } + }, + "fill": { + "$ref": "#/defs/Fill" + } + }, + "required": [ + "runs", + "fill" + ] + } + }, + { + "name": "test-runs-drop", + "class": "act", + "description": "Drop one test run by hand (design 47 G §728): the act and why are written in the hand-act log first, then the run's ask is dropped, each reader ended could not run naming the drop, and the key deleted with the run's output object.", + "input": { + "type": "object", + "properties": { + "key": { + "type": "string", + "description": "the run's key in test-runs" + }, + "why": { + "type": "string", + "description": "why, written with the caller the runtime proves in the hand-act log" + } + }, + "required": [ + "key", + "why" + ] + }, + "answer": { + "type": "object", + "properties": { + "run": { + "$ref": "#/defs/TestRun", + "description": "the entry deleted" + }, + "ended": { + "type": "array", + "items": { + "$ref": "#/defs/TestRunReader" + }, + "description": "the readers it ended" + } + }, + "required": [ + "run" + ] + } + } + ], + "events": [ + { + "name": "pipeline-started", + "subject": "pipeline-started..", + "body": { + "$ref": "#/defs/PipelineStarted" + }, + "keep": { + "max-age-days": 90, + "max-mib": 32 + } + }, + { + "name": "pipeline-ended", + "subject": "pipeline-ended..", + "body": { + "$ref": "#/defs/PipelineEnded" + }, + "keep": { + "max-age-days": 90, + "max-mib": 192 + } + }, + { + "name": "run-started", + "subject": "run-started...", + "body": { + "$ref": "#/defs/RunStarted" + }, + "keep": { + "max-age-days": 90, + "max-mib": 16 + } + }, + { + "name": "step-started", + "subject": "step-started...", + "body": { + "$ref": "#/defs/StepEvent" + }, + "keep": { + "max-age-days": 90, + "max-mib": 128 + } + }, + { + "name": "step-ended", + "subject": "step-ended...", + "body": { + "$ref": "#/defs/StepEvent" + }, + "keep": { + "max-age-days": 90, + "max-mib": 128 + } + }, + { + "name": "wait-held", + "subject": "wait-held..", + "body": { + "$ref": "#/defs/Wait" + }, + "keep": { + "max-age-days": 90, + "max-mib": 32 + } + }, + { + "name": "wait-ended", + "subject": "wait-ended..", + "body": { + "$ref": "#/defs/Wait" + }, + "keep": { + "max-age-days": 90, + "max-mib": 32 + } + }, + { + "name": "re-pinned", + "body": { + "type": "object", + "description": "a kept mover re-pinned at a revision (design 47 H2 §821; H-A-349 DQ-13)", + "properties": { + "pipeline": { + "type": "string", + "description": "the pipeline's key" + }, + "mover": { + "type": "string", + "description": "the mover's module" + }, + "revision": { + "type": "integer", + "description": "the catalog revision it is re-pinned at" + } + }, + "required": [ + "pipeline", + "mover", + "revision" + ] + } + }, + { + "name": "filling", + "body": { + "$ref": "#/defs/Fill" + } + }, + { + "name": "full", + "body": { + "$ref": "#/defs/Fill" + } + }, + { + "name": "room", + "body": { + "$ref": "#/defs/Fill" + } + } + ], + "family": { + "name": "mesh-pipeline-runner.events", + "max-age-days": 7 + }, + "states": [ + { + "name": "pipelines", + "template": "state", + "history": 1, + "description": "one key per pipeline, one per take-in, written by compare-and-set on its absence (design 47 H §746, §747), and one key per mover of a pipeline holding that mover's steps (H-A-187)", + "key-forms": [ + { + "key": ".", + "value": { + "$ref": "#/defs/Pipeline" + }, + "description": "a pipeline of one catalog revision, keyed by the revision's number (design 47 H-A-213 (4), H-A-219); one kept from before under its source has the same form" + }, + { + "key": "..published..", + "value": { + "$ref": "#/defs/Pipeline" + }, + "description": "a pipeline a package's publish started, its movers the package's kind-1 dependents the publishing pipeline does not move" + }, + { + "key": ".run.", + "value": { + "$ref": "#/defs/Pipeline" + }, + "description": "a run on request's pipeline, one per run, its movers the named commits' modules; it delivers nothing (H-A-83)" + }, + { + "key": "..unfinished.", + "value": { + "$ref": "#/defs/Pipeline" + }, + "description": "a pipeline of its own of one mover kept in the runner's state unfinished, put in it by place-unfinished; the revision the unfinished entry names (design 47 H2 §86, H-A-398)" + }, + { + "key": "steps..", + "value": { + "type": "object", + "properties": { + "steps": { + "type": "array", + "items": { + "$ref": "#/defs/StepResult" + }, + "description": "the mover's steps" + }, + "unlogged": { + "type": "array", + "items": { + "$ref": "#/defs/LogEntry" + }, + "description": "the step-log entries of the mover's steps that ended and are not appended to the step log yet, in order (H-A-189)" + } + }, + "required": [ + "steps" + ] + }, + "description": "one mover's steps and unlogged step-log entries in a pipeline, its key the pipeline's and the mover's module escaped as a key's name, written by compare-and-set as the pipeline's record is; the record's own steps stay empty, so a record never grows with the catalog (H-A-187)" + } + ] + }, + { + "name": "queue", + "template": "state", + "history": 1, + "description": "the pipelines waiting their turn under pipelines-at-once, the lowest sequence read first by key order, and the pipelines running (design 47 H §748, H-A-36, H-A-72, H-A-96)", + "key-forms": [ + { + "key": "queue.next", + "value": { + "type": "integer", + "description": "the runner's counter: the sequence the next waiting pipeline gets, written by compare-and-set" + }, + "description": "the queue's counter" + }, + { + "key": "waiting.", + "value": { + "$ref": "#/defs/Waiting" + }, + "description": "a waiting pipeline, its sequence 20 digits" + }, + { + "key": "running.", + "value": { + "type": "object", + "properties": { + "pipeline": { + "type": "string", + "description": "the pipeline's key, as the key names it" + } + }, + "required": [ + "pipeline" + ] + }, + "description": "a running pipeline, written by compare-and-set when it is placed and deleted at its end, its cancel or its approval wait; pipelines-at-once is judged against these keys alone (H-A-96)" + } + ] + }, + { + "name": "kept", + "template": "state", + "history": 1, + "description": "each package's passing result per level, under the digest of what its tests read; kept in the runner's old state section", + "key-forms": [ + { + "key": ".", + "value": { + "$ref": "#/defs/KeptResult" + }, + "description": "a kept result" + } + ] + }, + { + "name": "step-log", + "template": "log", + "max-age-days": 100, + "description": "one entry per step's start and end; a step's output is read where the build seat's holder keeps it (design 47 H2 §791); kept on the runner's old streams section", + "key-forms": [ + { + "key": "", + "value": { + "$ref": "#/defs/LogEntry" + }, + "description": "an entry of a pipeline's log" + } + ] + } + ], + "defs": { + "TestCounts": { + "type": "object", + "properties": { + "unit": { + "type": "integer" + }, + "integration": { + "type": "integer" + } + }, + "description": "how many tests a step ran and carried, per level (ADR 0666 section 8)" + }, + "ToolchainUsed": { + "type": "object", + "properties": { + "language": { + "type": "string" + }, + "version": { + "type": "string" + }, + "recipe": { + "type": "string" + } + }, + "required": [ + "language", + "version", + "recipe" + ], + "description": "the toolchain a build was made with: the language, the installed version and the build module's recipe" + }, + "KeptResult": { + "type": "object", + "properties": { + "package": { + "type": "string" + }, + "digest": { + "type": "string", + "description": "sha256: of what the package's tests read" + }, + "level": { + "type": "string", + "description": "unit or integration" + }, + "toolchain": { + "$ref": "#/defs/ToolchainUsed" + }, + "pipeline": { + "type": "string", + "description": "the pipeline whose step kept it" + }, + "last-read": { + "type": "string", + "format": "date-time", + "description": "when a carry last read it: the runner ages it 30 days after (design 47 G section 330)" + } + }, + "description": "a package's passing result the runner keeps, by the digest of what its tests read (ADR 0666 section 8)", + "required": [ + "package", + "digest", + "level" + ] + }, + "Fill": { + "type": "object", + "properties": { + "store": { + "type": "string", + "description": "the kept data, by its name in the holder's manifest" + }, + "bytes": { + "type": "integer" + }, + "cap": { + "type": "integer", + "description": "bytes" + }, + "at": { + "type": "string", + "format": "date-time" + } + }, + "description": "a store's fill against its cap", + "required": [ + "store", + "bytes", + "cap" + ] + }, + "Accepted": { + "type": "object", + "properties": { + "by": { + "type": "string", + "description": "who accepted it, as the runtime proved the caller" + }, + "why": { + "type": "string" + }, + "at": { + "type": "string", + "format": "date-time" + } + }, + "description": "a failed optional step a person accepted, recorded in the hand-act log", + "required": [ + "by", + "why" + ] + }, + "StepResult": { + "type": "object", + "properties": { + "module": { + "type": "string", + "description": "the mover the step is of (design 47 H-A-35)" + }, + "step": { + "type": "string", + "enum": [ + "validate", + "build", + "lint", + "unit", + "integration", + "composition", + "review", + "approval" + ] + }, + "artifact": { + "type": "string", + "description": "the artifact reference, :; absent for a step of the whole module (review, approval)" + }, + "status": { + "type": "string", + "enum": [ + "required", + "optional", + "bypass" + ], + "description": "the step's status in the module's policy" + }, + "outcome": { + "type": "string", + "enum": [ + "running", + "waiting", + "passed", + "failed", + "nothing to run", + "bypassed", + "carried", + "could not run", + "waiting for a build node", + "did not build", + "accepted", + "not asked", + "declined", + "superseded", + "cancelled" + ], + "description": "never passed for a step that ran nothing (design 47 *Pipeline steps and the policy*)" + }, + "why": { + "type": "string", + "description": "what the outcome rests on: the part not heard, the seat with no holder, what a waiting step waits on" + }, + "on": { + "type": "string", + "description": "for a waiting step, who or what it waits on: a person, a build node, the revision view" + }, + "fingerprint": { + "type": "string", + "description": "the artifact's build-input fingerprint the step ran at" + }, + "seat": { + "type": "string", + "description": "the seat that serves the step, whose holder alone may end it (ADR 0666 section 21)" + }, + "node": { + "type": "string", + "description": "the node whose holder ended it" + }, + "ask": { + "type": "string", + "description": "the step's ask on the serving seat's work queue" + }, + "tests": { + "$ref": "#/defs/TestCounts" + }, + "carried-from": { + "type": "string", + "description": "for a carried result, the pipeline it was carried from" + }, + "rerun": { + "type": "boolean", + "description": "the step was asked again by the runner's rerun" + }, + "by": { + "type": "string", + "description": "for a rerun, the caller the runtime proved" + }, + "accepted": { + "$ref": "#/defs/Accepted" + }, + "started": { + "type": "string", + "format": "date-time" + }, + "ended": { + "type": "string", + "format": "date-time" + }, + "shared": { + "type": "array", + "items": { + "$ref": "#/defs/Shared" + }, + "description": "the run the step read or asked for every step that reads it: a composition step's per machine, as mesh-wants' step-ended names them; a test step's one, with asked-by when another step asked it (design 47 G §726, §729; A-468d, A-468e, A-469t)" + }, + "asked-at": { + "type": "string", + "format": "date-time", + "description": "when the step's ask was put on its seat's work queue (design 47 H2 §778)" + }, + "taken-at": { + "type": "string", + "format": "date-time", + "description": "when the seat's holder took the ask, from its ask-taken; the unanswered bound counts from it (design 47 H2 §778)" + }, + "not-on": { + "type": "array", + "items": { + "type": "string" + }, + "description": "the nodes that declined the step's ask, lacking what it needs: its ask is put again under a new id naming them, each node declining at most once (design 47 H2 H-A-375, H-A-445)" + } + }, + "description": "one step of a pipeline: its status in the policy and its outcome (design 47 *Pipeline steps and the policy*)", + "required": [ + "step", + "status", + "outcome" + ] + }, + "Widening": { + "type": "object", + "properties": { + "what": { + "type": "string", + "enum": [ + "serving part", + "acting rule", + "grant", + "credential", + "could not run" + ] + }, + "name": { + "type": "string", + "description": "the part, rule, grant or credential" + }, + "said": { + "type": "string", + "description": "what the change alters in it; for \"could not run\" the grants part did not answer grants-for (design 47 G section 441)" + } + }, + "description": "a part serving a step the change alters, or an acting rule, a grant or a credential it widens, computed by the running runner (ADR 0666 section 23)", + "required": [ + "what", + "name" + ] + }, + "Published": { + "type": "object", + "properties": { + "artifact": { + "type": "string", + "description": "the package whose publish started the pipeline" + }, + "version": { + "type": "string", + "description": "the version that publish named" + } + }, + "description": "the publish of a package that started a pipeline: a newer publish of the same artifact on the same branch supersedes it (design 47 G section 31, A-202)", + "required": [ + "artifact", + "version" + ] + }, + "Pipeline": { + "type": "object", + "properties": { + "pipeline": { + "type": "string", + "description": "the pipeline's key, ., ..published.. for a pipeline a publish started, or .run. for a run on request; a branch escaped as a key's name is (design 47 H §746, §747; H-A-83)" + }, + "branch": { + "type": "string", + "description": "the watched branch, or the branch of a run on request" + }, + "revision": { + "type": "integer", + "description": "the catalog graph revision of the branch the pipeline runs for" + }, + "source": { + "type": "integer", + "description": "the source revision of the watched branch; the pipeline is keyed by it (ADR 0666 section 9)" + }, + "call": { + "type": "string", + "description": "the merge call whose source revision the take-in took in (design 47 H §746)" + }, + "movers": { + "type": "array", + "items": { + "$ref": "#/defs/Mover" + }, + "description": "the modules the pipeline moves, its take-in's entries (design 47 H §746)" + }, + "superseded": { + "type": "array", + "items": { + "type": "string" + }, + "description": "the modules a newer waiting take-in of the same branch took out of the pipeline, written with the movers by compare-and-set; at its placing the runner writes a mover's build-step why from it (design 47 H §748, H-A-107)" + }, + "seq": { + "type": "integer", + "description": "for a pipeline written over pipelines-at-once, its place in the queue: the runner's counter queue.next at its write (design 47 H §748)" + }, + "end-count": { + "type": "integer", + "description": "how often the pipeline has ended: 1 at its first end, one higher at each end again after a passing rerun, so the assigner takes a later end of the same source revision (design 47 G sections 29, 325)" + }, + "run": { + "type": "string", + "description": "the run on request's id, for a pipeline of a run; such a pipeline delivers nothing" + }, + "v": { + "type": "integer", + "description": "the layout version" + }, + "outcome": { + "type": "string", + "enum": [ + "waiting", + "running", + "passed", + "failed", + "refused", + "superseded", + "cancelled" + ] + }, + "why": { + "type": "string", + "description": "for a pipeline that did not pass, the steps that held it" + }, + "steps": { + "type": "array", + "items": { + "$ref": "#/defs/StepResult" + }, + "description": "every mover's steps, each naming its mover, approval last; in the state pipelines a record's own are empty, each mover's kept under its own key (H-A-187)" + }, + "started": { + "type": "string", + "format": "date-time" + }, + "ended": { + "type": "string", + "format": "date-time" + }, + "unlogged": { + "type": "array", + "items": { + "$ref": "#/defs/LogEntry" + }, + "description": "the step-log entries of the steps that ended and are not appended to the step log yet, in order; written with the step's end in one write and cleared when appended (design 47 G sections 486, 606); in the state pipelines a record's own are empty, each mover's kept under its own key beside its steps (H-A-189)" + }, + "mover-count": { + "type": "integer", + "description": "how many movers the pipeline has, set by list alone (H-A-191)" + }, + "step-count": { + "type": "integer", + "description": "how many steps the pipeline's movers have, set by list alone, which answers the record with its steps empty (H-A-191)" + }, + "published": { + "$ref": "#/defs/Published", + "description": "for a pipeline a package's publish started, that publish (A-202)" + } + }, + "description": "one pipeline of one take-in, its movers the take-in's entries, as the runner keeps it in its state pipelines by compare-and-set (design 47 H §746)", + "required": [ + "pipeline", + "branch", + "source", + "movers", + "outcome", + "steps" + ] + }, + "PipelineEnded": { + "type": "object", + "properties": { + "pipeline": { + "type": "string", + "description": "the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747)" + }, + "branch": { + "type": "string", + "description": "the watched branch, or the branch of a run on request" + }, + "source": { + "type": "integer", + "description": "the source revision of the watched branch; the pipeline is keyed by it (ADR 0666 section 9)" + }, + "call": { + "type": "string", + "description": "the merge call whose source revision the take-in took in (design 47 H §746)" + }, + "revision": { + "type": "integer", + "description": "the catalog graph revision of the branch the pipeline runs for" + }, + "end-count": { + "type": "integer", + "description": "how often the pipeline has ended: 1 at its first end, one higher at each end again after a passing rerun, so the assigner takes a later end of the same source revision (design 47 G sections 29, 325)" + }, + "run": { + "type": "string", + "description": "the run on request's id, for a pipeline of a run; such a pipeline delivers nothing" + }, + "v": { + "type": "integer" + }, + "outcome": { + "type": "string", + "enum": [ + "waiting", + "running", + "passed", + "failed", + "refused", + "superseded", + "cancelled" + ] + }, + "why": { + "type": "string" + }, + "movers": { + "type": "array", + "items": { + "$ref": "#/defs/MoverEnded" + }, + "description": "per mover its fingerprint, released version, step results and tier, in the delivery order (design 47 H §746, H-A-2)" + }, + "started": { + "type": "string", + "format": "date-time" + }, + "ended": { + "type": "string", + "format": "date-time" + }, + "phases": { + "type": "object", + "description": "the pipeline's phases: the take-in, its placing, its end, and for each step kind the earliest start and the latest end across its movers; each mover's own step times stay in the step log (design 47 H2 §785)", + "properties": { + "taken": { + "type": "string", + "format": "date-time", + "description": "when the take-in took the pipeline's revision in" + }, + "placed": { + "type": "string", + "format": "date-time", + "description": "when the pipeline was placed" + }, + "ended": { + "type": "string", + "format": "date-time", + "description": "when the pipeline ended" + }, + "build": { + "type": "object", + "properties": { + "start": { + "type": "string", + "format": "date-time", + "description": "the earliest start across the movers" + }, + "end": { + "type": "string", + "format": "date-time", + "description": "the latest end across the movers" + } + }, + "description": "the build steps' span across the movers" + }, + "lint": { + "type": "object", + "properties": { + "start": { + "type": "string", + "format": "date-time", + "description": "the earliest start across the movers" + }, + "end": { + "type": "string", + "format": "date-time", + "description": "the latest end across the movers" + } + }, + "description": "the lint steps' span across the movers" + }, + "unit": { + "type": "object", + "properties": { + "start": { + "type": "string", + "format": "date-time", + "description": "the earliest start across the movers" + }, + "end": { + "type": "string", + "format": "date-time", + "description": "the latest end across the movers" + } + }, + "description": "the unit steps' span across the movers" + }, + "integration": { + "type": "object", + "properties": { + "start": { + "type": "string", + "format": "date-time", + "description": "the earliest start across the movers" + }, + "end": { + "type": "string", + "format": "date-time", + "description": "the latest end across the movers" + } + }, + "description": "the integration steps' span across the movers" + }, + "validate": { + "type": "object", + "properties": { + "start": { + "type": "string", + "format": "date-time", + "description": "the earliest start across the movers" + }, + "end": { + "type": "string", + "format": "date-time", + "description": "the latest end across the movers" + } + }, + "description": "the validate steps' span across the movers" + }, + "composition": { + "type": "object", + "properties": { + "start": { + "type": "string", + "format": "date-time", + "description": "the earliest start across the movers" + }, + "end": { + "type": "string", + "format": "date-time", + "description": "the latest end across the movers" + } + }, + "description": "the composition steps' span across the movers" + } + } + }, + "unfinished": { + "type": "array", + "items": { + "type": "string" + }, + "description": "the movers left unfinished by another's failure, each kept in the state unfinished and put in a pipeline again (design 47 H2 §786)" + } + }, + "description": "a pipeline's end, said once, with per mover its released version, step results and tier (design 47 H §746); the assigner writes wants only from a passed one", + "required": [ + "v", + "pipeline", + "branch", + "source", + "outcome", + "movers" + ] + }, + "PipelineStarted": { + "type": "object", + "properties": { + "pipeline": { + "type": "string", + "description": "the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747)" + }, + "branch": { + "type": "string", + "description": "the watched branch, or the branch of a run on request" + }, + "source": { + "type": "integer", + "description": "the source revision of the watched branch; the pipeline is keyed by it (ADR 0666 section 9)" + }, + "call": { + "type": "string", + "description": "the merge call whose source revision the take-in took in (design 47 H §746)" + }, + "revision": { + "type": "integer", + "description": "the catalog graph revision of the branch the pipeline runs for" + }, + "movers": { + "type": "array", + "items": { + "$ref": "#/defs/Mover" + }, + "description": "the pipeline's movers" + }, + "run": { + "type": "string", + "description": "the run on request's id, for a pipeline of a run; such a pipeline delivers nothing" + }, + "v": { + "type": "integer" + }, + "at": { + "type": "string", + "format": "date-time" + }, + "published": { + "$ref": "#/defs/Published", + "description": "for a pipeline a package's publish started, that publish (A-202)" + } + }, + "description": "a pipeline's start, said at its write, a waiting one's too (design 47 H §746, H-A-52)", + "required": [ + "v", + "pipeline", + "branch", + "source", + "movers" + ] + }, + "Member": { + "type": "object", + "properties": { + "module": { + "type": "string" + }, + "commit": { + "type": "string" + }, + "branch": { + "type": "string", + "description": "the branch the commit is on" + } + }, + "description": "a module at a named commit", + "required": [ + "module", + "commit" + ] + }, + "RunStarted": { + "type": "object", + "properties": { + "v": { + "type": "integer" + }, + "run": { + "type": "string" + }, + "members": { + "type": "array", + "items": { + "$ref": "#/defs/Member" + } + }, + "by": { + "type": "string", + "description": "the caller the runtime proved" + }, + "at": { + "type": "string", + "format": "date-time" + } + }, + "description": "a run on request's start, with its members at their commits (ADR 0666 section 18)", + "required": [ + "v", + "run", + "members", + "by" + ] + }, + "StepEvent": { + "type": "object", + "properties": { + "pipeline": { + "type": "string", + "description": "the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747)" + }, + "module": { + "type": "string" + }, + "branch": { + "type": "string", + "description": "the watched branch, or the branch of a run on request" + }, + "revision": { + "type": "integer", + "description": "the catalog graph revision of the branch the pipeline runs for" + }, + "source": { + "type": "integer", + "description": "the source revision of the watched branch; the pipeline is keyed by it (ADR 0666 section 9)" + }, + "commit": { + "type": "string", + "description": "the commit the module's entry was last changed at" + }, + "run": { + "type": "string", + "description": "the run on request's id, for a pipeline of a run; such a pipeline delivers nothing" + }, + "fingerprint": { + "type": "string", + "description": "the module's build-input fingerprint at the commit" + }, + "v": { + "type": "integer" + }, + "result": { + "$ref": "#/defs/StepResult" + }, + "at": { + "type": "string", + "format": "date-time" + } + }, + "description": "a step's start or end: the step, its status, its outcome, its fingerprint, its test counts, what a carried result was carried from, and for a rerun that it is one and its proved caller; a bypassed step too", + "required": [ + "v", + "pipeline", + "module", + "branch", + "commit", + "result" + ] + }, + "Wait": { + "type": "object", + "properties": { + "pipeline": { + "type": "string", + "description": "the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747)" + }, + "module": { + "type": "string", + "description": "for a step's wait, the mover whose step waits (design 47 H-A-72)" + }, + "branch": { + "type": "string", + "description": "the watched branch, or the branch of a run on request" + }, + "revision": { + "type": "integer", + "description": "the catalog graph revision of the branch the pipeline runs for" + }, + "source": { + "type": "integer", + "description": "the source revision of the watched branch; the pipeline is keyed by it (ADR 0666 section 9)" + }, + "commit": { + "type": "string", + "description": "the commit the module's entry was last changed at" + }, + "run": { + "type": "string", + "description": "the run on request's id, for a pipeline of a run; such a pipeline delivers nothing" + }, + "fingerprint": { + "type": "string", + "description": "the module's build-input fingerprint at the commit" + }, + "v": { + "type": "integer" + }, + "step": { + "type": "string", + "enum": [ + "validate", + "build", + "lint", + "unit", + "integration", + "composition", + "review", + "approval", + "pipeline" + ], + "description": "the pipeline step a wait is in, or pipeline for a pipeline waiting its turn (design 47 H §748)" + }, + "artifact": { + "type": "string" + }, + "on": { + "type": "string", + "enum": [ + "person", + "build node", + "revision view", + "pipelines-at-once" + ], + "description": "what the step waits on (design 47 *Health and going back*)" + }, + "who": { + "type": "string", + "description": "for a wait on a person, who: the operator, a reviewer" + }, + "since": { + "type": "string", + "format": "date-time" + }, + "how": { + "type": "string", + "description": "for its end, how it ended: answered, ran, superseded" + }, + "ask": { + "type": "string", + "description": "the step's own ask, where it has one (design 47 G §731)" + }, + "shared": { + "type": "array", + "items": { + "$ref": "#/defs/Shared" + }, + "description": "the run the step waits on, where the runner knows it: a test step's run key and ask (design 47 G §731, A-469v)" + } + }, + "description": "a step's wait, held and ended; a wait on a person is said as waiting on that person, never a stall (ADR 0283); said on . (design 47 H-A-72, H-A-221)", + "required": [ + "v", + "pipeline", + "branch", + "step", + "on", + "since" + ] + }, + "LogEntry": { + "type": "object", + "properties": { + "pipeline": { + "type": "string" + }, + "module": { + "type": "string", + "description": "the module of the mover whose step the entry is of (H-A-190)" + }, + "step": { + "type": "string", + "enum": [ + "build", + "lint", + "unit", + "integration", + "composition", + "review", + "approval" + ] + }, + "artifact": { + "type": "string" + }, + "what": { + "type": "string", + "enum": [ + "started", + "ended" + ] + }, + "outcome": { + "type": "string", + "enum": [ + "running", + "waiting", + "passed", + "failed", + "nothing to run", + "bypassed", + "carried", + "could not run", + "waiting for a build node", + "did not build", + "accepted", + "declined", + "superseded", + "cancelled" + ] + }, + "at": { + "type": "string", + "format": "date-time" + }, + "kept": { + "type": "array", + "items": { + "type": "string" + }, + "description": "the keys of the kept results the step wrote or carried, ., which the runner ages by their last-read (design 47 G sections 330, 470)" + } + }, + "description": "one entry of the step log", + "required": [ + "pipeline", + "step", + "what", + "at" + ] + }, + "Shared": { + "type": "object", + "description": "a run whose result a step took or waits on, asked once for every step that reads it (design 47 G §729, A-469t): for a composition step one per machine composed, `run` the `composition-runs` key and `ask` its run ask's id; for a test step one with no machine, `run` the `test-runs` key, `ask` its build seat ask's id and `asked-by` the step that asked the run, left out for that step itself", + "properties": { + "machine": { + "type": "string", + "description": "the machine composed, for a composition run" + }, + "run": { + "type": "string", + "description": "the run's key: in the assigner's composition-runs, or the runner's test-runs" + }, + "ask": { + "type": "string", + "description": "the id of the run's ask of the build seat" + }, + "asked-by": { + "type": "object", + "description": "the step that asked the run, for a test step reading another's run", + "properties": { + "pipeline": { + "type": "string", + "description": "its pipeline's key" + }, + "artifact": { + "type": "string", + "description": "its artifact, :" + } + }, + "required": [ + "pipeline", + "artifact" + ] + } + }, + "required": [ + "run", + "ask" + ] + }, + "TestRunReader": { + "type": "object", + "description": "a test step reading a test run (design 47 G §725)", + "properties": { + "pipeline": { + "type": "string", + "description": "its pipeline's key" + }, + "artifact": { + "type": "string", + "description": "its artifact, :" + } + }, + "required": [ + "pipeline", + "artifact" + ] + }, + "TestRunPackage": { + "type": "object", + "description": "one package a test run ran, with its digest (design 47 G §724)", + "properties": { + "package": { + "type": "string" + }, + "digest": { + "type": "string" + } + }, + "required": [ + "package", + "digest" + ] + }, + "TestRun": { + "type": "object", + "description": "one test run of a candidate, level and tested packages, a key of the runner's state test-runs (design 47 G §725, §728; A-469u)", + "properties": { + "key": { + "type": "string", + "description": "the key, .., each part escaped as a key's name" + }, + "level": { + "type": "string", + "enum": [ + "lint", + "unit", + "integration" + ] + }, + "digest": { + "type": "string", + "description": "the share digest (design 47 G §724)" + }, + "seat": { + "type": "string", + "description": "the build seat asked" + }, + "ask": { + "type": "string", + "description": "the id of the build seat's ask" + }, + "node": { + "type": "string", + "description": "the node whose holder took the ask" + }, + "started": { + "type": "string", + "format": "date-time" + }, + "readers": { + "type": "array", + "items": { + "$ref": "#/defs/TestRunReader" + }, + "description": "the steps reading the run" + }, + "ended": { + "type": "string", + "format": "date-time" + }, + "outcome": { + "type": "string" + }, + "said": { + "type": "string", + "description": "the outcome's words" + }, + "tests": { + "$ref": "#/defs/TestCounts" + }, + "output": { + "type": "string", + "description": "the run's output object, by its name in test-runs-objects" + }, + "packages": { + "type": "array", + "items": { + "$ref": "#/defs/TestRunPackage" + }, + "description": "the packages it ran, read from its output object" + }, + "gone": { + "type": "string", + "description": "why the packages are not named: the run's output is gone" + } + }, + "required": [ + "key", + "level", + "digest" + ] + }, + "Mover": { + "type": "object", + "properties": { + "module": { + "type": "string" + }, + "commit": { + "type": "string", + "description": "the commit the module's entry was last changed at" + }, + "fingerprint": { + "type": "string", + "description": "the module's build-input fingerprint at the commit" + }, + "repository": { + "type": "string" + }, + "tier": { + "type": "integer", + "description": "the mover's tier in the pipeline's delivery order, from 1 (ADR 0666 section 14; design 47 H §746)" + }, + "version": { + "type": "string", + "description": "the version released for the mover in this pipeline (version-released), absent until one is (design 47 H-A-52)" + }, + "published": { + "type": "array", + "items": { + "type": "string" + }, + "description": "the artifact references, :, of the mover's dependencies whose package-published (and, for a kind-3 edge, whose build step's passed end) was heard at the pipeline's branch and source, written with the pipeline by compare-and-set; absent when none (H-A-84)" + }, + "widens": { + "type": "array", + "items": { + "$ref": "#/defs/Widening" + } + }, + "tests": { + "$ref": "#/defs/TestCounts" + }, + "unplaced": { + "type": "string", + "enum": [ + "kept-revision-unread" + ], + "description": "why the mover has no step placed yet, where the runner knows it: kept-revision-unread for a kept mover whose kept revision the runner's view does not hold yet (design 47 H2 §787); absent once placed" + }, + "revision": { + "type": "integer", + "description": "the catalog revision a kept-revision-unread mover waits for (design 47 H2 §787)" + }, + "over-budget": { + "type": "boolean", + "description": "yes when the module is over its test budget (design 47 G section 419)" + } + }, + "description": "one module a pipeline moves: an entry of its take-in.taken, with what the pipeline found of it (design 47 H §746, H-A-35, H-A-72)", + "required": [ + "module", + "commit", + "fingerprint", + "repository", + "tier" + ] + }, + "MoverEnded": { + "type": "object", + "properties": { + "module": { + "type": "string" + }, + "fingerprint": { + "type": "string", + "description": "the module's build-input fingerprint the pipeline ran at" + }, + "commit": { + "type": "string", + "description": "the commit the mover's entry was last changed at, from the pipeline's value (design 47 H-A-119)" + }, + "widens": { + "type": "array", + "items": { + "$ref": "#/defs/Widening" + }, + "description": "what the mover's change widens, from the pipeline's value; the teller says it (design 47 H-A-119)" + }, + "version": { + "type": "string", + "description": "the version released for the mover, absent when none was released" + }, + "steps": { + "type": "array", + "items": { + "$ref": "#/defs/StepResult" + }, + "description": "the mover's step results, each naming its module (design 47 H-A-72)" + }, + "tier": { + "type": "integer", + "description": "the mover's tier in the delivery order, from 1 (ADR 0666 section 14)" + } + }, + "description": "one mover at its pipeline's end (design 47 H §746)", + "required": [ + "module", + "fingerprint", + "steps", + "tier" + ] + }, + "Waiting": { + "type": "object", + "properties": { + "pipeline": { + "type": "string", + "description": "the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747)" + } + }, + "description": "a pipeline waiting its turn under pipelines-at-once (design 47 H §748, H-A-72)", + "required": [ + "pipeline" + ] + }, + "FailedStep": { + "type": "object", + "properties": { + "module": { + "type": "string" + }, + "step": { + "type": "string", + "enum": [ + "validate", + "build", + "lint", + "unit", + "integration", + "composition", + "review", + "approval" + ] + }, + "artifact": { + "type": "string" + }, + "outcome": { + "type": "string", + "description": "the step's outcome: failed, could not run or did not build" + }, + "why": { + "type": "string" + } + }, + "description": "the first step of a pipeline or mover that failed or could not run, with its module and why (design 47 H2 H-A-376)", + "required": [ + "module", + "step", + "outcome" + ] + }, + "PipelineRow": { + "type": "object", + "properties": { + "pipeline": { + "type": "string", + "description": "the pipeline's key" + }, + "call": { + "type": "string", + "description": "the merge call whose source revision the take-in took in" + }, + "started": { + "type": "string", + "format": "date-time" + }, + "ended": { + "type": "string", + "format": "date-time" + }, + "outcome": { + "type": "string", + "enum": [ + "waiting", + "running", + "passed", + "failed", + "refused", + "superseded", + "cancelled" + ] + }, + "mover-count": { + "type": "integer", + "description": "how many movers the pipeline has" + }, + "failed": { + "$ref": "#/defs/FailedStep" + } + }, + "description": "one pipeline in list's short form (design 47 H2 H-A-376)", + "required": [ + "pipeline", + "outcome", + "mover-count" + ] + }, + "MoverRow": { + "type": "object", + "properties": { + "module": { + "type": "string" + }, + "outcome": { + "type": "string", + "enum": [ + "waiting", + "running", + "passed", + "failed", + "superseded" + ], + "description": "waiting for a mover with no step placed; running while a required step is open; failed once a required step did not pass; superseded for one a newer pipeline took out" + }, + "failed": { + "$ref": "#/defs/FailedStep" + } + }, + "description": "one mover of a pipeline in show's short form (design 47 H2 H-A-376)", + "required": [ + "module", + "outcome" + ] + } + }, + "form": 6 +} diff --git a/go/seatlist/seatlisttest/seats/mesh-recovery.json b/go/seatlist/seatlisttest/seats/mesh-recovery.json new file mode 100644 index 00000000..6b7546d9 --- /dev/null +++ b/go/seatlist/seatlisttest/seats/mesh-recovery.json @@ -0,0 +1,1172 @@ +{ + "format": 1, + "seat": "mesh-recovery", + "scope": "mesh", + "grain": "whole", + "domain": "Core", + "decision": "novox/hq ADR 0672", + "description": "Takes, sets aside and restores a module's kept data on the live bus, each act in the hand-act log (design 47 G 554). Its verbs answer at once and the result is the event.", + "verbs": [ + { + "name": "snapshot-kept", + "description": "Take the restore point of one module's kept data on the bus before it moves across a form change: the data whole and the position of each durable consumer. Answers accepted, busy or error at once; the point comes on the event ended (design 47 G 516, 554, 560).", + "by-caller": true, + "class": "act", + "input": { + "type": "object", + "properties": { + "module": { + "type": "string", + "description": "the module", + "minLength": 1 + }, + "kept": { + "type": "string", + "description": "the kept data, its local name in the module's manifest", + "minLength": 1 + }, + "form": { + "type": "integer", + "description": "its form, the number the holder raises when it changes how it writes it (0 when undeclared)" + }, + "fingerprint": { + "type": "string", + "description": "the fingerprint the restore point is taken before, or reverted from", + "minLength": 1 + }, + "why": { + "type": "string", + "description": "why, which the hand-act log keeps" + }, + "catch-up": { + "$ref": "#/defs/CatchUp" + } + }, + "required": [ + "module", + "kept", + "form", + "fingerprint", + "why" + ], + "additionalProperties": false + }, + "answer": { + "type": "object", + "description": "the act accepted, or busy with the act that runs, or the error; one of the three is set (design 47 G 554)", + "properties": { + "accepted": { + "type": "string", + "description": "accepted: the act's id, which the ending event carries" + }, + "busy": { + "type": "string", + "description": "busy: the id of the act that runs; it keeps no queue, so the caller asks again on that act's ending event" + }, + "error": { + "type": "string", + "description": "why it was not taken; set alone" + } + } + } + }, + { + "name": "restore-kept", + "description": "Restore one module's kept data from its restore point, the live data set aside first and never over it. Answers accepted, busy or error at once; the result comes on the event restored. It refuses aside-only and discard-aside: those are hand acts alone (design 47 G 516, 554, 560).", + "by-caller": true, + "class": "act", + "input": { + "type": "object", + "properties": { + "module": { + "type": "string", + "description": "the module", + "minLength": 1 + }, + "kept": { + "type": "string", + "description": "the kept data, its local name in the module's manifest", + "minLength": 1 + }, + "form": { + "type": "integer", + "description": "its form, the number the holder raises when it changes how it writes it (0 when undeclared)" + }, + "fingerprint": { + "type": "string", + "description": "the fingerprint the restore point is taken before, or reverted from", + "minLength": 1 + }, + "why": { + "type": "string", + "description": "why, which the hand-act log keeps" + }, + "from-backup": { + "type": "boolean", + "description": "ask the backup holder for the points directory when the point is missing there (design 47 G 516)" + }, + "snapshot": { + "type": "string", + "description": "the restore point's name", + "minLength": 1 + } + }, + "required": [ + "module", + "kept", + "form", + "fingerprint", + "why" + ], + "additionalProperties": false + }, + "answer": { + "type": "object", + "description": "the act accepted, or busy with the act that runs, or the error; one of the three is set (design 47 G 554)", + "properties": { + "accepted": { + "type": "string", + "description": "accepted: the act's id, which the ending event carries" + }, + "busy": { + "type": "string", + "description": "busy: the id of the act that runs; it keeps no queue, so the caller asks again on that act's ending event" + }, + "error": { + "type": "string", + "description": "why it was not taken; set alone" + } + } + } + }, + { + "name": "kept-points", + "description": "Every restore point and aside the recovery module holds, with its module, kept data, form, fingerprint, size and when, and the act that runs, with since when and what it waits on (design 47 G 517, 554).", + "class": "read", + "input": { + "type": "object", + "properties": { + "module": { + "type": "string", + "description": "one module (optional)" + }, + "kept": { + "type": "string", + "description": "one kept data (optional)" + } + }, + "additionalProperties": false + }, + "answer": { + "type": "object", + "properties": { + "points": { + "type": "array", + "items": { + "$ref": "#/defs/Point" + } + }, + "asides": { + "type": "array", + "items": { + "$ref": "#/defs/Aside" + } + }, + "running": { + "$ref": "#/defs/Running" + }, + "unrestored": { + "type": "array", + "description": "the asides whose record ended at removed that no running act holds: the live data of those names is gone and only the aside holds it; the module's health check fails while one is listed (design 47 G 554, 657, A-129)", + "items": { + "$ref": "#/defs/Aside" + } + } + }, + "required": [ + "points", + "asides", + "unrestored" + ] + } + }, + { + "name": "restore-aside", + "description": "Restore an aside under the live names it was taken from, for an aside whose record ends at removed that no running act holds; it refuses while live objects of those names exist. Ends with restored (design 47 G 566).", + "by-caller": true, + "class": "act", + "input": { + "type": "object", + "properties": { + "module": { + "type": "string", + "description": "the module", + "minLength": 1 + }, + "kept": { + "type": "string", + "description": "the kept data, its local name in the module's manifest", + "minLength": 1 + }, + "id": { + "type": "string", + "description": "the aside's id", + "minLength": 1 + }, + "why": { + "type": "string", + "description": "why, which the hand-act log keeps" + }, + "asker": { + "type": "string", + "description": "who asked: the caller the tool runner proved and recorded for the call (who@node, or operator), which the tools bundle's recovery tool passes and a module's process leaves out; the hand-act log names it beside the account the bus proved (design 47 G 682)" + } + }, + "required": [ + "module", + "kept", + "id", + "why" + ], + "additionalProperties": false + }, + "answer": { + "type": "object", + "description": "the act accepted, or busy with the act that runs, or the error; one of the three is set (design 47 G 554)", + "properties": { + "accepted": { + "type": "string", + "description": "accepted: the act's id, which the ending event carries" + }, + "busy": { + "type": "string", + "description": "busy: the id of the act that runs; it keeps no queue, so the caller asks again on that act's ending event" + }, + "error": { + "type": "string", + "description": "why it was not taken; set alone" + } + } + } + }, + { + "name": "cancel", + "description": "End an accepted act that has not begun its work, with its ending event, outcome failed and error cancelled; it refuses an act that has begun, naming its step (design 47 G 566).", + "by-caller": true, + "class": "act", + "input": { + "type": "object", + "properties": { + "act": { + "type": "string", + "description": "the act's id", + "minLength": 1 + }, + "why": { + "type": "string", + "description": "why, which the hand-act log keeps" + }, + "asker": { + "type": "string", + "description": "who asked: the caller the tool runner proved and recorded for the call (who@node, or operator), which the tools bundle's recovery tool passes and a module's process leaves out; the hand-act log names it beside the account the bus proved (design 47 G 682)" + } + }, + "required": [ + "act", + "why" + ], + "additionalProperties": false + }, + "answer": { + "type": "object", + "description": "the act cancelled, or busy, or the error; one of the three is set (design 47 G 682, 691)", + "properties": { + "accepted": { + "type": "string", + "description": "accepted: the id of the act it cancelled, whose ending event follows under that id" + }, + "busy": { + "type": "string", + "description": "busy: the id of the act that runs; it keeps no queue, so the caller asks again on that act's ending event" + }, + "error": { + "type": "string", + "description": "why not, naming the step an act has begun; set alone" + } + } + } + }, + { + "name": "take", + "description": "Hand act: take a point of one module's kept data at that fingerprint's form, as snapshot-kept does with no catch-up. Answers accepted, busy or error at once; the point comes on the event ended (design 47 G 682, 554).", + "by-caller": true, + "class": "act", + "input": { + "type": "object", + "properties": { + "module": { + "type": "string", + "description": "the module", + "minLength": 1 + }, + "kept": { + "type": "string", + "description": "the kept data, its local name in the module's manifest", + "minLength": 1 + }, + "fingerprint": { + "type": "string", + "description": "the fingerprint whose form the point is taken at", + "minLength": 1 + }, + "why": { + "type": "string", + "description": "why, which the hand-act log keeps" + }, + "asker": { + "type": "string", + "description": "who asked: the caller the tool runner proved and recorded for the call (who@node, or operator), which the tools bundle's recovery tool passes and a module's process leaves out; the hand-act log names it beside the account the bus proved (design 47 G 682)" + } + }, + "required": [ + "module", + "kept", + "fingerprint", + "why" + ], + "additionalProperties": false + }, + "answer": { + "type": "object", + "description": "the act accepted, or busy with the act that runs, or the error; one of the three is set (design 47 G 554)", + "properties": { + "accepted": { + "type": "string", + "description": "accepted: the act's id, which the ending event carries" + }, + "busy": { + "type": "string", + "description": "busy: the id of the act that runs; it keeps no queue, so the caller asks again on that act's ending event" + }, + "error": { + "type": "string", + "description": "why it was not taken; set alone" + } + } + } + }, + { + "name": "restore", + "description": "Hand act: restore one module's kept data as restore-kept does, from a point by its name or from a keeper's snapshot by its request; exactly one of snapshot and request, and from-backup only with snapshot (design 47 G 682, 681). Answers accepted, busy or error at once; the result comes on the event restored.", + "by-caller": true, + "class": "act", + "input": { + "type": "object", + "properties": { + "module": { + "type": "string", + "description": "the module", + "minLength": 1 + }, + "kept": { + "type": "string", + "description": "the kept data, its local name in the module's manifest", + "minLength": 1 + }, + "snapshot": { + "type": "string", + "description": "a restore point's name", + "minLength": 1 + }, + "request": { + "type": "string", + "description": "a keeper's snapshot by its request (design 47 G 681)", + "minLength": 1 + }, + "from-backup": { + "type": "boolean", + "description": "ask the backup holder for the points directory when the point is missing there; only with snapshot" + }, + "why": { + "type": "string", + "description": "why, which the hand-act log keeps" + }, + "asker": { + "type": "string", + "description": "who asked: the caller the tool runner proved and recorded for the call (who@node, or operator), which the tools bundle's recovery tool passes and a module's process leaves out; the hand-act log names it beside the account the bus proved (design 47 G 682)" + } + }, + "required": [ + "module", + "kept", + "why" + ], + "additionalProperties": false + }, + "answer": { + "type": "object", + "description": "the act accepted, or busy with the act that runs, or the error; one of the three is set (design 47 G 554)", + "properties": { + "accepted": { + "type": "string", + "description": "accepted: the act's id, which the ending event carries" + }, + "busy": { + "type": "string", + "description": "busy: the id of the act that runs; it keeps no queue, so the caller asks again on that act's ending event" + }, + "error": { + "type": "string", + "description": "why it was not taken; set alone" + } + } + } + }, + { + "name": "retry", + "description": "Hand act: ask again the last failed act of that module and kept data, with its fields; the error nothing to retry when its last act did not fail. The result comes on that act's ending event (design 47 G 682).", + "by-caller": true, + "class": "act", + "input": { + "type": "object", + "properties": { + "module": { + "type": "string", + "description": "the module", + "minLength": 1 + }, + "kept": { + "type": "string", + "description": "the kept data, its local name in the module's manifest", + "minLength": 1 + }, + "why": { + "type": "string", + "description": "why, which the hand-act log keeps" + }, + "asker": { + "type": "string", + "description": "who asked: the caller the tool runner proved and recorded for the call (who@node, or operator), which the tools bundle's recovery tool passes and a module's process leaves out; the hand-act log names it beside the account the bus proved (design 47 G 682)" + } + }, + "required": [ + "module", + "kept", + "why" + ], + "additionalProperties": false + }, + "answer": { + "type": "object", + "description": "the act accepted, or busy with the act that runs, or the error; one of the three is set (design 47 G 554)", + "properties": { + "accepted": { + "type": "string", + "description": "accepted: the act's id, which the ending event carries" + }, + "busy": { + "type": "string", + "description": "busy: the id of the act that runs; it keeps no queue, so the caller asks again on that act's ending event" + }, + "error": { + "type": "string", + "description": "why it was not taken; set alone" + } + } + } + }, + { + "name": "aside", + "description": "Hand act: set the live kept data aside without restoring (design 47 G 516, 682). Answers accepted, busy or error at once; the result comes on the event set-aside.", + "by-caller": true, + "class": "act", + "input": { + "type": "object", + "properties": { + "module": { + "type": "string", + "description": "the module", + "minLength": 1 + }, + "kept": { + "type": "string", + "description": "the kept data, its local name in the module's manifest", + "minLength": 1 + }, + "why": { + "type": "string", + "description": "why, which the hand-act log keeps" + }, + "asker": { + "type": "string", + "description": "who asked: the caller the tool runner proved and recorded for the call (who@node, or operator), which the tools bundle's recovery tool passes and a module's process leaves out; the hand-act log names it beside the account the bus proved (design 47 G 682)" + } + }, + "required": [ + "module", + "kept", + "why" + ], + "additionalProperties": false + }, + "answer": { + "type": "object", + "description": "the act accepted, or busy with the act that runs, or the error; one of the three is set (design 47 G 554)", + "properties": { + "accepted": { + "type": "string", + "description": "accepted: the act's id, which the ending event carries" + }, + "busy": { + "type": "string", + "description": "busy: the id of the act that runs; it keeps no queue, so the caller asks again on that act's ending event" + }, + "error": { + "type": "string", + "description": "why it was not taken; set alone" + } + } + } + }, + { + "name": "discard-aside", + "description": "Hand act: remove one aside's directory (design 47 G 682). Answers accepted, busy or error at once; the result comes on the event aside-discarded.", + "by-caller": true, + "class": "act", + "input": { + "type": "object", + "properties": { + "module": { + "type": "string", + "description": "the module", + "minLength": 1 + }, + "kept": { + "type": "string", + "description": "the kept data, its local name in the module's manifest", + "minLength": 1 + }, + "id": { + "type": "string", + "description": "the aside's id", + "minLength": 1 + }, + "why": { + "type": "string", + "description": "why, which the hand-act log keeps" + }, + "asker": { + "type": "string", + "description": "who asked: the caller the tool runner proved and recorded for the call (who@node, or operator), which the tools bundle's recovery tool passes and a module's process leaves out; the hand-act log names it beside the account the bus proved (design 47 G 682)" + } + }, + "required": [ + "module", + "kept", + "id", + "why" + ], + "additionalProperties": false + }, + "answer": { + "type": "object", + "description": "the act accepted, or busy with the act that runs, or the error; one of the three is set (design 47 G 554)", + "properties": { + "accepted": { + "type": "string", + "description": "accepted: the act's id, which the ending event carries" + }, + "busy": { + "type": "string", + "description": "busy: the id of the act that runs; it keeps no queue, so the caller asks again on that act's ending event" + }, + "error": { + "type": "string", + "description": "why it was not taken; set alone" + } + } + } + }, + { + "name": "expire-points", + "description": "Hand act: remove the points past their 90 days now, as the module does by itself (design 47 G 551, 682). Answers accepted, busy or error at once; the result comes on the event expiry-ended.", + "by-caller": true, + "class": "act", + "input": { + "type": "object", + "properties": { + "why": { + "type": "string", + "description": "why, which the hand-act log keeps" + }, + "asker": { + "type": "string", + "description": "who asked: the caller the tool runner proved and recorded for the call (who@node, or operator), which the tools bundle's recovery tool passes and a module's process leaves out; the hand-act log names it beside the account the bus proved (design 47 G 682)" + } + }, + "required": [ + "why" + ], + "additionalProperties": false + }, + "answer": { + "type": "object", + "description": "the act accepted, or busy with the act that runs, or the error; one of the three is set (design 47 G 554)", + "properties": { + "accepted": { + "type": "string", + "description": "accepted: the act's id, which the ending event carries" + }, + "busy": { + "type": "string", + "description": "busy: the id of the act that runs; it keeps no queue, so the caller asks again on that act's ending event" + }, + "error": { + "type": "string", + "description": "why it was not taken; set alone" + } + } + } + } + ], + "events": [ + { + "name": "ended", + "subject": "ended..", + "keep": { + "max-age-days": 90, + "last-per-subject": true + }, + "body": { + "type": "object", + "description": "a point taken, or the act failed; on done the point whole, on failed the error (design 47 G 560)", + "properties": { + "act": { + "type": "string", + "description": "the id the act was answered accepted with, or a hand act's own id (design 47 G 560)" + }, + "at": { + "type": "string", + "description": "when it was said, RFC 3339", + "format": "date-time" + }, + "module": { + "type": "string" + }, + "kept": { + "type": "string" + }, + "form": { + "type": "integer" + }, + "fingerprint": { + "type": "string" + }, + "snapshot": { + "type": "string" + }, + "outcome": { + "type": "string", + "description": "how the act ended", + "enum": [ + "done", + "failed" + ] + }, + "error": { + "type": "string", + "description": "when failed" + }, + "why": { + "type": "string", + "description": "the words of the hand act that ended it, for a cancel (design 47 G A-117, 560)" + }, + "consumers": { + "type": "array", + "items": { + "$ref": "#/defs/Consumer" + } + }, + "size": { + "type": "integer", + "minimum": 0 + }, + "catch-up": { + "$ref": "#/defs/CatchUp" + }, + "digests": { + "type": "array", + "items": { + "$ref": "#/defs/Digest" + } + } + }, + "required": [ + "act", + "at", + "module", + "kept", + "outcome" + ] + } + }, + { + "name": "restore-started", + "subject": "restore-started..", + "keep": { + "max-age-days": 90, + "last-per-subject": true + }, + "body": { + "type": "object", + "description": "a restore begins, said before it reads the dump's item events and before any aside is taken (design 47 G 560, 563)", + "properties": { + "act": { + "type": "string", + "description": "the id the act was answered accepted with, or a hand act's own id (design 47 G 560)" + }, + "at": { + "type": "string", + "description": "when it was said, RFC 3339", + "format": "date-time" + }, + "module": { + "type": "string" + }, + "kept": { + "type": "string" + }, + "form": { + "type": "integer" + }, + "fingerprint": { + "type": "string" + }, + "snapshot": { + "type": "string" + } + }, + "required": [ + "act", + "at", + "module", + "kept" + ] + } + }, + { + "name": "restored", + "subject": "restored..", + "keep": { + "max-age-days": 90, + "last-per-subject": true + }, + "body": { + "type": "object", + "description": "a restore ended (design 47 G 560)", + "properties": { + "act": { + "type": "string", + "description": "the id the act was answered accepted with, or a hand act's own id (design 47 G 560)" + }, + "at": { + "type": "string", + "description": "when it was said, RFC 3339", + "format": "date-time" + }, + "module": { + "type": "string" + }, + "kept": { + "type": "string" + }, + "form": { + "type": "integer" + }, + "fingerprint": { + "type": "string" + }, + "snapshot": { + "type": "string" + }, + "outcome": { + "type": "string", + "description": "how the act ended", + "enum": [ + "done", + "failed" + ] + }, + "error": { + "type": "string" + }, + "why": { + "type": "string", + "description": "the words of the hand act that ended it, for a cancel (design 47 G A-117, 560)" + }, + "aside": { + "$ref": "#/defs/Aside" + }, + "missing": { + "type": "boolean", + "description": "the point is not in the recovery module's directory" + } + }, + "required": [ + "act", + "at", + "module", + "kept", + "outcome" + ] + } + }, + { + "name": "set-aside", + "subject": "set-aside..", + "keep": { + "max-age-days": 90, + "last-per-subject": true + }, + "body": { + "type": "object", + "description": "live data set aside by the hand act (design 47 G 560)", + "properties": { + "act": { + "type": "string", + "description": "the id the act was answered accepted with, or a hand act's own id (design 47 G 560)" + }, + "at": { + "type": "string", + "description": "when it was said, RFC 3339", + "format": "date-time" + }, + "id": { + "type": "string" + }, + "names": { + "type": "array", + "description": "the bus names it was live under, one per bus object", + "items": { + "type": "string" + }, + "minItems": 1 + }, + "size": { + "type": "integer", + "minimum": 0 + } + }, + "required": [ + "act", + "at", + "id", + "names", + "size" + ] + } + }, + { + "name": "aside-discarded", + "subject": "aside-discarded..", + "keep": { + "max-age-days": 90, + "last-per-subject": true + }, + "body": { + "type": "object", + "description": "an aside discarded by the hand act (design 47 G 560)", + "properties": { + "act": { + "type": "string", + "description": "the id the act was answered accepted with, or a hand act's own id (design 47 G 560)" + }, + "at": { + "type": "string", + "description": "when it was said, RFC 3339", + "format": "date-time" + }, + "id": { + "type": "string" + }, + "names": { + "type": "array", + "description": "the bus names it was live under, one per bus object", + "items": { + "type": "string" + }, + "minItems": 1 + }, + "size": { + "type": "integer", + "minimum": 0 + } + }, + "required": [ + "act", + "at", + "id", + "names", + "size" + ] + } + }, + { + "name": "point-removed", + "subject": "point-removed..", + "keep": { + "max-age-days": 90, + "last-per-subject": true + }, + "body": { + "type": "object", + "description": "a restore point removed after its days, never the newest (design 47 G 551, 560)", + "properties": { + "act": { + "type": "string", + "description": "the id the act was answered accepted with, or a hand act's own id (design 47 G 560)" + }, + "at": { + "type": "string", + "description": "when it was said, RFC 3339", + "format": "date-time" + }, + "module": { + "type": "string" + }, + "kept": { + "type": "string" + }, + "form": { + "type": "integer" + }, + "fingerprint": { + "type": "string" + }, + "snapshot": { + "type": "string" + } + }, + "required": [ + "act", + "at", + "module", + "kept" + ] + } + }, + { + "name": "expiry-ended", + "keep": { + "max-age-days": 90 + }, + "body": { + "type": "object", + "description": "a run of the removal ended, also when it removed nothing (design 47 G 560)", + "properties": { + "act": { + "type": "string", + "description": "the id the act was answered accepted with, or a hand act's own id (design 47 G 560)" + }, + "at": { + "type": "string", + "description": "when it was said, RFC 3339", + "format": "date-time" + }, + "removed": { + "type": "integer", + "description": "the number of points the run removed, 0 included", + "minimum": 0 + } + }, + "required": [ + "act", + "at", + "removed" + ] + } + } + ], + "family": { + "name": "mesh-recovery.events", + "max-age-days": 7 + }, + "defs": { + "Consumer": { + "type": "object", + "description": "one durable consumer of the kept data and the sequence it had acknowledged", + "properties": { + "stream": { + "type": "string" + }, + "name": { + "type": "string" + }, + "sequence": { + "type": "integer", + "minimum": 0 + } + }, + "required": [ + "stream", + "name", + "sequence" + ] + }, + "Digest": { + "type": "object", + "description": "the digest of one object of a point's bytes", + "properties": { + "name": { + "type": "string" + }, + "digest": { + "type": "string" + } + }, + "required": [ + "name", + "digest" + ] + }, + "Point": { + "type": "object", + "description": "a restore point: the kept data whole and its consumers' positions, in the recovery module's points directory (design 47 G 516, 560)", + "properties": { + "module": { + "type": "string", + "description": "the module", + "minLength": 1 + }, + "kept": { + "type": "string", + "description": "the kept data, its local name in the module's manifest", + "minLength": 1 + }, + "form": { + "type": "integer", + "description": "its form, the number the holder raises when it changes how it writes it (0 when undeclared)" + }, + "fingerprint": { + "type": "string", + "description": "the fingerprint it was taken before" + }, + "snapshot": { + "type": "string", + "description": "the point's name, also its directory's" + }, + "consumers": { + "type": "array", + "items": { + "$ref": "#/defs/Consumer" + } + }, + "size": { + "type": "integer", + "description": "its bytes", + "minimum": 0 + }, + "at": { + "type": "string", + "description": "when it was taken, RFC 3339", + "format": "date-time" + }, + "catch-up": { + "$ref": "#/defs/CatchUp" + }, + "digests": { + "type": "array", + "items": { + "$ref": "#/defs/Digest" + }, + "description": "the digest of each object's bytes, checked before a restore (design 47 G 567)" + }, + "names": { + "type": "array", + "description": "the bus names it took, one per bus object (design 47 G A-230)", + "items": { + "type": "string" + }, + "minItems": 1 + } + }, + "required": [ + "module", + "kept", + "form", + "fingerprint", + "snapshot", + "size", + "at", + "names" + ] + }, + "Aside": { + "type": "object", + "description": "the live data set aside before a restore: a directory under aside/ in the recovery module's directory, nothing of it on the bus (design 47 G 516)", + "properties": { + "id": { + "type": "string", + "description": "the directory's name" + }, + "names": { + "type": "array", + "description": "the bus names it was live under, one per bus object", + "items": { + "type": "string" + }, + "minItems": 1 + }, + "size": { + "type": "integer", + "description": "its bytes on disk", + "minimum": 0 + } + }, + "required": [ + "id", + "names", + "size" + ] + }, + "Running": { + "type": "object", + "description": "the act that runs, since when and what it waits on (design 47 G 517, 554)", + "properties": { + "act": { + "type": "string" + }, + "step": { + "type": "string" + }, + "since": { + "type": "string", + "format": "date-time" + }, + "waits-on": { + "type": "string", + "description": "what it waits on, when it waits" + } + }, + "required": [ + "act", + "since" + ] + }, + "CatchUp": { + "type": "object", + "description": "the catch-up of the newer manifest: a tool and its arguments, never text (design 47 G 451, 516, 631)", + "properties": { + "tool": { + "type": "string" + }, + "arguments": { + "type": "object" + } + }, + "required": [ + "tool" + ] + } + } +} diff --git a/go/seatlist/seatlisttest/seats/mesh-vault.json b/go/seatlist/seatlisttest/seats/mesh-vault.json index e2372428..57afdfc1 100644 --- a/go/seatlist/seatlisttest/seats/mesh-vault.json +++ b/go/seatlist/seatlisttest/seats/mesh-vault.json @@ -9,7 +9,7 @@ "verbs": [ { "name": "get", - "description": "One secret, sealed to the public key the asker sends (the sealed box of ADR 0206), with its generation; answered only to an asker the vault allows, as the bus proved it. Never the value in the clear, and never to a private key.", + "description": "One secret, sealed to the public key the asker sends (the sealed box of ADR 0206), with its generation; answered only to an asker the vault allows, as the bus proved it, and a module's own secret put through own-put only to a process of the same module (design 47 H2 §822). Never the value in the clear, and never to a private key.", "optional": true, "input": { "properties": { @@ -640,6 +640,67 @@ "required": [ "retired" ] + }, + "class": "act" + }, + { + "name": "licence-put", + "description": "Hand the vault one of a licence's two secrets, its grant or its access token, from the licence manager, in two steps: name the licence by its key, the part and its generation and get a call and a key made for it; then send the call and the value sealed to that key. Keeps the newest generation and refuses an older one; says secret-changed with the secret's name and generation, never the value (novox/hq design 39 bus-and-switch §15).", + "optional": true, + "input": { + "properties": { + "call": { + "description": "the second step: the call the first step answered", + "type": "string" + }, + "generation": { + "description": "the first step: the licence's generation", + "type": "number" + }, + "licence": { + "description": "the first step: the licence's key", + "type": "string" + }, + "part": { + "description": "the first step: grant or access", + "type": "string" + }, + "sealed": { + "description": "the second step: the value sealed to the call's key — never the value", + "type": "string" + } + }, + "type": "object" + }, + "answer": { + "type": "object", + "description": "the holder's answer" + } + }, + { + "name": "own-put", + "description": "Store a secret as the calling module's own, in two steps modelled on licence-put (design 47 H2 §822, H-A-345, H-A-349 DQ-7): name the secret and get a call and a key made for it (call, seal_to); then send the call and the value sealed to that key, and the vault keeps it under that name as the module's own, the module named by the caller's account as the bus proved it, never by an argument, keeping its newest generation only (name, generation). get answers it to a process of the same module, sealed to the process's key. The holder says each write in its journal without the value.", + "optional": true, + "input": { + "properties": { + "call": { + "description": "the second step: the call the first step answered", + "type": "string" + }, + "name": { + "description": "the first step: the secret's name, among the calling module's own", + "type": "string" + }, + "sealed": { + "description": "the second step: the value sealed to the call's key — never the value", + "type": "string" + } + }, + "type": "object" + }, + "answer": { + "type": "object", + "description": "the holder's answer: to the first step the call and the key to seal to (call, seal_to); to the second the secret's name and the generation kept (name, generation)" } } ], diff --git a/go/seatlist/seatlisttest/seats/mesh-wants.json b/go/seatlist/seatlisttest/seats/mesh-wants.json index 11f475d6..f9d92669 100644 --- a/go/seatlist/seatlisttest/seats/mesh-wants.json +++ b/go/seatlist/seatlisttest/seats/mesh-wants.json @@ -277,6 +277,66 @@ "run" ] } + }, + { + "name": "keep-now", + "description": "Keep the assignment graph's revisions now, by hand (design 47 H2 §813 (2); H-A-417): written in the hand-act log first by the module's tool assigner_keep_now, then the keeping runs at once on the pass's newest computed revision; answers the revision kept, or the fault the keeping met.", + "input": { + "type": "object", + "properties": { + "why": { + "type": "string", + "description": "why the keeping runs now" + } + }, + "required": [ + "why" + ] + }, + "answer": { + "type": "object", + "properties": { + "revision": { + "$ref": "#/defs/AssignmentRevision", + "description": "the revision kept" + }, + "fault": { + "type": "string", + "description": "the fault the keeping met; absent when it kept the revision" + } + } + }, + "class": "act" + }, + { + "name": "drop", + "description": "End an ask on the seat's work queue for a caller that no longer waits for it (its step ended by its bound): one waiting is ended dropped when it is pulled, one taken is cancelled; both end the step could-not-run, its why naming the drop and who asked it, so no holder works on it after (design 47 H2 §778; ADR 0664 §4).", + "input": { + "type": "object", + "properties": { + "id": { + "type": "string", + "description": "the ask's id" + }, + "by": { + "type": "string", + "description": "who drops it" + }, + "why": { + "type": "string", + "description": "why the caller no longer waits" + } + }, + "required": [ + "id", + "why" + ] + }, + "answer": { + "type": "object", + "description": "whether the ask was held here" + }, + "class": "act" } ], "accepts": [ @@ -284,7 +344,14 @@ "name": "step", "body": { "$ref": "#/defs/StepRun" - } + }, + "max-age": "24h", + "max-msgs": 10000, + "ack-wait": "60s", + "in-progress": "20s", + "max-ack-pending": 1000, + "max-deliveries": 5, + "duplicates-window": "2m" } ], "events": [ @@ -434,7 +501,7 @@ }, { "name": "pipeline-wanted", - "subject": "pipeline-wanted..", + "subject": "pipeline-wanted..", "keep": { "max-age-days": 30, "max-mib": 64 @@ -525,6 +592,34 @@ "at" ] } + }, + { + "name": "ask-taken", + "subject": "ask-taken.", + "body": { + "type": "object", + "properties": { + "ask": { + "type": "string", + "description": "the ask's id" + }, + "node": { + "type": "string", + "description": "the node whose holder took it" + }, + "at": { + "type": "string", + "format": "date-time", + "description": "when it was taken" + } + }, + "required": [ + "ask", + "node", + "at" + ], + "description": "the seat's holder took the ask from the seat's work queue, at the pull: only the holder says it; the runner arms the ask's bound from it (design 47 H2 §777, §778)" + } } ], "family": { @@ -597,6 +692,10 @@ "type": "integer", "minimum": 1, "description": "the mover's tier in that pipeline's delivery order, from 1; the node-engine applies one pipeline's wants by it; absent with pipeline (design 47 H §750, §771, H-A-37)" + }, + "run": { + "type": "string", + "description": "the pipeline sequence of the composition run that wrote this want, in decimal: the holder writes by compare-and-set refusing a lower run, and the node-engine applies a want again only when its run is newer than the one it applied (design 47 H2 §813 (1); H-A-355 (1))" } } }, @@ -1701,6 +1800,13 @@ "verdict": { "type": "object", "description": "that machine's composer verdict (design 47 G §721)" + }, + "unlisted": { + "type": "array", + "items": { + "type": "string" + }, + "description": "the module names of the assignments left out of the machine's composition, those no watched branch's newest revision lists, written when the run starts and read by every step's judgement and by the resume (design 47 H-A-152, H-A-153)" } }, "required": [ @@ -1718,7 +1824,7 @@ }, "pipeline": { "type": "string", - "description": "the runner's pipeline key, ., a branch escaped as a key's name is (design 47 H §746)" + "description": "the runner's pipeline key, ., a branch escaped as a key's name is (design 47 H §746, H-A-213 (4))" }, "branch": { "type": "string" @@ -1726,6 +1832,10 @@ "source": { "type": "integer" }, + "revision": { + "type": "integer", + "description": "the pipeline's catalog revision number, the number of its key; the indexer frees the pins of that revision, not of another revision of the same source (design 47 H-A-214); absent from an older event" + }, "outcome": { "type": "string", "enum": [ @@ -1775,5 +1885,6 @@ "step": "place", "on": "every" } - ] + ], + "form": 2 } diff --git a/go/seatlist/seatlisttest/seats/node-build-node.json b/go/seatlist/seatlisttest/seats/node-build-node.json index a04b6240..b49cbaa9 100644 --- a/go/seatlist/seatlisttest/seats/node-build-node.json +++ b/go/seatlist/seatlisttest/seats/node-build-node.json @@ -45,6 +45,12 @@ "body": { "$ref": "#/defs/StepAsk" } + }, + { + "name": "install", + "body": { + "$ref": "#/defs/StepAsk" + } } ], "description": "The Node build module on a node: it takes the asks of the builder and the checker, runs them as steps and answers.", diff --git a/go/seatlist/seatlisttest/seats/node-engine.json b/go/seatlist/seatlisttest/seats/node-engine.json index f33b6cbc..6214da6a 100644 --- a/go/seatlist/seatlisttest/seats/node-engine.json +++ b/go/seatlist/seatlisttest/seats/node-engine.json @@ -1,11 +1,12 @@ { "format": 1, + "form": 4, "seat": "node-engine", "scope": "node", "grain": "whole", "domain": "Core", "decision": "novox/hq ADR 0563", - "description": "The node seat every node's engine holds (novox/hq ADR 0563, ADR 0618 §3, §4): its hand acts on the operator's warrant, its read-only judgements, its compositions and what it reads of a manifest. Each verb is answered on this node's subject, mesh.seat.node-engine.tool.., and each event is published on mesh.seat.node-engine.event.., the node's name its last token, or, for an event whose subject names , there (composed., running..): the grants part grants a node's account only its own node's subjects (ADR 0363 §2).", + "description": "The node seat every node's engine holds (novox/hq ADR 0563, ADR 0618 \u00a73, \u00a74): its hand acts on the operator's warrant, its read-only judgements, its compositions and what it reads of a manifest. Each verb is answered on this node's subject, mesh.seat.node-engine.tool.., and each event is published on mesh.seat.node-engine.event.., the node's name its last token, or, for an event whose subject names , there (composed., running..): the grants part grants a node's account only its own node's subjects (ADR 0363 \u00a72).", "acts": [ { "step": "compose", @@ -15,7 +16,7 @@ "verbs": [ { "name": "hand-over", - "description": "Hand a directory this machine uses as found to the mesh (novox/hq ADR 0563 §3, §5): the engine judges the path as it does every hand-over (absolute, as it states it, used as found, reached through no link) and refuses in words, or asks the operator on their channel at the level approve, binding the owner, group and mode it will set. Nothing changes on the call: only the operator's yes hands it over. One ask is open per path; acts shows where it stands.", + "description": "Hand a directory this machine uses as found to the mesh (novox/hq ADR 0563 \u00a73, \u00a75): the engine judges the path as it does every hand-over (absolute, as it states it, used as found, reached through no link) and refuses in words, or asks the operator on their channel at the level approve, binding the owner, group and mode it will set. Nothing changes on the call: only the operator's yes hands it over. One ask is open per path; acts shows where it stands.", "input": { "properties": { "path": { @@ -39,7 +40,7 @@ }, { "name": "access-act", - "description": "Ask the operator for the take act on a library whose root holds data outside its group (novox/hq ADR 0328 §7, ADR 0563 §3, §4), on the library's home engine: the access as declared and its root as last declared, judged and refused in words or asked at the level approve. Only the operator's yes sets the library's group, group write and setgid under its root. One ask is open per access.", + "description": "Ask the operator for the take act on a library whose root holds data outside its group (novox/hq ADR 0328 \u00a77, ADR 0563 \u00a73, \u00a74), on the library's home engine: the access as declared and its root as last declared, judged and refused in words or asked at the level approve. Only the operator's yes sets the library's group, group write and setgid under its root. One ask is open per access.", "input": { "properties": { "access": { @@ -68,7 +69,7 @@ }, { "name": "judge-directories", - "description": "Which of this machine's declared directories the engine would refuse to make or keep, one line each (novox/hq ADR 0563 §2). Read-only: changes no owner or mode.", + "description": "Which of this machine's declared directories the engine would refuse to make or keep, one line each (novox/hq ADR 0563 \u00a72). Read-only: changes no owner or mode.", "input": { "properties": {}, "type": "object" @@ -80,7 +81,7 @@ }, { "name": "setuid-search", - "description": "Throw away the engine's last search for setuid programs and start a full one (novox/hq ADR 0563 §2); until it completes root-free answers that the machine is not judged yet. Refused while one runs, and when one was started within the hour, naming when the next may start.", + "description": "Throw away the engine's last search for setuid programs and start a full one (novox/hq ADR 0563 \u00a72); until it completes root-free answers that the machine is not judged yet. Refused while one runs, and when one was started within the hour, naming when the next may start.", "input": { "properties": { "why": { @@ -97,7 +98,7 @@ }, { "name": "root-free", - "description": "Whether this machine is root-free (novox/hq ADR 0259 §8, ADR 0563 §6): free only on the engine's own verdict, at most fifteen minutes old, that the account agents run as cannot become root without a person, and the login shell's execute not served here. Anything else is no pass, and says why. Answered by this machine's engine alone.", + "description": "Whether this machine is root-free (novox/hq ADR 0259 \u00a78, ADR 0563 \u00a76): free only on the engine's own verdict, at most fifteen minutes old, that the account agents run as cannot become root without a person, and the login shell's execute not served here. Anything else is no pass, and says why. Answered by this machine's engine alone.", "input": { "properties": {}, "type": "object" @@ -121,7 +122,7 @@ }, { "name": "acts", - "description": "Every act this machine's engine was asked through this seat (novox/hq ADR 0563 §9): each hand-over, access act and setuid search, open, done or refused, with who asked, the ask and the operator's warrant. Read-only.", + "description": "Every act this machine's engine was asked through this seat (novox/hq ADR 0563 \u00a79): each hand-over, access act and setuid search, open, done or refused, with who asked, the ask and the operator's warrant. Read-only.", "input": { "properties": { "id": { @@ -242,7 +243,7 @@ "type": "string" }, "digest": { - "description": "a composition's digest (sha256:…), to read that revision rather than the latest", + "description": "a composition's digest (sha256:\u2026), to read that revision rather than the latest", "type": "string" }, "module": { @@ -287,7 +288,7 @@ }, "self": { "type": "boolean", - "description": "the node composes the module itself (ADR 0547 §2)" + "description": "the node composes the module itself (ADR 0547 \u00a72)" }, "backup": { "type": "string", @@ -455,6 +456,32 @@ }, "description": "every resource that differs from what runs, when asked" }, + "drift": { + "type": "array", + "description": "when the diff is asked, each directory entry of what runs that the mesh made, as its record holds, whose owner or mode on disk differ from it (H-A-392 (b), H-A-458 (2)): read-only; the engine sets it right at its next start or apply of the module", + "items": { + "type": "object", + "required": [ + "id", + "path", + "drift" + ], + "properties": { + "id": { + "type": "string", + "description": "the directory entry's id in the composition" + }, + "path": { + "type": "string", + "description": "its path on the machine" + }, + "drift": { + "type": "string", + "description": "how its owner or mode on disk differ, in words" + } + } + } + }, "own": { "type": "object", "properties": { @@ -517,6 +544,27 @@ "withheld-unknown": { "type": "string", "description": "withheld unknown: the grants part does not answer, when its verb withheld did not answer" + }, + "withheld-parts": { + "type": "array", + "description": "each part a compose kept because its read did not answer, with why (design 08 control-verbs \u00a74, H-A-321): for the packet filter's holder the part node-packet-filter.allowed, its filter file kept as it runs", + "items": { + "type": "object", + "required": [ + "part", + "why" + ], + "properties": { + "part": { + "type": "string", + "description": "the read asked, as ." + }, + "why": { + "type": "string", + "description": "why it did not answer, naming the holder asked" + } + } + } } }, "required": [ @@ -529,6 +577,63 @@ }, "class": "read" }, + { + "name": "composition-diff", + "description": "One module's composed values read by resource (design 47 H2 \u00a7813 (3), H-A-349 DQ-12b): given a module and a revision or a digest of its composition, the entries that differ between the composition that runs and that one, per resource, environment entries by name, a secret's value masked. Read-only. Answered by this machine's engine alone.", + "class": "read", + "input": { + "type": "object", + "required": [ + "module" + ], + "properties": { + "module": { + "type": "string", + "description": "the module, as placed on this machine; node for the node's own" + }, + "revision": { + "type": "integer", + "description": "the revision of the module's composition key to compare with, from the revisions the bucket keeps; give it or digest" + }, + "digest": { + "type": "string", + "description": "the digest (sha256:\u2026) of the composition to compare with; give it or revision" + } + } + }, + "answer": { + "type": "object", + "required": [ + "node", + "module", + "resources" + ], + "properties": { + "node": { + "type": "string", + "description": "the node" + }, + "module": { + "type": "string", + "description": "the module" + }, + "running": { + "type": "string", + "description": "the digest of the composition that runs; absent when nothing runs" + }, + "other": { + "$ref": "#/defs/Revision" + }, + "resources": { + "type": "array", + "description": "each resource with an entry that differs; empty when none differs", + "items": { + "$ref": "#/defs/CompositionDiffResource" + } + } + } + } + }, { "name": "compose", "description": "Compose this machine's modules now and say how it went, so a composition that failed, or a module held, recovers without a restart: what the pass refused or held, each with why, and when the engine composes again on its own after a seat that did not answer (a deadline, no responders: retried with a growing wait; a refusal never). With module, that module's apply is also tried again. Changes only what a pass the engine makes on its own would change.", @@ -595,7 +700,7 @@ }, { "name": "compose-candidate", - "description": "Compose a check's candidate on this machine in shadow, with its own sealed values and the indexer's manifest verdicts, and say per module placed here whether it composes (with its digest and whether the candidate changes it), is refused by the candidate or is held without it, each with why, and the verdict of the machine's own composition (novox/hq ADR 0593 §7). Read-only: applies nothing and writes nothing. Answered by this machine's engine alone, within the asker's bound.", + "description": "Compose a check's candidate on this machine in shadow, with its own sealed values and the indexer's manifest verdicts, and say per module placed here whether it composes (with its digest and whether the candidate changes it), is refused by the candidate or is held without it, each with why, and the verdict of the machine's own composition (novox/hq ADR 0593 \u00a77). Read-only: applies nothing and writes nothing. Answered by this machine's engine alone, within the asker's bound.", "optional": true, "input": { "properties": { @@ -640,7 +745,7 @@ "unready", "not-judged" ], - "description": "the node's verdict (ADR 0593 §7, §9)" + "description": "the node's verdict (ADR 0593 \u00a77, \u00a79)" }, "modules": { "type": "array", @@ -686,7 +791,7 @@ { "name": "manifest-reads", "optional": true, - "description": "The reader capabilities this machine's engine states (manifest words, forms and seat names it reads, novox/hq ADR 0618 §3) and when it last stated them. An engine built before it said them states none, which a reader takes as the baseline. Read-only. Answered by this machine's engine alone.", + "description": "The reader capabilities this machine's engine states (manifest words, forms and seat names it reads, novox/hq ADR 0618 \u00a73) and when it last stated them. An engine built before it said them states none, which a reader takes as the baseline. Read-only. Answered by this machine's engine alone.", "input": { "type": "object", "properties": {} @@ -720,7 +825,7 @@ "node", "stated-at" ], - "description": "what this machine's engine reads of a manifest (ADR 0618 §3)" + "description": "what this machine's engine reads of a manifest (ADR 0618 \u00a73)" }, "class": "read" }, @@ -787,7 +892,7 @@ }, { "name": "seat-list", - "description": "The seat list this engine holds and what it waits for (novox/hq ADR 0636 §7): each seat's source (trunk or bootstrap) and digest, the seats held from the bootstrap alone, the definitions refused, the last error and whether the seat mesh-catalog has answered. Read-only.", + "description": "The seat list this engine holds and what it waits for (novox/hq ADR 0636 \u00a77): each seat's source (trunk or bootstrap) and digest, the seats held from the bootstrap alone, the definitions refused, the last error and whether the seat mesh-catalog has answered. Read-only.", "input": { "type": "object", "properties": {} @@ -798,7 +903,7 @@ }, { "name": "seat-list-again", - "description": "Make this engine read the seat list from the seat mesh-catalog now (novox/hq ADR 0636 §7), the recovery of a list unread; it answers what it holds afterwards, and the error when the read failed. A read that changed nothing still ends the engine's wait.", + "description": "Make this engine read the seat list from the seat mesh-catalog now (novox/hq ADR 0636 \u00a77), the recovery of a list unread; it answers what it holds afterwards, and the error when the read failed. A read that changed nothing still ends the engine's wait.", "input": { "type": "object", "properties": {} @@ -1057,6 +1162,52 @@ "$ref": "#/defs/ProcessBuilds" }, "class": "read" + }, + { + "name": "converged", + "description": "The node's convergence state, which the node-engine keeps (ADR 0168, ADR 0683; design 33 control-verbs \u00a710): adopted or converged, since when it became so, for an adopted node the kind of the firewall found, and the node's links as the engine classifies them: the tunnel's interface and the outward links (design 47 H-A-325). Read-only: the holder of node-packet-filter asks it at its start and at every allow and disallow, and refuses when it does not answer.", + "class": "read", + "input": { + "type": "object", + "properties": {} + }, + "answer": { + "type": "object", + "required": [ + "state", + "tunnel", + "outward" + ], + "properties": { + "state": { + "type": "string", + "enum": [ + "adopted", + "converged" + ], + "description": "the node's mode: adopted (the found firewall's terms) or converged (the mesh's own filter)" + }, + "since": { + "type": "string", + "description": "when the node became so, RFC 3339; absent where the engine recorded no time for it" + }, + "found": { + "type": "string", + "description": "the kind of the firewall found, for an adopted node" + }, + "tunnel": { + "type": "string", + "description": "the private network's tunnel interface on this node, as the engine's filter render names it" + }, + "outward": { + "type": "array", + "items": { + "type": "string" + }, + "description": "the links of this node that face outside, as the engine classifies them for its filter render (ADR 0140); an allowance with no interface never opens one" + } + } + } } ], "events": [ @@ -1205,6 +1356,14 @@ "type": "boolean", "description": "the move changes the build alone" }, + "run": { + "type": "string", + "description": "the composition run's pipeline sequence, in decimal, the module was applied from (design 47 H2 §813 (1), H-A-388): a restarted engine applies no older run again" + }, + "pipeline": { + "type": "string", + "description": "the pipeline of the want the module was applied from, its key ., beside its run (H-A-429 (1), H-A-458 (1)): a want's run is compared with this run only when the want's branch is the same, the key without its last .; absent for a want in none" + }, "kept": { "type": "array", "items": { @@ -1696,7 +1855,7 @@ "description": "one finding of the observer that wrote it" } ], - "description": "the findings this node's engine raises about itself, one key per finding, in the layout of the seat mesh-conditions (its findings state): a key written raises the condition of the same key, a key deleted clears it (novox/hq ADR 0564 §2, ADR 0636 §1); today `machine..seat-list-unread`" + "description": "the findings this node's engine raises about itself, one key per finding, in the layout of the seat mesh-conditions (its findings state): a key written raises the condition of the same key, a key deleted clears it (novox/hq ADR 0564 \u00a72, ADR 0636 \u00a71); today `machine..seat-list-unread`" }, { "name": "bounds", @@ -1708,7 +1867,7 @@ "key": ".", "value": { "type": "object", - "description": "one bound a wait of the node-engine runs against, kept with its own time to live: its expiry is heard as the bus's delete marker and ends the wait (design 47 G §560, A-77)", + "description": "one bound a wait of the node-engine runs against, kept with its own time to live: its expiry is heard as the bus's delete marker and ends the wait (design 47 G \u00a7560, A-77)", "properties": { "v": { "type": "integer", @@ -1815,7 +1974,7 @@ }, "auto": { "type": "boolean", - "description": "the engine asked it itself, on finding a library's root holding data outside its group (ADR 0563 §4)" + "description": "the engine asked it itself, on finding a library's root holding data outside its group (ADR 0563 \u00a74)" } }, "required": [ @@ -1825,7 +1984,7 @@ "at", "state" ], - "description": "one act asked through the seat, as acts reads it (ADR 0563 §9)" + "description": "one act asked through the seat, as acts reads it (ADR 0563 \u00a79)" }, "ActSaid": { "type": "object", @@ -1870,7 +2029,7 @@ "state", "node" ], - "description": "what the seat says of an act: the act, the path or access, the caller as the runtime proved it and the ask (ADR 0563 §1)" + "description": "what the seat says of an act: the act, the path or access, the caller as the runtime proved it and the ask (ADR 0563 \u00a71)" }, "Acts": { "type": "object", @@ -1961,7 +2120,7 @@ "at", "why" ], - "description": "what a node-engine says when what it reads changed or its node returned (ADR 0618 §4)" + "description": "what a node-engine says when what it reads changed or its node returned (ADR 0618 \u00a74)" }, "Module": { "type": "object", @@ -2061,7 +2220,7 @@ "why", "judged" ], - "description": "the engine's root-free verdict (ADR 0563 §6)" + "description": "the engine's root-free verdict (ADR 0563 \u00a76)" }, "Said": { "type": "object", @@ -2106,7 +2265,7 @@ }, "SeatList": { "type": "object", - "description": "what an engine holds of the seat list (novox/hq ADR 0636 §7)", + "description": "what an engine holds of the seat list (novox/hq ADR 0636 \u00a77)", "properties": { "status": { "type": "object", @@ -2451,6 +2610,10 @@ "type": "string" } }, + "declaration-reads": { + "type": "integer", + "description": "the declaration Version its engine reads (design 47 H2 §813 (12), H-A-416 (2), H-A-419): the composer writes this node's declaration at the lower of its own Version and this one, 1 when none is stated" + }, "runner-capabilities": { "type": "array", "items": { @@ -3160,7 +3323,7 @@ }, "Running": { "type": "object", - "description": "a module's build running on a node, and its health: what the node-engines of other nodes and the grants part decide a seat's pass from (ADR 0660 §2), and the assigner reads a provider's and a canary's health and a put's end from (design 47 G 520); said again on the same subject each time healthy changes and for every module the have holds at the engine's start, kept last per subject with no age limit", + "description": "a module's build running on a node, and its health: what the node-engines of other nodes and the grants part decide a seat's pass from (ADR 0660 \u00a72), and the assigner reads a provider's and a canary's health and a put's end from (design 47 G 520); said again on the same subject each time healthy changes and for every module the have holds at the engine's start, kept last per subject with no age limit", "properties": { "v": { "type": "integer", @@ -3331,6 +3494,52 @@ "step", "since" ] + }, + "CompositionDiffEntry": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "the entry: an environment entry by its name, else the part of the resource that differs" + }, + "running": { + "type": "string", + "description": "its value in the running composition; absent where the running one lacks it or the value is a secret" + }, + "other": { + "type": "string", + "description": "its value in the other composition; absent where the other lacks it or the value is a secret" + }, + "masked": { + "type": "boolean", + "description": "the value is a secret, so neither value is given" + } + }, + "description": "one entry that differs between the running composition and the other" + }, + "CompositionDiffResource": { + "type": "object", + "required": [ + "resource", + "entries" + ], + "properties": { + "resource": { + "type": "string", + "description": "the resource, as the composition names it" + }, + "entries": { + "type": "array", + "items": { + "$ref": "#/defs/CompositionDiffEntry" + }, + "description": "the entries of the resource that differ" + } + }, + "description": "one resource with the entries of it that differ" } } } diff --git a/go/seatlist/seatlisttest/seats/node-files.json b/go/seatlist/seatlisttest/seats/node-files.json index 6a43cf4e..12e62958 100644 --- a/go/seatlist/seatlisttest/seats/node-files.json +++ b/go/seatlist/seatlisttest/seats/node-files.json @@ -59,7 +59,7 @@ }, { "name": "list", - "description": "Every entry of one directory with its type, owner, group, mode, size and, for a link, its target. At most 2000 entries; a longer directory is cut and says so. A place a secret is kept is refused by name. Read as the node's operator account, through sudo -n where its rights do not reach; a place a secret is kept, or a known credential file, is refused by name.", + "description": "Every entry of one directory with its type, owner, group, mode, size and, for a link, its target. At most 2000 entries; a longer directory is cut and says so. A place a secret is kept, at any depth beneath it, is listed too and marked secret-place: its entries, never their content; what reaches a process's memory or files (/proc//environ, mem, fd, map_files, root, cwd) is refused by name. Read as the node's operator account, through sudo -n where its rights do not reach.", "replaces": [ "ls", "ls -la" diff --git a/go/seatlist/seatlisttest/seats/node-packet-filter.json b/go/seatlist/seatlisttest/seats/node-packet-filter.json index 005b50ce..10bc82b5 100644 --- a/go/seatlist/seatlisttest/seats/node-packet-filter.json +++ b/go/seatlist/seatlisttest/seats/node-packet-filter.json @@ -2,13 +2,14 @@ "format": 1, "seat": "node-packet-filter", "scope": "node", - "grain": "seat-list", + "grain": "whole", "domain": "Connectivity", - "decision": "novox/hq ADR 0121", + "decision": "novox/hq ADR 0121, ADR 0170, ADR 0683", + "description": "The node seat of the machine's packet filter (ADR 0121, ADR 0170, ADR 0683): what the machine enforces, the mesh's own filter loaded again, one rule set the mesh did not write removed, and the operator's allowances, a port opened by hand beside the ports derived from the modules' listens (design 33 control-verbs). An allowance is the holder's own state, kept one bucket per machine under its node's keys; the node-engine asks allowed at every compose of a converged node's filter and renders it into the inet mesh table's input chain (design 08 control-verbs). Each verb is answered on this node's subject; the event allowances-changed is published on mesh.seat.node-packet-filter.event.allowances-changed..", "verbs": [ { "name": "rules", - "description": "The packet filter as this machine enforces it now: the nftables ruleset and, where the tool exists, the legacy filter's listings. Narrowed to one table or chain when asked.", + "description": "The packet filter as this machine enforces it now: the nftables ruleset and, where the tool exists, the legacy filter's listings. Narrowed to one table or chain when asked. Beside it every allowance, marked loaded only when a rule of the mesh's table carries its comment allowance with an equal at, and every loaded allowance comment whose key is not kept, listed as loaded, not kept.", "replaces": [ "nft list ruleset", "iptables -L", @@ -26,6 +27,42 @@ } }, "type": "object" + }, + "answer": { + "type": "object", + "required": [ + "nftables", + "legacy", + "allowances", + "loaded-not-kept" + ], + "properties": { + "nftables": { + "type": "string", + "description": "the nftables ruleset, whole or narrowed" + }, + "legacy": { + "type": "object", + "additionalProperties": { + "type": "string" + }, + "description": "each legacy tool's listing, by tool" + }, + "allowances": { + "type": "array", + "items": { + "$ref": "#/defs/Loaded" + }, + "description": "allowed's answer, each marked loaded only on a rule comment allowance with an equal at" + }, + "loaded-not-kept": { + "type": "array", + "items": { + "$ref": "#/defs/LoadedNotKept" + }, + "description": "every allowance comment of the mesh's table whose key is not kept: loaded, not kept" + } + } } }, { @@ -37,6 +74,23 @@ "input": { "properties": {}, "type": "object" + }, + "answer": { + "type": "object", + "required": [ + "loaded", + "table" + ], + "properties": { + "loaded": { + "type": "string", + "description": "the filter file loaded" + }, + "table": { + "type": "string", + "description": "the mesh's table as loaded" + } + } } }, { @@ -57,7 +111,501 @@ "where" ], "type": "object" + }, + "answer": { + "type": "object", + "required": [ + "where", + "did" + ], + "properties": { + "where": { + "type": "string" + }, + "did": { + "type": "array", + "items": { + "type": "string" + }, + "description": "each command run" + } + } + } + }, + { + "name": "allow", + "description": "Open a port by hand on this machine, beside the ports derived from the modules' listens: one allowance per protocol, port and interface, kept as the holder's state and rendered by the node-engine into the inet mesh table's input chain at its next compose of the filter. Who set it is the caller the bus proved (for a session's call, the tool runner's account). A key that exists is replaced whole and answered. A kept allowance whose named interface is an outward link, as the node-engine's converged names them, is answered with outward, that link. Written in the hand-act log. Refuses, naming the argument and writing nothing, an unproven caller, an adopted node (naming the adoption) or a node-engine converged that does not answer within 10 s, a bad port, protocol, interface, source or until, an empty or longer why, an unknown argument, a port a derived rule of the loaded mesh table admits from the private network only unless a source is given, and more allowances on the node than the holder's setting allowances-max (default 100). Whether the loaded chain carries it is read with rules.", + "replaces": [ + "nft add rule inet mesh input" + ], + "class": "act", + "input": { + "type": "object", + "additionalProperties": false, + "required": [ + "port", + "why" + ], + "properties": { + "port": { + "type": "integer", + "minimum": 1, + "maximum": 65535, + "description": "the port, 1-65535" + }, + "protocol": { + "type": "string", + "enum": [ + "tcp", + "udp" + ], + "description": "tcp (the default) or udp" + }, + "interfaces": { + "type": "array", + "items": { + "type": "string", + "pattern": "^[A-Za-z0-9_.-]{1,15}$" + }, + "description": "the input interfaces it is opened on, each 1-15 letters, digits, _ . - (optional; none: every input interface that is neither the private network's tunnel nor an outward link, so an outward link is opened only when named)" + }, + "source": { + "type": "string", + "description": "one IPv4 or IPv6 CIDR it is opened to (optional; none: any)" + }, + "until": { + "type": "string", + "description": "when it expires, an RFC 3339 time in the future (optional)" + }, + "why": { + "type": "string", + "minLength": 1, + "maxLength": 500, + "description": "why, in words the operator reads (1-500 characters)" + } + } + }, + "answer": { + "type": "object", + "required": [ + "allowances", + "said" + ], + "properties": { + "allowances": { + "type": "array", + "items": { + "$ref": "#/defs/Kept" + } + }, + "said": { + "type": "boolean", + "description": "allowances-changed was said with change allowed" + } + } + }, + "by-caller": true + }, + { + "name": "disallow", + "description": "Take back allowances: the keys the protocol, port and interfaces name (none: the default set's key) are deleted, or, when one does not exist, it is refused by name and none is deleted. The source is no part of an allowance's identity and is refused as an argument. Written in the hand-act log. Refuses as allow does an unproven caller, an adopted node and a converged that does not answer.", + "replaces": [ + "nft delete rule inet mesh input" + ], + "class": "act", + "input": { + "type": "object", + "additionalProperties": false, + "required": [ + "port", + "why" + ], + "properties": { + "port": { + "type": "integer", + "minimum": 1, + "maximum": 65535, + "description": "the port, 1-65535" + }, + "protocol": { + "type": "string", + "enum": [ + "tcp", + "udp" + ], + "description": "tcp (the default) or udp" + }, + "interfaces": { + "type": "array", + "items": { + "type": "string", + "pattern": "^[A-Za-z0-9_.-]{1,15}$" + }, + "description": "the input interfaces it is opened on, each 1-15 letters, digits, _ . - (optional; none: every input interface that is neither the private network's tunnel nor an outward link, so an outward link is opened only when named)" + }, + "why": { + "type": "string", + "minLength": 1, + "maxLength": 500, + "description": "why, in words the operator reads (1-500 characters)" + } + } + }, + "answer": { + "type": "object", + "required": [ + "allowances", + "said" + ], + "properties": { + "allowances": { + "type": "array", + "items": { + "$ref": "#/defs/Allowance" + }, + "description": "each allowance as removed" + }, + "said": { + "type": "boolean", + "description": "allowances-changed was said with change disallowed" + } + } + }, + "by-caller": true + }, + { + "name": "allowed", + "description": "Every allowance kept for this machine only, with its key, port, protocol, interface, source, until, why, by and at; none on an adopted node. An allowance is removed at its until by the holder, so none answered has expired. The node-engine asks it at every compose of the filter, bound 10 s.", + "class": "read", + "input": { + "type": "object", + "additionalProperties": false, + "properties": {} + }, + "answer": { + "type": "object", + "required": [ + "allowances" + ], + "properties": { + "allowances": { + "type": "array", + "items": { + "$ref": "#/defs/Allowance" + } + } + } + }, + "timeout": 10 + } + ], + "events": [ + { + "name": "allowances-changed", + "subject": "allowances-changed.", + "body": { + "$ref": "#/defs/AllowancesChanged" } } - ] + ], + "family": { + "name": "node-packet-filter.events", + "max-age-days": 7 + }, + "states": [ + { + "name": "allowances", + "template": "state", + "description": "the operator's allowances, one bucket per machine whose keys carry the node: each holder reads and writes its own node's keys only (design 33 control-verbs §9; design 47 H-A-315 (1), H-A-317)", + "key-forms": [ + { + "key": ".allow..", + "value": { + "$ref": "#/defs/AllowanceValue" + }, + "description": "the default set: every input interface that is neither the private network's tunnel nor an outward link" + }, + { + "key": ".allow...", + "value": { + "$ref": "#/defs/AllowanceValue" + }, + "description": "one named input interface: every token after the fourth, since a name may hold a dot" + } + ], + "history": 1 + } + ], + "defs": { + "AllowanceValue": { + "type": "object", + "description": "one allowance as the holder keeps it", + "required": [ + "port", + "protocol", + "why", + "by", + "at" + ], + "properties": { + "port": { + "type": "integer" + }, + "protocol": { + "type": "string", + "enum": [ + "tcp", + "udp" + ] + }, + "interface": { + "type": "string", + "description": "the one input interface it is opened on; absent: every input interface that is neither the private network's tunnel nor an outward link" + }, + "source": { + "type": "string", + "description": "the IPv4 or IPv6 CIDR it is opened to; absent: any" + }, + "until": { + "type": "string", + "description": "when it expires, RFC 3339; absent: never" + }, + "why": { + "type": "string" + }, + "by": { + "type": "string", + "description": "the caller as the bus named it on the request" + }, + "at": { + "type": "string", + "description": "when it was written, RFC 3339, the holder's clock" + } + } + }, + "Allowance": { + "type": "object", + "description": "one allowance: a port opened by hand on this machine, with its key", + "required": [ + "key", + "port", + "protocol", + "why", + "by", + "at" + ], + "properties": { + "key": { + "type": "string", + "description": "its key, allow..[.]: its key in the holder's state without the node" + }, + "port": { + "type": "integer" + }, + "protocol": { + "type": "string", + "enum": [ + "tcp", + "udp" + ] + }, + "interface": { + "type": "string", + "description": "the one input interface it is opened on; absent: every input interface that is neither the private network's tunnel nor an outward link" + }, + "source": { + "type": "string", + "description": "the IPv4 or IPv6 CIDR it is opened to; absent: any" + }, + "until": { + "type": "string", + "description": "when it expires, RFC 3339; absent: never" + }, + "why": { + "type": "string" + }, + "by": { + "type": "string", + "description": "the caller as the bus named it on the request" + }, + "at": { + "type": "string", + "description": "when it was written, RFC 3339, the holder's clock" + } + } + }, + "Kept": { + "type": "object", + "description": "one allowance as allow kept it, with what it replaced", + "required": [ + "key", + "port", + "protocol", + "why", + "by", + "at" + ], + "properties": { + "key": { + "type": "string", + "description": "its key, allow..[.]: its key in the holder's state without the node" + }, + "port": { + "type": "integer" + }, + "protocol": { + "type": "string", + "enum": [ + "tcp", + "udp" + ] + }, + "interface": { + "type": "string", + "description": "the one input interface it is opened on; absent: every input interface that is neither the private network's tunnel nor an outward link" + }, + "source": { + "type": "string", + "description": "the IPv4 or IPv6 CIDR it is opened to; absent: any" + }, + "until": { + "type": "string", + "description": "when it expires, RFC 3339; absent: never" + }, + "why": { + "type": "string" + }, + "by": { + "type": "string", + "description": "the caller as the bus named it on the request" + }, + "at": { + "type": "string", + "description": "when it was written, RFC 3339, the holder's clock" + }, + "outward": { + "type": "string", + "description": "the outward link this allowance opens, when its named interface is one of the outward links the node-engine's converged answered for this call; absent: it opens none" + }, + "replaced": { + "$ref": "#/defs/Allowance" + } + } + }, + "Loaded": { + "type": "object", + "description": "one allowance, marked loaded only when a rule of the mesh's table carries the comment allowance with an equal at", + "required": [ + "key", + "port", + "protocol", + "why", + "by", + "at", + "loaded" + ], + "properties": { + "key": { + "type": "string", + "description": "its key, allow..[.]: its key in the holder's state without the node" + }, + "port": { + "type": "integer" + }, + "protocol": { + "type": "string", + "enum": [ + "tcp", + "udp" + ] + }, + "interface": { + "type": "string", + "description": "the one input interface it is opened on; absent: every input interface that is neither the private network's tunnel nor an outward link" + }, + "source": { + "type": "string", + "description": "the IPv4 or IPv6 CIDR it is opened to; absent: any" + }, + "until": { + "type": "string", + "description": "when it expires, RFC 3339; absent: never" + }, + "why": { + "type": "string" + }, + "by": { + "type": "string", + "description": "the caller as the bus named it on the request" + }, + "at": { + "type": "string", + "description": "when it was written, RFC 3339, the holder's clock" + }, + "loaded": { + "type": "boolean" + } + } + }, + "AllowancesChanged": { + "type": "object", + "required": [ + "node", + "change", + "allowances", + "at" + ], + "properties": { + "node": { + "type": "string" + }, + "change": { + "type": "string", + "enum": [ + "allowed", + "disallowed", + "expired", + "started" + ] + }, + "allowances": { + "type": "array", + "items": { + "$ref": "#/defs/Allowance" + }, + "description": "the allowances concerned; all of them for started" + }, + "by": { + "type": "string", + "description": "who, for allowed and disallowed" + }, + "why": { + "type": "string", + "description": "why, for allowed and disallowed" + }, + "at": { + "type": "string", + "description": "when it was said, RFC 3339" + } + } + }, + "LoadedNotKept": { + "type": "object", + "description": "one allowance comment allowance of the mesh's table whose key is not kept", + "required": [ + "key", + "at" + ], + "properties": { + "key": { + "type": "string", + "description": "the key the comment names, allow..[.]" + }, + "at": { + "type": "string", + "description": "the at the comment names" + } + } + } + } } diff --git a/go/seatlist/seatlisttest/seats/node-service-manager.json b/go/seatlist/seatlisttest/seats/node-service-manager.json index c46eeff3..d84ef496 100644 --- a/go/seatlist/seatlisttest/seats/node-service-manager.json +++ b/go/seatlist/seatlisttest/seats/node-service-manager.json @@ -208,6 +208,44 @@ "type": "object" } }, + { + "name": "kernel-log", + "description": "The kernel's messages (what journalctl -k gives) of this boot or an earlier one, at most 2000 lines, in a time window and narrowed to a priority and to lines holding a text when asked. A secret a line held is shown as [redacted: \u003cwhat it was\u003e].", + "replaces": [ + "journalctl -k", + "journalctl -k -b -1", + "dmesg" + ], + "input": { + "properties": { + "boot": { + "description": "which boot: 0 this boot (the default), -1 the one before, -2 the one before that, and so on (optional)", + "type": "string" + }, + "lines": { + "description": "how many lines from the end of what matches (default 100, at most 2000)", + "type": "string" + }, + "match": { + "description": "only the lines holding this text, as written — a fixed string, not a pattern (optional)", + "type": "string" + }, + "priority": { + "description": "only entries this severe or more: 0-7 or emerg, alert, crit, err, warning, notice, info, debug (optional)", + "type": "string" + }, + "since": { + "description": "the window's start: an RFC 3339 time (2026-10-07T09:30:00Z) or relative to now (-30min, -2h, yesterday) (optional)", + "type": "string" + }, + "until": { + "description": "the window's end, in the same forms (optional; now when absent)", + "type": "string" + } + }, + "type": "object" + } + }, { "name": "failed", "description": "Every failed unit on this machine, in the system manager and in the operator account's; a manager that does not answer is reported with its error, never as nothing failed.", diff --git a/go/seatlist/seatlisttest/seats/oci-artifact-registry.json b/go/seatlist/seatlisttest/seats/oci-artifact-registry.json index a4fda915..4dfaf454 100644 --- a/go/seatlist/seatlisttest/seats/oci-artifact-registry.json +++ b/go/seatlist/seatlisttest/seats/oci-artifact-registry.json @@ -12,7 +12,7 @@ "verbs": [ { "name": "hold", - "description": "Keep one untagged bundle, archive or copied base from removal by retention, as the artifact sweep's rules keep it (ADR 0617 \u00a72); never an image of the holder's own. Emits held.", + "description": "Keep one untagged bundle, archive or copied base from removal by retention, as the artifact sweep's rules keep it (ADR 0617 §2); never an image of the holder's own. Emits held.", "input": { "type": "object", "properties": { @@ -94,13 +94,21 @@ }, { "name": "list", - "description": "What the store holds of one repository, or of every repository the seat holds when none is named: every manifest keeping one bundle or archive, each with its digest, whether it is an index (and the manifests an index names), the tags naming it, its size and whether it is held. Read from the store itself, so a manifest no tag names is listed too; it reads only (ADR 0632 \u00a73).", + "description": "What the store holds of one repository, or of every repository the seat holds when none is named: every manifest keeping one bundle or archive, each with its digest, whether it is an index (and the manifests an index names), the tags naming it, its size and whether it is held. Read from the store itself, so a manifest no tag names is listed too; it reads only (ADR 0632 §3). Filtered by name or module while it reads and paged by the cursor next, at most 768 KB a page, capped saying the rest is read from next (design 47 H2 §779, §811, §813 (7)).", "input": { "type": "object", "properties": { "name": { "type": "string", "description": "the repository, /; every repository of the seat when not given" + }, + "module": { + "type": "string", + "description": "only the repositories of this module, /*, applied while reading (design 47 H2 §813 (7))" + }, + "next": { + "type": "string", + "description": "the cursor the previous page answered, opaque; absent for the first page (design 47 H2 §779)" } } }, @@ -113,6 +121,18 @@ "$ref": "#/defs/Repository" }, "description": "each repository named, in order of its name" + }, + "capped": { + "type": "boolean", + "description": "yes when the answer stopped at 768 KB; the rest is read from next (design 47 H2 §811)" + }, + "total": { + "type": "integer", + "description": "how many repositories the filters hold in all (design 47 H2 §811)" + }, + "next": { + "type": "string", + "description": "the cursor of the next page, opaque; empty at the end (design 47 H2 §779)" } }, "required": [ @@ -376,5 +396,6 @@ } ] } - ] + ], + "form": 2 } diff --git a/go/settings.go b/go/settings.go new file mode 100644 index 00000000..fad54cc3 --- /dev/null +++ b/go/settings.go @@ -0,0 +1,33 @@ +package stdio + +import ( + "fmt" + "os" + "strings" + "time" +) + +// SettingWord is the environment word the node-engine composes a module's config value key into, in a process's +// environment and in the module's slice of the tool runner's MESH_TOOL_ENV alike (design 47 H2 §781; H-A-370): +// MESH_CONFIG_, the key upper-cased with '-' and '.' written '_', holding the value as text. +func SettingWord(key string) string { + return "MESH_CONFIG_" + strings.ToUpper(strings.NewReplacer("-", "_", ".", "_").Replace(key)) +} + +// Setting is the module's config value key as the node-engine composed it, and whether it composed one: the one +// settings reader of a process and a bundle (§781). +func Setting(key string) (string, bool) { return os.LookupEnv(SettingWord(key)) } + +// SettingDuration is the config value key read as a Go duration ("10m"): def when none is composed or it is empty, an +// error naming the key for a value that is no duration above zero. +func SettingDuration(key string, def time.Duration) (time.Duration, error) { + v, ok := Setting(key) + if v = strings.TrimSpace(v); !ok || v == "" { + return def, nil + } + d, err := time.ParseDuration(v) + if err != nil || d <= 0 { + return 0, fmt.Errorf("the setting %s (%s) holds %q, which is not a duration above zero", key, SettingWord(key), v) + } + return d, nil +} diff --git a/go/setupstarved.go b/go/setupstarved.go new file mode 100644 index 00000000..1aa7702e --- /dev/null +++ b/go/setupstarved.go @@ -0,0 +1,6 @@ +package stdio + +// SetupStarved opens the one line a test prints when its setup wait on the bus passed the setup bound (design 47 H2 +// §800, H-A-368): a build module reads it in the test output, never an assertion's words, and runs the level again. It +// lives in the root package so the delivery SDK reads it without importing the test helpers (H-A-399). +const SetupStarved = "mesh: setup starved:" diff --git a/go/state.go b/go/state.go index 43b7c30d..92883f83 100644 --- a/go/state.go +++ b/go/state.go @@ -22,6 +22,13 @@ type StateEntry struct { Value json.RawMessage `json:"value"` Revision uint64 `json:"revision"` At time.Time `json:"at,omitzero"` + // SeenAgain is set in a page (Page) on a key written again since an earlier page of the same read handed it over, + // so a reader knows this value replaces the one it holds (design 47 H2 §779). Get never sets it. + SeenAgain bool `json:"seen-again,omitempty"` + // Deleted is set in a page (Page) on a key's delete marker, at its sequence, with no value: a reader following pages + // removes the key, so a deleted key never comes back alive from a bucket with history (design 47 H2 §779, H-A-451 + // (1)). Get never sets it. + Deleted bool `json:"deleted,omitempty"` } // The operations a watch hands over (StateChange.Op). @@ -107,18 +114,35 @@ func (s KeptState) Get(key string) (*StateEntry, error) { return &e, json.Unmarshal(raw, &e) } -// StatePage is the values of one bulk read: those after a key, in key order, and the key to ask after for the rest. +// StatePage is the values of one page of a read: those the bus wrote after the cursor's sequence, in sequence order, +// and the cursor to hand back for the rest (design 47 H2 §779). type StatePage struct { Entries []StateEntry `json:"entries"` - // Next is the last key of Entries when more follow it, else empty. + // Next is the opaque cursor that continues the read, empty at its end. Next string `json:"next,omitempty"` } -// Page is the current values in key order after the key `after`, as many as fit maxBytes (keys and values) or the time budget allows, at least -// one: one request for many values, where Get is one request each. A runtime that does not answer it gives an error -// satisfying errors.Is(err, errors.ErrUnsupported), and the reader asks Get per key instead. -func (s KeptState) Page(after string, maxBytes int, budget time.Duration) (StatePage, error) { - raw, err := s.ask("page", map[string]any{"after": after, "max_bytes": maxBytes, "budget_ms": budget.Milliseconds()}) +// Page is one page of the state's values, in the order the bus wrote them, after the opaque cursor (empty: from the +// start), as many as fit maxBytes (keys and values; the runtime holds a page to 768 KB at most) or the time budget +// allows, at least one: runtime verb `mesh/state.page`, read from an ordered consumer after the cursor's sequence, so a +// state is read once however many pages it takes (design 47 H2 §779). A key written again between two pages comes +// again in a later page with SeenAgain set, and a key deleted comes as an entry with Deleted set and no value. A keep (KeepPrefix) is not an entry and is left out. Hand Next back for the next page. A runtime that does not answer it gives an +// error satisfying errors.Is(err, errors.ErrUnsupported), and the reader asks Get per key instead. +func (s KeptState) Page(cursor string, maxBytes int, budget time.Duration) (StatePage, error) { + p, err := s.page(cursor, maxBytes, budget) + kept := p.Entries[:0] + for _, e := range p.Entries { + if !IsKeep(e.Key) { + kept = append(kept, e) + } + } + p.Entries = kept + return p, err +} + +// page is one page as the runtime answers it, keeps included (KeepPrefix). +func (s KeptState) page(cursor string, maxBytes int, budget time.Duration) (StatePage, error) { + raw, err := s.ask("page", map[string]any{"cursor": cursor, "max_bytes": maxBytes, "budget_ms": budget.Milliseconds()}) if err != nil { return StatePage{}, err } @@ -257,14 +281,23 @@ func (s KeptState) Increment(key string, attempts int) (uint64, error) { return 0, fmt.Errorf("%s was written by another writer first %d times: %w", key, attempts, ErrChanged) } -// Keys is every key with a current value, sorted. +// Keys is every key with a current value, sorted; a keep (KeepPrefix) is not an entry and is left out. func (s KeptState) Keys() ([]string, error) { raw, err := s.ask("keys", map[string]any{}) if err != nil { return nil, err } - var keys []string - return keys, json.Unmarshal(raw, &keys) + var all []string + if err := json.Unmarshal(raw, &all); err != nil { + return nil, err + } + keys := all[:0] + for _, k := range all { + if !IsKeep(k) { + keys = append(keys, k) + } + } + return keys, nil } // Watch hands handle every current value — none that is deleted — and then every change, in order, @@ -279,7 +312,14 @@ func (s KeptState) Watch(key string, handle func(StateChange) error) error { return err } id := c.newID() - f := newFollower("the watch "+id+" of the state "+s.name, handle, func(ch StateChange) string { return ch.Op + " " + ch.Key }) + // A keep (KeepPrefix) is not an entry of the state: no watcher hears it. + entries := func(ch StateChange) error { + if IsKeep(ch.Key) { + return nil + } + return handle(ch) + } + f := newFollower("the watch "+id+" of the state "+s.name, entries, func(ch StateChange) string { return ch.Op + " " + ch.Key }) c.mu.Lock() c.watches[id] = f c.mu.Unlock() diff --git a/go/state_history_test.go b/go/state_history_test.go index d490531d..a14c2919 100644 --- a/go/state_history_test.go +++ b/go/state_history_test.go @@ -38,7 +38,7 @@ func historying(t *testing.T, answer func(message) map[string]any, run func()) [ if json.Unmarshal(lines.Bytes(), &m) != nil { continue } - if strings.HasPrefix(m.Method, "mesh/state.") { + if strings.HasPrefix(m.Method, "mesh/state.") || strings.HasPrefix(m.Method, "mesh/queue.") { amu.Lock() asked = append(asked, m) amu.Unlock() @@ -143,13 +143,23 @@ func TestHistoryIsEveryKeptValueOldestFirst(t *testing.T) { } // Info asks mesh/state.info with the state alone and hands back the bucket's fill against its caps, as the tool -// runner answers it (design 47 G §623); a refusal is the runtime's error. +// runner answers it (design 47 G §623); a refusal is the runtime's error. QueueWaiting asks mesh/queue.waiting with the +// seat alone and hands back its waiting asks and whether they were cut (design 47 H2 §777, H-A-410). func TestInfoIsTheBucketsFillAgainstItsCaps(t *testing.T) { var got *StateInfo var refused error + var waiting Waiting + var waitingRefused error asked := historying(t, func(m message) map[string]any { var p map[string]any _ = json.Unmarshal(m.Params, &p) + if m.Method == "mesh/queue.waiting" { + if p["seat"] != "mesh-wants" || len(p) != 1 { + return map[string]any{"error": map[string]any{"code": -32000, "message": "issued no queue of that seat"}} + } + return map[string]any{"result": map[string]any{"asks": []map[string]any{{"id": "main.9", "module": "mesh-delivery-pipeline", + "age_seconds": 42}, {"id": "main.10", "module": "mesh-delivery-pipeline", "age_seconds": 1.5, "again": true}}, "capped": true}} + } if p["state"] != "issues" { return map[string]any{"error": map[string]any{"code": -32000, "message": "issues was issued no state"}} } @@ -164,8 +174,19 @@ func TestInfoIsTheBucketsFillAgainstItsCaps(t *testing.T) { if i, refused = State("elsewhere.issues").Info(); i != nil { t.Errorf("a refused info handed back %v", i) } + if waiting, err = QueueWaiting("mesh-wants"); err != nil { + t.Errorf("waiting: %v", err) + } + _, waitingRefused = QueueWaiting("node-build-go") }) - if len(asked) != 2 || asked[0].Method != "mesh/state.info" { + if w := waiting; !w.Capped || len(w.Asks) != 2 || w.Asks[0] != (WaitingAsk{ID: "main.9", Module: "mesh-delivery-pipeline", AgeSeconds: 42}) || + w.Asks[1] != (WaitingAsk{ID: "main.10", Module: "mesh-delivery-pipeline", AgeSeconds: 1.5, Again: true}) { + t.Errorf("the waiting asks are %+v", w) + } + if waitingRefused == nil || !strings.Contains(waitingRefused.Error(), "issued no queue") { + t.Errorf("a refused waiting read answered %v", waitingRefused) + } + if len(asked) != 4 || asked[0].Method != "mesh/state.info" || asked[2].Method != "mesh/queue.waiting" { t.Fatalf("asked %+v", asked) } if got == nil || got.Bytes != 300 || got.MaxBytes != 1000 || got.Keys != 4 || got.Bucket != "issues_issues" || !got.KeyTTL { diff --git a/go/state_test.go b/go/state_test.go index d5f8455b..85a80848 100644 --- a/go/state_test.go +++ b/go/state_test.go @@ -6,7 +6,9 @@ import ( "errors" "fmt" "io" + "sort" "strings" + "sync" "testing" "time" ) @@ -56,6 +58,12 @@ func TestABundleKeepsAndWatchesStateThroughTheRuntime(t *testing.T) { } return State("servers").PutTTL(a["key"].(string), 1, 90*time.Second) }}, + {Name: "take", Run: func(a map[string]any) (any, error) { + return State("servers").TakeOnce(a["seat"].(string), a["ask"].(string), 10*time.Minute) + }}, + {Name: "page", Run: func(a map[string]any) (any, error) { + return State("servers").Page(a["cursor"].(string), 1000, time.Second) + }}, }, inR, outW) }() lines := bufio.NewScanner(outR) @@ -219,6 +227,48 @@ func TestABundleKeepsAndWatchesStateThroughTheRuntime(t *testing.T) { t.Fatalf("the put answered %v", r) } } + // Design 47 H2 §775, H-A-359: a bundle takes an ask once: the create of . with the ask's bound as its time + // to live answers true, the runtime's refusal as changed answers false, and an ask with no id is refused unasked. + for i, exists := range []bool{false, true} { + send(map[string]any{"jsonrpc": "2.0", "id": 40 + i, "method": "tools/call", "params": map[string]any{"name": "take", + "arguments": map[string]any{"seat": "git", "ask": "a-7"}}}) + asked := next() + p := asked["params"].(map[string]any) + if asked["method"] != "mesh/state.create" || p["key"] != "git.a-7" || p["ttl_seconds"] != float64(600) { + t.Fatalf("asked %v", asked) + } + if exists { + send(map[string]any{"jsonrpc": "2.0", "id": asked["id"], "error": map[string]any{"code": CodeChanged, "message": "exists"}}) + } else { + send(map[string]any{"jsonrpc": "2.0", "id": asked["id"], "result": map[string]any{"revision": 5}}) + } + r := next() + if text := fmt.Sprint(r["result"]); r["id"] != float64(40+i) || !strings.Contains(text, map[bool]string{false: "true", true: "false"}[exists]) { + t.Fatalf("a take, the key existing %v, answered %v", exists, r) + } + } + send(map[string]any{"jsonrpc": "2.0", "id": 42, "method": "tools/call", "params": map[string]any{"name": "take", + "arguments": map[string]any{"seat": "git", "ask": ""}}}) + if r := next(); r["id"] != float64(42) || !strings.Contains(fmt.Sprint(r), "is not an ask id") { + t.Fatalf("a take with no ask id answered %v", r) + } + // Design 47 H2 §779: a page read hands the runtime the opaque cursor and answers the cursor that continues it, and a + // key written again since an earlier page comes marked seen-again; a kept delete comes as an entry marked deleted with + // no value (H-A-451 (1)). + send(map[string]any{"jsonrpc": "2.0", "id": 43, "method": "tools/call", "params": map[string]any{"name": "page", + "arguments": map[string]any{"cursor": "c.17"}}}) + if asked := next(); asked["method"] != "mesh/state.page" || asked["params"].(map[string]any)["cursor"] != "c.17" || asked["params"].(map[string]any)["after"] != nil { + t.Fatalf("asked %v", asked) + } else { + send(map[string]any{"jsonrpc": "2.0", "id": asked["id"], "result": map[string]any{"next": "c.19", "entries": []any{ + map[string]any{"key": "a", "value": 1, "revision": 18}, map[string]any{"key": "b", "value": 2, "revision": 19, "seen-again": true}, + map[string]any{"key": "c", "revision": 20, "deleted": true}}}}) + } + if r := next(); r["id"] != float64(43) || !strings.Contains(fmt.Sprint(r), `"next":"c.19"`) || !strings.Contains(fmt.Sprint(r), `"revision":19,"seen-again":true`) || + strings.Count(fmt.Sprint(r), "seen-again") != 1 || !strings.Contains(fmt.Sprint(r), `"key":"c","value":null,"revision":20,"deleted":true`) || + strings.Count(fmt.Sprint(r), "deleted") != 1 { + t.Fatalf("the page answered %v", r) + } if !errors.Is(&refused{code: CodeChanged, message: "x"}, ErrChanged) { t.Fatal("a refusal coded -32010 is not ErrChanged") } @@ -307,4 +357,256 @@ func TestABundleKeepsAndWatchesStateThroughTheRuntime(t *testing.T) { if v, _, _ := w.Get("placement", "gone"); string(v) != "again" { t.Fatalf("a change after its own delete is not taken: %s", v) } + // Design 47 H2 §782: one bundle's state calls through its one runtime channel are safe at once: gets, puts, pages, + // watches and bus-state calls from many goroutines, the runtime answering each, under the race detector. + t.Run("one runtime channel serves concurrent state calls", func(t *testing.T) { + inR, inW := io.Pipe() + outR, outW := io.Pipe() + t.Cleanup(func() { _ = inW.Close(); _ = outR.Close() }) + const many = 16 + go func() { + _ = ServeOn("", []Tool{{Name: "many", Run: func(map[string]any) (any, error) { + var wg sync.WaitGroup + errs := make(chan error, many*6) + for i := range many { + wg.Add(1) + go func() { + defer wg.Done() + s, key := State("servers"), fmt.Sprintf("k%d", i) + _, err := s.Put(key, i) + errs <- err + _, err = s.Get(key) + errs <- err + _, err = s.Page("", 1000, time.Second) + errs <- err + errs <- s.Watch(key, func(StateChange) error { return nil }) + _, err = BusState("mesh/objects.info", map[string]any{"objects": "o"}) + errs <- err + _, err = s.Info() + errs <- err + }() + } + wg.Wait() + close(errs) + for err := range errs { + if err != nil { + return nil, err + } + } + return "done", nil + }}}, inR, outW) + }() + var wmu sync.Mutex + send := func(m map[string]any) { + b, _ := json.Marshal(m) + wmu.Lock() + defer wmu.Unlock() + _, _ = inW.Write(append(b, '\n')) + } + send(map[string]any{"jsonrpc": "2.0", "id": 1, "method": "tools/call", "params": map[string]any{"name": "many"}}) + lines := bufio.NewScanner(outR) + answered := make(chan map[string]any, 1) + go func() { + n := 1000 + for lines.Scan() { + var m map[string]any + _ = json.Unmarshal(lines.Bytes(), &m) + method, _ := m["method"].(string) + switch { + case m["id"] == float64(1): + answered <- m + return + case method == "mesh/state.watch": + n++ + id := m["id"] + key := m["params"].(map[string]any)["key"] + // The runtime hands the current value over, then answers the watch, on a goroutine of its own, as + // the tool runner's answers come in any order. + go func(n int) { + send(map[string]any{"jsonrpc": "2.0", "id": n, "method": "mesh/state", "params": map[string]any{"watch": id, + "change": map[string]any{"state": "servers", "key": key, "op": "put", "value": 1, "current": true}}}) + send(map[string]any{"jsonrpc": "2.0", "id": id, "result": map[string]any{}}) + }(n) + case method == "mesh/state.page": + go send(map[string]any{"jsonrpc": "2.0", "id": m["id"], "result": map[string]any{"entries": []any{}}}) + case method == "mesh/state.get": + go send(map[string]any{"jsonrpc": "2.0", "id": m["id"], "result": map[string]any{"key": "k", "value": 1, "revision": 2}}) + case method != "": + go send(map[string]any{"jsonrpc": "2.0", "id": m["id"], "result": map[string]any{"revision": 1}}) + } + } + }() + select { + case m := <-answered: + if m["error"] != nil || fmt.Sprint(m["result"]) != "map[content:[map[text:\"done\" type:text]]]" { + t.Fatalf("the concurrent calls answered %v", m) + } + case <-time.After(10 * time.Second): + t.Fatal("the concurrent calls did not end") + } + }) + // Design 47 H2 §812 (H-A-380, H-A-386): prune-mark and prune-target are read through the settings reader with their + // defaults, and a bundle's Put judges the fill the runtime's put answer carries: past 70 % the oldest keys by revision + // are deleted to 50 %, a key kept under _keep/ survives, and no reader (Keys, Page) counts the keep. + t.Run("a write past its mark prunes to its target and a kept key survives", func(t *testing.T) { + t.Run("settings", func(t *testing.T) { + if p, err := PruneSettings(); err != nil || p != (Prune{Mark: 70, Target: 50}) { + t.Fatalf("the defaults are %+v, %v", p, err) + } + t.Setenv(SettingWord(PruneMarkSetting), "80%") + t.Setenv(SettingWord(PruneTargetSetting), "60") + if p, err := PruneSettings(); err != nil || p != (Prune{Mark: 80, Target: 60}) { + t.Fatalf("the settings read %+v, %v", p, err) + } + t.Setenv(SettingWord(PruneTargetSetting), "90") + if _, err := PruneSettings(); err == nil || !strings.Contains(err.Error(), "prune-target") { + t.Fatalf("a target above the mark was taken: %v", err) + } + t.Setenv(SettingWord(PruneTargetSetting), "half") + if _, err := PruneSettings(); err == nil || !strings.Contains(err.Error(), "MESH_CONFIG_PRUNE_TARGET") { + t.Fatalf("a target that is no percentage was taken: %v", err) + } + }) + + servesAlone(t) + t.Setenv(ServedAs, "bridge") + inR, inW := io.Pipe() + outR, outW := io.Pipe() + pruned := State("performed").Pruned(Prune{Mark: 70, Target: 50}) + go func() { + _ = ServeOn("", []Tool{ + {Name: "write", Run: func(a map[string]any) (any, error) { return pruned.Put(a["key"].(string), a["value"]) }}, + {Name: "keep", Run: func(a map[string]any) (any, error) { + return nil, pruned.Keep(a["key"].(string), time.Now().Add(time.Hour)) + }}, + {Name: "keys", Run: func(map[string]any) (any, error) { return State("performed").Keys() }}, + {Name: "page", Run: func(map[string]any) (any, error) { return State("performed").Page("", 1<<20, time.Second) }}, + }, inR, outW) + }() + // The runtime: a state of a cap of 10000 bytes whose fill is what its current values put. + const limit = 10_000 + var mu sync.Mutex + values, revs, next := map[string]json.RawMessage{}, map[string]uint64{}, uint64(0) + held := func() uint64 { + var n uint64 + for k, v := range values { + n += PutBytes(k, v) + } + return n + } + var mostHeld uint64 + answers := make(chan map[string]any, 1) + send := func(m map[string]any) { b, _ := json.Marshal(m); _, _ = inW.Write(append(b, '\n')) } + go func() { + lines := bufio.NewScanner(outR) + lines.Buffer(make([]byte, 1<<20), 1<<22) + for lines.Scan() { + var m struct { + ID any `json:"id"` + Method string `json:"method"` + Params map[string]any `json:"params"` + Result any `json:"result"` + Error any `json:"error"` + } + _ = json.Unmarshal(lines.Bytes(), &m) + if m.Method == "" { + answers <- map[string]any{"result": m.Result, "error": m.Error} + continue + } + mu.Lock() + var result any = map[string]any{} + key, _ := m.Params["key"].(string) + switch m.Method { + case "mesh/state.put": + v, _ := json.Marshal(m.Params["value"]) + next++ + values[key], revs[key] = v, next + mostHeld = max(mostHeld, held()) + result = map[string]any{"revision": next, "fill": map[string]any{"count": len(values), "bytes": held(), "max_bytes": limit}} + case "mesh/state.delete": + delete(values, key) + case "mesh/state.keys": + keys := []string{} + for k := range values { + keys = append(keys, k) + } + sort.Strings(keys) + result = keys + case "mesh/state.page": + var entries []StateEntry + for k, v := range values { + entries = append(entries, StateEntry{Key: k, Value: v, Revision: revs[k]}) + } + sort.Slice(entries, func(i, j int) bool { return entries[i].Revision < entries[j].Revision }) + result = StatePage{Entries: entries} + } + mu.Unlock() + if m.ID != nil { + send(map[string]any{"jsonrpc": "2.0", "id": m.ID, "result": result}) + } + } + }() + call := func(name string, args map[string]any) any { + t.Helper() + send(map[string]any{"jsonrpc": "2.0", "id": 1, "method": "tools/call", "params": map[string]any{"name": name, "arguments": args}}) + select { + case a := <-answers: + r, _ := a["result"].(map[string]any) + if a["error"] != nil || r == nil || r["isError"] == true { + t.Fatalf("%s answered %v %v", name, a["error"], r) + } + var out any + _ = json.Unmarshal([]byte(r["content"].([]any)[0].(map[string]any)["text"].(string)), &out) + return out + case <-time.After(3 * time.Second): + t.Fatalf("%s answered nothing", name) + return nil + } + } + value := strings.Repeat("x", 300) + call("write", map[string]any{"key": "k00", "value": value}) + call("keep", map[string]any{"key": "k00"}) + for _, k := range []string{"k01", "k02", "k03", "k04", "k05", "k06", "k07", "k08", "k09", "k10", "k11", "k12", "k13", "k14", "k15", "k16", "k17", "k18", "k19"} { + call("write", map[string]any{"key": k, "value": value}) + mu.Lock() + over := held()*100 > 70*limit + mu.Unlock() + if over { + t.Fatalf("the state holds more than 70 %% of its cap after %s: no prune", k) + } + } + mu.Lock() + _, k00 := values["k00"] + _, k01 := values["k01"] + _, k19 := values["k19"] + _, keep := values[KeepKey("k00")] + after := held() + mu.Unlock() + if mostHeld*100 <= 70*limit { + t.Fatalf("the test never took the state past its mark (%d bytes at most)", mostHeld) + } + if !k00 || !keep || k01 || !k19 { + t.Fatalf("after the prunes k00 %v, its keep %v, k01 %v, k19 %v: want the kept key and its keep, the oldest other gone, the newest there", k00, keep, k01, k19) + } + if after*100 > 70*limit { + t.Fatalf("the state holds %d of %d bytes", after, limit) + } + if keys := call("keys", nil); strings.Contains(strings.Join(strs(keys), " "), KeepPrefix) { + t.Fatalf("Keys counted the keep: %v", keys) + } + page := call("page", nil).(map[string]any) + for _, e := range page["entries"].([]any) { + if IsKeep(e.(map[string]any)["key"].(string)) { + t.Fatalf("Page counted the keep: %v", e) + } + } + }) +} + +func strs(v any) []string { + var out []string + for _, s := range v.([]any) { + out = append(out, s.(string)) + } + return out } diff --git a/go/stateobjects/prune.go b/go/stateobjects/prune.go new file mode 100644 index 00000000..c4e48db5 --- /dev/null +++ b/go/stateobjects/prune.go @@ -0,0 +1,109 @@ +package stateobjects + +import ( + "errors" + "fmt" + "strings" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/views" +) + +// Pruning is one state a module prunes toward its cap (design 47 H2 §812): its local name in the manifest, and the +// pruned state (a process's process.PrunedState or a bundle's stdio.PrunedState). +type Pruning struct { + Name string + State stdio.Prunable +} + +// PruneTools are the two generic tools of a module that prunes its states (design 47 H2 §812; H-A-380 (3)), named for +// the module as Tools names its own: `_state_prune_now` (entry, why) prunes an entry to its target now, and +// `_state_keep` (entry, key, until, why) keeps a key from the prune until a time. Each is written in the hand-act +// log before anything is done, and refused when it cannot be. +func PruneTools(module string, entries []Pruning, acts views.HandActs) []stdio.Tool { + if len(entries) == 0 { + return nil + } + var names []string + for _, e := range entries { + names = append(names, e.Name) + } + entryArg := map[string]any{"type": "string", "description": "the state pruned: its local name in the manifest, one of " + strings.Join(names, ", ")} + whyArg := map[string]any{"type": "string", "description": "why, in plain words"} + pick := func(args map[string]any) (Pruning, string, error) { + why, _ := args["why"].(string) + if why = strings.TrimSpace(why); why == "" { + return Pruning{}, "", errors.New("a prune or a keep names why: nothing was done") + } + name, _ := args["entry"].(string) + name = strings.TrimSpace(name) + for _, e := range entries { + if e.Name == name || (name == "" && len(entries) == 1) { + return e, why, nil + } + } + return Pruning{}, "", fmt.Errorf("entry names one of %s, not %q: nothing was done", strings.Join(names, ", "), name) + } + recorded := func(tool, what, why, cause string) error { + if acts == nil { + return errors.New("no way to record a hand act is given, so nothing was done") + } + if err := acts.RecordHandAct(views.HandAct{What: what, Why: why, Cause: cause, Verb: tool}); err != nil { + return fmt.Errorf("the hand act was not recorded, so nothing was done: %w", err) + } + return nil + } + pruneNow, keep := module+"_state_prune_now", module+"_state_keep" + return []stdio.Tool{ + { + Name: pruneNow, + Description: "Prune one of " + module + "'s states to its prune-target now, oldest keys first, whatever its fill (design 47 §812). " + + "A key kept by " + keep + " stays. Written in the hand-act log first; needs why. Answers the keys removed and the fill before and after.", + Input: map[string]any{"type": "object", "required": []string{"why"}, "properties": map[string]any{"entry": entryArg, "why": whyArg}}, + Run: func(args map[string]any) (any, error) { + e, why, err := pick(args) + if err != nil { + return nil, err + } + if err := recorded(pruneNow, "pruned "+module+"'s state "+e.Name+" to its target now", why, "state-pruned"); err != nil { + return nil, err + } + return e.State.PruneNow() + }, + }, + { + Name: keep, + Description: "Keep one key of one of " + module + "'s states from the prune until a time (design 47 §812): the keep is kept in the state " + + "itself under " + stdio.KeepPrefix + ", never pruned and never counted as an entry. Written in the hand-act log first; needs key, until and why.", + Input: map[string]any{"type": "object", "required": []string{"key", "until", "why"}, "properties": map[string]any{ + "entry": entryArg, + "key": map[string]any{"type": "string", "description": "the key kept"}, + "until": map[string]any{"type": "string", "description": "until when, an RFC 3339 time still to come"}, + "why": whyArg, + }}, + Run: func(args map[string]any) (any, error) { + e, why, err := pick(args) + if err != nil { + return nil, err + } + key, _ := args["key"].(string) + text, _ := args["until"].(string) + until, err := time.Parse(time.RFC3339, strings.TrimSpace(text)) + if err != nil { + return nil, fmt.Errorf("until %q is not an RFC 3339 time: nothing was kept", text) + } + if err := stdio.KeepProblem(key, until); err != nil { + return nil, fmt.Errorf("%w: nothing was kept", err) + } + if err := recorded(keep, "kept the key "+key+" of "+module+"'s state "+e.Name+" from the prune until "+until.UTC().Format(time.RFC3339), why, "state-kept"); err != nil { + return nil, err + } + if err := e.State.Keep(key, until); err != nil { + return nil, err + } + return map[string]any{"entry": e.Name, "key": key, "kept-as": stdio.KeepKey(key), "until": until.UTC()}, nil + }, + }, + } +} diff --git a/go/stateobjects/prune_test.go b/go/stateobjects/prune_test.go new file mode 100644 index 00000000..77c1bdda --- /dev/null +++ b/go/stateobjects/prune_test.go @@ -0,0 +1,59 @@ +package stateobjects + +import ( + "errors" + "strings" + "testing" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" +) + +type prunable struct { + pruned int + kept map[string]time.Time +} + +func (p *prunable) PruneNow() (stdio.Pruned, error) { + p.pruned++ + return stdio.Pruned{State: "performed"}, nil +} +func (p *prunable) Keep(key string, until time.Time) error { + p.kept[key] = until + return nil +} + +// Design 47 H2 §812 (H-A-380 (3)): `_state_prune_now` (entry, why) and `_state_keep` (entry, key, until, +// why) are written in the hand-act log before they act, and do nothing without why or when the act is not recorded. +func subThePruneToolsPruneAndKeepAsHandActs(t *testing.T) { + p := &prunable{kept: map[string]time.Time{}} + a := &acts{} + tools := map[string]stdio.Tool{} + for _, tool := range PruneTools("mesh-forge-bridge", []Pruning{{Name: "performed", State: p}}, a) { + tools[tool.Name] = tool + } + now, keep := tools["mesh-forge-bridge_state_prune_now"], tools["mesh-forge-bridge_state_keep"] + if now.Run == nil || keep.Run == nil { + t.Fatalf("the tools are %v", tools) + } + if _, err := now.Run(map[string]any{"entry": "performed"}); err == nil || p.pruned != 0 { + t.Fatalf("a prune without why ran: %v", err) + } + if _, err := now.Run(map[string]any{"entry": "performed", "why": "full before a delivery"}); err != nil || p.pruned != 1 { + t.Fatalf("prune now: %v, pruned %d", err, p.pruned) + } + until := time.Now().Add(24 * time.Hour).UTC().Format(time.RFC3339) + if _, err := keep.Run(map[string]any{"entry": "performed", "key": "pr.42", "until": until, "why": "the proof reads it"}); err != nil || p.kept["pr.42"].IsZero() { + t.Fatalf("keep: %v, kept %v", err, p.kept) + } + if _, err := keep.Run(map[string]any{"key": "pr.43", "until": "2000-01-01T00:00:00Z", "why": "late"}); err == nil || !p.kept["pr.43"].IsZero() { + t.Fatalf("a keep until a time past was taken: %v", err) + } + if len(a.done) != 2 || a.done[0].Verb != "mesh-forge-bridge_state_prune_now" || !strings.Contains(a.done[1].What, "pr.42") || a.done[1].Why != "the proof reads it" { + t.Fatalf("the hand acts are %+v", a.done) + } + a.err = errors.New("the hand-act log refused") + if _, err := now.Run(map[string]any{"why": "again"}); err == nil || p.pruned != 1 { + t.Fatalf("a prune not recorded ran: %v", err) + } +} diff --git a/go/stateobjects/ref_test.go b/go/stateobjects/ref_test.go index b8f29382..181f5e3f 100644 --- a/go/stateobjects/ref_test.go +++ b/go/stateobjects/ref_test.go @@ -40,4 +40,5 @@ func TestABodyIsAReferenceOnlyWhenItIsExactlyOne(t *testing.T) { t.Run("ValuesOfAnySizeThroughTheRuntime", subValuesOfAnySizeThroughTheRuntime) t.Run("AWriteLostLeavesNoObjectBehind", subAWriteLostLeavesNoObjectBehind) t.Run("AValueLargerThanTheStoreTakesIsRefused", subAValueLargerThanTheStoreTakesIsRefused) + t.Run("ThePruneToolsPruneAndKeepAsHandActs", subThePruneToolsPruneAndKeepAsHandActs) } diff --git a/go/stdio.go b/go/stdio.go index b6bf51ad..4fb2ee17 100644 --- a/go/stdio.go +++ b/go/stdio.go @@ -26,6 +26,7 @@ import ( "sync" "time" + "git.novox.be/novox/mesh-sdk/go/busretry" "git.novox.be/novox/mesh-sdk/go/calladdress" ) @@ -191,6 +192,9 @@ func ServeOn(name string, tools []Tool, in io.Reader, out io.Writer) error { if name == "" { name = "bundle" } + if err := busretry.ReadPace(SettingDuration); err != nil { + return fmt.Errorf("the kept-call pace of %s: %w", name, err) + } byName := map[string]Tool{} for _, t := range tools { byName[t.Name] = t @@ -268,18 +272,18 @@ func ServeOn(name string, tools []Tool, in io.Reader, out io.Writer) error { } // Beside the loop, never in it: a tool that emits waits for the runtime's answer, which // arrives on the line this loop reads next — and calls do not queue behind a slow one. - go func(id any, t Tool, args map[string]any, caller, call string) { + go func(id any, t Tool, args map[string]any, caller, call, ask string) { if args == nil { args = map[string]any{} } - result, err := runAs(t, caller, call, args) + result, err := runAs(t, inFlight{caller, call, ask}, args) if err != nil { c.say(map[string]any{"id": id, "result": map[string]any{"content": []map[string]any{{"type": "text", "text": err.Error()}}, "isError": true}}) return } text, _ := json.Marshal(result) c.say(map[string]any{"id": id, "result": map[string]any{"content": []map[string]any{{"type": "text", "text": string(text)}}}}) - }(id, t, p.Arguments, callerIn(p.Meta), callIn(p.Meta)) + }(id, t, p.Arguments, callerIn(p.Meta), callIn(p.Meta), askIn(p.Meta)) case "mesh/event": // One event the runtime read from this module's consumer (ADR 0198), answered once every // handler it is for has taken it; an error is answered as one, so the runtime does not diff --git a/go/testbus/testbus.go b/go/testbus/testbus.go index d061ef82..34142905 100644 --- a/go/testbus/testbus.go +++ b/go/testbus/testbus.go @@ -19,12 +19,15 @@ package testbus import ( + "fmt" "os" "sync" "testing" "time" "github.com/nats-io/nats-server/v2/server" + + stdio "git.novox.be/novox/mesh-sdk/go" ) // Version is the release of the server the tests run. @@ -64,9 +67,9 @@ func Start(t testing.TB) *server.Server { t.Fatalf("a bus for this test could not be made: %v", err) } go s.Start() - if !s.ReadyForConnections(30 * time.Second) { + if bound := SetupBound(t); !s.ReadyForConnections(bound) { s.Shutdown() - t.Fatal("a bus for this test did not come up within 30s") + Starved(t, fmt.Sprintf("a bus for this test did not come up within %s", bound)) } t.Cleanup(func() { s.Shutdown() @@ -74,3 +77,34 @@ func Start(t testing.TB) *server.Server { }) return s } + +// SetupStarved opens the one line a test prints when its setup wait on the bus passed the setup bound (design 47 H2 +// §800, H-A-368): a build module reads it in the test output, never an assertion's words, and runs the level again. +const SetupStarved = stdio.SetupStarved + +// levelBoundWord is the environment word a build step gives each test level its bound in, as a duration ("10m"). +const levelBoundWord = "MESH_TEST_LEVEL_BOUND" + +// unsetSetupBound is the setup bound of a run whose level has no bound (a developer's own run). +const unsetSetupBound = 30 * time.Second + +// SetupBound is how long a test's setup may wait on the bus: half of the level's bound (MESH_TEST_LEVEL_BOUND), 30 s when +// it is unset, never a value in the test. A bound that is no duration fails the test, naming it. +func SetupBound(t testing.TB) time.Duration { + t.Helper() + word := os.Getenv(levelBoundWord) + if word == "" { + return unsetSetupBound + } + level, err := time.ParseDuration(word) + if err != nil || level <= 0 { + t.Fatalf("%s=%q is not a duration such as 10m", levelBoundWord, word) + } + return level / 2 +} + +// Starved fails the test with the line SetupStarved and its words: a setup wait on the bus passed SetupBound. +func Starved(t testing.TB, words string) { + t.Helper() + t.Fatalf("%s %s", SetupStarved, words) +} diff --git a/go/testbus/testbus_integration_test.go b/go/testbus/testbus_integration_test.go index 3b9ab19f..4ffb2351 100644 --- a/go/testbus/testbus_integration_test.go +++ b/go/testbus/testbus_integration_test.go @@ -50,6 +50,28 @@ func TestBusAnswersPublishAndKeyValueCreate(t *testing.T) { if string(entry.Value()) != "v" { t.Errorf("read back %q, want %q", entry.Value(), "v") } + // Design 47 H2 §800, H-A-368: the setup bound is half of the level's bound, 30 s when the level has none, and a starved + // setup fails with the one line a build module reads. + t.Run("the setup bound is half the level's and a starved setup says so", func(t *testing.T) { + for _, c := range []struct { + level string + want time.Duration + fails bool + }{{"", 30 * time.Second, false}, {"10m", 5 * time.Minute, false}, {"ten", 0, true}, {"-1s", 0, true}} { + t.Setenv(levelBoundWord, c.level) + f := &failed{TB: t} + var got time.Duration + catch(f, func() { got = SetupBound(f) }) + if c.fails != (f.words != "") || (!c.fails && got != c.want) { + t.Errorf("level %q: bound %s, failed %q", c.level, got, f.words) + } + } + f := &failed{TB: t} + catch(f, func() { Starved(f, "the stream did not answer within 5s") }) + if f.words != "mesh: setup starved: the stream did not answer within 5s" { + t.Errorf("a starved setup said %q", f.words) + } + }) } // 1175/AC3, 1171/AC1: the bus is gone when the test that started it ends. diff --git a/go/testbus/testbus_test.go b/go/testbus/testbus_test.go new file mode 100644 index 00000000..5738fa2d --- /dev/null +++ b/go/testbus/testbus_test.go @@ -0,0 +1,27 @@ +package testbus + +import ( + "fmt" + "testing" +) + +// failed is a testing.TB that keeps what a helper failed with, so the helper's own failure is read, not suffered. +type failed struct { + testing.TB + words string +} + +func (f *failed) Helper() {} +func (f *failed) Fatalf(format string, args ...any) { + f.words = fmt.Sprintf(format, args...) + panic(f) +} + +func catch(f *failed, run func()) { + defer func() { + if r := recover(); r != nil && r != f { + panic(r) + } + }() + run() +} diff --git a/go/testdata/beside/CAPTURED b/go/testdata/beside/CAPTURED index fcac974b..1b078d7a 100644 --- a/go/testdata/beside/CAPTURED +++ b/go/testdata/beside/CAPTURED @@ -21,6 +21,13 @@ mesh-delivery f8bbde9627255fe37bd6603a40fcce14fb61d97b modules/mesh-delivery-co mesh-delivery d24d6a08905996841b62c723736339e809ca4625 modules/mesh-delivery-contracts/seats/git/seat.json, taken again as seat.json.captured (milestone 47/H1, C2 amended, H-A-71) mesh-delivery 6e027de59773affe80d7bf45cf49d81292893ff1 modules/mesh-delivery-contracts/seats/mesh-delivery/seat.json, taken again as seat.json.captured (milestone 47/H1, C3) mesh-delivery 9a55e6e8c0faa36dd96be2d90c13e5bb42be0bf3 modules/mesh-delivery-contracts/seats/git, mesh-delivery and mesh-catalog, taken again as seat.json.captured (milestone 47/H1, through C1d) +mesh-delivery d2d3ebbf44fbf0362edaf78d991615f564644615 modules/mesh-delivery-contracts/seats/git/seat.json, taken again as seat.json.captured (milestone 47/H2, D5) +mesh-contracts e6ed560e48216024a0ab6851074c134e69a9bc2b seats/node-engine/seat.json, taken again as seat.json.captured (milestone 47/H2, branch feat/47-H2, H-A-452) +mesh-delivery 80ff1afacca4c1d2e61b0e31139b140645c8088b modules/mesh-delivery-contracts/seats/mesh-delivery/seat.json, taken again as seat.json.captured (milestone 47/H2, branch feat/47-H2, H-A-452) +mesh-delivery 5e94328066198d72cf4fd5d6f1c57b6ad1c8e967 modules/mesh-delivery-contracts/seats/mesh-catalog/seat.json, taken again as seat.json.captured (milestone 47/H2, branch feat/47-H2-i, its form 5, H-A-452) +mesh-contracts 8f2bed18e5ae219489573984fdfddee2d9550288 seats/node-engine/seat.json, taken again as seat.json.captured (milestone 47/H2, branch feat/47-H2, its form 4, H-A-452) +mesh-delivery abfe37e690885956a16fca7c31a5792849e40818 modules/mesh-delivery-contracts/seats/mesh-delivery/seat.json, taken again as seat.json.captured (milestone 47/H2, branch feat/47-H2, H-A-452) +mesh-contracts ba5a253888ca0d7767b76ecb8b1ad7367c577662 seats/node-engine/seat.json, taken again as seat.json.captured (milestone 47/H2, branch feat/47-H2, the have's pipeline description, H-A-452) mesh-host cdcba9ca73755854d21e4e657d7e46c32b9b5265 internal/apply/placement_guard.go The two seat copies are put back the same way (git show : > .captured, the path kept). To move the others, from this repository's go/ directory, with the two repositories checked out beside mesh-sdk: diff --git a/go/testdata/beside/mesh-contracts/seats/node-engine/seat.json.captured b/go/testdata/beside/mesh-contracts/seats/node-engine/seat.json.captured index f33b6cbc..6214da6a 100644 --- a/go/testdata/beside/mesh-contracts/seats/node-engine/seat.json.captured +++ b/go/testdata/beside/mesh-contracts/seats/node-engine/seat.json.captured @@ -1,11 +1,12 @@ { "format": 1, + "form": 4, "seat": "node-engine", "scope": "node", "grain": "whole", "domain": "Core", "decision": "novox/hq ADR 0563", - "description": "The node seat every node's engine holds (novox/hq ADR 0563, ADR 0618 §3, §4): its hand acts on the operator's warrant, its read-only judgements, its compositions and what it reads of a manifest. Each verb is answered on this node's subject, mesh.seat.node-engine.tool.., and each event is published on mesh.seat.node-engine.event.., the node's name its last token, or, for an event whose subject names , there (composed., running..): the grants part grants a node's account only its own node's subjects (ADR 0363 §2).", + "description": "The node seat every node's engine holds (novox/hq ADR 0563, ADR 0618 \u00a73, \u00a74): its hand acts on the operator's warrant, its read-only judgements, its compositions and what it reads of a manifest. Each verb is answered on this node's subject, mesh.seat.node-engine.tool.., and each event is published on mesh.seat.node-engine.event.., the node's name its last token, or, for an event whose subject names , there (composed., running..): the grants part grants a node's account only its own node's subjects (ADR 0363 \u00a72).", "acts": [ { "step": "compose", @@ -15,7 +16,7 @@ "verbs": [ { "name": "hand-over", - "description": "Hand a directory this machine uses as found to the mesh (novox/hq ADR 0563 §3, §5): the engine judges the path as it does every hand-over (absolute, as it states it, used as found, reached through no link) and refuses in words, or asks the operator on their channel at the level approve, binding the owner, group and mode it will set. Nothing changes on the call: only the operator's yes hands it over. One ask is open per path; acts shows where it stands.", + "description": "Hand a directory this machine uses as found to the mesh (novox/hq ADR 0563 \u00a73, \u00a75): the engine judges the path as it does every hand-over (absolute, as it states it, used as found, reached through no link) and refuses in words, or asks the operator on their channel at the level approve, binding the owner, group and mode it will set. Nothing changes on the call: only the operator's yes hands it over. One ask is open per path; acts shows where it stands.", "input": { "properties": { "path": { @@ -39,7 +40,7 @@ }, { "name": "access-act", - "description": "Ask the operator for the take act on a library whose root holds data outside its group (novox/hq ADR 0328 §7, ADR 0563 §3, §4), on the library's home engine: the access as declared and its root as last declared, judged and refused in words or asked at the level approve. Only the operator's yes sets the library's group, group write and setgid under its root. One ask is open per access.", + "description": "Ask the operator for the take act on a library whose root holds data outside its group (novox/hq ADR 0328 \u00a77, ADR 0563 \u00a73, \u00a74), on the library's home engine: the access as declared and its root as last declared, judged and refused in words or asked at the level approve. Only the operator's yes sets the library's group, group write and setgid under its root. One ask is open per access.", "input": { "properties": { "access": { @@ -68,7 +69,7 @@ }, { "name": "judge-directories", - "description": "Which of this machine's declared directories the engine would refuse to make or keep, one line each (novox/hq ADR 0563 §2). Read-only: changes no owner or mode.", + "description": "Which of this machine's declared directories the engine would refuse to make or keep, one line each (novox/hq ADR 0563 \u00a72). Read-only: changes no owner or mode.", "input": { "properties": {}, "type": "object" @@ -80,7 +81,7 @@ }, { "name": "setuid-search", - "description": "Throw away the engine's last search for setuid programs and start a full one (novox/hq ADR 0563 §2); until it completes root-free answers that the machine is not judged yet. Refused while one runs, and when one was started within the hour, naming when the next may start.", + "description": "Throw away the engine's last search for setuid programs and start a full one (novox/hq ADR 0563 \u00a72); until it completes root-free answers that the machine is not judged yet. Refused while one runs, and when one was started within the hour, naming when the next may start.", "input": { "properties": { "why": { @@ -97,7 +98,7 @@ }, { "name": "root-free", - "description": "Whether this machine is root-free (novox/hq ADR 0259 §8, ADR 0563 §6): free only on the engine's own verdict, at most fifteen minutes old, that the account agents run as cannot become root without a person, and the login shell's execute not served here. Anything else is no pass, and says why. Answered by this machine's engine alone.", + "description": "Whether this machine is root-free (novox/hq ADR 0259 \u00a78, ADR 0563 \u00a76): free only on the engine's own verdict, at most fifteen minutes old, that the account agents run as cannot become root without a person, and the login shell's execute not served here. Anything else is no pass, and says why. Answered by this machine's engine alone.", "input": { "properties": {}, "type": "object" @@ -121,7 +122,7 @@ }, { "name": "acts", - "description": "Every act this machine's engine was asked through this seat (novox/hq ADR 0563 §9): each hand-over, access act and setuid search, open, done or refused, with who asked, the ask and the operator's warrant. Read-only.", + "description": "Every act this machine's engine was asked through this seat (novox/hq ADR 0563 \u00a79): each hand-over, access act and setuid search, open, done or refused, with who asked, the ask and the operator's warrant. Read-only.", "input": { "properties": { "id": { @@ -242,7 +243,7 @@ "type": "string" }, "digest": { - "description": "a composition's digest (sha256:…), to read that revision rather than the latest", + "description": "a composition's digest (sha256:\u2026), to read that revision rather than the latest", "type": "string" }, "module": { @@ -287,7 +288,7 @@ }, "self": { "type": "boolean", - "description": "the node composes the module itself (ADR 0547 §2)" + "description": "the node composes the module itself (ADR 0547 \u00a72)" }, "backup": { "type": "string", @@ -455,6 +456,32 @@ }, "description": "every resource that differs from what runs, when asked" }, + "drift": { + "type": "array", + "description": "when the diff is asked, each directory entry of what runs that the mesh made, as its record holds, whose owner or mode on disk differ from it (H-A-392 (b), H-A-458 (2)): read-only; the engine sets it right at its next start or apply of the module", + "items": { + "type": "object", + "required": [ + "id", + "path", + "drift" + ], + "properties": { + "id": { + "type": "string", + "description": "the directory entry's id in the composition" + }, + "path": { + "type": "string", + "description": "its path on the machine" + }, + "drift": { + "type": "string", + "description": "how its owner or mode on disk differ, in words" + } + } + } + }, "own": { "type": "object", "properties": { @@ -517,6 +544,27 @@ "withheld-unknown": { "type": "string", "description": "withheld unknown: the grants part does not answer, when its verb withheld did not answer" + }, + "withheld-parts": { + "type": "array", + "description": "each part a compose kept because its read did not answer, with why (design 08 control-verbs \u00a74, H-A-321): for the packet filter's holder the part node-packet-filter.allowed, its filter file kept as it runs", + "items": { + "type": "object", + "required": [ + "part", + "why" + ], + "properties": { + "part": { + "type": "string", + "description": "the read asked, as ." + }, + "why": { + "type": "string", + "description": "why it did not answer, naming the holder asked" + } + } + } } }, "required": [ @@ -529,6 +577,63 @@ }, "class": "read" }, + { + "name": "composition-diff", + "description": "One module's composed values read by resource (design 47 H2 \u00a7813 (3), H-A-349 DQ-12b): given a module and a revision or a digest of its composition, the entries that differ between the composition that runs and that one, per resource, environment entries by name, a secret's value masked. Read-only. Answered by this machine's engine alone.", + "class": "read", + "input": { + "type": "object", + "required": [ + "module" + ], + "properties": { + "module": { + "type": "string", + "description": "the module, as placed on this machine; node for the node's own" + }, + "revision": { + "type": "integer", + "description": "the revision of the module's composition key to compare with, from the revisions the bucket keeps; give it or digest" + }, + "digest": { + "type": "string", + "description": "the digest (sha256:\u2026) of the composition to compare with; give it or revision" + } + } + }, + "answer": { + "type": "object", + "required": [ + "node", + "module", + "resources" + ], + "properties": { + "node": { + "type": "string", + "description": "the node" + }, + "module": { + "type": "string", + "description": "the module" + }, + "running": { + "type": "string", + "description": "the digest of the composition that runs; absent when nothing runs" + }, + "other": { + "$ref": "#/defs/Revision" + }, + "resources": { + "type": "array", + "description": "each resource with an entry that differs; empty when none differs", + "items": { + "$ref": "#/defs/CompositionDiffResource" + } + } + } + } + }, { "name": "compose", "description": "Compose this machine's modules now and say how it went, so a composition that failed, or a module held, recovers without a restart: what the pass refused or held, each with why, and when the engine composes again on its own after a seat that did not answer (a deadline, no responders: retried with a growing wait; a refusal never). With module, that module's apply is also tried again. Changes only what a pass the engine makes on its own would change.", @@ -595,7 +700,7 @@ }, { "name": "compose-candidate", - "description": "Compose a check's candidate on this machine in shadow, with its own sealed values and the indexer's manifest verdicts, and say per module placed here whether it composes (with its digest and whether the candidate changes it), is refused by the candidate or is held without it, each with why, and the verdict of the machine's own composition (novox/hq ADR 0593 §7). Read-only: applies nothing and writes nothing. Answered by this machine's engine alone, within the asker's bound.", + "description": "Compose a check's candidate on this machine in shadow, with its own sealed values and the indexer's manifest verdicts, and say per module placed here whether it composes (with its digest and whether the candidate changes it), is refused by the candidate or is held without it, each with why, and the verdict of the machine's own composition (novox/hq ADR 0593 \u00a77). Read-only: applies nothing and writes nothing. Answered by this machine's engine alone, within the asker's bound.", "optional": true, "input": { "properties": { @@ -640,7 +745,7 @@ "unready", "not-judged" ], - "description": "the node's verdict (ADR 0593 §7, §9)" + "description": "the node's verdict (ADR 0593 \u00a77, \u00a79)" }, "modules": { "type": "array", @@ -686,7 +791,7 @@ { "name": "manifest-reads", "optional": true, - "description": "The reader capabilities this machine's engine states (manifest words, forms and seat names it reads, novox/hq ADR 0618 §3) and when it last stated them. An engine built before it said them states none, which a reader takes as the baseline. Read-only. Answered by this machine's engine alone.", + "description": "The reader capabilities this machine's engine states (manifest words, forms and seat names it reads, novox/hq ADR 0618 \u00a73) and when it last stated them. An engine built before it said them states none, which a reader takes as the baseline. Read-only. Answered by this machine's engine alone.", "input": { "type": "object", "properties": {} @@ -720,7 +825,7 @@ "node", "stated-at" ], - "description": "what this machine's engine reads of a manifest (ADR 0618 §3)" + "description": "what this machine's engine reads of a manifest (ADR 0618 \u00a73)" }, "class": "read" }, @@ -787,7 +892,7 @@ }, { "name": "seat-list", - "description": "The seat list this engine holds and what it waits for (novox/hq ADR 0636 §7): each seat's source (trunk or bootstrap) and digest, the seats held from the bootstrap alone, the definitions refused, the last error and whether the seat mesh-catalog has answered. Read-only.", + "description": "The seat list this engine holds and what it waits for (novox/hq ADR 0636 \u00a77): each seat's source (trunk or bootstrap) and digest, the seats held from the bootstrap alone, the definitions refused, the last error and whether the seat mesh-catalog has answered. Read-only.", "input": { "type": "object", "properties": {} @@ -798,7 +903,7 @@ }, { "name": "seat-list-again", - "description": "Make this engine read the seat list from the seat mesh-catalog now (novox/hq ADR 0636 §7), the recovery of a list unread; it answers what it holds afterwards, and the error when the read failed. A read that changed nothing still ends the engine's wait.", + "description": "Make this engine read the seat list from the seat mesh-catalog now (novox/hq ADR 0636 \u00a77), the recovery of a list unread; it answers what it holds afterwards, and the error when the read failed. A read that changed nothing still ends the engine's wait.", "input": { "type": "object", "properties": {} @@ -1057,6 +1162,52 @@ "$ref": "#/defs/ProcessBuilds" }, "class": "read" + }, + { + "name": "converged", + "description": "The node's convergence state, which the node-engine keeps (ADR 0168, ADR 0683; design 33 control-verbs \u00a710): adopted or converged, since when it became so, for an adopted node the kind of the firewall found, and the node's links as the engine classifies them: the tunnel's interface and the outward links (design 47 H-A-325). Read-only: the holder of node-packet-filter asks it at its start and at every allow and disallow, and refuses when it does not answer.", + "class": "read", + "input": { + "type": "object", + "properties": {} + }, + "answer": { + "type": "object", + "required": [ + "state", + "tunnel", + "outward" + ], + "properties": { + "state": { + "type": "string", + "enum": [ + "adopted", + "converged" + ], + "description": "the node's mode: adopted (the found firewall's terms) or converged (the mesh's own filter)" + }, + "since": { + "type": "string", + "description": "when the node became so, RFC 3339; absent where the engine recorded no time for it" + }, + "found": { + "type": "string", + "description": "the kind of the firewall found, for an adopted node" + }, + "tunnel": { + "type": "string", + "description": "the private network's tunnel interface on this node, as the engine's filter render names it" + }, + "outward": { + "type": "array", + "items": { + "type": "string" + }, + "description": "the links of this node that face outside, as the engine classifies them for its filter render (ADR 0140); an allowance with no interface never opens one" + } + } + } } ], "events": [ @@ -1205,6 +1356,14 @@ "type": "boolean", "description": "the move changes the build alone" }, + "run": { + "type": "string", + "description": "the composition run's pipeline sequence, in decimal, the module was applied from (design 47 H2 §813 (1), H-A-388): a restarted engine applies no older run again" + }, + "pipeline": { + "type": "string", + "description": "the pipeline of the want the module was applied from, its key ., beside its run (H-A-429 (1), H-A-458 (1)): a want's run is compared with this run only when the want's branch is the same, the key without its last .; absent for a want in none" + }, "kept": { "type": "array", "items": { @@ -1696,7 +1855,7 @@ "description": "one finding of the observer that wrote it" } ], - "description": "the findings this node's engine raises about itself, one key per finding, in the layout of the seat mesh-conditions (its findings state): a key written raises the condition of the same key, a key deleted clears it (novox/hq ADR 0564 §2, ADR 0636 §1); today `machine..seat-list-unread`" + "description": "the findings this node's engine raises about itself, one key per finding, in the layout of the seat mesh-conditions (its findings state): a key written raises the condition of the same key, a key deleted clears it (novox/hq ADR 0564 \u00a72, ADR 0636 \u00a71); today `machine..seat-list-unread`" }, { "name": "bounds", @@ -1708,7 +1867,7 @@ "key": ".", "value": { "type": "object", - "description": "one bound a wait of the node-engine runs against, kept with its own time to live: its expiry is heard as the bus's delete marker and ends the wait (design 47 G §560, A-77)", + "description": "one bound a wait of the node-engine runs against, kept with its own time to live: its expiry is heard as the bus's delete marker and ends the wait (design 47 G \u00a7560, A-77)", "properties": { "v": { "type": "integer", @@ -1815,7 +1974,7 @@ }, "auto": { "type": "boolean", - "description": "the engine asked it itself, on finding a library's root holding data outside its group (ADR 0563 §4)" + "description": "the engine asked it itself, on finding a library's root holding data outside its group (ADR 0563 \u00a74)" } }, "required": [ @@ -1825,7 +1984,7 @@ "at", "state" ], - "description": "one act asked through the seat, as acts reads it (ADR 0563 §9)" + "description": "one act asked through the seat, as acts reads it (ADR 0563 \u00a79)" }, "ActSaid": { "type": "object", @@ -1870,7 +2029,7 @@ "state", "node" ], - "description": "what the seat says of an act: the act, the path or access, the caller as the runtime proved it and the ask (ADR 0563 §1)" + "description": "what the seat says of an act: the act, the path or access, the caller as the runtime proved it and the ask (ADR 0563 \u00a71)" }, "Acts": { "type": "object", @@ -1961,7 +2120,7 @@ "at", "why" ], - "description": "what a node-engine says when what it reads changed or its node returned (ADR 0618 §4)" + "description": "what a node-engine says when what it reads changed or its node returned (ADR 0618 \u00a74)" }, "Module": { "type": "object", @@ -2061,7 +2220,7 @@ "why", "judged" ], - "description": "the engine's root-free verdict (ADR 0563 §6)" + "description": "the engine's root-free verdict (ADR 0563 \u00a76)" }, "Said": { "type": "object", @@ -2106,7 +2265,7 @@ }, "SeatList": { "type": "object", - "description": "what an engine holds of the seat list (novox/hq ADR 0636 §7)", + "description": "what an engine holds of the seat list (novox/hq ADR 0636 \u00a77)", "properties": { "status": { "type": "object", @@ -2451,6 +2610,10 @@ "type": "string" } }, + "declaration-reads": { + "type": "integer", + "description": "the declaration Version its engine reads (design 47 H2 §813 (12), H-A-416 (2), H-A-419): the composer writes this node's declaration at the lower of its own Version and this one, 1 when none is stated" + }, "runner-capabilities": { "type": "array", "items": { @@ -3160,7 +3323,7 @@ }, "Running": { "type": "object", - "description": "a module's build running on a node, and its health: what the node-engines of other nodes and the grants part decide a seat's pass from (ADR 0660 §2), and the assigner reads a provider's and a canary's health and a put's end from (design 47 G 520); said again on the same subject each time healthy changes and for every module the have holds at the engine's start, kept last per subject with no age limit", + "description": "a module's build running on a node, and its health: what the node-engines of other nodes and the grants part decide a seat's pass from (ADR 0660 \u00a72), and the assigner reads a provider's and a canary's health and a put's end from (design 47 G 520); said again on the same subject each time healthy changes and for every module the have holds at the engine's start, kept last per subject with no age limit", "properties": { "v": { "type": "integer", @@ -3331,6 +3494,52 @@ "step", "since" ] + }, + "CompositionDiffEntry": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "the entry: an environment entry by its name, else the part of the resource that differs" + }, + "running": { + "type": "string", + "description": "its value in the running composition; absent where the running one lacks it or the value is a secret" + }, + "other": { + "type": "string", + "description": "its value in the other composition; absent where the other lacks it or the value is a secret" + }, + "masked": { + "type": "boolean", + "description": "the value is a secret, so neither value is given" + } + }, + "description": "one entry that differs between the running composition and the other" + }, + "CompositionDiffResource": { + "type": "object", + "required": [ + "resource", + "entries" + ], + "properties": { + "resource": { + "type": "string", + "description": "the resource, as the composition names it" + }, + "entries": { + "type": "array", + "items": { + "$ref": "#/defs/CompositionDiffEntry" + }, + "description": "the entries of the resource that differ" + } + }, + "description": "one resource with the entries of it that differ" } } } diff --git a/go/testdata/beside/mesh-delivery/modules/mesh-delivery-contracts/seats/git/seat.json.captured b/go/testdata/beside/mesh-delivery/modules/mesh-delivery-contracts/seats/git/seat.json.captured index db42887f..b038f415 100644 --- a/go/testdata/beside/mesh-delivery/modules/mesh-delivery-contracts/seats/git/seat.json.captured +++ b/go/testdata/beside/mesh-delivery/modules/mesh-delivery-contracts/seats/git/seat.json.captured @@ -5,7 +5,7 @@ "grain": "whole", "domain": "Delivery", "decision": "novox/hq ADR 0111", - "description": "The forge: every read and act a delivery part makes of the mesh's git repositories, and what the forge says happened (design 47 *The seat git*, ADR 0596 \u00a71, \u00a72). A failed read answers an error, never an empty answer.", + "description": "The forge: every read and act a delivery part makes of the mesh's git repositories, and what the forge says happened (design 47 *The seat git*, ADR 0596 §1, §2). A failed read answers an error, never an empty answer.", "delivers": "git", "verbs": [ { @@ -48,6 +48,14 @@ }, "error-kind": { "$ref": "#/defs/ErrorKind" + }, + "part": { + "type": "string", + "description": "with the error kind away, what did not answer (design 47 H2 §776)" + }, + "words": { + "type": "string", + "description": "with the error kind away, the holder's words (design 47 H2 §776)" } }, "description": "what pull answers; error and error-kind alone when it fails" @@ -96,6 +104,14 @@ }, "error-kind": { "$ref": "#/defs/ErrorKind" + }, + "part": { + "type": "string", + "description": "with the error kind away, what did not answer (design 47 H2 §776)" + }, + "words": { + "type": "string", + "description": "with the error kind away, the holder's words (design 47 H2 §776)" } }, "description": "what branch answers; error and error-kind alone when it fails" @@ -126,6 +142,14 @@ }, "error-kind": { "$ref": "#/defs/ErrorKind" + }, + "part": { + "type": "string", + "description": "with the error kind away, what did not answer (design 47 H2 §776)" + }, + "words": { + "type": "string", + "description": "with the error kind away, the holder's words (design 47 H2 §776)" } }, "description": "what open-pulls answers; pulls always said, empty when none is open; error and error-kind beside an empty pulls when it fails", @@ -171,9 +195,9 @@ "tags": { "type": "array", "items": { - "type": "string" + "$ref": "#/defs/Tag" }, - "description": "the tags of this page" + "description": "the tags of this page, each with the commit it names (design 47 H2 §796)" }, "page": { "type": "integer", @@ -193,6 +217,14 @@ }, "error-kind": { "$ref": "#/defs/ErrorKind" + }, + "part": { + "type": "string", + "description": "with the error kind away, what did not answer (design 47 H2 §776)" + }, + "words": { + "type": "string", + "description": "with the error kind away, the holder's words (design 47 H2 §776)" } }, "description": "what tags answers; error and error-kind alone when it fails" @@ -201,7 +233,7 @@ }, { "name": "trees", - "description": "The trees of one commit: the root tree and the tree of every directory that holds a module.json, by their git hashes; with files, every blob under each named directory; with paths, the object of each named path. A listing the forge cut is refused, never answered in part.", + "description": "The trees of one commit: the root tree and the tree of every directory that holds a module.json, by their git hashes; with files, every blob under each named directory; with paths, the object of each named path. A listing the forge cut is refused, never answered in part. A directory named in files or paths that the commit does not hold answers the error kind not-found (design 47 H2 §796).", "input": { "type": "object", "properties": { @@ -284,6 +316,14 @@ }, "error-kind": { "$ref": "#/defs/ErrorKind" + }, + "part": { + "type": "string", + "description": "with the error kind away, what did not answer (design 47 H2 §776)" + }, + "words": { + "type": "string", + "description": "with the error kind away, the holder's words (design 47 H2 §776)" } }, "description": "what trees answers; error and error-kind alone when it fails" @@ -350,6 +390,14 @@ }, "error-kind": { "$ref": "#/defs/ErrorKind" + }, + "part": { + "type": "string", + "description": "with the error kind away, what did not answer (design 47 H2 §776)" + }, + "words": { + "type": "string", + "description": "with the error kind away, the holder's words (design 47 H2 §776)" } }, "description": "what file answers; error and error-kind alone when it fails" @@ -410,6 +458,14 @@ }, "error-kind": { "$ref": "#/defs/ErrorKind" + }, + "part": { + "type": "string", + "description": "with the error kind away, what did not answer (design 47 H2 §776)" + }, + "words": { + "type": "string", + "description": "with the error kind away, the holder's words (design 47 H2 §776)" } }, "description": "what contains answers; error and error-kind alone when it fails" @@ -464,6 +520,14 @@ }, "error-kind": { "$ref": "#/defs/ErrorKind" + }, + "part": { + "type": "string", + "description": "with the error kind away, what did not answer (design 47 H2 §776)" + }, + "words": { + "type": "string", + "description": "with the error kind away, the holder's words (design 47 H2 §776)" } }, "description": "what statuses answers; error and error-kind alone when it fails" @@ -546,6 +610,14 @@ }, "error-kind": { "$ref": "#/defs/ErrorKind" + }, + "part": { + "type": "string", + "description": "with the error kind away, what did not answer (design 47 H2 §776)" + }, + "words": { + "type": "string", + "description": "with the error kind away, the holder's words (design 47 H2 §776)" } }, "description": "what status answers; error and error-kind alone when it fails" @@ -607,6 +679,14 @@ }, "error-kind": { "$ref": "#/defs/ErrorKind" + }, + "part": { + "type": "string", + "description": "with the error kind away, what did not answer (design 47 H2 §776)" + }, + "words": { + "type": "string", + "description": "with the error kind away, the holder's words (design 47 H2 §776)" } }, "description": "what comment answers; error and error-kind alone when it fails" @@ -668,6 +748,14 @@ }, "error-kind": { "$ref": "#/defs/ErrorKind" + }, + "part": { + "type": "string", + "description": "with the error kind away, what did not answer (design 47 H2 §776)" + }, + "words": { + "type": "string", + "description": "with the error kind away, the holder's words (design 47 H2 §776)" } }, "description": "what view answers; error and error-kind alone when it fails" @@ -730,6 +818,14 @@ }, "error-kind": { "$ref": "#/defs/ErrorKind" + }, + "part": { + "type": "string", + "description": "with the error kind away, what did not answer (design 47 H2 §776)" + }, + "words": { + "type": "string", + "description": "with the error kind away, the holder's words (design 47 H2 §776)" } }, "description": "what note answers; error and error-kind alone when it fails" @@ -738,7 +834,7 @@ }, { "name": "merge-pull", - "description": "Merge one or more pull requests with the forge tools' account, each at the head the caller names: one merge call, a delivery's start and, with two or more members, its group (design 47 H \u00a7734, \u00a7735, \u00a7766; ADR 0682 \u00a71, \u00a72). Every member is judged before any is merged: still the pull request's head, its base a watched branch, no run at the head's build-input fingerprint failed a required step (judged from its own state failed-steps), registrations heard; no branch protection is read. When one member is refused, none is merged. Then it merges them in the order named, writing the call in its state calls before the first merge, and says merge-called once. It answers once, at the call's end, naming the call and each member's outcome; a member already merged is refused already-merged, naming its commit.", + "description": "Merge one or more pull requests with the forge tools' account, each at the head the caller names: one merge call, a delivery's start and, with two or more members, its group (design 47 H §734, §735, §766; ADR 0682 §1, §2). Every member is judged before any is merged: still the pull request's head, its base a watched branch, no run at the head's build-input fingerprint failed a required step (judged from its own state failed-steps), registrations heard; no branch protection is read. When one member is refused, none is merged. Then it merges them in the order named, writing the call in its state calls before the first merge, and says merge-called once. It answers once, at the call's end, naming the call and each member's outcome; a member already merged is refused already-merged, naming its commit.", "input": { "type": "object", "properties": { @@ -805,6 +901,14 @@ }, "error-kind": { "$ref": "#/defs/ErrorKind" + }, + "part": { + "type": "string", + "description": "with the error kind away, what did not answer (design 47 H2 §776)" + }, + "words": { + "type": "string", + "description": "with the error kind away, the holder's words (design 47 H2 §776)" } }, "description": "what merge-pull answers at the call's end; error and error-kind alone when it fails" @@ -815,7 +919,7 @@ { "name": "tag", "class": "act", - "description": "Write the git tag of a released version, go/vMAJOR.MINOR.PATCH, at the commit its build was made from; the builder writes it only on the catalog's version-released (design 47 G section 370). A tag already at that commit is answered performed. An error answer carries error and error-kind.", + "description": "Write the git tag of a released version, /vMAJOR.MINOR.PATCH for a Go package (design 47 H2 §796), at the commit its build was made from; the builder writes it only on the catalog's version-released (design 47 G section 370). A tag already at that commit is answered performed. An error answer carries error and error-kind.", "input": { "type": "object", "properties": { @@ -856,6 +960,14 @@ }, "error-kind": { "$ref": "#/defs/ErrorKind" + }, + "part": { + "type": "string", + "description": "with the error kind away, what did not answer (design 47 H2 §776)" + }, + "words": { + "type": "string", + "description": "with the error kind away, the holder's words (design 47 H2 §776)" } }, "description": "what tag answers; error and error-kind alone when it fails" @@ -1259,7 +1371,7 @@ "name": "calls", "template": "state", "history": 1, - "description": "every merge call, written before its first merge; one whose merge-called is not said is said at the holder's start; deleted past the stream merge-called's age (design 47 H \u00a7766, H-A-47)", + "description": "every merge call, written before its first merge; one whose merge-called is not said is said at the holder's start; deleted past the stream merge-called's age (design 47 H §766, H-A-47)", "key-forms": [ { "key": "call.", @@ -1328,7 +1440,7 @@ "description": "its page on the forge" } }, - "description": "one pull request as the forge holds it; never its description (ADR 0659 \u00a71)", + "description": "one pull request as the forge holds it; never its description (ADR 0659 §1)", "required": [ "repository", "number", @@ -1361,7 +1473,7 @@ "description": "whether the forge's merging account may write the repository: false, its statuses and its merge are refused" } }, - "description": "what the forge would merge now, for an open pull request (issues 1275, 1945; ADR 0441 \u00a71): answered whole or not at all, and a read the forge could not make answers the verb's error, never a guess; absent for a merged or closed one", + "description": "what the forge would merge now, for an open pull request (issues 1275, 1945; ADR 0441 §1): answered whole or not at all, and a read the forge could not make answers the verb's error, never a guess; absent for a merged or closed one", "required": [ "mergeable", "protected", @@ -1471,9 +1583,10 @@ "enum": [ "refused", "not-found", - "failed" + "failed", + "away" ], - "description": "which kind of error, set with error alone: refused, the forge or the holder refused and error says why in the forge's own words; not-found, the forge said that what was asked about (a repository, a commit, a pull request, a branch or a tag) does not exist, so a caller never reads it out of error's words; failed, the forge failed or could not be reached, so the act or read could not be performed" + "description": "which kind of error, set with error alone: refused, the forge or the holder refused and error says why in the forge's own words; not-found, the forge said that what was asked about (a repository, a commit, a pull request, a branch or a tag) does not exist, so a caller never reads it out of error's words; failed, the forge failed or could not be reached, so the act or read could not be performed; away, a part the holder depends on (the forge, its store) is away, with part naming what did not answer and words: a caller reads it as a transport fault and calls again (design 47 H2 §776)" }, "FailedStep": { "type": "object", @@ -1529,7 +1642,7 @@ }, "MemberOutcome": { "type": "object", - "description": "one member of a merge call and where it stands (design 47 H \u00a7766, H-A-22)", + "description": "one member of a merge call and where it stands (design 47 H §766, H-A-22)", "properties": { "repository": { "type": "string" @@ -1564,7 +1677,7 @@ }, "Call": { "type": "object", - "description": "one merge call, written before its first merge and updated as each member merges (design 47 H \u00a7766)", + "description": "one merge call, written before its first merge and updated as each member merges (design 47 H §766)", "properties": { "by": { "type": "string", @@ -1640,12 +1753,30 @@ "head", "state" ] + }, + "Tag": { + "type": "object", + "description": "a tag and the commit it names (design 47 H2 §796)", + "properties": { + "tag": { + "type": "string" + }, + "commit": { + "type": "string", + "description": "the commit it names, its full 40-hex sha", + "pattern": "^[0-9a-f]{40}$" + } + }, + "required": [ + "tag", + "commit" + ] } }, "bodies": { "MergeCalled": { "type": "object", - "description": "a merge call's members merged or refused, said once by the forge module (design 47 H \u00a7736, \u00a7767)", + "description": "a merge call's members merged or refused, said once by the forge module (design 47 H §736, §767)", "properties": { "call": { "type": "string", @@ -1702,7 +1833,7 @@ }, "refused": { "type": "array", - "description": "one per member not merged after every member was judged (design 47 H \u00a7767)", + "description": "one per member not merged after every member was judged (design 47 H §767)", "items": { "type": "object", "properties": { @@ -1732,5 +1863,6 @@ "members" ] } - } + }, + "form": 2 } diff --git a/go/testdata/beside/mesh-delivery/modules/mesh-delivery-contracts/seats/mesh-catalog/seat.json.captured b/go/testdata/beside/mesh-delivery/modules/mesh-delivery-contracts/seats/mesh-catalog/seat.json.captured index a6d62b1a..f40e9799 100644 --- a/go/testdata/beside/mesh-delivery/modules/mesh-delivery-contracts/seats/mesh-catalog/seat.json.captured +++ b/go/testdata/beside/mesh-delivery/modules/mesh-delivery-contracts/seats/mesh-catalog/seat.json.captured @@ -253,6 +253,49 @@ }, "class": "read" }, + { + "name": "seat-list", + "class": "read", + "description": "The trunk's seat list as the mesh holds it, the object of the holder's objects seat-lists of that digest (design 47 H2 §814, §820; H-A-441, H-A-443): its digest and size, then its bytes (the definitions resolved, a JSON array in the form of a composer request's seats, sorted by seat) in chunks of at most 512 KiB on the chunk subject of the call, each acknowledged by the caller on the ack subject within 30 seconds. A validate run's asker fills the request's seats from these bytes. Reads only.", + "input": { + "type": "object", + "properties": { + "call": { + "type": "string", + "description": "128 random bits the caller chooses: the subject token of this send's chunks and acknowledgements", + "minLength": 1 + } + }, + "required": [ + "call" + ] + }, + "answer": { + "type": "object", + "properties": { + "digest": { + "type": "string", + "description": "the seat list's sha256, the object's name in seat-lists" + }, + "size": { + "type": "integer", + "minimum": 0, + "description": "the seat list's size in bytes" + }, + "chunks": { + "type": "integer", + "minimum": 0, + "description": "how many chunks are sent" + } + }, + "required": [ + "digest", + "size", + "chunks" + ] + }, + "chunks": true + }, { "name": "revision", "description": "The catalog half of the revision view (ADR 0656 §6): one catalog graph revision with its entries whole, the trunk heads, the seat definitions by digest, and each manifest's verdict and fingerprint; the trunk's newest when no revision and no candidate is named, or a candidate's for its group and heads. Answered by the holder from what it holds, in its own runtime (ADR 0663), in pages of entries. Reads only. A watched branch with no revision yet answers {none-yet: true} alone (design 47 G section 530).", @@ -312,7 +355,7 @@ }, { "name": "registrations", - "description": "Every module's registration, or one with every version and who changed it on whose warrant (design 47 *A module's registration*). Its readers take a first view here, then hear registration-changed.", + "description": "Every module's registration, or one with every version and who changed it on whose warrant (design 47 *A module's registration*). Its readers take a first view here, then hear registration-changed. Filtered by module or repository while it reads and paged by the cursor next, at most 768 KB a page (design 47 H2 §806, §811).", "input": { "type": "object", "properties": { @@ -323,6 +366,14 @@ "history": { "type": "boolean", "description": "with every version of the one module" + }, + "repository": { + "type": "string", + "description": "only the registrations of this repository, /, applied while reading (design 47 H2 §806)" + }, + "next": { + "type": "string", + "description": "the cursor the previous page answered, opaque; absent for the first page (design 47 H2 §779)" } } }, @@ -344,6 +395,18 @@ }, "fill": { "$ref": "#/defs/Fill" + }, + "capped": { + "type": "boolean", + "description": "yes when the answer stopped at 768 KB; the rest is read from next (design 47 H2 §806)" + }, + "total": { + "type": "integer", + "description": "how many registrations the filters hold in all (design 47 H2 §811)" + }, + "next": { + "type": "string", + "description": "the cursor of the next page, opaque; empty at the end (design 47 H2 §779)" } }, "required": [ @@ -657,6 +720,32 @@ "source": { "type": "integer" }, + "call": { + "type": "string", + "description": "what made the revision, as the revision's call names it (design 47 H-A-212, H-A-213); absent for a candidate's" + }, + "call-kind": { + "type": "string", + "enum": [ + "take-in", + "release", + "take-in-again", + "recompute" + ], + "description": "the kind of what made the revision, which a reader reads instead of call's words (design 47 H-A-213 (3), H-A-214, H-A-215, H-A-217); absent for a candidate's" + }, + "module": { + "type": "string", + "description": "for call-kind release, the module released (H-A-215)" + }, + "version": { + "type": "string", + "description": "for call-kind release, the version released (H-A-215)" + }, + "repository": { + "type": "string", + "description": "for call-kind take-in-again, the repository taken in again (H-A-215)" + }, "group": { "type": "string" }, @@ -1448,6 +1537,13 @@ "$ref": "#/defs/Registration" }, "description": "a module's registration" + }, + { + "key": "source..", + "value": { + "$ref": "#/defs/Registration" + }, + "description": "a source-only registration of a repository with no module (design 47 H2 §815)" } ] }, @@ -1462,6 +1558,13 @@ "$ref": "#/defs/Registration" }, "description": "one version of a module's registration" + }, + { + "key": "source...", + "value": { + "$ref": "#/defs/Registration" + }, + "description": "one version of a source-only registration (design 47 H2 §815)" } ] }, @@ -1594,6 +1697,10 @@ "type": "string", "description": "its manifest's verdict: accepted, refused or held" }, + "class": { + "type": "string", + "description": "core or leaf: core when its manifest claims a seat, leaf when it claims none; an entry without it reads as leaf (design 47 H2 H-A-394, H-A-431)" + }, "fp": { "type": "string", "description": "its fingerprint (src2: and 64 hex digits); absent when it has none" @@ -1871,6 +1978,32 @@ "type": "integer", "description": "the source revision it was taken in from; for a candidate, the source revision its heads are merged onto" }, + "call": { + "type": "string", + "description": "what made the revision (design 47 H-A-212, H-A-213): a take-in's the gate's call its source revision names, a re-pin's release of , a take-in again's take-in again of ; absent for a candidate's" + }, + "call-kind": { + "type": "string", + "enum": [ + "take-in", + "release", + "take-in-again", + "recompute" + ], + "description": "the kind of what made the revision, which a reader reads instead of call's words (design 47 H-A-213 (3), H-A-214, H-A-215, H-A-217): a take-in's, a re-pin's on a module's release, a take-in again's, a recompute's that moved a fingerprint (its call naming the reason); absent for a candidate's" + }, + "module": { + "type": "string", + "description": "for call-kind release, the module released (H-A-215)" + }, + "version": { + "type": "string", + "description": "for call-kind release, the version released (H-A-215)" + }, + "repository": { + "type": "string", + "description": "for call-kind take-in-again, the repository taken in again (H-A-215)" + }, "group": { "type": "string", "description": "a candidate's group" @@ -1927,6 +2060,10 @@ "$ref": "#/defs/TakeInEntry" }, "description": "the movers of the take-in that wrote it, so a redelivered source-moved whose revision is kept says take-in.taken again from them, recomputing nothing (design 47 H §743, H-A-92); absent for one with no mover" + }, + "base": { + "type": "integer", + "description": "the newest kept revision of its branch below it, which the take-in that wrote it pinned for its pipeline: the base of the validate step's case before (design 47 H-A-147, H-A-151, H-A-154); absent on the branch's first take-in" } }, "required": [ @@ -2078,14 +2215,15 @@ }, "Registration": { "type": "object", - "description": "a module's registration (ADR 0666 section 2; design 47 *A module's registration*): its git origin, its manifest's path, its watched branches and its policy, every version with who changed it and the warrant of the operator's Approve", + "description": "a module's registration (ADR 0666 section 2; design 47 *A module's registration*): its git origin, its manifest's path, its watched branches and its policy, every version with who changed it and the warrant of the operator's Approve; or a source-only registration of a repository with no module (design 47 H2 §815): its origin, one watched branch, its policy and its check, naming no module and no path, kept under source..", "properties": { "v": { "type": "integer", "description": "the layout version" }, "module": { - "type": "string" + "type": "string", + "description": "the module; absent for a source-only registration" }, "origin": { "type": "string", @@ -2093,7 +2231,7 @@ }, "path": { "type": "string", - "description": "the path of its manifest in the repository" + "description": "the path of its manifest in the repository; absent for a source-only registration" }, "branches": { "type": "array", @@ -2121,12 +2259,14 @@ "at": { "type": "string", "format": "date-time" + }, + "check": { + "$ref": "#/defs/SourceCheck", + "description": "for a source-only registration, the one step its heads run (design 47 H2 §815)" } }, "required": [ - "module", "origin", - "path", "branches", "policy" ] @@ -2157,9 +2297,10 @@ "placed", "review required", "unknown module", - "nothing moves" + "nothing moves", + "holds-manifest" ], - "description": "why no ask was raised" + "description": "why no ask was raised; holds-manifest: a source-only registration whose repository holds a module.json at its root or under modules/ at the branch's head (design 47 H2 §815; H-A-450)" }, "words": { "type": "string", @@ -2469,6 +2610,34 @@ "ref", "language" ] + }, + "SourceCheck": { + "type": "object", + "description": "the check of a source-only registration: a build seat and a step kind, run at the repository's root, never a free command (design 47 H2 §815)", + "properties": { + "seat": { + "type": "string", + "enum": [ + "node-build-go", + "node-build-node", + "node-build-image" + ], + "description": "the build seat that runs it" + }, + "step": { + "type": "string", + "enum": [ + "lint", + "unit", + "integration" + ], + "description": "the step kind" + } + }, + "required": [ + "seat", + "step" + ] } }, "bodies": { @@ -2567,5 +2736,6 @@ "call" ] } - } + }, + "form": 5 } diff --git a/go/testdata/beside/mesh-delivery/modules/mesh-delivery-contracts/seats/mesh-delivery/seat.json.captured b/go/testdata/beside/mesh-delivery/modules/mesh-delivery-contracts/seats/mesh-delivery/seat.json.captured index 015f9657..08e74ab4 100644 --- a/go/testdata/beside/mesh-delivery/modules/mesh-delivery-contracts/seats/mesh-delivery/seat.json.captured +++ b/go/testdata/beside/mesh-delivery/modules/mesh-delivery-contracts/seats/mesh-delivery/seat.json.captured @@ -9,7 +9,7 @@ "verbs": [ { "name": "deliveries", - "description": "Every delivery not final, and those that ended in the last day, one line each: its id (owner/repository@commit), its state, its group, what it waits for and since when. Narrowed by state, repository or group.", + "description": "Every delivery not final, and those that ended in the last day, one line each: its id (owner/repository@commit), its state, its group, what it waits for and since when. Narrowed by state, repository or group. Filtered by repository, module, outcome and since while it reads, at most 768 KB a page, capped saying the rest is read from the cursor next (design 47 H2 §811).", "input": { "properties": { "all": { @@ -31,13 +31,55 @@ "state": { "description": "one state, e.g. delivering or held", "type": "string" + }, + "module": { + "type": "string", + "description": "only the deliveries that move this module, applied while reading (design 47 H2 §811)" + }, + "outcome": { + "type": "string", + "description": "only the deliveries that ended with this outcome, applied while reading (design 47 H2 §811)" + }, + "since": { + "type": "string", + "format": "date-time", + "description": "only the deliveries opened at or after this time, applied while reading (design 47 H2 §811)" + }, + "next": { + "type": "string", + "description": "the cursor the previous page answered, opaque; absent for the first page (design 47 H2 §779)" } }, "type": "object" }, "answer": { "type": "object", - "description": "what deliveries answers, as its description says" + "description": "what deliveries answers, as its description says", + "properties": { + "deliveries": { + "type": "array", + "items": { + "type": "object", + "description": "one delivery's record, as the gate keeps it" + }, + "description": "this page's deliveries, the open ones first, then the newest" + }, + "capped": { + "type": "boolean", + "description": "yes when the answer stopped at 768 KB; the rest is read from next (design 47 H2 §811)" + }, + "total": { + "type": "integer", + "description": "how many deliveries the filters hold in all (design 47 H2 §811)" + }, + "next": { + "type": "string", + "description": "the cursor of the next page, opaque; empty at the end (design 47 H2 §779)" + } + }, + "required": [ + "deliveries" + ] }, "class": "read" }, @@ -196,7 +238,7 @@ "name": "record-opened", "body": { "type": "object", - "description": "a delivery record opened for one source revision of a merge call (design 47 H \u00a7737)", + "description": "a delivery record opened for one source revision of a merge call (design 47 H §737)", "properties": { "record": { "type": "string", @@ -319,7 +361,7 @@ "subject": "source-moved..", "body": { "type": "object", - "description": "a source revision of a watched branch, named once per merge call (design 47 H \u00a7737)", + "description": "a source revision of a watched branch, named once per merge call (design 47 H §737)", "properties": { "v": { "type": "integer" @@ -337,7 +379,7 @@ }, "start": { "type": "boolean", - "description": "true only for a branch's first revision (design 47 H \u00a7768)" + "description": "true only for a branch's first revision (design 47 H §768)" }, "heads": { "type": "array", @@ -448,7 +490,7 @@ }, "call": { "type": "string", - "description": "the merge call that moved it; a call the newest revision already carries writes nothing (design 47 H \u00a7737, H-A-33)" + "description": "the merge call that moved it; a call the newest revision already carries writes nothing (design 47 H §737, H-A-33)" } }, "required": [ @@ -512,7 +554,7 @@ "name": "heads", "template": "state", "history": 1, - "description": "each head the forge bridge's head-moved said that source. does not name for that repository; deleted when source. names it (design 47 H \u00a7740)", + "description": "each head the forge bridge's head-moved said that source. does not name for that repository; deleted when source. names it (design 47 H §740)", "key-forms": [ { "key": ".", @@ -607,6 +649,10 @@ "type": "string", "format": "date-time", "description": "when the gate first said it" + }, + "record": { + "type": "string", + "description": "the delivery record the act is for, by its source revision, .; absent on an act said before it was named, which bridge_owed_again by record never finds (design 47 H2 §807, H-A-432)" } }, "required": [ @@ -672,5 +718,6 @@ "head" ] } - } + }, + "form": 2 } diff --git a/go/testdata/seat-names.txt b/go/testdata/seat-names.txt index cb5940f1..cf329454 100644 --- a/go/testdata/seat-names.txt +++ b/go/testdata/seat-names.txt @@ -1,3 +1,4 @@ +buscompose/nats.go: mesh-grants catalogue/environment_into.go: node-environment catalogue/environment_into.go: node-login-shell catalogue/environment_into.go: node-power diff --git a/go/views/tools.go b/go/views/tools.go index 8ab9061a..2d811666 100644 --- a/go/views/tools.go +++ b/go/views/tools.go @@ -25,9 +25,10 @@ func (k KeptSource) Get(key string) (*Entry, bool, error) { return &Entry{Value: e.Value, Revision: e.Revision, Written: e.At}, true, nil } -// Page is the values after a key in one request (stdio.KeptState.Page), as FirstView prefers them to Get per key. -func (k KeptSource) Page(after string, maxBytes int, budget time.Duration) ([]KeyedEntry, string, error) { - p, err := k.State.Page(after, maxBytes, budget) +// Page is the values after the opaque cursor in one request (stdio.KeptState.Page, in the order the bus wrote them), +// as FirstView prefers them to Get per key, and the cursor that continues them. +func (k KeptSource) Page(cursor string, maxBytes int, budget time.Duration) ([]KeyedEntry, string, error) { + p, err := k.State.Page(cursor, maxBytes, budget) if err != nil { return nil, "", err } @@ -61,10 +62,10 @@ func FirstViewTool(owner, name, state, words string, values, marks Source, now f Name: name, Description: fmt.Sprintf("The first view of %s's state %s (novox/hq ADR 0615, ADR 0621): %s Each value whole by key, at the revision "+ "the owner said its views family up to, so a reader then follows the events on that family and asks again only on a gap in "+ - "revisions. One page at a time: next is the key to ask after for the rest.", + "revisions. One page at a time: hand next back as after for the rest.", owner, state, words), Input: map[string]any{"type": "object", "properties": map[string]any{ - "after": map[string]any{"type": "string", "description": "the key the page before ended at (its next); none starts at the first"}}}, + "after": map[string]any{"type": "string", "description": "the next of the page before; none starts at the first"}}}, Run: func(args map[string]any) (any, error) { after, _ := args["after"].(string) return FirstView(owner, state, values, marks, strings.TrimSpace(after), now()) @@ -87,7 +88,7 @@ func ReadTool(owner, name, state, words string, values, marks Source, now func() "key": map[string]any{"type": "string", "description": "one key: its value and revision"}, "history": map[string]any{"type": "boolean", "description": "with key: every value the state keeps for it, oldest first"}, "values": map[string]any{"type": "boolean", "description": "a page of values whole by key"}, - "after": map[string]any{"type": "string", "description": "with values: the key the page before ended at"}}}, + "after": map[string]any{"type": "string", "description": "with values: the next of the page before"}}}, Run: func(args map[string]any) (any, error) { key, _ := args["key"].(string) after, _ := args["after"].(string) diff --git a/go/views/views.go b/go/views/views.go index 379f7a6e..a3fbbe73 100644 --- a/go/views/views.go +++ b/go/views/views.go @@ -174,7 +174,7 @@ func DecodeMark(data []byte) (Mark, error) { } // Answer is a first view, or one page of an ordinary read: the state's values by key, from After on, at the revision -// of the state the owner said up to (Mark) before it read them; Next is the key to ask after for the rest, empty when +// of the state the owner said up to (Mark) before it read them; Next is handed back as after for the rest, empty when // the answer is whole. A reader keeps the first page's revision: a value on a later page may be newer, and the events // after that revision bring every key to its newest in order. type Answer struct { @@ -222,15 +222,17 @@ type KeyedEntry struct { Entry } -// Pager is a Source that answers many values in one request: the values in key order after a key, as many as fit -// maxBytes or the time budget and at least one, and the key to ask after for the rest. FirstView uses it when its source has it, unless +// Pager is a Source that answers many values in one request: the values after an opaque cursor (empty: from the +// start), as many as fit maxBytes or the time budget and at least one, and the cursor that continues them, empty at +// the end (design 47 H2 §779). FirstView uses it when its source has it, unless // it answers errors.Is(err, errors.ErrUnsupported), when it reads per key. type Pager interface { - Page(after string, maxBytes int, budget time.Duration) ([]KeyedEntry, string, error) + Page(cursor string, maxBytes int, budget time.Duration) ([]KeyedEntry, string, error) } // FirstView answers one page of state from its owner's source: the mark first, so no change after the revision it -// answers at is missed, then the values in key order after after, until PageBytes or PageBudget. A state not said yet (no mark) +// answers at is missed, then the values after after (the next of the page before: the Pager's cursor, or with a source +// that reads per key the last key, in key order), until PageBytes or PageBudget. A state not said yet (no mark) // answers revision 0: a first view of whatever it holds, which the reader takes as a gap at the first event. func FirstView(owner, state string, values, marks Source, after string, now time.Time) (Answer, error) { a := Answer{V: Version, Owner: owner, State: state, Values: map[string]json.RawMessage{}, At: now.UTC()} diff --git a/go/views/views_test.go b/go/views/views_test.go index 031b44c7..d7b4af9f 100644 --- a/go/views/views_test.go +++ b/go/views/views_test.go @@ -1441,6 +1441,52 @@ func (s *pagedSource) Page(after string, _ int, _ time.Duration) ([]KeyedEntry, func TestViews(t *testing.T) { t.Run("Views", subViews) t.Run("StdioWriterPublishesAndListensThroughTheModulesOwnPlumbing", subStdioWriterPublishesAndListensThroughTheModulesOwnPlumbing) + // Design 47 H2 §782: one process reader's pictures are safe at once: gets, keys and watches from many goroutines while + // the owner's events change the picture and further views are followed, under the race detector. + t.Run("one process reader serves concurrent reads", func(t *testing.T) { + made := time.Date(2026, 11, 22, 0, 0, 0, 0, time.UTC) + r := &ProcessReader{Module: "reader", Call: func(string, any) (json.RawMessage, error) { + return json.Marshal(Answer{V: Version, Owner: "o", State: "s", Revision: 1, Created: made, Values: map[string]json.RawMessage{"a": json.RawMessage(`1`)}}) + }} + p, err := r.Follow("o", "s", "o.o_read") + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + if err := p.Follower().Start(ctx); err != nil { + t.Fatal(err) + } + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + for i := range 200 { + p.Follower().Apply(Event{V: Version, Owner: "o", State: "s", Key: fmt.Sprintf("k%d", i%7), Revision: uint64(i + 2), Created: made, Value: json.RawMessage(`2`)}) + } + }() + for i := range 16 { + wg.Add(1) + go func() { + defer wg.Done() + w, err := p.Watch(ctx, ">") + if err != nil { + t.Error(err) + return + } + defer w.Stop() + for range 20 { + _, _ = p.Get(ctx, "a") + _, _ = p.Keys(ctx) + _ = r.Followers() + } + if _, err := r.Follow("o", fmt.Sprintf("s%d", i), "o.o_read"); err != nil { + t.Error(err) + } + }() + } + wg.Wait() + }) } type putRecorder struct{ puts map[string][]byte } diff --git a/package-lock.json b/package-lock.json index 395b47b7..d65254c8 100644 --- a/package-lock.json +++ b/package-lock.json @@ -7,6 +7,10 @@ "": { "name": "@novox/mesh-sdk", "version": "0.1.19", + "dependencies": { + "blakejs": "1.2.1", + "tweetnacl": "1.0.3" + }, "devDependencies": { "@types/node": "^22.0.0", "typescript": "^5.6.0" @@ -22,6 +26,18 @@ "undici-types": "~6.21.0" } }, + "node_modules/blakejs": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/blakejs/-/blakejs-1.2.1.tgz", + "integrity": "sha512-QXUSXI3QVc/gJME0dBpXrag1kbzOqCjCX8/b54ntNyW6sjtoqxqRk3LTmXzaJoh71zMsDCjM+47jS7XiwN/+fQ==", + "license": "MIT" + }, + "node_modules/tweetnacl": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-1.0.3.tgz", + "integrity": "sha512-6rt+RN7aOi1nGMyC4Xa5DdYiukl2UWCbcJft7YhxReBGQD7OAM8Pbxw6YMo4r2diNEA8FEmu32YOn9rhaiE5yw==", + "license": "Unlicense" + }, "node_modules/typescript": { "version": "5.9.3", "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", diff --git a/package.json b/package.json index b2285b6a..47125069 100644 --- a/package.json +++ b/package.json @@ -15,7 +15,8 @@ "./state": "./dist/state/index.js", "./callers": "./dist/callers/index.js", "./runtime": "./dist/runtime/index.js", - "./views": "./dist/views/index.js" + "./views": "./dist/views/index.js", + "./seal": "./dist/seal/index.js" }, "files": [ "dist" @@ -29,5 +30,9 @@ "devDependencies": { "@types/node": "^22.0.0", "typescript": "^5.6.0" + }, + "dependencies": { + "blakejs": "1.2.1", + "tweetnacl": "1.0.3" } } diff --git a/src/callers/index.ts b/src/callers/index.ts index 8a913775..b6e3e34b 100644 --- a/src/callers/index.ts +++ b/src/callers/index.ts @@ -17,6 +17,12 @@ export const META_CALLER = "mesh/caller"; /** Where in a `tools/call`'s `_meta` the runtime names the id of the call in flight (`buscalls.MetaCall`). */ export const META_CALL = "mesh/call"; +/** + * Where in a `tools/call`'s `_meta` the runtime names the ask id a kept call carried (the Go side's `stdio.MetaAsk`; + * design 47 H2 §775, H-A-351): what a holder takes the ask once by. + */ +export const META_ASK = "mesh/ask"; + // A bus user's name: `.`, `person.`, `controller`; anything else is no caller. const busUser = /^[A-Za-z0-9_-]+(\.[A-Za-z0-9_-]+)*$/; // A call's id: one subject-safe token. @@ -25,6 +31,7 @@ const callID = /^[A-Za-z0-9_-]{1,128}$/; interface InFlight { readonly caller: string; readonly call: string; + readonly ask: string; } // The caller and the call's id of every call while its tool runs, by the arguments object that call was handed. @@ -42,12 +49,18 @@ export function callIn(meta: unknown): string { return typeof s === "string" && callID.test(s) ? s : ""; } +/** The ask id a `tools/call`'s `_meta` names, or empty: one subject-safe token, as a call's id is. */ +export function askIn(meta: unknown): string { + const s = meta && typeof meta === "object" ? (meta as Record)[META_ASK] : undefined; + return typeof s === "string" && callID.test(s) ? s : ""; +} + /** * Run `run` with the caller and the call's id `meta` names held for `args` while it runs: the stdio session does this * for every `tools/call`. */ export async function runAs(meta: unknown, args: object, run: () => Promise): Promise { - calls.set(args, { caller: callerIn(meta), call: callIn(meta) }); + calls.set(args, { caller: callerIn(meta), call: callIn(meta), ask: askIn(meta) }); try { return await run(); } finally { @@ -65,6 +78,11 @@ export function callIdOf(args: object | undefined | null): string { return (args && calls.get(args)?.call) || ""; } +/** The ask id the call that was handed `args` carried (H-A-351), or empty. Valid while the tool runs. */ +export function askIdOf(args: object | undefined | null): string { + return (args && calls.get(args)?.ask) || ""; +} + /** * A module's trusted list of who may call it (ADR 0459 §2): an entry is a module (on any machine), exactly * `.`, or exactly `person.`. A wildcard is no entry and an unproven caller never matches. @@ -212,3 +230,32 @@ export function readDefinitionCallers(verb: string, value: unknown, cls?: string } return value as string[]; } + +/** + * The seat error kind a holder answers when a part it depends on is away (design 47 H2 §776; H-A-349 DQ-15): its + * answer's `error-kind`, beside `part` (what did not answer) and `words`; a caller reads it as a transport fault and + * calls again at the kept-call pace. The Go side's `busretry.ErrorKindAway`. + */ +export const ERROR_KIND_AWAY = "away"; + +/** A holder's answer of the kind away: `error` and `error-kind` as each seat's error answer has them, with `part` and `words`. */ +export interface AwayAnswer { + readonly error: string; + readonly "error-kind": typeof ERROR_KIND_AWAY; + readonly part: string; + readonly words: string; +} + +/** The answer of the kind away a holder gives when `part` does not answer (the Go side's `busretry.AwayAnswer`). */ +export function awayAnswer(part: string, words: string): AwayAnswer { + return { error: `away: ${part} does not answer: ${words}`, "error-kind": ERROR_KIND_AWAY, part, words }; +} + +/** The part and words a verb's answer says when its `error-kind` is away, else undefined (the Go side's `busretry.AwayIn`). */ +export function awayIn(answer: unknown): { part: string; words: string } | undefined { + if (!answer || typeof answer !== "object" || Array.isArray(answer)) return undefined; + const a = answer as Record; + if (a["error-kind"] !== ERROR_KIND_AWAY) return undefined; + const str = (v: unknown) => (typeof v === "string" ? v : ""); + return { part: str(a.part), words: str(a.words) || str(a.error) }; +} diff --git a/src/index.ts b/src/index.ts index e07f2aa2..9b5ca520 100644 --- a/src/index.ts +++ b/src/index.ts @@ -16,3 +16,4 @@ export * as state from "./state/index.js"; export * as runtime from "./runtime/index.js"; export * as callers from "./callers/index.js"; export * as views from "./views/index.js"; +export * as seal from "./seal/index.js"; diff --git a/src/seal/index.ts b/src/seal/index.ts new file mode 100644 index 00000000..b1e3e28f --- /dev/null +++ b/src/seal/index.ts @@ -0,0 +1,181 @@ +// The mesh's sealed box in TypeScript (novox/hq ADR 0183, ADR 0206; design 47 H2, lead answer H-A-407), the same +// box as the Go package seal (go/seal) byte for byte: a value sealed to one recipient's public key, which only that +// recipient's private key opens. X25519 for the agreement, HKDF-SHA256 for the key, AES-256-GCM for the box. +// +// A box is `{ v: 1, eph, iv, tag, ct }`, every field standard base64; `eph` is the one-time public key as SPKI DER. +// The key is derived with the salt eph-DER ‖ the recipient's raw public key and the info "novox-mesh sealed box v1", +// so a box cannot be re-addressed to another key. Keys travel as PEM: the public key as SPKI ("PUBLIC KEY"), the +// private key as PKCS #8 ("PRIVATE KEY"). Node's crypto only. +// +// sealAnonymous is the other seal, the NaCl anonymous box (libsodium's crypto_box_seal) the vault's intake opens: +// X25519 and XSalsa20-Poly1305 under a one-time key, the nonce BLAKE2b-24 of the one-time public key ‖ the recipient's, +// the box the one-time public key ‖ the sealed bytes. From two pinned libraries, tweetnacl and blakejs (H-A-407). + +import { + createCipheriv, + createDecipheriv, + createPrivateKey, + createPublicKey, + diffieHellman, + generateKeyPairSync, + hkdfSync, + randomBytes, + type KeyObject, +} from "node:crypto"; +import nacl from "tweetnacl"; +import blakejs from "blakejs"; + +/** A value sealed to one recipient, as it travels in JSON (go/seal's Box). */ +export interface Box { + v: number; + eph: string; + iv: string; + tag: string; + ct: string; +} + +/** A recipient's keypair as the two PEM strings it is kept and sent as (go/seal's KeyPair). */ +export interface KeyPair { + publicKey: string; + privateKey: string; +} + +/** The only box version open opens. */ +export const VERSION = 1; + +const INFO = "novox-mesh sealed box v1"; + +// PKCS #8 DER of an X25519 private key is this prefix and the key's 32 raw bytes (RFC 8410). +const X25519_PKCS8_PREFIX = Buffer.from("302e020100300506032b656e04220420", "hex"); + +/** A fresh X25519 keypair, PEM-encoded. */ +export function generateKeyPair(): KeyPair { + const { publicKey, privateKey } = generateKeyPairSync("x25519"); + return { + publicKey: publicKey.export({ type: "spki", format: "pem" }).toString(), + privateKey: privateKey.export({ type: "pkcs8", format: "pem" }).toString(), + }; +} + +/** Seals plaintext to the recipient's PEM public key, with a fresh one-time key and iv. */ +export function seal(plaintext: string, recipientPEM: string): Box { + const eph = generateKeyPairSync("x25519").privateKey; + return sealWith(plaintext, recipientPEM, eph, randomBytes(12)); +} + +/** + * sealWith is seal with its two sources of randomness given: the one-time key (a KeyObject, or its 32 raw bytes) and + * the iv. Exported for the byte-for-byte tests against the Go package's vectors; a caller seals with seal. + */ +export function sealWith(plaintext: string, recipientPEM: string, eph: KeyObject | Uint8Array, iv: Uint8Array): Box { + const recipient = publicFromPEM(recipientPEM); + const ephKey = eph instanceof Uint8Array ? privateFromRaw(eph) : eph; + const secret = diffieHellman({ privateKey: ephKey, publicKey: recipient }); + const ephDER = createPublicKey(ephKey).export({ type: "spki", format: "der" }); + const cipher = createCipheriv("aes-256-gcm", boxKey(secret, ephDER, rawPublic(recipient)), iv); + const ct = Buffer.concat([cipher.update(plaintext, "utf8"), cipher.final()]); + const b64 = (b: Uint8Array) => Buffer.from(b).toString("base64"); + return { v: VERSION, eph: b64(ephDER), iv: b64(iv), tag: b64(cipher.getAuthTag()), ct: b64(ct) }; +} + +/** + * Opens a box with the recipient's PEM private key. It throws for a box sealed to another key, one changed in any + * field, or one of another version. + */ +export function open(box: Box, privatePEM: string): string { + if (box.v !== VERSION) { + throw new Error(`a sealed box of version ${box.v}, not ${VERSION}`); + } + const priv = privateFromPEM(privatePEM); + const ephDER = Buffer.from(box.eph, "base64"); + let eph: KeyObject; + try { + eph = createPublicKey({ key: ephDER, format: "der", type: "spki" }); + } catch (e) { + throw new Error(`the box's eph: ${(e as Error).message}`); + } + if (eph.asymmetricKeyType !== "x25519") { + throw new Error("the box's eph is not an X25519 key"); + } + const secret = diffieHellman({ privateKey: priv, publicKey: eph }); + const iv = Buffer.from(box.iv, "base64"); + const tag = Buffer.from(box.tag, "base64"); + if (iv.length !== 12 || tag.length !== 16) { + throw new Error("the box does not open with this key"); + } + try { + const decipher = createDecipheriv("aes-256-gcm", boxKey(secret, ephDER, rawPublic(createPublicKey(priv))), iv); + decipher.setAuthTag(tag); + return Buffer.concat([decipher.update(Buffer.from(box.ct, "base64")), decipher.final()]).toString("utf8"); + } catch { + throw new Error("the box does not open with this key"); + } +} + +function publicFromPEM(pem: string): KeyObject { + let k: KeyObject; + try { + k = createPublicKey(pem); + } catch { + throw new Error("not a PEM public key"); + } + if (k.asymmetricKeyType !== "x25519") { + throw new Error("not an X25519 public key"); + } + return k; +} + +function privateFromPEM(pem: string): KeyObject { + let k: KeyObject; + try { + k = createPrivateKey(pem); + } catch { + throw new Error("not a PEM private key"); + } + if (k.asymmetricKeyType !== "x25519") { + throw new Error("not an X25519 private key"); + } + return k; +} + +function privateFromRaw(raw: Uint8Array): KeyObject { + if (raw.length !== 32) { + throw new Error("an X25519 private key is 32 bytes"); + } + return createPrivateKey({ key: Buffer.concat([X25519_PKCS8_PREFIX, raw]), format: "der", type: "pkcs8" }); +} + +// rawPublic is an X25519 public key's 32 raw bytes: the end of its SPKI DER. +function rawPublic(k: KeyObject): Buffer { + const der = k.export({ type: "spki", format: "der" }); + return der.subarray(der.length - 32); +} + +// boxKey is HKDF-SHA256 (RFC 5869) for one 32-byte block, salted with eph-DER ‖ the recipient's raw public key. +function boxKey(secret: Uint8Array, ephDER: Uint8Array, recipientRaw: Uint8Array): Buffer { + return Buffer.from(hkdfSync("sha256", secret, Buffer.concat([ephDER, recipientRaw]), INFO, 32)); +} + +/** + * Seals value to sealTo, a raw X25519 public key in standard base64 (the vault's `seal_to`), in the NaCl anonymous + * box, answered in standard base64: what golang.org/x/crypto/nacl/box's SealAnonymous makes and OpenAnonymous opens. + */ +export function sealAnonymous(value: string | Uint8Array, sealTo: string): string { + return sealAnonymousWith(value, sealTo, nacl.randomBytes(nacl.box.secretKeyLength)); +} + +/** + * sealAnonymousWith is sealAnonymous with its one-time private key given (32 bytes). Exported for the byte-for-byte + * tests against the Go package's bytes; a caller seals with sealAnonymous. + */ +export function sealAnonymousWith(value: string | Uint8Array, sealTo: string, ephSecret: Uint8Array): string { + const recipient = new Uint8Array(Buffer.from(sealTo.trim(), "base64")); + if (recipient.length !== nacl.box.publicKeyLength) { + throw new Error("the key to seal to is not a raw X25519 public key in base64"); + } + const eph = nacl.box.keyPair.fromSecretKey(new Uint8Array(ephSecret)); + const nonce = blakejs.blake2b(Buffer.concat([eph.publicKey, recipient]), undefined, nacl.box.nonceLength); + const message = typeof value === "string" ? new Uint8Array(Buffer.from(value, "utf8")) : new Uint8Array(value); + const sealed = nacl.box(message, nonce, recipient, eph.secretKey); + return Buffer.concat([eph.publicKey, sealed]).toString("base64"); +} diff --git a/src/state/index.ts b/src/state/index.ts index cf7c16f3..6cdd6a1e 100644 --- a/src/state/index.ts +++ b/src/state/index.ts @@ -133,6 +133,50 @@ function withinOneCall(key: string, value: unknown): void { } } +/** The state a holder of a kept call keeps the ids of the asks it took in (design 47 H2 §775, §820): Go's `stdio.TakenState`. */ +export const TAKEN_STATE = "taken"; + +/** The key an ask taken from `seat` is kept under in TAKEN_STATE: `.`. */ +export function takenKey(seat: string, ask: string): string { + return `${seat}.${ask}`; +} + +/** The code of the runtime's refusal of a create whose key has a value (Go's `stdio.CodeChanged`). */ +const CODE_CHANGED = -32010; + +/** + * Whether this holder takes the ask `ask` of `seat` now (design 47 H2 §775; H-A-359), as Go's `KeptState.TakeOnce`: the + * first time it is given it is created under takenKey with the ask's bound as its time to live (the runtime's + * `mesh/state.create`) and this resolves true; given again within the bound, false, and the holder acts once. + * `stateName` is the holder's TAKEN_STATE, declared `key-ttl`. + */ +export async function takeOnce(seat: string, ask: string, boundMs: number, stateName = TAKEN_STATE): Promise { + if (!ask || /[.*> ]/.test(ask)) throw new Error(`taking an ask of ${seat}: ${JSON.stringify(ask)} is not an ask id: an ask id holds no dot and no wildcard, naming one key and no other`); + if (!seat || /[.*> ]/.test(seat)) throw new Error(`taking the ask ${ask}: ${JSON.stringify(seat)} is not a seat's name`); + refusedUnannounced(stateName); + try { + await boundChannel().ask("mesh/state.create", { + state: stateName, + key: takenKey(seat, ask), + value: { seat, ask }, + ttl_seconds: Math.max(1, Math.floor(boundMs / 1000)), + }); + return true; + } catch (e) { + if ((e as { code?: number }).code === CODE_CHANGED) return false; + throw e; + } +} + +/** + * Remove what the deleted object `name` left behind in the object store `objects` (by its local name), its chunks, + * through the runtime's `mesh/objects.delete`, which purges them again when asked again (H-A-366): the Go side's `busstate.Objects.Purge`. A bundle never opens a + * connection of its own for it. + */ +export async function purgeObject(objects: string, name: string): Promise { + await boundChannel().ask("mesh/objects.delete", { objects, name }); +} + /** A state this module keeps (by its local name) or reads (`.`). */ export function state(name: string): State { const bound = (): StateChannel => { diff --git a/test/retire.test.ts b/test/retire.test.ts index 4b6f4e71..2a5d4391 100644 --- a/test/retire.test.ts +++ b/test/retire.test.ts @@ -154,7 +154,9 @@ test("asked for again, a retired consumer is created again and announced re-enab } }); -test("five passes are not enough: the same set must also hold for stableForMs", async () => { +test("five passes are not enough: the same set must also hold for stableForMs, ten minutes by default", async () => { + const { STABLE_FOR_MS } = await import("../dist/provider-loop/index.js"); + assert.equal(STABLE_FOR_MS, 10 * 60_000); const HOLD = 400; const w = await world(["a", "b", "c"], { stableForMs: HOLD }); try { @@ -170,11 +172,6 @@ test("five passes are not enough: the same set must also hold for stableForMs", } }); -test("the default hold is ten minutes", async () => { - const { STABLE_FOR_MS } = await import("../dist/provider-loop/index.js"); - assert.equal(STABLE_FOR_MS, 10 * 60_000); -}); - test("an adapter without retire is MARK-ONLY: nothing changes in the backend, remove only on delete", async () => { const w = await world(["a", "b", "c"], { retire: false, del: false }); try { diff --git a/test/sdk.test.ts b/test/sdk.test.ts index 6127376e..411c3363 100644 --- a/test/sdk.test.ts +++ b/test/sdk.test.ts @@ -670,7 +670,7 @@ test("the provider loop is named for what it is, and its old name stays an alias await newTest6(); // a TypeScript bundle's value over one call's limit fails at once, namin }); -import { allows, refusal, parseCallers, parseVerbCallers, parseConfiguredVerbCallers, readDefinitionCallers, moduleOf, UNPROVEN } from "../dist/callers/index.js"; +import { allows, refusal, parseCallers, parseVerbCallers, parseConfiguredVerbCallers, readDefinitionCallers, moduleOf, UNPROVEN, runAs, askIdOf, META_ASK, awayAnswer, awayIn } from "../dist/callers/index.js"; import { Follower, statusTools, keepStatus, statusKey, viewWaitsFinding, type Answer, type Status } from "../dist/views/index.js"; // callers and verb callers judge a proved caller by the Go side's rules (ADR 0459 §2, ADR 0627 §2) @@ -698,6 +698,17 @@ async function newTest1(): Promise { for (const [cls, want] of byClass) assert.deepEqual(readDefinitionCallers("list", undefined, cls), want, String(cls)); assert.deepEqual(readDefinitionCallers("register", ["seat:mesh-broker"], "act"), ["seat:mesh-broker"]); assert.throws(() => parseCallers('{"callers":["a"],"more":1}'), /unknown field "more"/); + // design 47 H2 §775, H-A-351: the ask id a kept call carried is the runtime's to name under mesh/ask, never the arguments' + for (const [meta, args, want] of [[{ [META_ASK]: "build-7Hq2" }, {}, "build-7Hq2"], [{}, { [META_ASK]: "forged" }, ""], [{ [META_ASK]: "a.b" }, {}, ""]] as const) { + const a = { ...args }; + assert.equal(await runAs(meta, a, async () => askIdOf(a)), want, JSON.stringify(meta)); + assert.equal(askIdOf(a), "", "no ask once the call ended"); + } + // design 47 H2 §776, DQ-15: the kind away is answered and read as the Go side does + assert.deepEqual(awayIn(JSON.parse(JSON.stringify(awayAnswer("the forge's store", "401")))), { part: "the forge's store", words: "401" }); + assert.equal(awayAnswer("p", "w")["error-kind"], "away"); + assert.equal(awayIn({ error: "no", "error-kind": "refused" }), undefined); + assert.deepEqual(awayIn({ error: "the store is down", "error-kind": "away" }), { part: "", words: "the store is down" }); const verbs = parseVerbCallers('{"pull":["*"],"merge-pull":["mesh-delivery-forge-bridge"],"comment":["operator"]}', ["pull", "merge-pull", "comment"], {}); assert.equal(refusal(verbs, "pull", "ace.anyone"), undefined); @@ -1023,4 +1034,27 @@ async function newTest5(): Promise { async function newTest6(): Promise { const { state, CHUNK_BYTES } = await import("../dist/state/index.js"); await assert.rejects(state("s").put("k", { text: "x".repeat(CHUNK_BYTES) }), /512 KiB/); + // design 47 H2 §775, H-A-359: a holder given the same ask id twice acts once + const { takeOnce, useStateChannel } = await import("../dist/state/index.js"); + const created = new Set(); + const asked: unknown[] = []; + useStateChannel({ + ask: async (method: string, params: unknown) => { + asked.push([method, params]); + const p = params as { key: string }; + if (created.has(p.key)) throw Object.assign(new Error("exists"), { code: -32010 }); + created.add(p.key); + return { revision: 1 }; + }, + watch: async () => {}, + } as never); + assert.equal(await takeOnce("git", "a-7", 600_000), true); + assert.equal(await takeOnce("git", "a-7", 600_000), false); + assert.deepEqual(asked[0], ["mesh/state.create", { state: "taken", key: "git.a-7", value: { seat: "git", ask: "a-7" }, ttl_seconds: 600 }]); + await assert.rejects(takeOnce("git", "", 1000), /is not an ask id/); + // H-A-366: a bundle purges a deleted object's leftovers through the runtime + const { purgeObject } = await import("../dist/state/index.js"); + asked.length = 0; + await purgeObject("artifacts", "pkg-1.tgz"); + assert.deepEqual(asked[0], ["mesh/objects.delete", { objects: "artifacts", name: "pkg-1.tgz" }]); } diff --git a/test/seal.test.ts b/test/seal.test.ts new file mode 100644 index 00000000..9f3e1322 --- /dev/null +++ b/test/seal.test.ts @@ -0,0 +1,49 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import { join } from "node:path"; + +import { seal, open, sealWith, sealAnonymous, sealAnonymousWith, generateKeyPair, type Box } from "../dist/seal/index.js"; + +// The seals are proven against the Go package's bytes (design 47 H2, H-A-407): the fixtures go/seal's tests make and +// check on every run. +const testdata = join(import.meta.dirname, "..", "go", "seal", "testdata"); +const fixture = (name: string) => JSON.parse(readFileSync(join(testdata, name), "utf8")); + +// ADR 0206's box: every box Go's seal opens (the licence manager's, the agent module's TypeScript) opens here; given +// the same one-time key and iv, seal makes Go's box byte for byte, so go/seal's Open opens it; another key and a +// changed field are refused. +test("seal and open make and open go/seal's box, and sealAnonymous Go's anonymous box, byte for byte", () => { + const f = fixture("sealed-by-licence-manager.json"); + for (const s of f.sealed as { plaintext: string; box: Box }[]) { + assert.equal(open(s.box, f.recipient.privateKey), s.plaintext); + } + for (const v of f.vectors as { plaintext: string; ephPrivate: string; iv: string; box: Box }[]) { + const got = sealWith(v.plaintext, f.recipient.publicKey, Buffer.from(v.ephPrivate, "base64"), Buffer.from(v.iv, "base64")); + assert.deepEqual(got, v.box); + } + const ts = fixture("sealed-by-typescript.json"); + assert.equal(open(ts.box, ts.privateKey), ts.plaintext); + + const a = generateKeyPair(); + const b = generateKeyPair(); + const box = seal("a token", a.publicKey); + assert.equal(open(box, a.privateKey), "a token"); + assert.notDeepEqual(seal("a token", a.publicKey), box); + assert.throws(() => open(box, b.privateKey), /does not open with this key/); + assert.throws(() => open({ ...box, ct: Buffer.from("changed").toString("base64") }, a.privateKey), /does not open/); + assert.throws(() => open({ ...box, v: 2 }, a.privateKey), /version 2/); + + // The NaCl anonymous box the vault's intake opens: given the same one-time key, sealAnonymous makes what + // golang.org/x/crypto/nacl/box's SealAnonymous made, byte for byte; a fresh one differs and is as long. + const an = fixture("sealed-anonymous-by-go.json"); + assert.ok(an.vectors.length > 0); + for (const v of an.vectors as { plaintext: string; ephSecret: string; sealed: string }[]) { + assert.equal(sealAnonymousWith(v.plaintext, an.recipient.public, Buffer.from(v.ephSecret, "base64")), v.sealed); + assert.equal(sealAnonymousWith(Buffer.from(v.plaintext, "utf8"), an.recipient.public, Buffer.from(v.ephSecret, "base64")), v.sealed); + } + const fresh = sealAnonymous("a token", an.recipient.public); + assert.notEqual(fresh, sealAnonymous("a token", an.recipient.public)); + assert.equal(Buffer.from(fresh, "base64").length, 32 + 16 + "a token".length); + assert.throws(() => sealAnonymous("x", "c2hvcnQ="), /not a raw X25519 public key/); +}); diff --git a/test/standing.test.ts b/test/standing.test.ts index 658ec6b4..90bc3610 100644 --- a/test/standing.test.ts +++ b/test/standing.test.ts @@ -224,7 +224,7 @@ test("a failing consumer the mesh stopped asking for is said recovered when it i } }); -test("an error is classed by its words", () => { +test("an error is classed by its words, and an adapter that classes its own errors is believed", async () => { const table: Record = { 'Keycloak token request failed: 401 {"error":"invalid_grant"}': CLASS_CREDENTIALS, 'FATAL: password authentication failed for user "postgres"': CLASS_CREDENTIALS, @@ -235,9 +235,7 @@ test("an error is classed by its words", () => { 'extension "nope" is not available': CLASS_REFUSED, }; for (const [text, want] of Object.entries(table)) assert.equal(classOf(text), want, text); -}); -test("an adapter that classes its own errors is believed", async () => { const w = await world({ classify: true }); try { w.failing = new Error("anything");