From c5a593eb05bb990cdfaa2e14b63356c8c6c6bad6 Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 21 Nov 2026 23:04:31 +0100 Subject: [PATCH 01/54] N1: the bus module's nats_consumers lists consumers by delivered-rate or created MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The flow per consumer is measured within one call, so a busy bus says with what. Builds: design 47 H2 §813 (5), §819 --- modules/nats/cmd/nats-tools/consumer.go | 100 ++++++++++++++++++- modules/nats/cmd/nats-tools/consumer_test.go | 40 +++++++- modules/nats/module.json | 1 + 3 files changed, 139 insertions(+), 2 deletions(-) diff --git a/modules/nats/cmd/nats-tools/consumer.go b/modules/nats/cmd/nats-tools/consumer.go index ccaa14555..bb37e3d42 100644 --- a/modules/nats/cmd/nats-tools/consumer.go +++ b/modules/nats/cmd/nats-tools/consumer.go @@ -11,6 +11,7 @@ import ( "encoding/json" "errors" "fmt" + "sort" "strings" "time" @@ -254,7 +255,7 @@ func decodeStored(b []byte) (*Stored, error) { return &Stored{Seq: doc.Message.Seq, Subject: doc.Message.Subject, Time: doc.Message.Time, Data: doc.Message.Data}, nil } -// readTools are the two reads that locate a lost event. +// readTools are the reads that locate a lost event, and the consumers' flow. func readTools(jsz func() ([]byte, error), open MsgOpen) []stdio.Tool { return []stdio.Tool{ {Name: "nats_consumer", @@ -270,6 +271,7 @@ func readTools(jsz func() ([]byte, error), open MsgOpen) []stdio.Tool { } return Consumer(doc, arg(a, "stream"), arg(a, "consumer")) }}, + flowTool(jsz, time.Sleep), {Name: "nats_stream_messages", Description: "The messages a stream keeps for one subject (a filter with * or > is allowed), newest first: sequence, subject, time and size of each, and the body only when asked for, no larger than 4 KiB, and only on the stream EVENTS (every kept event is there); on every other stream (kept events, asks, calls and replies, mail, dead letters, records, logs) only sequence, subject, time and size. Bounded by count (default 20, at most 200) and bytes (default 1 MiB); says when it stopped before the oldest the stream keeps. Read on the bus module's own credential. Refuses the streams of a module's kept state (KV_*, OBJ_*): their values are read through their owners' tools; this tool is for lost events.", Input: map[string]any{"stream": str("the stream, e.g. EVENTS"), "subject": str("a subject or filter, e.g. mesh.mod.mesh-delivery-indexer.event.view-changed"), @@ -313,3 +315,99 @@ func readTools(jsz func() ([]byte, error), open MsgOpen) []stdio.Tool { }}, } } + +// defaultRateWindow is how long nats_consumers measures each consumer's deliveries when the call names no rate-window +// (design 47 H2 §813 (5)). +const defaultRateWindow = 10 * time.Second + +// ConsumerFlow is one consumer's flow: how many messages a second it was delivered over the window, and when it was made. +type ConsumerFlow struct { + Stream string `json:"stream"` + Consumer string `json:"consumer"` + Created string `json:"created"` + DeliveredRate float64 `json:"delivered-rate"` + Delivered int64 `json:"delivered"` + Pending int64 `json:"pending"` + Unacknowledged int64 `json:"unacknowledged"` +} + +// ConsumerFlows is every consumer of every stream, its delivered-rate the deliveries between the two monitoring documents +// before and after, read window apart, a second; sorted by delivered-rate, the fastest first, or by created, the newest +// first (design 47 H2 §813 (5)). A consumer the second document does not hold is left out: it was removed meanwhile. +func ConsumerFlows(before, after []byte, window time.Duration, by string) (map[string]any, error) { + if by == "" { + by = "delivered-rate" + } + if by != "delivered-rate" && by != "created" { + return nil, fmt.Errorf("sort is delivered-rate or created, not %q", by) + } + var a, b jszDoc + if err := json.Unmarshal(before, &b); err != nil { + return nil, err + } + if err := json.Unmarshal(after, &a); err != nil { + return nil, err + } + was := map[string]int64{} + for _, acc := range b.Accounts { + for _, s := range acc.Streams { + for _, c := range s.Consumers { + was[s.Name+"\x00"+c.Name] = c.Delivered.Consumer + } + } + } + rows := []ConsumerFlow{} + for _, acc := range a.Accounts { + for _, s := range acc.Streams { + for _, c := range s.Consumers { + rate := 0.0 + if start, ok := was[s.Name+"\x00"+c.Name]; ok && window > 0 && c.Delivered.Consumer > start { + rate = float64(c.Delivered.Consumer-start) / window.Seconds() + } + rows = append(rows, ConsumerFlow{Stream: s.Name, Consumer: c.Name, Created: c.Created, DeliveredRate: rate, + Delivered: c.Delivered.Consumer, Pending: c.NumPending, Unacknowledged: c.NumAckPending}) + } + } + } + sort.SliceStable(rows, func(i, j int) bool { + if by == "created" { + return createdAt(rows[i].Created).After(createdAt(rows[j].Created)) + } + return rows[i].DeliveredRate > rows[j].DeliveredRate + }) + return map[string]any{"sort": by, "rate-window-seconds": window.Seconds(), "consumers": rows}, nil +} + +func createdAt(s string) time.Time { t, _ := time.Parse(time.RFC3339Nano, s); return t } + +// flowTool is nats_consumers: the monitoring document read twice, rate-window apart, within the one call; wait is how the +// window passes (time.Sleep live, a stand-in in a test). +func flowTool(jsz func() ([]byte, error), wait func(time.Duration)) stdio.Tool { + return stdio.Tool{Name: "nats_consumers", + Description: "Every consumer of every stream with its flow: delivered-rate (messages delivered a second, measured over rate-window within this one call), delivered, pending and unacknowledged, and when it was created; sorted by delivered-rate (the fastest first, the default) or by created (the newest first). The question behind a bus that is busy and no one knows with what.", + Input: map[string]any{"sort": str("delivered-rate (default) or created"), + "rate-window": num("seconds the deliveries are measured over, within this call (default 10)")}, + Run: func(a map[string]any) (any, error) { + window := defaultRateWindow + if v, ok := a["rate-window"].(float64); ok { + if v <= 0 { + return nil, errors.New("rate-window is a number of seconds above 0") + } + window = time.Duration(v * float64(time.Second)) + } + by := arg(a, "sort") + if by != "" && by != "delivered-rate" && by != "created" { + return nil, fmt.Errorf("sort is delivered-rate or created, not %q", by) + } + before, err := jsz() + if err != nil { + return nil, err + } + wait(window) + after, err := jsz() + if err != nil { + return nil, err + } + return ConsumerFlows(before, after, window, by) + }} +} diff --git a/modules/nats/cmd/nats-tools/consumer_test.go b/modules/nats/cmd/nats-tools/consumer_test.go index b8adf93d5..e0ca69ccf 100644 --- a/modules/nats/cmd/nats-tools/consumer_test.go +++ b/modules/nats/cmd/nats-tools/consumer_test.go @@ -47,6 +47,42 @@ func TestAConsumerIsReadWithItsConfigAndItsState(t *testing.T) { t.Fatalf("%v answered", c) } } + t.Run("nats_consumers measures the flow within the call and sorts it", consumersFlow) +} + +// consumersFlow: design 47 H2 §813 (5). Two consumers, the older delivered 40 messages over the 4 s window, the newer +// 4; the window passes through the stand-in wait, never a real sleep. +func consumersFlow(t *testing.T) { + doc := func(oldSeq, newSeq int) []byte { + return []byte(fmt.Sprintf(`{"account_details":[{"stream_detail":[{"name":"EVENTS","consumer_detail":[ + {"name":"old","created":"2026-11-01T10:00:00Z","delivered":{"consumer_seq":%d},"num_pending":7}, + {"name":"new","created":"2026-11-21T10:00:00Z","delivered":{"consumer_seq":%d}}]}]}]}`, oldSeq, newSeq)) + } + reads, waited := 0, time.Duration(0) + tool := flowTool(func() ([]byte, error) { + reads++ + if reads%2 == 1 { + return doc(100, 10), nil + } + return doc(140, 14), nil + }, func(d time.Duration) { waited += d }) + out, err := tool.Run(map[string]any{"rate-window": float64(4)}) + if err != nil { + t.Fatal(err) + } + rows := out.(map[string]any)["consumers"].([]ConsumerFlow) + if waited != 4*time.Second || len(rows) != 2 || rows[0].Consumer != "old" || rows[0].DeliveredRate != 10 || rows[1].DeliveredRate != 1 || rows[0].Pending != 7 { + t.Fatalf("waited %s: %+v", waited, rows) + } + out, _ = tool.Run(map[string]any{"sort": "created"}) + if rows := out.(map[string]any)["consumers"].([]ConsumerFlow); waited != 14*time.Second || rows[0].Consumer != "new" { + t.Fatalf("the default window or the sort by created: waited %s, %+v", waited, rows) + } + for _, bad := range []map[string]any{{"sort": "name"}, {"rate-window": float64(0)}} { + if _, err := tool.Run(bad); err == nil { + t.Fatalf("%v answered", bad) + } + } } func TestMessagesComeNewestFirstBoundedAndBodiesOnlyWhenAsked(t *testing.T) { @@ -82,9 +118,11 @@ func TestMessagesComeNewestFirstBoundedAndBodiesOnlyWhenAsked(t *testing.T) { if r := got["messages"].([]MessageRow)[0]; r.Body != "" || r.BodyOmitted == "" { t.Fatalf("%+v", r) } + t.Run("messages scan back across blocks and say an unanswered read", messagesScanBack) } -func TestMessagesScanBackAcrossBlocksAndSayAnUnansweredRead(t *testing.T) { +// messagesScanBack is run by TestMessagesComeNewestFirstBoundedAndBodiesOnlyWhenAsked (design 47 H2 §819: fewer tests). +func messagesScanBack(t *testing.T) { ctx := context.Background() got, _ := NewestMessages(ctx, func(_ context.Context, _ string, seq uint64, _ string) (*Stored, error) { if seq <= 3 { diff --git a/modules/nats/module.json b/modules/nats/module.json index 75e59564f..105d9a30d 100644 --- a/modules/nats/module.json +++ b/modules/nats/module.json @@ -164,6 +164,7 @@ "nats_streams", "nats_backlog", "nats_consumer", + "nats_consumers", "nats_stream_messages", "nats_buckets", "nats_users", -- 2.54.0 From 1ad170705ee7a0c237ec3cc3b22cd3a7a7e24e10 Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 21 Nov 2026 23:06:43 +0100 Subject: [PATCH 02/54] N2: the keeper withdraws a refused drop's or recreate's ask on operator-channel MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A refused request leaves nothing for the operator to answer, so its ask is cancelled in the same pass. Builds: design 47 H2 §813 (6) --- modules/nats/cmd/nats-tools/keep.go | 3 +++ modules/nats/internal/keeper/keeper.go | 7 +++++-- modules/nats/internal/keeper/keeper_test.go | 20 ++++++++++++++++++++ modules/nats/internal/keeper/requests.go | 17 +++++++++++++++++ 4 files changed, 45 insertions(+), 2 deletions(-) diff --git a/modules/nats/cmd/nats-tools/keep.go b/modules/nats/cmd/nats-tools/keep.go index 366aacda6..1a946c07a 100644 --- a/modules/nats/cmd/nats-tools/keep.go +++ b/modules/nats/cmd/nats-tools/keep.go @@ -219,6 +219,9 @@ func runKeeper(ctx context.Context, getenv func(string) string, logOut io.Writer return err }, Warrant: WarrantFrom(conn.Ask), Finding: writer, + Cancel: func(id string) error { + return conn.Accept("operator-channel", "cancel", map[string]string{"id": id}, stdio.EmitOptions{MessageID: "cancel." + id}) + }, Logf: logger.Printf, Raise: percent(getenv, "MESH_NATS_FULL_RAISE", 75), Clear: percent(getenv, "MESH_NATS_FULL_CLEAR", 70), // The settings snapshots-cap (20 GiB) and snapshot-keep-days (90) of the manifest, read from the file it declares diff --git a/modules/nats/internal/keeper/keeper.go b/modules/nats/internal/keeper/keeper.go index 1cf484be7..41428e100 100644 --- a/modules/nats/internal/keeper/keeper.go +++ b/modules/nats/internal/keeper/keeper.go @@ -349,8 +349,11 @@ type Keeper struct { // Act records a hand act through the seat mesh-hand-acts; Warrant reads the router's record of an ask. Act func(what, why, cause string) error Warrant func(askID string) (*oc.Ask, *oc.Warrant, error) - Now func() time.Time - Logf func(string, ...any) + // Cancel takes an ask back on the seat operator-channel (ADR 0259 §3's cancel): a drop or recreate the keeper refuses at + // once withdraws the ask its tool raised, in the same pass (design 47 H2 §813 (6)). + Cancel func(askID string) error + Now func() time.Time + Logf func(string, ...any) // Raise and Clear are the fill marks in percent (ADR 0595 §6): settings of the bus module, 75 and 70 by default. Raise, Clear int64 // Files are the snapshot files (the disk when nil); SnapshotsCap the cap of their directory in bytes and diff --git a/modules/nats/internal/keeper/keeper_test.go b/modules/nats/internal/keeper/keeper_test.go index 76c8a5edc..176994029 100644 --- a/modules/nats/internal/keeper/keeper_test.go +++ b/modules/nats/internal/keeper/keeper_test.go @@ -846,6 +846,26 @@ func TestTheKeeperMakesWhatAVerdictAsksForAndSaysIt(t *testing.T) { if got.Done.IsZero() || got.Waiting != "" || !strings.Contains(got.Error, "refused") || !strings.Contains(got.Error, w.Words) || len(bus.deleted) != 0 { t.Errorf("ended %+v, deleted %v: the router's refusal and its words, nothing removed", got, bus.deleted) } + // Design 47 H2 §813 (6): the refused request withdraws its ask in the same pass; a cancel that fails is said. + var cancelled []string + k.Cancel = func(id string) error { + cancelled = append(cancelled, id) + if id == "bus-2" { + return errors.New("no responders") + } + return nil + } + for _, r := range []Request{{ID: "rc", Verb: VerbRecreate, Module: "m", Local: "drafts", Ask: "bus-1", Caller: "person.jo"}, + {ID: "rd", Verb: VerbDrop, Module: "m", Local: "drafts", Ask: "bus-2", Caller: "person.jo"}} { + raw, _ = json.Marshal(r) + _, _ = store.Put(StateRequests, r.ID, raw) + } + k.ActOnRequests() + raw, _, _, _ = store.Get(StateRequests, "rd") + _ = json.Unmarshal(raw, &got) + if fmt.Sprint(cancelled) != "[bus-1 bus-2]" || !strings.Contains(got.Error, "bus-2 could not be withdrawn: no responders") { + t.Errorf("cancelled %v, ended %+v: each refused request's ask withdrawn once, a failed cancel said", cancelled, got) + } }}, {"a drop waits for the warrant, refuses another act's, then acts after a snapshot", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) { store := k.Store.(*memStore) diff --git a/modules/nats/internal/keeper/requests.go b/modules/nats/internal/keeper/requests.go index 48b7a3f8d..69be4ae0a 100644 --- a/modules/nats/internal/keeper/requests.go +++ b/modules/nats/internal/keeper/requests.go @@ -169,6 +169,7 @@ func (k *Keeper) ActOnRequests() { r.Done, r.Outcome = k.now(), outcome if err != nil { r.Error = err.Error() + k.withdraw(&r) } data, _ := json.Marshal(r) if _, err := k.Store.Put(StateRequests, key, data); err != nil { @@ -367,3 +368,19 @@ func (k *Keeper) record(r *Request, what string) { k.logf("the hand act %q could not be recorded: %v", what, err) } } + +// withdraw takes back the operator's ask of a drop or recreate the keeper refused (design 47 H2 §813 (6)): nothing is left +// for the operator to answer once the request ended. A cancel that cannot be sent is said on the request's end. +func (k *Keeper) withdraw(r *Request) { + if r.Ask == "" || (r.Verb != VerbDrop && r.Verb != VerbRecreate) { + return + } + if k.Cancel == nil { + r.Error += "; its ask " + r.Ask + " is not withdrawn: the keeper holds no cancel" + return + } + if err := k.Cancel(r.Ask); err != nil { + k.logf("the ask %s of the refused request %s cannot be withdrawn: %v", r.Ask, r.ID, err) + r.Error += fmt.Sprintf("; its ask %s could not be withdrawn: %v", r.Ask, err) + } +} -- 2.54.0 From 51c27203de73891ea6cdadac01f1f66ffac35468 Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 21 Nov 2026 23:06:52 +0100 Subject: [PATCH 03/54] N3: the keeper's approve-level bend names the research that ends it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 47 H2 §813 (11) --- modules/nats/cmd/nats-tools/requests.go | 3 ++- modules/nats/internal/keeper/requests.go | 7 ++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/modules/nats/cmd/nats-tools/requests.go b/modules/nats/cmd/nats-tools/requests.go index 7c9d0d47e..b0902d809 100644 --- a/modules/nats/cmd/nats-tools/requests.go +++ b/modules/nats/cmd/nats-tools/requests.go @@ -173,7 +173,8 @@ func askExpiry(a map[string]any) (time.Duration, error) { // askFor is the operator's ask for a recreate or a drop of one entry, its approve option binding exactly that act: at // the approve level until a channel carries a destroy answer, ending at the request's AskExpires, the caller's (design -// 47 H-A-237). +// 47 H-A-237); the bend ends with the research effort that designs the channel carrying the destroy proof (design 47 H2 +// §813 (11)). func askFor(r keeper.Request) (oc.Ask, error) { binds, err := occ.ActDigest(r.Act()) if err != nil { diff --git a/modules/nats/internal/keeper/requests.go b/modules/nats/internal/keeper/requests.go index 69be4ae0a..0f1b08821 100644 --- a/modules/nats/internal/keeper/requests.go +++ b/modules/nats/internal/keeper/requests.go @@ -72,8 +72,8 @@ type Request struct { } // AskLasts is how long the operator's ask for a recreate or a drop waits for an answer when its caller names no expiry: -// sixty minutes (design 47 H-A-237: the ask is at the approve level until a channel carries a destroy answer, with the -// asker's expiry). It is shorter than the day the state requests keeps a key, so a request whose ask ended unanswered is +// sixty minutes (design 47 H-A-237: the ask is at the approve level, with the asker's expiry, a bend that ends with the +// research effort designing the channel that carries the destroy proof, design 47 H2 §813 (11)). It is shorter than the day the state requests keeps a key, so a request whose ask ended unanswered is // ended by the keeper, saying so, before its key ages out, and the warrant's acted key (ActedKey) outlives the ask it // spends. const AskLasts = 60 * time.Minute @@ -333,7 +333,8 @@ func (k *Keeper) warranted(r *Request) error { return fmt.Errorf("no warrant: %w", err) } // An approve-level choice warrants a recreate or a drop (design 47 H-A-237): no channel carries a destroy answer yet, - // so the ask is at the approve level; a destroy-level choice still warrants it. + // so the ask is at the approve level; a destroy-level choice still warrants it. The bend ends with the research effort + // that designs the channel carrying the destroy proof (design 47 H2 §813 (11)); this check then asks the destroy level. if occ.Rank(o.Level) < occ.Rank(occ.Approve) { return fmt.Errorf("the operator chose %s, so nothing is removed", o.Label) } -- 2.54.0 From 1cbe0b12947591bdfa661f9ce8186423260b80fe Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 21 Nov 2026 23:07:54 +0100 Subject: [PATCH 04/54] N1: nats_connections carries each connection's five busiest subscriptions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Also fixes the trunk race in the request test: the watch hands its handler over a channel. Builds: design 47 H2 §813 (5); H-A-356 --- modules/nats/cmd/nats-tools/main.go | 4 ++-- modules/nats/cmd/nats-tools/requests_test.go | 17 ++++---------- modules/nats/cmd/nats-tools/tools.go | 24 ++++++++++++++++++++ modules/nats/cmd/nats-tools/tools_test.go | 8 ++++++- 4 files changed, 38 insertions(+), 15 deletions(-) diff --git a/modules/nats/cmd/nats-tools/main.go b/modules/nats/cmd/nats-tools/main.go index d84305c97..ac2774ac3 100644 --- a/modules/nats/cmd/nats-tools/main.go +++ b/modules/nats/cmd/nats-tools/main.go @@ -57,10 +57,10 @@ func tools(monitor Monitor, users func(context.Context) ([]byte, error), written return Server(varz, health) }}, {Name: "nats_connections", - Description: "Every client connected to the bus, named by the mesh user it is (a machine's host `node.`, a machine's runtime `.node-tools`, the controller, a person): address, uptime, round trip, traffic and pending bytes. Narrowed by a user or connection name, sorted by out_bytes (default), in_bytes, pending, msgs or subscriptions.", + Description: "Every client connected to the bus, named by the mesh user it is (a machine's host `node.`, a machine's runtime `.node-tools`, the controller, a person): address, uptime, round trip, traffic and pending bytes, and its five busiest subscriptions by messages (subject, queue group, messages). Narrowed by a user or connection name, sorted by out_bytes (default), in_bytes, pending, msgs or subscriptions.", Input: map[string]any{"user": str("part of a user or connection name, e.g. g14"), "sort": str("out_bytes, in_bytes, pending, msgs or subscriptions"), "limit": num("at most this many (default 50)")}, Run: func(a map[string]any) (any, error) { - connz, err := get("/connz?auth=1&limit=1024") + connz, err := get("/connz?auth=1&limit=1024&subs=detail") if err != nil { return nil, err } diff --git a/modules/nats/cmd/nats-tools/requests_test.go b/modules/nats/cmd/nats-tools/requests_test.go index 006bb1cc8..b7c380fbb 100644 --- a/modules/nats/cmd/nats-tools/requests_test.go +++ b/modules/nats/cmd/nats-tools/requests_test.go @@ -2,7 +2,6 @@ package main import ( "errors" - "runtime" "strings" "testing" "time" @@ -133,20 +132,14 @@ func TestAPersonsRequestIsHandedToTheKeeperAsTheProvedCaller(t *testing.T) { t.Errorf("an ask not raised: %v, %d requests held (was %d)", err, len(held), before) } // A request's end is heard on the one watch of the state, handed to the call that waits for it. - var handle func(string, []byte) + // The watch hands its handler over a channel, never a variable both goroutines touch (the race on main, H-A-356); the + // call registered its wait before it took the watch, so the handler's arrival means the call waits. + handles := make(chan func(string, []byte), 1) watches := 0 - e := newEnds(func(h func(string, []byte)) error { watches++; handle = h; return nil }) + e := newEnds(func(h func(string, []byte)) error { watches++; handles <- h; return nil }) got := make(chan *keeper.Request, 1) go func() { got <- e.wait("r1", time.Minute) }() - for { - e.mu.Lock() - ready := len(e.waiting["r1"]) == 1 && handle != nil - e.mu.Unlock() - if ready { - break - } - runtime.Gosched() - } + handle := <-handles handle("r1", []byte(`{"id":"r1"}`)) handle("r1", []byte(`{"id":"r1","done":"2026-11-04T12:00:01Z","outcome":"created"}`)) if r := <-got; r == nil || r.Outcome != "created" || watches != 1 { diff --git a/modules/nats/cmd/nats-tools/tools.go b/modules/nats/cmd/nats-tools/tools.go index 8235f97de..60a3107e1 100644 --- a/modules/nats/cmd/nats-tools/tools.go +++ b/modules/nats/cmd/nats-tools/tools.go @@ -57,13 +57,30 @@ type Connection struct { OutBytes int64 `json:"out_bytes"` PendingBytes int64 `json:"pending_bytes"` Subscriptions int64 `json:"subscriptions"` + // Busiest is the connection's busiest subscriptions by messages, at most busiestKept (design 47 H2 §813 (5), H-A-356). + Busiest []ConnSubscription `json:"busiest_subscriptions,omitempty"` } +// ConnSubscription is one subscription of a connection: its subject, queue group and the messages it received. +type ConnSubscription struct { + Subject string `json:"subject"` + Queue string `json:"queue,omitempty"` + Msgs int64 `json:"msgs"` +} + +// busiestKept is how many of a connection's subscriptions it carries, the busiest by messages (H-A-356: five). +const busiestKept = 5 + func connectionsOf(connz []byte) ([]Connection, error) { var doc struct { Connections []struct { Connection AuthorizedUser string `json:"authorized_user"` + Subs []struct { + Subject string `json:"subject"` + Queue string `json:"qgroup"` + Msgs int64 `json:"msgs"` + } `json:"subscriptions_list_detail"` } `json:"connections"` } if err := json.Unmarshal(connz, &doc); err != nil { @@ -73,6 +90,13 @@ func connectionsOf(connz []byte) ([]Connection, error) { for _, c := range doc.Connections { conn := c.Connection conn.User = c.AuthorizedUser + for _, sub := range c.Subs { + conn.Busiest = append(conn.Busiest, ConnSubscription{Subject: sub.Subject, Queue: sub.Queue, Msgs: sub.Msgs}) + } + sort.SliceStable(conn.Busiest, func(i, j int) bool { return conn.Busiest[i].Msgs > conn.Busiest[j].Msgs }) + if len(conn.Busiest) > busiestKept { + conn.Busiest = conn.Busiest[:busiestKept] + } out = append(out, conn) } return out, nil diff --git a/modules/nats/cmd/nats-tools/tools_test.go b/modules/nats/cmd/nats-tools/tools_test.go index bc233d76e..ebbbe91ad 100644 --- a/modules/nats/cmd/nats-tools/tools_test.go +++ b/modules/nats/cmd/nats-tools/tools_test.go @@ -17,7 +17,9 @@ import ( const connz = `{"connections":[ {"cid":1,"ip":"192.0.2.1","name":"mesh-host/anchor","authorized_user":"node.anchor","out_bytes":10,"in_bytes":5,"pending_bytes":0,"subscriptions":2,"in_msgs":3,"out_msgs":4}, - {"cid":2,"ip":"192.0.2.2","name":"laptop.node-tools","authorized_user":"laptop.node-tools","out_bytes":900,"in_bytes":50,"pending_bytes":7,"subscriptions":300,"in_msgs":30,"out_msgs":40}]}` + {"cid":2,"ip":"192.0.2.2","name":"laptop.node-tools","authorized_user":"laptop.node-tools","out_bytes":900,"in_bytes":50,"pending_bytes":7,"subscriptions":300,"in_msgs":30,"out_msgs":40, + "subscriptions_list_detail":[{"subject":"a","msgs":1},{"subject":"b","qgroup":"q","msgs":60},{"subject":"c","msgs":5}, + {"subject":"d","msgs":40},{"subject":"e","msgs":3},{"subject":"f","msgs":2}]}]}` const subsz = `{"num_subscriptions":3,"subscriptions_list":[ {"subject":"$SRV.INFO","msgs":4,"cid":2}, @@ -59,6 +61,10 @@ func TestConnectionsAreNamedByTheirUserAndSorted(t *testing.T) { if len(cs) != 2 || cs[0].User != "laptop.node-tools" || cs[1].User != "node.anchor" { t.Fatalf("%+v", cs) } + // Design 47 H2 §813 (5), H-A-356: a connection's five busiest subscriptions by messages, its queue group named. + if b := cs[0].Busiest; fmt.Sprint(b) != "[{b q 60} {d 40} {c 5} {e 3} {f 2}]" || cs[1].Busiest != nil { + t.Fatalf("busiest %v, %v", b, cs[1].Busiest) + } narrowed, _ := Connections([]byte(connz), "anchor", "pending", 10) if n := narrowed["total"].(int); n != 1 { t.Fatalf("narrowed to %d", n) -- 2.54.0 From 0206ec825ec1f1f6ee8e4d48464e24f323095a03 Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 21 Nov 2026 23:14:43 +0100 Subject: [PATCH 05/54] VA1: the vault takes a module's own secret through own-put and answers it through get MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The forge module needs a place for its token that only its own processes read: own-put keeps it as module//, named by the caller's account, its newest generation only. Builds: design 47 H2 §822, §808; H-A-345, H-A-349 DQ-7, H-A-360 --- .../cmd/mesh-vault/licences_test.go | 58 ++++++++- .../cmd/mesh-vault/mint_integration_test.go | 41 +++++++ modules/mesh-vault/cmd/mesh-vault/ownput.go | 110 ++++++++++++++++++ modules/mesh-vault/cmd/mesh-vault/tools.go | 20 ++++ modules/mesh-vault/cmd/mesh-vault/vault.go | 8 ++ modules/mesh-vault/module.json | 3 +- 6 files changed, 238 insertions(+), 2 deletions(-) create mode 100644 modules/mesh-vault/cmd/mesh-vault/ownput.go diff --git a/modules/mesh-vault/cmd/mesh-vault/licences_test.go b/modules/mesh-vault/cmd/mesh-vault/licences_test.go index 6368f4572..0ffd6add6 100644 --- a/modules/mesh-vault/cmd/mesh-vault/licences_test.go +++ b/modules/mesh-vault/cmd/mesh-vault/licences_test.go @@ -2,6 +2,7 @@ package main import ( "encoding/json" + "fmt" "os" "strings" "testing" @@ -59,6 +60,61 @@ func TestLicencePutKeepsTheNewestGenerationAndRefusesAnOlder(t *testing.T) { if _, err := v.LicencePutOpen("", "jane", LicenceAccess, 1); err == nil { t.Error("licence-put took an unproven caller") } + t.Run("own-put keeps a module's own secret, handed to that module alone", ownPutKeepsAModulesOwn) + t.Run("secret_holders lists every record by name and fingerprint, never the value", secretHoldersByNameAndFingerprint) +} + +// ownPutKeepsAModulesOwn: design 47 H2 §822, H-A-360. A module puts a secret in two steps and reads it back through get +// by the full name own-put answered; another module's account and a non-module account are refused it; a second put +// keeps only the newer generation; the journal says each write, never the value. +func ownPutKeepsAModulesOwn(t *testing.T) { + v := newVault(t, nil) + var journal strings.Builder + v.Log = func(format string, args ...any) { fmt.Fprintf(&journal, format+"\n", args...) } + put := func(caller, name, value string) (Changed, error) { + t.Helper() + o, err := v.OwnPutOpen(caller, name) + if err != nil { + return Changed{}, err + } + if o.Secret != "module/gitea/admin-token" { + t.Fatalf("own-put names %q, want module/gitea/admin-token", o.Secret) + } + return v.OwnPutTake(caller, o.Call, sealToCall(t, o.SealTo, value)) + } + first, err := put("novox.gitea", "admin-token", "tok-1") + if err != nil || first.Name != "module/gitea/admin-token" { + t.Fatalf("%+v %v", first, err) + } + if got := handedTo(t, v, "ace.gitea", "module/gitea/admin-token"); got != "tok-1" { + t.Fatalf("handed %q to another process of the module, want tok-1", got) + } + second, err := put("ace.gitea", "admin-token", "tok-2") + if err != nil || second.Generation <= first.Generation { + t.Fatalf("second put %+v (%v) after generation %d", second, err, first.Generation) + } + rec, _, _ := v.Store.Read("module/gitea/admin-token") + if rec.Previous != nil || handedTo(t, v, "novox.gitea", "module/gitea/admin-token") != "tok-2" { + t.Fatalf("the newer generation alone is kept: %+v", rec) + } + key, _ := seal.GenerateKeyPair() + for _, other := range []string{"novox.mesh-config", "person.jo", "novox.node-tools", "provisioner.novox"} { + if _, err := v.Get(other, "module/gitea/admin-token", key.PublicKey); err == nil { + t.Errorf("%s was handed gitea's own secret", other) + } + } + for _, c := range []struct{ caller, name string }{{"person.jo", "x"}, {"novox.node-tools", "x"}, {"node.ace", "x"}, {"novox.gitea", ""}, {"novox.gitea", "A B"}} { + if _, err := v.OwnPutOpen(c.caller, c.name); err == nil { + t.Errorf("own-put opened for %s naming %q", c.caller, c.name) + } + } + o, _ := v.OwnPutOpen("novox.gitea", "admin-token") + if _, err := v.OwnPutTake("ace.gitea", o.Call, sealToCall(t, o.SealTo, "tok-3")); err == nil { + t.Error("a call opened by one caller was taken by another") + } + if all := journal.String() + everything(t, v); strings.Contains(all, "tok-") || !strings.Contains(all, "own-put: module/gitea/admin-token kept at generation") { + t.Errorf("the journal says each write without the value: %s", all) + } } // design 39 bus-and-switch §17/AC1, §16, B3, B8, B25: a licence's access token is refused to a machine no binding names @@ -121,7 +177,7 @@ func TestATokenIsGivenOnlyToAMachineABindingNames(t *testing.T) { // issue 1409, design 39 bus-and-switch §20, parts §12: secret_holders lists every record of the store by name and // fingerprint, a licence's two secrets with their key and part, a record it cannot open with an error, and no value. -func TestSecretHoldersListsEveryStoreRecordByNameAndFingerprintNeverTheValue(t *testing.T) { +func secretHoldersByNameAndFingerprint(t *testing.T) { v := newVault(t, nil) const manager = "claude-licence-manager" grant, access := `{"refreshToken":"rt-SECRET-grant"}`, `{"accessToken":"at-SECRET-access"}` diff --git a/modules/mesh-vault/cmd/mesh-vault/mint_integration_test.go b/modules/mesh-vault/cmd/mesh-vault/mint_integration_test.go index 63e9364c1..a14cd33bd 100644 --- a/modules/mesh-vault/cmd/mesh-vault/mint_integration_test.go +++ b/modules/mesh-vault/cmd/mesh-vault/mint_integration_test.go @@ -646,4 +646,45 @@ func TestTheVaultWholeMintsOnTheBusWhoeverHoldsTheLease(t *testing.T) { t.Errorf("a waiter on %s heard %s", other, extra.Subject) } } + // Design 47 H2 §822: a module puts its own secret in two steps through the seat and reads it back through get; a + // second module's get of that name is refused; a second put keeps the newer generation alone. + putOwn := func(caller, value string) Changed { + t.Helper() + raw, err := p.call(Seat+"."+OwnPutVerb, caller, map[string]any{"name": "admin-token"}) + if err != nil { + t.Fatal(err) + } + var o Opened + _ = json.Unmarshal(raw, &o) + raw, err = p.call(Seat+"."+OwnPutVerb, caller, map[string]any{"call": o.Call, "sealed": sealToCall(t, o.SealTo, value)}) + if err != nil { + t.Fatal(err) + } + var c Changed + _ = json.Unmarshal(raw, &c) + return c + } + getOwn := func(caller string) (string, error) { + key, _ := seal.GenerateKeyPair() + raw, err := p.call(Seat+".get", caller, map[string]any{"name": "module/gitea/admin-token", "public_key": key.PublicKey}) + if err != nil { + return "", err + } + var a Answer + _ = json.Unmarshal(raw, &a) + return seal.Open(a.Sealed, key.PrivateKey) + } + first := putOwn("novox.gitea", "tok-1") + if got, err := getOwn("ace.gitea"); err != nil || got != "tok-1" || first.Name != "module/gitea/admin-token" { + t.Fatalf("own-put %+v, get answered %q (%v)", first, got, err) + } + if _, err := getOwn("novox.mesh-config"); err == nil { + t.Error("another module was handed gitea's own secret") + } + if second := putOwn("ace.gitea", "tok-2"); second.Generation <= first.Generation { + t.Errorf("the second put kept generation %d after %d", second.Generation, first.Generation) + } + if got, _ := getOwn("novox.gitea"); got != "tok-2" { + t.Errorf("get answered %q after the second put, want tok-2", got) + } } diff --git a/modules/mesh-vault/cmd/mesh-vault/ownput.go b/modules/mesh-vault/cmd/mesh-vault/ownput.go new file mode 100644 index 000000000..ac5d00b9b --- /dev/null +++ b/modules/mesh-vault/cmd/mesh-vault/ownput.go @@ -0,0 +1,110 @@ +package main + +// A module's own secret, written by the module itself (design 47 H2 §822, H-A-345, H-A-349 DQ-7, H-A-360): `own-put` in +// two steps modelled on `licence-put` — name the secret and get a call and a key made for it, then send the call and the +// value sealed to that key. The vault keeps it as `module//`, the module named by the caller's account as +// the bus proved it (`.`, a bundle's call named so by the runtime), never by an argument, and keeps the +// newest generation only. `get` answers that full name to a caller of the same module alone, sealed to its key. Each +// write is said in the vault's journal, never the value. + +import ( + "errors" + "fmt" + "strings" + + stdio "git.novox.be/novox/mesh-sdk/go" +) + +// OwnPutVerb is the verb a module writes its own secret with. +const OwnPutVerb = "own-put" + +// modulePrefix starts the name of every secret a module wrote itself through own-put. +const modulePrefix = "module/" + +// callerModule is the module a proved caller is an account of, `.`, or why it is none: a person, a +// machine's provisioner client or node-engine, and the tool runner's own account answer no module's secret (H-A-360). +func callerModule(caller string) (string, error) { + machine, module, ok := strings.Cut(caller, ".") + switch { + case caller == "" || caller == stdio.Unproven: + return "", errors.New("a module's own secret answers a caller the runtime proves, and it proved none") + case !ok || machine == "" || module == "" || strings.Contains(module, "."), + machine == "person", machine == "node", machine+"." == ProvisionerPrefix, + module == toolRunnerModule: + return "", fmt.Errorf("%s is no module's account (.), so it has no own secret here", caller) + } + return stdio.ModuleOf(caller), nil +} + +// ModuleSecret is the vault's name of module's own secret name: `module//`. +func ModuleSecret(module, name string) (string, error) { + full := modulePrefix + module + "/" + name + if strings.TrimSpace(name) == "" { + return "", errors.New("own-put names the secret; nothing is taken") + } + if err := validName(full); err != nil { + return "", err + } + return full, nil +} + +// moduleSecretRefusal says whether name is a module's own secret written through own-put and, when it is, why caller +// may not have it ("" when caller is an account of that module). +func moduleSecretRefusal(caller, name string) (bool, string) { + rest, ok := strings.CutPrefix(name, modulePrefix) + if !ok { + return false, "" + } + owner, _, _ := strings.Cut(rest, "/") + module, err := callerModule(caller) + if err != nil { + return true, err.Error() + } + if module != owner { + return true, fmt.Sprintf("%s is %s's own secret, handed only to an account of %s", name, owner, owner) + } + return true, "" +} + +// OwnPutOpen opens a call to take a secret of the caller's module. +func (v *Vault) OwnPutOpen(caller, name string) (Opened, error) { + module, err := callerModule(caller) + if err != nil { + return Opened{}, err + } + secret, err := ModuleSecret(module, name) + if err != nil { + return Opened{}, err + } + id, key, err := v.openCall(OwnPutVerb, secret, caller, nil) + if err != nil { + return Opened{}, err + } + return Opened{Call: id, SealTo: key, Secret: secret}, nil +} + +// OwnPutTake opens the value sealed to the call's key and keeps it as the module's own, its newest generation only. +func (v *Vault) OwnPutTake(caller, call, sealed string) (Changed, error) { + c, err := v.takeCall(OwnPutVerb, call) + if err != nil { + return Changed{}, err + } + if c.by != caller { + return Changed{}, fmt.Errorf("the own-put call %q was opened by another caller; nothing was taken", call) + } + value, err := c.open(sealed) + if err != nil { + return Changed{}, err + } + defer zero(value) + if len(value) == 0 { + return Changed{}, errors.New("the value handed in is empty; nothing was taken") + } + changed, err := v.keepWith(c.secret, string(value), caller, OwnPutVerb, OriginAccepted, keepOptions{}) + if err != nil { + v.log("own-put of %s by %s refused: %v", c.secret, caller, err) + return Changed{}, err + } + v.log("own-put: %s kept at generation %d, written by %s", c.secret, changed.Generation, caller) + return changed, nil +} diff --git a/modules/mesh-vault/cmd/mesh-vault/tools.go b/modules/mesh-vault/cmd/mesh-vault/tools.go index 930905e7c..3b78a65e2 100644 --- a/modules/mesh-vault/cmd/mesh-vault/tools.go +++ b/modules/mesh-vault/cmd/mesh-vault/tools.go @@ -231,6 +231,26 @@ func definitions(env *Env) []toolDef { g, _ := a["generation"].(float64) return v.LicencePutOpen(caller, arg(a, "licence"), arg(a, "part"), int(g)) }}, + {Name: Seat + "." + OwnPutVerb, + Description: "Store a secret as the calling module's own (design 47 H2 §822), in two steps: name the secret and " + + "get a call and a key made for it; then send the call and the value sealed to that key. Kept as " + + "module//, the module named by the caller's account as the bus proved it, never by an " + + "argument, its newest generation only; get answers that name to an account of the same module alone. " + + "Answers the call and the key, then the secret's name and generation; never the value.", + Input: map[string]any{"type": "object", "properties": map[string]any{ + "name": str("the first step: the secret's name, among the calling module's own"), + "call": str("the second step: the call the first step answered"), + "sealed": str("the second step: the value sealed to the call's key (NaCl anonymous box, base64) — never the value")}}, + run: func(caller string, a map[string]any) (any, error) { + v, err := env.ready() + if err != nil { + return nil, err + } + if call := arg(a, "call"); call != "" { + return v.OwnPutTake(caller, call, arg(a, "sealed")) + } + return v.OwnPutOpen(caller, arg(a, "name")) + }}, {Name: Seat + ".move-in", Description: "Take one value of the controller's store into the vault, as it is (design 57's V3): from the " + "node-engine of a machine it was sealed to, and no other caller, in two steps — name the row (node, " + diff --git a/modules/mesh-vault/cmd/mesh-vault/vault.go b/modules/mesh-vault/cmd/mesh-vault/vault.go index 35c4b5d9b..bdccc343f 100644 --- a/modules/mesh-vault/cmd/mesh-vault/vault.go +++ b/modules/mesh-vault/cmd/mesh-vault/vault.go @@ -383,8 +383,16 @@ func (v *Vault) Get(caller, name, publicKey string) (Answer, error) { return Answer{}, fmt.Errorf("%s may not ask for %s: %s", caller, name, why) } isLicence, licenceWhy := v.licenceRefusal(caller, name) + isModule, moduleWhy := moduleSecretRefusal(caller, name) if isLogin { // Judged above: the node's own engine, sealed to its own key. + } else if isModule { + // A module's own secret written through own-put, to an account of that module alone (design 47 H2 §822). + if moduleWhy != "" { + v.log("refused %s to %s: %s", name, caller, moduleWhy) + _ = v.recordHanding(Handing{To: caller, Name: name, Outcome: "refused", Why: moduleWhy}) + return Answer{}, fmt.Errorf("%s may not ask for %s: %s", caller, name, moduleWhy) + } } else if isLicence { // A licence's secret, by the licence rule (design 39 bus-and-switch §16, §17). if licenceWhy != "" { diff --git a/modules/mesh-vault/module.json b/modules/mesh-vault/module.json index c49df2aa9..9332157b0 100644 --- a/modules/mesh-vault/module.json +++ b/modules/mesh-vault/module.json @@ -233,7 +233,8 @@ "node-account-confirm", "abandon", "retire", - "licence-put" + "licence-put", + "own-put" ] } ], -- 2.54.0 From 0bc03bba35460a0736584b3756fac376d281d9d7 Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 21 Nov 2026 23:20:11 +0100 Subject: [PATCH 06/54] =?UTF-8?q?=C2=A7780=20readers:=20take=20the=20form?= =?UTF-8?q?=20the=20tolerant=20seat=20decoders=20answer?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit mesh-contracts' K1 regenerated every seat package to answer (value, form, error); the catalog's four callers take the form. Builds: design 47 H2 §780; H-A-350 DMC-4 --- modules/mesh-build-go/cmd/mesh-build-go/steps.go | 2 +- modules/mesh-build-node/cmd/mesh-build-node/steps.go | 2 +- modules/mesh-conditions/cmd/mesh-conditions/keeper_test.go | 2 +- modules/mesh-delivery-gate/cmd/mesh-delivery-gate/heads.go | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/mesh-build-go/cmd/mesh-build-go/steps.go b/modules/mesh-build-go/cmd/mesh-build-go/steps.go index c1aac6760..7f97be58d 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/steps.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/steps.go @@ -172,7 +172,7 @@ func (m *Module) steps(ctx context.Context, args map[string]any) (any, error) { // and by again (§326). if again, _ := args["again"].(bool); !again && fp != "" && m.ReadDetected != nil { if raw, err := m.ReadDetected(key); err == nil && len(raw) > 0 { - if kept, err := seat.DecodeDetectedStepsByModuleArtifactValue(raw); err == nil && kept.Fingerprint == fp { + if kept, _, err := seat.DecodeDetectedStepsByModuleArtifactValue(raw); err == nil && kept.Fingerprint == fp { return StepsAnswer{Artifact: artifact, Fingerprint: fp, Steps: kept.Steps, Kept: true}, nil } } diff --git a/modules/mesh-build-node/cmd/mesh-build-node/steps.go b/modules/mesh-build-node/cmd/mesh-build-node/steps.go index 43fd63c83..0e0d236cb 100644 --- a/modules/mesh-build-node/cmd/mesh-build-node/steps.go +++ b/modules/mesh-build-node/cmd/mesh-build-node/steps.go @@ -114,7 +114,7 @@ func (m *Module) steps(ctx context.Context, args map[string]any) (any, error) { key := seat.DetectedStepsByModuleArtifactKey(str("module"), name) if again, _ := args["again"].(bool); !again && fp != "" && m.ReadDetected != nil { if raw, err := m.ReadDetected(key); err == nil && len(raw) > 0 { - if kept, err := seat.DecodeDetectedStepsByModuleArtifactValue(raw); err == nil && kept.Fingerprint == fp { + if kept, _, err := seat.DecodeDetectedStepsByModuleArtifactValue(raw); err == nil && kept.Fingerprint == fp { return StepsAnswer{Artifact: artifact, Fingerprint: fp, Steps: kept.Steps, Kept: true}, nil } } diff --git a/modules/mesh-conditions/cmd/mesh-conditions/keeper_test.go b/modules/mesh-conditions/cmd/mesh-conditions/keeper_test.go index d6331c34e..9e872fdff 100644 --- a/modules/mesh-conditions/cmd/mesh-conditions/keeper_test.go +++ b/modules/mesh-conditions/cmd/mesh-conditions/keeper_test.go @@ -156,7 +156,7 @@ func (r *rig) open(key string) (Condition, bool) { var c Condition if ok { // The seat's layout, read strictly: a field the seat does not name is a fault of the keeper's write. - v, err := seat.DecodeConditionsByKeyValue(raw) + v, _, err := seat.DecodeConditionsByKeyValue(raw) if err != nil { panic(fmt.Sprintf("the condition %s is not the seat's layout: %v: %s", key, err, raw)) } diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/heads.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/heads.go index 7e400fd1d..ea4d1d7bd 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/heads.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/heads.go @@ -53,7 +53,7 @@ func (kvHeads) Get(key string) (*HeldHead, error) { if err != nil || e == nil { return nil, err } - v, err := seat.DecodeHeadsByRepositoryBranchValue(e.Value) + v, _, err := seat.DecodeHeadsByRepositoryBranchValue(e.Value) if err != nil { return nil, fmt.Errorf("heads %s is not readable: %w", key, err) } -- 2.54.0 From d545299a71f5f066b61d6dc08fdeb7c174512a7b Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 22 Nov 2026 00:02:56 +0100 Subject: [PATCH 07/54] RG3: the registry purges a deleted chunk's leftovers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every delete of a chunk in the incoming object store is followed by the runtime's purge of what it left (busstate.Objects.Purge), so the store's kept descriptions equal its live objects; a failed purge is said. Builds: design 47 H2 §813 (8); H-A-227, H-A-366 --- .../cmd/distribution-tools/bounded.go | 19 +++++++-- .../cmd/distribution-tools/bounded_test.go | 39 ++++++++++++++++++- .../cmd/distribution-tools/main.go | 3 +- 3 files changed, 55 insertions(+), 6 deletions(-) diff --git a/modules/distribution/cmd/distribution-tools/bounded.go b/modules/distribution/cmd/distribution-tools/bounded.go index 7bf77f0dd..c7211e4e7 100644 --- a/modules/distribution/cmd/distribution-tools/bounded.go +++ b/modules/distribution/cmd/distribution-tools/bounded.go @@ -46,8 +46,13 @@ func stepErr(bound time.Duration, what string, f func() error) error { return err } -// boundedObjects is the incoming object store with each operation bounded and logged. -type boundedObjects struct{ parts.Objects } +// boundedObjects is the incoming object store with each operation bounded and logged. Purge is the store's purge of a +// deleted object's chunks (busstate.Objects.Purge): every delete of a chunk is followed by it, so what a delete left +// behind is dropped and the store's kept descriptions equal its live objects (design 47 H2 §813 (8); H-A-227, H-A-366). +type boundedObjects struct { + parts.Objects + Purge func(name string) error +} func (b boundedObjects) Used() (uint64, error) { return step(stepBound, "the incoming object store's fill (objects)", b.Objects.Used) @@ -59,7 +64,15 @@ func (b boundedObjects) Get(name string) ([]byte, error) { return step(stepBound, "the get of a chunk from the incoming object store (objects)", func() ([]byte, error) { return b.Objects.Get(name) }) } func (b boundedObjects) Delete(name string) error { - return stepErr(stepBound, "the delete of a chunk in the incoming object store (objects)", func() error { return b.Objects.Delete(name) }) + err := stepErr(stepBound, "the delete of a chunk in the incoming object store (objects)", func() error { return b.Objects.Delete(name) }) + if b.Purge == nil { + return err + } + perr := stepErr(stepBound, "the purge of a deleted chunk's leftovers in the incoming object store (objects)", func() error { return b.Purge(name) }) + if err != nil { + return err + } + return perr } // boundedTransfers is the transfers state with its reads and writes bounded and logged. diff --git a/modules/distribution/cmd/distribution-tools/bounded_test.go b/modules/distribution/cmd/distribution-tools/bounded_test.go index 64cec0079..47acf3d3d 100644 --- a/modules/distribution/cmd/distribution-tools/bounded_test.go +++ b/modules/distribution/cmd/distribution-tools/bounded_test.go @@ -8,8 +8,15 @@ import ( "time" ) +// The incoming store's steps (design 47 H2 §819: one test function for the bounded steps). +func TestTheBoundedSteps(t *testing.T) { + t.Run("a hung step answers with its name", hungStepAnswersWithItsName) + t.Run("a given-up step says when its call ended", givenUpStepSaysWhenItsCallEnded) + t.Run("a deleted chunk is purged", deletedChunkIsPurged) +} + // A bus operation that never answers becomes an error naming its step, within the bound. -func TestAHungStepAnswersWithItsName(t *testing.T) { +func hungStepAnswersWithItsName(t *testing.T) { block := make(chan struct{}) defer close(block) _, err := step(10*time.Millisecond, "the write of the idle key", func() (int, error) { <-block; return 0, nil }) @@ -22,7 +29,7 @@ func TestAHungStepAnswersWithItsName(t *testing.T) { } // A step given up on is said given up by name, and the log says later when the call behind it ended and how (A-443a). -func TestAGivenUpStepSaysWhenItsCallEnded(t *testing.T) { +func givenUpStepSaysWhenItsCallEnded(t *testing.T) { var mu sync.Mutex var lines []string ended := make(chan struct{}) @@ -68,3 +75,31 @@ func TestAFindingIsSaidOffThePath(t *testing.T) { t.Fatal("the call was held by the finding") } } + +// fakeObjects is an object store that counts deletes. +type fakeObjects struct{ deleted []string } + +func (f *fakeObjects) Put(string, []byte) error { return nil } +func (f *fakeObjects) Get(string) ([]byte, error) { return nil, nil } +func (f *fakeObjects) Delete(name string) error { f.deleted = append(f.deleted, name); return nil } +func (f *fakeObjects) Used() (uint64, error) { return 0, nil } + +// Every delete of a chunk purges what it left behind, so the store's kept descriptions equal its live objects (design 47 +// H2 §813 (8)); a purge's failure is said, never swallowed. +func deletedChunkIsPurged(t *testing.T) { + f := &fakeObjects{} + var purged []string + b := boundedObjects{f, func(name string) error { purged = append(purged, name); return nil }} + for i := range 1000 { + if err := b.Delete(fmt.Sprintf("chunk.%d", i)); err != nil { + t.Fatal(err) + } + } + if len(f.deleted) != 1000 || len(purged) != 1000 || purged[999] != "chunk.999" { + t.Fatalf("deleted %d, purged %d", len(f.deleted), len(purged)) + } + b.Purge = func(string) error { return fmt.Errorf("no responders") } + if err := b.Delete("chunk.x"); err == nil || !strings.Contains(err.Error(), "no responders") { + t.Fatalf("a failed purge answered %v", err) + } +} diff --git a/modules/distribution/cmd/distribution-tools/main.go b/modules/distribution/cmd/distribution-tools/main.go index ae4a503b4..bbecf4d67 100644 --- a/modules/distribution/cmd/distribution-tools/main.go +++ b/modules/distribution/cmd/distribution-tools/main.go @@ -24,6 +24,7 @@ import ( reg "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/containerimageregistry" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/parts" stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/busstate" "git.novox.be/novox/mesh-sdk/go/seatlist" "git.novox.be/novox/mesh-sdk/go/views" @@ -488,7 +489,7 @@ func main() { }() base, _ := url.Parse("http://127.0.0.1:5001") store := &holder.OCI{Base: base, Upstream: &holder.Upstream{}} - h := &holder.Holder{Logf: logf, Objects: boundedObjects{parts.BusObjects(holder.IncomingObjects)}, Builder: holder.Reserved{Ask: stdio.Ask}, Store: store, Say: say, Finding: boundedFinding(findings.Say), + h := &holder.Holder{Logf: logf, Objects: boundedObjects{parts.BusObjects(holder.IncomingObjects), busstate.ObjectsOf(holder.IncomingObjects).Purge}, Builder: holder.Reserved{Ask: stdio.Ask}, Store: store, Say: say, Finding: boundedFinding(findings.Say), Transfers: boundedTransfers{parts.BusTransfers[holder.Opening](holder.TransfersState)}, Cap: boundedCap(parts.BusCap(holder.IncomingObjects)), Record: recordAct, Idle: boundedArm(parts.BusArm(holder.IdleState))} files := &holder.Files{Container: or(os.Getenv("MESH_REGISTRY_CONTAINER"), "mesh-registry"), Run: holder.ExecRunner, UID: os.Getuid()} v := &holder.Verbs{Holder: h, Catalog: store, Holds: kept[holder.Hold]{state: "holds", key: holder.Hold.Key}, Mirrors: mirrors{}, -- 2.54.0 From babcd1bb3153c90f48797a7aee86b252cc8cca3a Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 22 Nov 2026 00:05:58 +0100 Subject: [PATCH 08/54] RG1: the registry takes a publish ask once by its id MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A publish carrying an ask id is kept in the state taken for ask-taken-bound (10m): a repeat while the act runs joins it, a repeat after it ended answers the act's state read afresh (the transfer's lacking parts, a held version ended, a transfer gone refused), never publishing twice. Builds: design 47 H2 §775, §820; H-A-359, H-A-381 --- .../cmd/distribution-tools/contract_test.go | 2 +- .../cmd/distribution-tools/main.go | 14 +++- .../cmd/distribution-tools/taken.go | 75 +++++++++++++++++++ .../cmd/distribution-tools/taken_test.go | 63 ++++++++++++++++ .../internal/holder/holder_test.go | 19 +++++ .../distribution/internal/holder/transfer.go | 32 ++++++++ modules/distribution/module.json | 18 ++++- 7 files changed, 217 insertions(+), 6 deletions(-) create mode 100644 modules/distribution/cmd/distribution-tools/taken.go create mode 100644 modules/distribution/cmd/distribution-tools/taken_test.go diff --git a/modules/distribution/cmd/distribution-tools/contract_test.go b/modules/distribution/cmd/distribution-tools/contract_test.go index 9fadaf0bc..28c175e05 100644 --- a/modules/distribution/cmd/distribution-tools/contract_test.go +++ b/modules/distribution/cmd/distribution-tools/contract_test.go @@ -36,7 +36,7 @@ func TestTheVerbsSpeakTheSeatsContract(t *testing.T) { reg.VerbVersions: reg.VersionsInput{}, reg.VerbRemove: reg.RemoveInput{}, reg.VerbHold: reg.HoldInput{}, reg.VerbRelease: reg.ReleaseInput{}, reg.VerbMirror: reg.MirrorInput{}, reg.VerbTags: reg.TagsInput{}} seen := 0 - for _, tool := range tools(&holder.Verbs{}, kept[holder.Window]{}) { + for _, tool := range tools(&holder.Verbs{}, kept[holder.Window]{}, nil) { seat, verb, ok := strings.Cut(tool.Name, ".") in, isVerb := inputs[verb] if !ok || !isVerb || (seat != holder.ImageSeat && seat != holder.ArtifactSeat) { diff --git a/modules/distribution/cmd/distribution-tools/main.go b/modules/distribution/cmd/distribution-tools/main.go index bbecf4d67..2e5fb3b45 100644 --- a/modules/distribution/cmd/distribution-tools/main.go +++ b/modules/distribution/cmd/distribution-tools/main.go @@ -244,7 +244,7 @@ func arg[T any](args map[string]any) (T, error) { func str(args map[string]any, k string) string { s, _ := args[k].(string); return strings.TrimSpace(s) } // tools are distribution's tools over v. -func tools(v *holder.Verbs, windows kept[holder.Window]) []stdio.Tool { +func tools(v *holder.Verbs, windows kept[holder.Window], taken *asks) []stdio.Tool { obj := func(props map[string]any, req ...string) map[string]any { return map[string]any{"type": "object", "required": req, "properties": props} } @@ -262,7 +262,9 @@ func tools(v *holder.Verbs, windows kept[holder.Window]) []stdio.Tool { caller := stdio.CallerOf(args) start := time.Now() logf("%s.publish received from %s: %s %s (%s, %d parts)", seat, caller, in.Name, in.Version, in.Kind, len(in.Parts)) - a, err := v.Publish(seat, caller, publishOf(in)) + r := publishOf(in) + a, err := taken.once(seat, stdio.AskIDOf(args), func() (holder.PublishAnswer, error) { return v.Publish(seat, caller, r) }, + func() (holder.PublishAnswer, error) { return v.Repeat(seat, caller, r) }) if err == nil { // The window's learning asks the delivery view (up to 60 s, past the runner's 30 s call bound), // so it never holds the answer. @@ -503,7 +505,13 @@ func main() { logf("transfer %s had no part for %s: dropped with its parts", id, holder.TransferIdle) }) windows := kept[holder.Window]{state: "windows", key: func(w holder.Window) string { return w.Name }} - if err := stdio.Serve("", tools(v, windows)); err != nil { + bound, err := stdio.SettingDuration(askTakenBound, askTakenDefault) + if err != nil { + logf("the setting %s: %v", askTakenBound, err) + os.Exit(1) + } + taken := &asks{take: stdio.State(stdio.TakenState).TakeOnce, bound: bound} + if err := stdio.Serve("", tools(v, windows, taken)); err != nil { logf("%v", err) os.Exit(1) } diff --git a/modules/distribution/cmd/distribution-tools/taken.go b/modules/distribution/cmd/distribution-tools/taken.go new file mode 100644 index 000000000..4c5ec9f8e --- /dev/null +++ b/modules/distribution/cmd/distribution-tools/taken.go @@ -0,0 +1,75 @@ +package main + +import ( + "fmt" + "sync" + "time" + + "distribution/internal/holder" +) + +// askTakenBound is the setting the taken key of a publish ask lives (design 47 H2 §775, H-A-381 (2)): the holder never +// receives the caller's bound, so it keeps an ask's id this long, the largest caller's bound by default. +const askTakenBound = "ask-taken-bound" + +// askTakenDefault is ask-taken-bound's default, the runner's ask-unanswered-bound (H-A-381 (2)). +const askTakenDefault = 10 * time.Minute + +// asks takes each publish ask once by its id (design 47 H2 §775; ADR 0668 §2; H-A-381): the first call with an id is +// kept in the state taken and acts; a repeat while that act runs joins it and gets its answer; a repeat after it ended +// answers the act's state read afresh (holder.Repeat), never acting twice. No answer is kept in taken. A call that +// carries no ask id is no kept call, and acts as it did before. +type asks struct { + take func(seat, ask string, bound time.Duration) (bool, error) + bound time.Duration + mu sync.Mutex + running map[string]*flight + // joined, when set, is told each repeat that joins a running act (a test's hook). + joined func() +} + +// flight is one act of a taken ask while it runs, which a repeat joins. +type flight struct { + done chan struct{} + a holder.PublishAnswer + err error +} + +func (k *asks) once(seat, ask string, act, repeat func() (holder.PublishAnswer, error)) (holder.PublishAnswer, error) { + if ask == "" || k == nil || k.take == nil { + return act() + } + key := seat + "." + ask + k.mu.Lock() + if f, ok := k.running[key]; ok { + k.mu.Unlock() + if k.joined != nil { + k.joined() + } + <-f.done + return f.a, f.err + } + if k.running == nil { + k.running = map[string]*flight{} + } + f := &flight{done: make(chan struct{})} + k.running[key] = f + k.mu.Unlock() + defer func() { + k.mu.Lock() + delete(k.running, key) + k.mu.Unlock() + close(f.done) + }() + took, err := k.take(seat, ask, k.bound) + switch { + case err != nil: + f.err = fmt.Errorf("publish: the ask %s of %s cannot be taken: %w", ask, seat, err) + case took: + f.a, f.err = act() + default: + logf("%s.publish: the ask %s was taken before; it answers the act's state, acting not again", seat, ask) + f.a, f.err = repeat() + } + return f.a, f.err +} diff --git a/modules/distribution/cmd/distribution-tools/taken_test.go b/modules/distribution/cmd/distribution-tools/taken_test.go new file mode 100644 index 000000000..2107a7265 --- /dev/null +++ b/modules/distribution/cmd/distribution-tools/taken_test.go @@ -0,0 +1,63 @@ +package main + +import ( + "errors" + "sync" + "sync/atomic" + "testing" + "time" + + "distribution/internal/holder" +) + +// A publish ask given twice acts once (design 47 H2 §775; H-A-381): a repeat while the act runs joins it and gets its +// answer, a repeat after it ended answers the act's state, a call with no ask id acts as before, a take that fails is said. +func TestAPublishAskIsTakenOnce(t *testing.T) { + taken := map[string]bool{} + var mu sync.Mutex + k := &asks{bound: time.Minute, take: func(seat, ask string, bound time.Duration) (bool, error) { + mu.Lock() + defer mu.Unlock() + if ask == "broken" { + return false, errors.New("no responders") + } + if bound != time.Minute || taken[seat+"."+ask] { + return false, nil + } + taken[seat+"."+ask] = true + return true, nil + }} + var acts, repeats atomic.Int32 + release, started, joined := make(chan struct{}), make(chan struct{}, 1), make(chan struct{}, 1) + k.joined = func() { joined <- struct{}{} } + act := func() (holder.PublishAnswer, error) { + acts.Add(1) + started <- struct{}{} + <-release + return holder.PublishAnswer{Transfer: "t", Missing: []string{"d"}}, nil + } + repeat := func() (holder.PublishAnswer, error) { repeats.Add(1); return holder.PublishAnswer{Transfer: "t"}, nil } + answers := make(chan holder.PublishAnswer, 2) + go func() { a, _ := k.once(holder.ArtifactSeat, "a1", act, repeat); answers <- a }() + <-started + go func() { a, _ := k.once(holder.ArtifactSeat, "a1", act, repeat); answers <- a }() + <-joined + close(release) + for range 2 { + if a := <-answers; len(a.Missing) != 1 { + t.Errorf("an answer of the running act = %+v", a) + } + } + if a, _ := k.once(holder.ArtifactSeat, "a1", act, repeat); acts.Load() != 1 || repeats.Load() != 1 || a.Transfer != "t" { + t.Errorf("acts %d, repeats %d after a repeat of an ended act", acts.Load(), repeats.Load()) + } + release = make(chan struct{}) + close(release) + act2 := func() (holder.PublishAnswer, error) { acts.Add(1); return holder.PublishAnswer{}, nil } + if _, err := k.once(holder.ArtifactSeat, "", act2, repeat); err != nil || acts.Load() != 2 { + t.Errorf("a call with no ask id: acts %d, %v", acts.Load(), err) + } + if _, err := k.once(holder.ArtifactSeat, "broken", act2, repeat); err == nil || acts.Load() != 2 { + t.Errorf("a failed take: acts %d, %v", acts.Load(), err) + } +} diff --git a/modules/distribution/internal/holder/holder_test.go b/modules/distribution/internal/holder/holder_test.go index 9f2f0f77e..62aa77ca3 100644 --- a/modules/distribution/internal/holder/holder_test.go +++ b/modules/distribution/internal/holder/holder_test.go @@ -251,6 +251,25 @@ func TestTheHolderOfTheTwoImageSeats(t *testing.T) { } }) + // A publish ask taken before answers the act's state read afresh and never publishes again (design 47 H2 §775; + // H-A-381): an open transfer its lacking parts, a transfer gone with nothing held a refusal naming it. + t.Run("a repeated ask answers the act's state", func(t *testing.T) { + b := newBed(true) + if _, err := b.h.Repeat(ArtifactSeat, builder, req("tr")); err == nil || !strings.Contains(err.Error(), "transfer tr is gone") { + t.Errorf("a repeat of an act that never happened answered %v", err) + } + if _, err := b.h.Publish(ArtifactSeat, builder, req("tr")); err != nil { + t.Fatal(err) + } + opened := len(b.h.Transfers.(memTransfers)) + if a, err := b.h.Repeat(ArtifactSeat, builder, req("tr")); err != nil || !reflect.DeepEqual(a.Missing, []string{part.Digest}) { + t.Errorf("a repeat of an open transfer = %+v, %v", a, err) + } + if len(b.h.Transfers.(memTransfers)) != opened || len(b.said) != 0 { + t.Errorf("a repeat acted: transfers %d, said %+v", len(b.h.Transfers.(memTransfers)), b.said) + } + }) + t.Run("a transfer in parts, cut and resumed, hashed, numbered by the builder, published", func(t *testing.T) { b := newBed(true) a, err := b.h.Publish(ArtifactSeat, builder, req("t1")) diff --git a/modules/distribution/internal/holder/transfer.go b/modules/distribution/internal/holder/transfer.go index 7b3bcba67..11574ddf2 100644 --- a/modules/distribution/internal/holder/transfer.go +++ b/modules/distribution/internal/holder/transfer.go @@ -397,6 +397,38 @@ func (h *Holder) DropIdle() ([]string, error) { return h.receiver().DropIdle() } // already gone. It answers whether it dropped the transfer. func (h *Holder) Expired(id string) (bool, error) { return h.receiver().Expired(id) } +// Repeat is `publish` given again with an ask id the holder already took and whose first act ended (design 47 H2 §775; +// H-A-381): it never publishes twice and keeps no answer, but answers the act's state as it stands, read afresh, which is +// what the act answers when done: a version the store holds ended, an open transfer the parts it lacks; a transfer that +// is gone, with nothing held, says the act never happened, and is refused naming it. +func (h *Holder) Repeat(seat, caller string, r PublishRequest) (PublishAnswer, error) { + if err := checkPublish(seat, r); err != nil { + return PublishAnswer{}, err + } + if r.Version != "" { + var ok bool + err := h.timed("the store's read of what it holds (the program's registry)", func() (err error) { + _, ok, err = h.Store.Held(r.Name, r.Version, r.Digest) + return err + }) + if err != nil { + return PublishAnswer{}, fmt.Errorf("publish: the store cannot say whether it holds %s@%s: %w", r.Name, r.Version, err) + } else if ok { + return PublishAnswer{Transfer: r.Transfer, Ended: true}, nil + } + } + t, found, err := h.Transfers.Get(r.Transfer) + switch { + case err != nil: + return PublishAnswer{}, fmt.Errorf("publish: the transfer %s of a repeated ask cannot be read: %w", r.Transfer, err) + case !found: + return PublishAnswer{}, fmt.Errorf("publish: refused: the ask was taken before, and transfer %s is gone with nothing of %s held: the act did not happen", r.Transfer, r.Name) + case t.Opener != caller: + return PublishAnswer{}, fmt.Errorf("publish: refused for %s: transfer %s was opened by another caller", caller, r.Transfer) + } + return PublishAnswer{Transfer: t.ID, Missing: t.Lacks()}, nil +} + func checkPublish(seat string, r PublishRequest) error { switch { case !slices.Contains(Seats, seat): diff --git a/modules/distribution/module.json b/modules/distribution/module.json index 0aca09da3..501826bf4 100644 --- a/modules/distribution/module.json +++ b/modules/distribution/module.json @@ -257,6 +257,20 @@ "holds", "windows", "mirrors", - "findings" - ] + "findings", + { + "name": "taken", + "key-ttl": true, + "max-mib": 8 + } + ], + "config": { + "ask-taken-bound": { + "type": "string", + "description": "How long the registry keeps the id of a publish ask it took, as a duration such as 10m (design 47 H2 §775): a publish given again with that id within it acts not again but answers the transfer as it stands. The largest caller's bound, the pipeline runner's ask-unanswered-bound, by default.", + "trusted": false, + "runtime": true, + "default": "10m" + } + } } -- 2.54.0 From 34f55bdf7926f08224efe2b4a358d29131402ec2 Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 22 Nov 2026 00:06:07 +0100 Subject: [PATCH 09/54] RG1: the bounded steps' finding test joins their one test function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 47 H2 §819 --- modules/distribution/cmd/distribution-tools/bounded_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/distribution/cmd/distribution-tools/bounded_test.go b/modules/distribution/cmd/distribution-tools/bounded_test.go index 47acf3d3d..04b1c0331 100644 --- a/modules/distribution/cmd/distribution-tools/bounded_test.go +++ b/modules/distribution/cmd/distribution-tools/bounded_test.go @@ -13,6 +13,7 @@ func TestTheBoundedSteps(t *testing.T) { t.Run("a hung step answers with its name", hungStepAnswersWithItsName) t.Run("a given-up step says when its call ended", givenUpStepSaysWhenItsCallEnded) t.Run("a deleted chunk is purged", deletedChunkIsPurged) + t.Run("a finding is said off the path", findingIsSaidOffThePath) } // A bus operation that never answers becomes an error naming its step, within the bound. @@ -59,7 +60,7 @@ func givenUpStepSaysWhenItsCallEnded(t *testing.T) { } // The finding is said off the caller's path: a writer that never answers does not hold the call that says it. -func TestAFindingIsSaidOffThePath(t *testing.T) { +func findingIsSaidOffThePath(t *testing.T) { block := make(chan struct{}) defer close(block) say := boundedFinding(func(string, string, string, bool) { <-block }) -- 2.54.0 From 410451a2a2eda3e6bb6d2c60a97bfa0f4aa681e1 Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 22 Nov 2026 00:14:55 +0100 Subject: [PATCH 10/54] GT3: the gate's deliveries filters while it reads and pages at 768 KB MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit deliveries takes module, outcome, since and the cursor next beside state, repository and group, applies them while it reads, and answers at most 768 KB a page with capped, total and the next page's opaque cursor. Builds: design 47 H2 §811, §779 --- .../cmd/mesh-delivery-gate/main.go | 18 +++- .../cmd/mesh-delivery-gate/manifest_test.go | 2 +- .../cmd/mesh-delivery-gate/record_test.go | 63 +++++++++++ .../cmd/mesh-delivery-gate/verbs.go | 100 ++++++++++++++++-- 4 files changed, 169 insertions(+), 14 deletions(-) diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/main.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/main.go index 546065b58..2391c55ee 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/main.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/main.go @@ -198,12 +198,24 @@ func tools(h *Holder, l *listening) []stdio.Tool { {Name: seat + "deliveries", Description: "Every delivery record the gate keeps, as it is kept (design 47 G A-24): the open ones, and those that ended in " + "the last thirty days (all: every one kept), each with its milestone or group, its pull requests, its entries' outcomes " + - "per node, its acts owed the forge and why it is not ended. Narrowed by state (open or ended), repository or group.", + "per node, its acts owed the forge and why it is not ended. Narrowed by state (open or ended), repository, group, module, " + + "outcome or since while it reads, at most 768 KB a page with capped, total and the next page's cursor (design 47 H2 §811).", Input: map[string]any{"type": "object", "properties": map[string]any{"state": str("open or ended"), "repository": str("owner/repository"), "group": str("a group's id (its branch name)"), - "all": map[string]any{"type": "string", "enum": []string{"true", "false"}}}}, + "all": map[string]any{"type": "string", "enum": []string{"true", "false"}}, + "module": str("only the deliveries that move this module"), "outcome": str("only the deliveries that ended with this outcome"), + "since": str("only the deliveries opened at or after this time, RFC 3339"), "next": str("the cursor the previous page answered")}}, Run: func(a map[string]any) (any, error) { - return h.Deliveries(strArg(a, "state"), strArg(a, "repository"), strArg(a, "group"), boolArg(a, "all")) + f := DeliveriesFilter{State: strArg(a, "state"), Repository: strArg(a, "repository"), Group: strArg(a, "group"), + Module: strArg(a, "module"), Outcome: strArg(a, "outcome"), All: boolArg(a, "all"), Next: strArg(a, "next")} + if since := strArg(a, "since"); since != "" { + t, err := time.Parse(time.RFC3339, since) + if err != nil { + return nil, fmt.Errorf("deliveries: since %q is no RFC 3339 time", since) + } + f.Since = t + } + return h.Deliveries(f) }}, {Name: seat + "show", Description: "One delivery record whole, as it is kept: by its instance, its milestone, its group's id or the id of one of " + diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/manifest_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/manifest_test.go index 90c0d1751..d238d801d 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/manifest_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/manifest_test.go @@ -371,7 +371,7 @@ func TestTheGroupsFromBranchNamesAreGone(t *testing.T) { if _, _, err := h.Records.Open(opening("main", 7, map[string]string{"novox/app#3": strings.Repeat("a", 40)})); err != nil { t.Fatal(err) } - d, err := h.Deliveries("", "", "", true) + d, err := h.Deliveries(DeliveriesFilter{All: true}) if err != nil || len(d["deliveries"].([]recordView)) != 1 { t.Fatalf("deliveries from the records: %v %v", d, err) } diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record_test.go index 487886b12..aeb067199 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record_test.go @@ -569,6 +569,69 @@ func TestTheRecordsToolsReadThemAndEndOneByHand(t *testing.T) { t.Fatalf("%+v %v", lines, err) } }) + // deliveries filters while it reads and pages at 768 KB with capped, total and a cursor (design 47 H2 §811, §779). + t.Run("deliveries by repository over 4 MB of records answers under 768 KB, page by page", func(t *testing.T) { + rs := &Records{Ledger: newLedgerFake(t), Now: func() time.Time { return at }} + big := make([]string, 400) + for i := range big { + big[i] = strings.Repeat("c", 40) + } + for i := range 300 { + repo := "novox/app" + if i%2 == 1 { + repo = "novox/other" + } + r, _, err := rs.Open(opening("main", int64(i+1), map[string]string{fmt.Sprintf("%s@%d", repo, i): strings.Repeat("1", 40)})) + if err != nil { + t.Fatal(err) + } + if err := rs.SetTrace(r, RecordTrace{Commits: big}); err != nil { + t.Fatal(err) + } + if i == 4 { + if err := rs.AddEntry(r, RecordEntry{Module: "mesh-x", Fingerprint: "src2:f1", Node: "ace"}); err != nil { + t.Fatal(err) + } + } + } + h := &Holder{Records: rs, Now: func() time.Time { return at }} + kept, _ := rs.Kept() + raw, _ := json.Marshal(kept) + if whole, _ := h.Deliveries(DeliveriesFilter{All: true}); len(raw) < 4<<20 || !whole["capped"].(bool) || whole["total"].(int) != 300 { + t.Fatalf("the records hold %d bytes in all, total %v", len(raw), whole["total"]) + } + seen, next := map[string]bool{}, "" + for pages := 0; ; pages++ { + page, err := h.Deliveries(DeliveriesFilter{Repository: "novox/app", All: true, Next: next}) + if err != nil { + t.Fatal(err) + } + if raw, _ := json.Marshal(page); len(raw) > 768<<10 || page["total"].(int) != 150 { + t.Fatalf("a page of %d bytes, total %v", len(raw), page["total"]) + } + for _, v := range page["deliveries"].([]recordView) { + if seen[v.Instance] || !holdsRepository(v.Record, "novox/app") { + t.Fatalf("%s answered twice or of another repository", v.Instance) + } + seen[v.Instance] = true + } + if next = page["next"].(string); next == "" || pages > 20 { + break + } + } + if len(seen) != 150 { + t.Errorf("the pages answered %d of 150 records", len(seen)) + } + if m, _ := h.Deliveries(DeliveriesFilter{Module: "mesh-x", All: true}); m["total"].(int) != 1 { + t.Errorf("by module: %v", m["total"]) + } + if s, _ := h.Deliveries(DeliveriesFilter{Since: at.Add(time.Hour), All: true}); s["total"].(int) != 0 { + t.Errorf("since an hour after every opening: %v", s["total"]) + } + if _, err := h.Deliveries(DeliveriesFilter{Next: "not-a-cursor"}); err == nil { + t.Error("a cursor this verb never answered was taken") + } + }) t.Run("gate_record_end ends a record only once the hand-act log took it", func(t *testing.T) { rs, r, said := setup(t) log := &handActsFake{refuse: errors.New("no responders")} diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/verbs.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/verbs.go index ecf5170d9..78a9be406 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/verbs.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/verbs.go @@ -1,10 +1,14 @@ package main import ( + "encoding/base64" + "encoding/json" "fmt" "sort" "strings" "time" + + "git.novox.be/novox/mesh-sdk/go/views" ) // The seat's verbs that read (novox/hq ADR 0239; design 47 G §431, §172, A-24): from G's delivery the gate keeps no merge @@ -111,9 +115,66 @@ func firstOf(a, b string) string { return b } +// DeliveriesFilter narrows `deliveries` (design 47 H2 §811): state (open or ended), repository, group, the module a record +// moves, the outcome it ended with and the time it opened at or after; All lists the ended ones of every day kept; Next is +// the cursor the previous page answered. +type DeliveriesFilter struct { + State, Repository, Group, Module, Outcome string + Since time.Time + All bool + Next string +} + +// movesModule is whether one of the record's entries is the module's. +func movesModule(r *Record, module string) bool { + for _, e := range r.Entries { + if e.Module == module { + return true + } + } + return false +} + +// endedWith is whether the record ended with the outcome: its why, or the why's first word before a colon ("failed" for +// "failed: "). +func endedWith(r *Record, outcome string) bool { + if r.Ended == nil { + return false + } + return r.Ended.Why == outcome || strings.HasPrefix(r.Ended.Why, outcome+":") +} + +// deliveriesCursor is the opaque cursor of a page of `deliveries` (§779): the place of the last record it answered in the +// order open first, then the newest. +func deliveriesCursor(v recordView) string { + return base64.RawURLEncoding.EncodeToString([]byte(fmt.Sprintf("%t|%d|%s", v.Record.Ended == nil, v.Record.Opened.UnixNano(), v.Instance))) +} + +// beforeOrAt is whether v comes at or before the place cursor names, in the order of `deliveries`. +func beforeOrAt(v recordView, cursor string) (bool, error) { + raw, err := base64.RawURLEncoding.DecodeString(cursor) + parts := strings.SplitN(string(raw), "|", 3) + if err != nil || len(parts) != 3 { + return false, fmt.Errorf("deliveries: %q is no cursor this verb answered", cursor) + } + var opened int64 + if _, err := fmt.Sscanf(parts[1], "%d", &opened); err != nil { + return false, fmt.Errorf("deliveries: %q is no cursor this verb answered", cursor) + } + open := v.Record.Ended == nil + switch cOpen := parts[0] == "true"; { + case open != cOpen: + return open, nil + case v.Record.Opened.UnixNano() != opened: + return v.Record.Opened.UnixNano() > opened, nil + } + return v.Instance >= parts[2], nil +} + // Deliveries is the `deliveries` verb: every record kept as it is kept, the open ones first, then the newest; the ended ones -// of the last thirty days (all: every one kept), narrowed by state (open or ended), repository or group. -func (h *Holder) Deliveries(state, repository, group string, all bool) (map[string]any, error) { +// of the last thirty days (all: every one kept), narrowed by the filter while it reads, at most views.PageBytes a page +// with capped, total and the next page's cursor (design 47 H2 §811, §779), never refused as too big. +func (h *Holder) Deliveries(f DeliveriesFilter) (map[string]any, error) { kept, err := h.keptRecords() if err != nil { return nil, err @@ -121,10 +182,11 @@ func (h *Holder) Deliveries(state, repository, group string, all bool) (map[stri now := h.Now() var out []recordView for _, r := range kept { - if state != "" && state != stateOf(r) || repository != "" && !holdsRepository(r, repository) || group != "" && groupOf(r) != group { - continue - } - if r.Ended != nil && !all && now.Sub(r.Ended.At) > listedFor { + switch { + case f.State != "" && f.State != stateOf(r), f.Repository != "" && !holdsRepository(r, f.Repository), + f.Group != "" && groupOf(r) != f.Group, f.Module != "" && !movesModule(r, f.Module), + f.Outcome != "" && !endedWith(r, f.Outcome), !f.Since.IsZero() && r.Opened.Before(f.Since), + r.Ended != nil && !f.All && now.Sub(r.Ended.At) > listedFor: continue } out = append(out, viewOf(r)) @@ -133,12 +195,30 @@ func (h *Holder) Deliveries(state, repository, group string, all bool) (map[stri if (out[i].Record.Ended == nil) != (out[j].Record.Ended == nil) { return out[i].Record.Ended == nil } - return out[i].Record.Opened.After(out[j].Record.Opened) + if !out[i].Record.Opened.Equal(out[j].Record.Opened) { + return out[i].Record.Opened.After(out[j].Record.Opened) + } + return out[i].Instance > out[j].Instance }) - if out == nil { - out = []recordView{} + page, size, capped, next := []recordView{}, 0, false, "" + for _, v := range out { + if f.Next != "" { + at, err := beforeOrAt(v, f.Next) + if err != nil { + return nil, err + } + if at { + continue + } + } + raw, _ := json.Marshal(v) + if size+len(raw) > views.PageBytes && len(page) > 0 { + capped, next = true, deliveriesCursor(page[len(page)-1]) + break + } + page, size = append(page, v), size+len(raw) } - return map[string]any{"deliveries": out}, nil + return map[string]any{"deliveries": page, "capped": capped, "total": len(out), "next": next}, nil } func (h *Holder) keptRecords() ([]*Record, error) { -- 2.54.0 From 6f744d6602daf846caa8dd32cf4b8930f25d8feb Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 22 Nov 2026 00:19:16 +0100 Subject: [PATCH 11/54] GT1: the gate's ledger keeps one entry per module and revision MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A record's head entry record/ keeps the record-wide fields and the count of its movers; each mover, the catalog half's too, has its entry mover/ with its pipeline, version and step outcomes per node, found through the index movers on its record's key. An entry over 64 KiB is refused with record-too-large, cleared when written within it or at the record's end; the ledger is capped at 1 GiB. Builds: design 47 H2 §809, §820; H-A-385, H-A-391 --- .../cmd/mesh-delivery-gate/findings.go | 33 ++- .../cmd/mesh-delivery-gate/main.go | 2 +- .../cmd/mesh-delivery-gate/movers.go | 221 ++++++++++++++++++ .../cmd/mesh-delivery-gate/record.go | 101 +++++++- .../cmd/mesh-delivery-gate/record_test.go | 97 +++++++- .../cmd/mesh-delivery-gate/recordevents.go | 1 + .../cmd/mesh-delivery-gate/recordpipeline.go | 6 + .../cmd/mesh-delivery-gate/recordtools.go | 13 +- modules/mesh-delivery-gate/module.json | 127 +++++++--- 9 files changed, 552 insertions(+), 49 deletions(-) create mode 100644 modules/mesh-delivery-gate/cmd/mesh-delivery-gate/movers.go diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/findings.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/findings.go index 194f71865..1ac846fc4 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/findings.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/findings.go @@ -2,6 +2,7 @@ package main import ( "context" + "fmt" "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" stdio "git.novox.be/novox/mesh-sdk/go" @@ -28,5 +29,35 @@ type FindingWriter interface { // generated contract of the seat that defines it (A-17). It is started once the runtime serves (A-40). func NewFindingWriter() (*views.HeldWriter[meshconditions.Finding, meshconditions.FindingCleared, meshconditions.FindingsStated, meshconditions.FindingsAskedEvent], error) { return views.NewStdioHeldWriter(views.WriterConfig[meshconditions.Finding, meshconditions.FindingCleared, meshconditions.FindingsStated, meshconditions.FindingsAskedEvent]{ - Asked: meshconditions.SubjectFindingsAsked, Module: Module, State: stdio.State("findings"), Kinds: []string{HeadUntaken}}) + Asked: meshconditions.SubjectFindingsAsked, Module: Module, State: stdio.State("findings"), Kinds: []string{HeadUntaken, RecordTooLarge}}) +} + +// recordTooLarge raises record-too-large for a delivery record's entry the gate refused over EntryLimit, or clears it when +// the writer holds it (design 47 H2 §809): the record's records hook. +func (h *Holder) recordTooLarge(instance string, bytes int, raised bool) { + if h.Findings == nil { + return + } + held := false + for _, f := range h.Findings.Held() { + if f.Scope == FindingScope && f.ID == instance && f.Kind == RecordTooLarge { + held = true + } + } + var err error + switch { + case raised && !held: + now := h.Now().UTC() + err = h.Findings.Write(context.Background(), meshconditions.Finding{Scope: FindingScope, ID: instance, Kind: RecordTooLarge, + Observer: Module, Severity: meshconditions.FindingSeverityWarning, Resolver: "operator", Since: now, At: now, + Summary: fmt.Sprintf("%s would hold %d bytes, over the %d one entry holds: the write is refused", instance, bytes, EntryLimit), + Headline: "A delivery record's entry is too large", Subject: meshconditions.Subject{Scope: FindingScope, ID: instance}, + Explanation: "The gate keeps one entry per record and one per mover, each within 64 KiB (design 47 H2 §809); " + + "this entry would pass it, so the write is refused. It clears when the entry is next written within 64 KiB or the record ends."}) + case !raised && held: + err = h.Findings.Clear(context.Background(), FindingScope, instance, RecordTooLarge, "the entry is written within 64 KiB, or its record ended") + } + if err != nil && h.Logf != nil { + h.Logf("[mesh-delivery] record-too-large of %s is not said: %v", instance, err) + } } diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/main.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/main.go index 2391c55ee..509b4e256 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/main.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/main.go @@ -60,7 +60,7 @@ func main() { } h.init() h.SaySeat = stdio.SeatPublish - h.Records = &Records{Ledger: sdkLedger{}, Now: h.Now, Retention: h.recordRetention, Say: stdio.SeatPublish, Logf: logf} + h.Records = &Records{Ledger: sdkLedger{}, Now: h.Now, Retention: h.recordRetention, Say: stdio.SeatPublish, Logf: logf, TooLarge: h.recordTooLarge} h.Records.Changed = h.recordChanged h.Forge = seatGit{ask: stdio.Ask} h.Catalog = seatCatalog{ask: stdio.Ask} diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/movers.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/movers.go new file mode 100644 index 000000000..8b3fdbf4c --- /dev/null +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/movers.go @@ -0,0 +1,221 @@ +package main + +// A record's entries, one per module (design 47 H2 §809; H-A-385, H-A-391): the head entry `record/` keeps +// every record-wide field, its outcome, the count of its movers and §810's times, nothing per module; each mover, the +// catalog half's too, has its own entry `mover/` with its pipeline, version and the module's step +// outcomes per node (its machines and their running times among them). No entry names two modules. A record is joined +// from its head and the movers the ledger's index `movers` names under the record's key, one query. The gate refuses an +// entry over 64 KiB with the finding record-too-large, cleared when that entry is next written within 64 KiB or at the +// record's end. + +import ( + "encoding/json" + "fmt" + "sort" + "strings" + "time" + + "git.novox.be/novox/mesh-sdk/go/buscalls" +) + +// TypeMover is the ledger type of a mover's entry. +const TypeMover = "mover" + +// MoversIndex is the ledger's index of the movers' entries on their record's key (manifest `requires` records' indexes). +const MoversIndex = "movers" + +// The operations of a mover's entry; the manifest's type mover declares exactly these (record_test.go). +const ( + OpMover = "mover" // set-fields: its record's key, its module, its pipeline and its version + // OpEntry and OpOutcome write its entries and outcomes as they did on the whole record. +) + +// MoverOperations are a mover's operations, in the order the manifest's type allows them. +var MoverOperations = []string{OpMover, OpEntry, OpOutcome} + +// OpMovers is the head entry's count of its movers (set-fields movers). +const OpMovers = "movers" + +// EntryLimit is the most one entry of the ledger holds (§809). +const EntryLimit = 64 << 10 + +// RecordTooLarge is the finding of an entry refused over EntryLimit (scope delivery). +const RecordTooLarge = "record-too-large" + +// Mover is one mover's entry, as apply derives it from its operations. +type Mover struct { + // Instance and Sequence are its path in the ledger and its last operation; not in the state. + Instance string `json:"-"` + Sequence uint64 `json:"-"` + + // Record is its record's key, `.`, the index's field. + Record string `json:"record"` + Module string `json:"module"` + Pipeline string `json:"pipeline,omitempty"` + Version string `json:"version,omitempty"` + Entries map[string]RecordEntry `json:"entries,omitempty"` + Outcomes map[string]RecordOutcome `json:"outcomes,omitempty"` +} + +// Running are the machines the mover runs healthy on, each with when it was first said. +func (m *Mover) Running() map[string]time.Time { + out := map[string]time.Time{} + for _, o := range m.Outcomes { + if o.Outcome == OutcomeRunning && o.Node != "" { + if at, ok := out[o.Node]; !ok || o.At.Before(at) { + out[o.Node] = o.At + } + } + } + return out +} + +// MoverInstance is the entry of a record's mover: `mover/`, each escaped as a name. +func MoverInstance(milestone, module string) (string, error) { + if strings.TrimSpace(module) == "" { + return "", fmt.Errorf("a mover of the record %s names its module", milestone) + } + inst := TypeMover + "/" + milestoneToken(milestone+"."+module) + if !segmentRe.MatchString(strings.TrimPrefix(inst, TypeMover+"/")) { + return "", fmt.Errorf("the mover %s of %s cannot be named: %q", module, milestone, inst) + } + return inst, nil +} + +// moversKey is the index key that names every mover of a record: its key's token, then the rest. +func moversKey(milestone string) string { + return strings.TrimSuffix(buscalls.IndexKey([]string{milestone}, "x"), ".x") + ".>" +} + +// movers reads the movers' entries of a record through the index, one query, and each entry. +func (rs *Records) movers(milestone string) ([]*Mover, error) { + insts, err := rs.Ledger.Query(MoversIndex, moversKey(milestone)) + if err != nil { + return nil, fmt.Errorf("the movers of the record %s cannot be read: %w", milestone, err) + } + var out []*Mover + for _, inst := range insts { + raw, seq, err := rs.Ledger.Get(inst) + if err != nil { + return nil, err + } + if raw == nil { + continue + } + var m Mover + if err := json.Unmarshal(raw, &m); err != nil { + return nil, fmt.Errorf("the mover %s is not readable: %w", inst, err) + } + if m.Record != milestone || m.Module == "" { + continue + } + m.Instance, m.Sequence = inst, seq + out = append(out, &m) + } + sort.Slice(out, func(i, j int) bool { return out[i].Module < out[j].Module }) + return out, nil +} + +// join lays a record's movers into it: their entries and outcomes, by module. +func (r *Record) join(ms []*Mover) { + r.movers = map[string]*Mover{} + for _, m := range ms { + r.movers[m.Module] = m + for k, e := range m.Entries { + if r.Entries == nil { + r.Entries = map[string]RecordEntry{} + } + r.Entries[k] = e + } + for k, o := range m.Outcomes { + if r.Outcomes == nil { + r.Outcomes = map[string]RecordOutcome{} + } + r.Outcomes[k] = o + } + } +} + +// Movers are the record's movers' entries, by module. +func (r *Record) Movers() map[string]*Mover { return r.movers } + +// mover is the record's entry of module, opened in the ledger when it has none, and counted on the head entry. +func (rs *Records) mover(r *Record, module string) (*Mover, error) { + if m := r.movers[module]; m != nil { + return m, nil + } + inst, err := MoverInstance(r.Milestone, module) + if err != nil { + return nil, err + } + m := &Mover{Instance: inst, Record: r.Milestone, Module: module} + seq, err := rs.Ledger.Apply(inst, OpMover, map[string]any{"record": r.Milestone, "module": module}) + if err != nil { + return nil, err + } + m.Sequence = seq + if r.movers == nil { + r.movers = map[string]*Mover{} + } + r.movers[module] = m + hseq, err := rs.Ledger.Apply(r.Instance, OpMovers, map[string]any{"movers": len(r.movers)}) + if err != nil { + return m, fmt.Errorf("the mover %s is written and the record %s's count of its movers is not: %w", module, r.Instance, err) + } + r.MoverCount, r.Sequence = len(r.movers), hseq + return m, nil +} + +// SetMover writes a mover's pipeline and version, unless its entry holds them already. +func (rs *Records) SetMover(r *Record, module, pipeline, version string) error { + m, err := rs.mover(r, module) + if err != nil { + return err + } + next := *m + next.Pipeline, next.Version = firstOf(pipeline, m.Pipeline), firstOf(version, m.Version) + if next.Pipeline == m.Pipeline && next.Version == m.Version { + return nil + } + if err := rs.fits(m.Instance, next); err != nil { + return err + } + seq, err := rs.Ledger.Apply(m.Instance, OpMover, map[string]any{"record": r.Milestone, "module": module, + "pipeline": next.Pipeline, "version": next.Version}) + if err != nil { + return err + } + m.Pipeline, m.Version, m.Sequence = next.Pipeline, next.Version, seq + rs.fitted(m.Instance) + return nil +} + +// fits refuses an entry whose state would pass EntryLimit, raising record-too-large for it (§809). +func (rs *Records) fits(instance string, state any) error { + raw, err := json.Marshal(state) + if err != nil { + return err + } + if len(raw) <= EntryLimit { + return nil + } + if rs.TooLarge != nil { + rs.TooLarge(instance, len(raw), true) + } + return fmt.Errorf("the delivery record's entry %s would hold %d bytes, over the %d an entry holds: refused (%s)", instance, len(raw), EntryLimit, RecordTooLarge) +} + +// fitted clears record-too-large for an entry written within EntryLimit (TooLarge clears only one it holds). +func (rs *Records) fitted(instance string) { + if rs.TooLarge != nil { + rs.TooLarge(instance, 0, false) + } +} + +// clearTooLarge clears record-too-large for every entry of a record at its end. +func (rs *Records) clearTooLarge(r *Record) { + rs.fitted(r.Instance) + for _, m := range r.movers { + rs.fitted(m.Instance) + } +} diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record.go index 9a1c5da6a..6c095e89c 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record.go @@ -31,7 +31,7 @@ const RecordsLedger = "records" // RecordForm is the form of the records' layout (design 47 *A module's kept data on the bus*): raised whenever the gate // changes how it writes them. -const RecordForm = 1 +const RecordForm = 2 // The ledger's two types: a record, and the list of the records opened in one month (the ledger is listed through it, // since a bundle has no listing of a ledger's instances; a month of about 600 records keeps it far under a value's 256 @@ -47,8 +47,8 @@ const ( OpOpen = "open" // set-fields: the milestone, its branch, source revision and merge call, the heads, when, the form OpGroup = "group" // set-fields: the group's state OpTrace = "trace" // set-fields: the trace - OpEntry = "entry" // set-entry into entries: one entry of the graph difference, by its module, fingerprint and node - OpOutcome = "outcome" // set-entry into outcomes, by the entry, the node and the step + OpEntry = "entry" // a mover's: set-entry into entries, one entry of the graph difference, by its module, fingerprint and node + OpOutcome = "outcome" // a mover's: set-entry into outcomes, by the entry, the node and the step OpAway = "away" // set-entry into away, by node OpBack = "back" // remove from away, by node OpEnd = "end" // set-fields: ended @@ -58,7 +58,7 @@ const ( ) // RecordOperations are a record's operations, in the order the manifest's type allows them. -var RecordOperations = []string{OpOpen, OpGroup, OpTrace, OpEntry, OpOutcome, OpAway, OpBack, OpAct, OpEnd} +var RecordOperations = []string{OpOpen, OpGroup, OpTrace, OpMovers, OpAway, OpBack, OpAct, OpEnd} // HeldOperations are the month's list's operations. var HeldOperations = []string{OpHold, OpDrop} @@ -99,6 +99,11 @@ type Record struct { Ended *RecordEnded `json:"ended,omitempty"` // Recovery marks the merge its author named a recovery by the seat's verb recovery (design 47 G §81, A-23). Recovery *RecordRecovery `json:"recovery,omitempty"` + // MoverCount is the count of its movers, each with an entry of its own (design 47 H2 §809); Entries and Outcomes above + // are joined from those entries when read. + MoverCount int `json:"movers,omitempty"` + + movers map[string]*Mover } // RecordRecovery is the field recovery of a record: who named the merge a recovery (the caller the runtime proved), why, @@ -267,6 +272,8 @@ type RecordLedger interface { Get(instance string) (json.RawMessage, uint64, error) // Remove removes a record with its operations and its snapshot through the tool runner's ledger removal. Remove(instance string) error + // Query is the instances a declared index names under key (`>` for the rest). + Query(index, key string) ([]string, error) } // sdkLedger is the gate's ledger through the runtime: the tool runner's ledger calls, on the gate's own ledger only @@ -316,6 +323,22 @@ func (sdkLedger) Remove(instance string) error { return nil } +func (sdkLedger) Query(index, key string) ([]string, error) { + out, err := ledgerCall(buscalls.LedgerQuery, buscalls.QueryRequest{Ledger: RecordsLedger, Index: index, Key: key}) + if err != nil { + return nil, err + } + var got buscalls.QueryAnswer + if err := json.Unmarshal(out, &got); err != nil { + return nil, fmt.Errorf("the ledger's answer to the query of %s is not readable: %w", index, err) + } + insts := make([]string, 0, len(got.Entries)) + for _, e := range got.Entries { + insts = append(insts, e.Instance) + } + return insts, nil +} + func (sdkLedger) Get(instance string) (json.RawMessage, uint64, error) { out, err := ledgerCall(buscalls.LedgerGet, buscalls.InstanceRequest{Ledger: RecordsLedger, Instance: instance}) if err != nil { @@ -345,6 +368,8 @@ type Records struct { // Changed is told of every change of a record that owes the forge an act (A-39): a step's outcome applied, the record // ended (owedacts.go); nil owes none. Changed func(r *Record, c RecordChange) + // TooLarge raises (raised) or clears record-too-large for an entry (§809); nil raises none. + TooLarge func(instance string, bytes int, raised bool) // held are the records kept, by instance, read once from the ledger (Current) and kept as the gate writes them: the // gate is their one writer. @@ -454,6 +479,11 @@ func (rs *Records) Read(instance string) (*Record, error) { return nil, nil } r.Instance, r.Sequence = instance, seq + ms, err := rs.movers(r.Milestone) + if err != nil { + return nil, err + } + r.join(ms) return &r, nil } @@ -628,6 +658,11 @@ func (rs *Records) Sweep(now time.Time) error { return err } if r != nil { + for _, m := range r.movers { + if err := rs.Ledger.Remove(m.Instance); err != nil { + return err + } + } if err := rs.Ledger.Remove(e.Instance); err != nil { return err } @@ -646,11 +681,17 @@ func (rs *Records) SetGroup(r *Record, g RecordGroup) error { if r.Group != nil && reflect.DeepEqual(*r.Group, g) { return nil } + next := *r + next.Group = &g + if err := rs.fits(r.Instance, next.head()); err != nil { + return err + } seq, err := rs.Ledger.Apply(r.Instance, OpGroup, map[string]any{"group": g}) if err != nil { return err } r.Group, r.Sequence = &g, seq + rs.fitted(r.Instance) return nil } @@ -659,11 +700,17 @@ func (rs *Records) SetTrace(r *Record, t RecordTrace) error { if r.Trace != nil && reflect.DeepEqual(*r.Trace, t) { return nil } + next := *r + next.Trace = &t + if err := rs.fits(r.Instance, next.head()); err != nil { + return err + } seq, err := rs.Ledger.Apply(r.Instance, OpTrace, map[string]any{"trace": t}) if err != nil { return err } r.Trace, r.Sequence = &t, seq + rs.fitted(r.Instance) return nil } @@ -677,14 +724,25 @@ func (rs *Records) Outcome(r *Record, o RecordOutcome) (applied bool, err error) if had, ok := r.Outcomes[o.Key]; ok && (had.Event != "" && had.Event == o.Event || sameOutcome(had, o)) { return false, nil } - seq, err := rs.Ledger.Apply(r.Instance, OpOutcome, o) + m, err := rs.mover(r, o.Module) if err != nil { return false, err } + next := *m + next.Outcomes = withEntry(m.Outcomes, o.Key, o) + if err := rs.fits(m.Instance, next); err != nil { + return false, err + } + seq, err := rs.Ledger.Apply(m.Instance, OpOutcome, o) + if err != nil { + return false, err + } + m.Outcomes, m.Sequence = next.Outcomes, seq + rs.fitted(m.Instance) if r.Outcomes == nil { r.Outcomes = map[string]RecordOutcome{} } - r.Outcomes[o.Key], r.Sequence = o, seq + r.Outcomes[o.Key] = o if rs.Changed != nil { rs.Changed(r, RecordChange{What: outcomeWords(o), Why: firstOf(o.Why, "the "+o.Step+" step's event")}) } @@ -697,17 +755,44 @@ func (rs *Records) AddEntry(r *Record, e RecordEntry) error { if had, ok := r.Entries[e.Key]; ok && had == e { return nil } - seq, err := rs.Ledger.Apply(r.Instance, OpEntry, e) + m, err := rs.mover(r, e.Module) if err != nil { return err } + next := *m + next.Entries = withEntry(m.Entries, e.Key, e) + if err := rs.fits(m.Instance, next); err != nil { + return err + } + seq, err := rs.Ledger.Apply(m.Instance, OpEntry, e) + if err != nil { + return err + } + m.Entries, m.Sequence = next.Entries, seq + rs.fitted(m.Instance) if r.Entries == nil { r.Entries = map[string]RecordEntry{} } - r.Entries[e.Key], r.Sequence = e, seq + r.Entries[e.Key] = e return nil } +// withEntry is a copy of m with v under k. +func withEntry[V any](m map[string]V, k string, v V) map[string]V { + out := make(map[string]V, len(m)+1) + for key, val := range m { + out[key] = val + } + out[k] = v + return out +} + +// head is the record's head entry as the ledger holds it: every record-wide field, none of a mover's. +func (r Record) head() Record { + r.Entries, r.Outcomes, r.movers = nil, nil, nil + return r +} + // sameOutcome is whether two outcomes say the same, whenever and from whichever event they were taken. func sameOutcome(a, b RecordOutcome) bool { a.At, b.At, a.Event, b.Event = time.Time{}, time.Time{}, "", "" diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record_test.go index aeb067199..da028b6e5 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record_test.go @@ -7,6 +7,7 @@ import ( "os" "path/filepath" "slices" + "sort" "strings" "testing" "time" @@ -150,6 +151,33 @@ func (f *ledgerFake) Remove(instance string) error { return nil } +// Query answers a declared index as the tool runner's watcher keeps it: the key of each instance whose type declares the +// index, from the values of its fields and its path (buscalls.IndexKey), matched with `>` for the rest. +func (f *ledgerFake) Query(index, key string) ([]string, error) { + var out []string + for inst, raw := range f.states { + ty := f.types[strings.SplitN(inst, "/", 2)[0]] + for _, ix := range ty.Indexes { + if ix.Name != index || string(raw) == "null" { + continue + } + state := map[string]any{} + _ = json.Unmarshal(raw, &state) + var vals []string + for _, on := range ix.On { + v, _ := state[on].(string) + vals = append(vals, v) + } + k := buscalls.IndexKey(vals, inst) + if k == key || strings.HasSuffix(key, ">") && strings.HasPrefix(k, strings.TrimSuffix(key, ">")) { + out = append(out, inst) + } + } + } + sort.Strings(out) + return out, nil +} + func (f *ledgerFake) Get(instance string) (json.RawMessage, uint64, error) { s := f.states[instance] if s == nil || string(s) == "null" { @@ -165,6 +193,64 @@ func TestTheDeliveryRecordIsALedgerInstanceKeyedByItsSourceRevision(t *testing.T same := map[string]string{"novox/lib@4": strings.Repeat("2", 40), "novox/app@3": strings.Repeat("1", 40)} at := time.Date(2026, 11, 9, 20, 0, 0, 0, time.UTC) + // One entry per module and revision (design 47 H2 §809; H-A-385, H-A-391): 126 movers write 127 entries, none over + // 64 KiB, the head counting 126; an entry made too large is refused with record-too-large, cleared at the record's end. + t.Run("a record of 126 movers writes 127 entries, each within 64 KiB", func(t *testing.T) { + l := newLedgerFake(t) + found := map[string]bool{} + rs := &Records{Ledger: l, Now: func() time.Time { return at }, TooLarge: func(inst string, _ int, raised bool) { + if raised { + found[inst] = true + } else { + delete(found, inst) + } + }} + r, _, err := rs.Open(opening("main", 90, heads)) + if err != nil { + t.Fatal(err) + } + for i := range 126 { + module := fmt.Sprintf("mod-%03d", i) + if err := rs.AddEntry(r, RecordEntry{Module: module, Fingerprint: "src2:f" + module}); err != nil { + t.Fatal(err) + } + if err := rs.SetMover(r, module, "main.90", "0.1.7"); err != nil { + t.Fatal(err) + } + for _, node := range []string{"ace", "g14", "novox", "shanks"} { + if _, err := rs.Outcome(r, RecordOutcome{Module: module, Fingerprint: "src2:f" + module, Node: node, Step: "running", + Outcome: OutcomeRunning, At: at}); err != nil { + t.Fatal(err) + } + } + } + entries := 0 + for inst, raw := range l.states { + if strings.HasPrefix(inst, TypeRecord+"/") || strings.HasPrefix(inst, TypeMover+"/") { + entries++ + if len(raw) > EntryLimit { + t.Errorf("%s holds %d bytes", inst, len(raw)) + } + if strings.HasPrefix(inst, TypeMover+"/") && strings.Count(string(raw), `"module":"mod-`) != 1+1+4 { + t.Errorf("%s names another module: %s", inst, raw) + } + } + } + read, err := rs.Read(r.Instance) + if err != nil || entries != 127 || read.MoverCount != 126 || len(read.Movers()) != 126 || len(read.Outcomes) != 504 || + len(read.Movers()["mod-007"].Running()) != 4 || read.Movers()["mod-007"].Version != "0.1.7" { + t.Fatalf("%d entries, the head counts %d, %d joined, %d outcomes, %v", entries, read.MoverCount, len(read.Movers()), len(read.Outcomes), err) + } + why := strings.Repeat("w", EntryLimit) + if _, err := rs.Outcome(read, RecordOutcome{Module: "mod-001", Fingerprint: "src2:fmod-001", Node: "ace", Step: "build", + Outcome: OutcomeFailed, Why: why, At: at}); err == nil || !strings.Contains(err.Error(), RecordTooLarge) || len(found) != 1 { + t.Fatalf("an entry over 64 KiB was taken (%v), findings %v", err, found) + } + if err := rs.End(read, "every present node runs its entries", ""); err != nil || len(found) != 0 { + t.Fatalf("the record's end left %v (%v)", found, err) + } + }) + t.Run("an instance per source revision, a path the tool runner takes", func(t *testing.T) { a, err := RecordInstance(recordMilestone("feat/x", 7)) if err != nil { @@ -322,7 +408,7 @@ func TestTheDeliveryRecordIsALedgerInstanceKeyedByItsSourceRevision(t *testing.T // W11.1: the manifest's ledger types allow exactly the operations the gate applies, and the ledger is capped at 768 MiB. func TestTheManifestsLedgerTypesAreTheRecordsOperations(t *testing.T) { l := newLedgerFake(t) - for name, want := range map[string][]string{TypeRecord: RecordOperations, TypeHeld: HeldOperations} { + for name, want := range map[string][]string{TypeRecord: RecordOperations, TypeHeld: HeldOperations, TypeMover: MoverOperations} { var got []string for _, op := range l.types[name].Operations { got = append(got, op.Name) @@ -332,8 +418,11 @@ func TestTheManifestsLedgerTypesAreTheRecordsOperations(t *testing.T) { } } raw, _ := os.ReadFile(filepath.Join("..", "..", "module.json")) - if !strings.Contains(string(raw), `"max-mib": 768`) { - t.Error("the records ledger is not capped at 768 MiB") + if !strings.Contains(string(raw), `"max-mib": 1024`) { + t.Error("the records ledger is not capped at 1 GiB (design 47 H2 §809)") + } + if ix := l.types[TypeMover].Indexes; len(ix) != 1 || ix[0].Name != MoversIndex || !slices.Equal(ix[0].On, []string{"record"}) { + t.Errorf("the type mover's indexes are %+v, not one %s on its record's key", ix, MoversIndex) } } @@ -555,7 +644,7 @@ func TestTheRecordsToolsReadThemAndEndOneByHand(t *testing.T) { } got, err := rs.RecordOf(r.Instance, "") if err != nil || len(got) != 1 || got[0].Record.Milestone != recordMilestone("main", 47) || got[0].Waits != "waiting for mesh-x on ace" || - got[0].Bytes == 0 || got[0].Of != 256<<10 { + got[0].Bytes == 0 || got[0].Of != EntryLimit { t.Fatalf("%+v %v", got, err) } if byMilestone, err := rs.RecordOf("", recordMilestone("main", 47)); err != nil || len(byMilestone) != 1 { diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/recordevents.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/recordevents.go index 5546b825f..7b90a3d5a 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/recordevents.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/recordevents.go @@ -137,6 +137,7 @@ func (rs *Records) end(r *Record, why, by string) error { return err } r.Ended, r.Sequence = &e, seq + rs.clearTooLarge(r) rs.keep(r) if rs.Changed != nil { rs.Changed(r, RecordChange{What: "ended", Why: why, Ended: true}) diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/recordpipeline.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/recordpipeline.go index be806d965..af495c710 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/recordpipeline.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/recordpipeline.go @@ -186,6 +186,9 @@ func (h *Holder) pipelineStarted(p runner.PipelineStartedEvent) error { if err := h.Records.AddEntry(r, RecordEntry{Module: m.Module, Fingerprint: m.Fingerprint}); err != nil { return err } + if err := h.Records.SetMover(r, m.Module, p.Pipeline, m.Version); err != nil { + return err + } } return nil } @@ -206,6 +209,9 @@ func (h *Holder) pipelineEnded(p runner.PipelineEndedEvent) error { if err := h.Records.AddEntry(r, RecordEntry{Module: m.Module, Fingerprint: m.Fingerprint}); err != nil { return err } + if err := h.Records.SetMover(r, m.Module, p.Pipeline, m.Version); err != nil { + return err + } for _, s := range m.Steps { if s.Step == "" || s.Outcome == "" { continue diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/recordtools.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/recordtools.go index 3437a9907..6b39b2cbc 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/recordtools.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/recordtools.go @@ -15,12 +15,9 @@ import ( stdio "git.novox.be/novox/mesh-sdk/go" ) -// valueLimit is the bus's limit of one value, which a record's state is held under (design 47: a record of 30 modules on -// four nodes is about 30 KiB of it). -const valueLimit = 256 << 10 - -// recordView is one record as gate_record answers it: its state, the sequence of its last operation, its size against -// the value limit, and while open why it has not ended. +// recordView is one record as gate_record answers it, joined from its head entry and its movers' entries (design 47 H2 +// §809): its state, the sequence of its head's last operation, its head entry's size against the 64 KiB an entry holds, +// and while open why it has not ended. type recordView struct { Instance string `json:"instance"` Sequence uint64 `json:"sequence"` @@ -31,8 +28,8 @@ type recordView struct { } func viewOf(r *Record) recordView { - raw, _ := json.Marshal(r) - v := recordView{Instance: r.Instance, Sequence: r.Sequence, Bytes: len(raw), Of: valueLimit, Record: r} + raw, _ := json.Marshal(r.head()) + v := recordView{Instance: r.Instance, Sequence: r.Sequence, Bytes: len(raw), Of: EntryLimit, Record: r} if r.Ended == nil { v.Waits = allRun(r) } diff --git a/modules/mesh-delivery-gate/module.json b/modules/mesh-delivery-gate/module.json index 94b485b45..b702a5e67 100644 --- a/modules/mesh-delivery-gate/module.json +++ b/modules/mesh-delivery-gate/module.json @@ -36,7 +36,7 @@ "storage": "ledger", "template": "ledger", "options": { - "max-mib": 768 + "max-mib": 1024 }, "types": [ { @@ -125,6 +125,9 @@ }, "owed": { "type": "object" + }, + "movers": { + "type": "integer" } }, "operations": [ @@ -160,34 +163,11 @@ "sequence": true }, { - "name": "entry", - "effect": "set-entry", + "name": "movers", + "effect": "set-fields", "fields": [ - "key", - "module", - "fingerprint", - "node" + "movers" ], - "into": "entries", - "key": "key", - "sequence": true - }, - { - "name": "outcome", - "effect": "set-entry", - "fields": [ - "key", - "module", - "fingerprint", - "node", - "step", - "outcome", - "why", - "at", - "event" - ], - "into": "outcomes", - "key": "key", "sequence": true }, { @@ -234,6 +214,99 @@ } ] }, + { + "name": "mover", + "schema": { + "record": { + "type": "string" + }, + "module": { + "type": "string" + }, + "pipeline": { + "type": "string" + }, + "version": { + "type": "string" + }, + "key": { + "type": "string" + }, + "fingerprint": { + "type": "string" + }, + "node": { + "type": "string" + }, + "step": { + "type": "string" + }, + "outcome": { + "type": "string" + }, + "why": { + "type": "string" + }, + "at": { + "type": "string" + }, + "event": { + "type": "string" + } + }, + "operations": [ + { + "name": "mover", + "effect": "set-fields", + "fields": [ + "record", + "module", + "pipeline", + "version" + ], + "sequence": true + }, + { + "name": "entry", + "effect": "set-entry", + "fields": [ + "key", + "module", + "fingerprint", + "node" + ], + "into": "entries", + "key": "key", + "sequence": true + }, + { + "name": "outcome", + "effect": "set-entry", + "fields": [ + "key", + "module", + "fingerprint", + "node", + "step", + "outcome", + "why", + "at", + "event" + ], + "into": "outcomes", + "key": "key", + "sequence": true + } + ], + "indexes": [ + { + "name": "movers", + "on": [ + "record" + ] + } + ] + }, { "name": "held", "schema": { -- 2.54.0 From e866a64be30c8b6725e838c5925961787e3c9bcd Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 22 Nov 2026 00:21:06 +0100 Subject: [PATCH 12/54] GT4: the gate watches a source-only registration's branch as any MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The gate reads the registrations page by page from the cursor each page answers, so a source-only registration (no module) past the first page is watched as any branch; its merge's take-in with no movers ends the record nothing to deliver as before. Builds: design 47 H2 §815, §806, §779 --- .../cmd/mesh-delivery-gate/sourcestart.go | 37 +++++++++++++------ .../mesh-delivery-gate/sourcetools_test.go | 26 +++++++++++++ 2 files changed, 51 insertions(+), 12 deletions(-) diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/sourcestart.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/sourcestart.go index 1535f950b..ccefaaf20 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/sourcestart.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/sourcestart.go @@ -30,20 +30,33 @@ type Registered interface { // seatCatalog reads the seat mesh-catalog through the tool runner. type seatCatalog struct{ ask asking } +// Registrations reads every registration, a module's and a source-only one alike (design 47 H2 §815), page by page from +// the cursor each page answers (§779, §806), so a registration past the first 768 KB is watched too. func (s seatCatalog) Registrations() (meshcatalog.RegistrationsAnswer, error) { - var a meshcatalog.RegistrationsAnswer - raw, err := s.ask("seat:"+meshcatalog.Seat+"."+meshcatalog.VerbRegistrations, meshcatalog.RegistrationsInput{}) - if err != nil { - return a, err + var all meshcatalog.RegistrationsAnswer + seen := map[string]bool{} + for next := ""; ; { + var a meshcatalog.RegistrationsAnswer + raw, err := s.ask("seat:"+meshcatalog.Seat+"."+meshcatalog.VerbRegistrations, meshcatalog.RegistrationsInput{Next: next}) + if err != nil { + return all, err + } + answer, said, ok := answerOf(raw) + if !ok { + return all, fmt.Errorf("registrations refused: %s", lastLine(said)) + } + if err := json.Unmarshal(answer, &a); err != nil { + return all, fmt.Errorf("the answer of registrations is not readable: %w", err) + } + all.Registrations = append(all.Registrations, a.Registrations...) + if a.Next == "" { + return all, nil + } + if seen[a.Next] { + return all, fmt.Errorf("registrations answered the cursor %q twice: its pages do not end", a.Next) + } + seen[a.Next], next = true, a.Next } - answer, said, ok := answerOf(raw) - if !ok { - return a, fmt.Errorf("registrations refused: %s", lastLine(said)) - } - if err := json.Unmarshal(answer, &a); err != nil { - return a, fmt.Errorf("the answer of registrations is not readable: %w", err) - } - return a, nil } // Branch is the seat git's verb branch: a repository's branch head. diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/sourcetools_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/sourcetools_test.go index ddbf95276..5c0ba7fb7 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/sourcetools_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/sourcetools_test.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "reflect" + "slices" "strings" "testing" "time" @@ -97,6 +98,31 @@ func (f fakeCatalog) Registrations() (meshcatalog.RegistrationsAnswer, error) { // Design 47 G A-401: a watched branch starts from the forge as a clone does: each registered repository at the branch's head, // the group empty, source-moved said; a repository whose head cannot be read is left out; a key held is never rewritten. func TestAWatchedBranchStartsFromTheForge(t *testing.T) { + // A source-only registration (no module, design 47 H2 §815) is watched as any branch, its registrations read page by + // page from the cursor (§779, §806). + t.Run("a source-only registration past the first page is watched", func(t *testing.T) { + pages := map[string]meshcatalog.RegistrationsAnswer{ + "": {Registrations: []meshcatalog.Registration{{Module: "mesh-a", Origin: "o/mesh", Branches: []string{"main"}}}, Next: "p2", Capped: true}, + "p2": {Registrations: []meshcatalog.Registration{{Origin: "o/hq", Branches: []string{"main"}, Check: &meshcatalog.SourceCheck{Seat: "node-build-go", Step: "unit"}}}}, + } + var asked []string + c := seatCatalog{ask: func(key string, body any) (json.RawMessage, error) { + in := body.(meshcatalog.RegistrationsInput) + asked = append(asked, in.Next) + return json.Marshal(pages[in.Next]) + }} + a, err := c.Registrations() + if err != nil || !slices.Equal(asked, []string{"", "p2"}) { + t.Fatalf("asked %v: %v", asked, err) + } + if w := watchedRepositories(a.Registrations); !slices.Equal(w["main"], []string{"o/hq", "o/mesh"}) { + t.Errorf("main watches %v", w["main"]) + } + pages["p2"] = meshcatalog.RegistrationsAnswer{Next: "p2"} + if _, err := c.Registrations(); err == nil { + t.Error("pages that never end were read on") + } + }) regs := []meshcatalog.Registration{ {Module: "car-hunter", Origin: "o/car-hunter", Branches: []string{"main"}}, {Module: "mesh-a", Origin: "o/mesh", Branches: []string{"main"}}, -- 2.54.0 From eac56f9990c583ee008cd03ad90817731eb365f6 Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 22 Nov 2026 00:22:05 +0100 Subject: [PATCH 13/54] GT1: the gate's whole test over a real bus reads the mover's entry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The test bus serves the ledger's query the movers index answers, and the take-in's mover is looked for in its own entry, the head counting it. Builds: design 47 H2 §809; H-A-391 --- .../cmd/mesh-delivery-gate/bus_test.go | 14 +++++++++++++- .../cmd/mesh-delivery-gate/mergecall_bus_test.go | 3 ++- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/bus_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/bus_test.go index 4e29de34f..2cbb06ac0 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/bus_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/bus_test.go @@ -387,13 +387,25 @@ func (r *runtime) answer(m rpcMessage) { return nil, fmt.Errorf("the runtime answers no ask %s in this test", a.Key) } switch m.Method { - case buscalls.LedgerApply, buscalls.LedgerGet: + case buscalls.LedgerApply, buscalls.LedgerGet, buscalls.LedgerQuery: // The gate's ledger `records` (W11.1), as the tool runner keeps it: the record test's ledger in memory. r.b.mu.Lock() defer r.b.mu.Unlock() if r.b.ledger == nil { r.b.ledger = newLedgerFake(r.t) } + if m.Method == buscalls.LedgerQuery { + var in buscalls.QueryRequest + if err := json.Unmarshal(m.Params, &in); err != nil { + return nil, err + } + insts, err := r.b.ledger.Query(in.Index, in.Key) + var a buscalls.QueryAnswer + for _, inst := range insts { + a.Entries = append(a.Entries, buscalls.IndexEntry{Instance: inst}) + } + return a, err + } if m.Method == buscalls.LedgerGet { var in buscalls.InstanceRequest if err := json.Unmarshal(m.Params, &in); err != nil { diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/mergecall_bus_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/mergecall_bus_test.go index 129567db4..899aa9b22 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/mergecall_bus_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/mergecall_bus_test.go @@ -75,7 +75,8 @@ func TestAGateWholeTakesAMergeCallAndWatchesTheHeadsOverARealBus(t *testing.T) { rt.until("the take-in's mover in the record", eventWithin, func() bool { b.mu.Lock() defer b.mu.Unlock() - return strings.Contains(string(b.ledger.states[instance]), `"module":"app"`) + mover, _ := MoverInstance(recordMilestone(branch, 1), "app") + return strings.Contains(string(b.ledger.states[mover]), `"module":"app"`) && strings.Contains(string(b.ledger.states[instance]), `"movers":1`) }) if e, _ := b.kvs["source"].Get(ctx, sourceKey); e.Revision() != 1 { t.Fatalf("the merge call heard twice wrote source.%s %d times", escaped, e.Revision()) -- 2.54.0 From 41610e903f4ed4ae3217967e8da51a06d70e6fc4 Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 22 Nov 2026 00:27:04 +0100 Subject: [PATCH 14/54] BM1: the build modules give a step met away again by again-delays MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Each build module declares again-delays (15s,60s) and gives its puller the delays steprun.AgainDelays reads from it; a publish's no responders is a transport fault by steprun.PublishFault with no seat call to learn why, so the modules no longer invoke the assigner's assignments. Builds: design 47 H2 §799; H-A-350 DMC-7, H-A-390 --- .../cmd/mesh-build-go/kit_test.go | 26 +++++++++++++++++ .../mesh-build-go/cmd/mesh-build-go/main.go | 8 ++++++ .../cmd/mesh-build-go/publish_test.go | 28 +++++-------------- modules/mesh-build-go/module.json | 8 +++++- .../cmd/mesh-build-image/compose_test.go | 2 +- .../cmd/mesh-build-image/copy.go | 4 +-- .../cmd/mesh-build-image/kit_test.go | 26 +++++++++++++++++ .../cmd/mesh-build-image/main.go | 8 ++++++ modules/mesh-build-image/module.json | 8 +++++- .../cmd/mesh-build-node/kit_test.go | 26 +++++++++++++++++ .../cmd/mesh-build-node/main.go | 8 ++++++ .../cmd/mesh-build-node/push_test.go | 14 ++-------- modules/mesh-build-node/module.json | 8 +++++- 13 files changed, 136 insertions(+), 38 deletions(-) diff --git a/modules/mesh-build-go/cmd/mesh-build-go/kit_test.go b/modules/mesh-build-go/cmd/mesh-build-go/kit_test.go index 4cbf1de79..3f1962c7d 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/kit_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/kit_test.go @@ -5,6 +5,8 @@ import ( "encoding/json" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/steprun" "git.novox.be/novox/mesh-sdk/go/forgekey" + "os" + "path/filepath" "strings" "testing" "time" @@ -40,6 +42,30 @@ func TestTheStartSweepEndsWhatAPreviousProcessLeft(t *testing.T) { // ADR 0664 §1 (issue 1748): the module's own paths from a failed fetch or download to its step's error say a network that did // not answer as a transport fault, and a failure of the source (an unknown revision, a checksum) as none. func TestTheStepsSayAFailedNetworkAsATransportFault(t *testing.T) { + // A step whose fetch or publish met its holder away is given again after the module's again-delays, 15 s after its + // first run and 60 s after its second by default (design 47 H2 §799; H-A-390). + t.Run("again-delays is declared and read", func(t *testing.T) { + raw, err := os.ReadFile(filepath.Join("..", "..", "module.json")) + if err != nil { + t.Fatal(err) + } + var m struct { + Config map[string]json.RawMessage `json:"config"` + } + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatal(err) + } + var c struct { + Default string `json:"default"` + Runtime bool `json:"runtime"` + } + decl, ok := m.Config[steprun.AgainDelaysSetting] + _ = json.Unmarshal(decl, &c) + delay, err := steprun.AgainDelays(c.Default) + if !ok || !c.Runtime || err != nil || delay(1) != 15*time.Second || delay(2) != 60*time.Second { + t.Fatalf("again-delays declared %v as %+v: %v", ok, c, err) + } + }) conformance.Passing(t, []string{conformance.GitFetch, conformance.GoDownload}, func(runner, words string) error { switch runner { case conformance.GitFetch: diff --git a/modules/mesh-build-go/cmd/mesh-build-go/main.go b/modules/mesh-build-go/cmd/mesh-build-go/main.go index 7886aaca6..79a538789 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/main.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/main.go @@ -102,6 +102,14 @@ func main() { steps := m.StepTools() booted := time.Now() puller := m.Puller(steps, nil) + // A step whose fetch or publish met its holder away is given again after again-delays (design 47 H2 §799; H-A-390). + againSetting, _ := stdio.Setting(steprun.AgainDelaysSetting) + againDelay, againErr := steprun.AgainDelays(againSetting) + if againErr != nil { + logf("the setting %s: %v", steprun.AgainDelaysSetting, againErr) + os.Exit(1) + } + puller.Delay = againDelay m.Held = puller if m.Node != "" { // A step a previous process left running is ended (issue 1890): one that names another owner, or none and diff --git a/modules/mesh-build-go/cmd/mesh-build-go/publish_test.go b/modules/mesh-build-go/cmd/mesh-build-go/publish_test.go index 92b0cc160..06ac02449 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/publish_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/publish_test.go @@ -17,7 +17,6 @@ import ( builds "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" registry "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/containerimageregistry" sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/steprun" ) // pushed are the blobs the fake artifact store took, by digest, for the tests that read an archive back. @@ -147,26 +146,13 @@ func TestARefusedPushFailsTheStepNotStored(t *testing.T) { !strings.Contains(st.Reason, "oci-artifact-registry") { t.Errorf("no holder: the step ended %s: %q", st.State, st.Reason) } - // ADR 0664 §1 (issue 1748): "no responders" is the same words for a seat nothing holds and for a holder whose tool runner is - // being replaced; the assigner says which. A named holder makes it a transport fault (the bus gives the ask again), none - // fails at once naming it. - for _, c := range []struct { - name, holders string - passing bool - words string - }{ - {"a holder named", `{"seats":[{"seat":"oci-artifact-registry","holders":[{"module":"mesh-registry","node":"novox"}]}],"holds":1}`, true, "the holder is silent: seat oci-artifact-registry, held by mesh-registry on novox"}, - {"nothing holds it", `{"seats":[{"seat":"go-package-registry","holders":[{"module":"mesh-registry","node":"novox"}]}],"holds":1}`, false, "nothing holds oci-artifact-registry"}, - } { - m.Call = func(key string, _ any) (json.RawMessage, error) { - if key == steprun.AssignerHolders { - return json.RawMessage(c.holders), nil - } - return nil, errors.New("nats: no responders available for request") - } - if _, st := runBuild(t, m, states, "tools"); st.State != sdkstep.StepFailed || st.Passing != c.passing || !strings.Contains(st.Reason, c.words) { - t.Errorf("%s: the step ended %s: %q (passing %v)", c.name, st.State, st.Reason, st.Passing) - } + // A publish met by no responders is a transport fault with no seat call to learn why (design 47 H2 §799, ADR 0664 §1): + // the run ends again, its why opening "publish: ". + m.Call = func(key string, _ any) (json.RawMessage, error) { + return nil, errors.New("nats: no responders available for request") + } + if _, st := runBuild(t, m, states, "tools"); st.State != sdkstep.StepFailed || !st.Passing || !strings.Contains(st.Reason, "publish: ") { + t.Errorf("the step ended %s: %q (passing %v)", st.State, st.Reason, st.Passing) } } diff --git a/modules/mesh-build-go/module.json b/modules/mesh-build-go/module.json index 48acd5e4f..3198c8f59 100644 --- a/modules/mesh-build-go/module.json +++ b/modules/mesh-build-go/module.json @@ -45,7 +45,6 @@ "seat:go-package-registry.publish-part", "seat:mesh-catalog.input", "seat:mesh-wants.input", - "seat:mesh-module-assigner.assignments", "seat:mesh-catalog.module", "seat:mesh-catalog.seat-definitions", "seat:mesh-catalog.entry" @@ -68,6 +67,13 @@ "description": "How many processors one Go step of this module needs (a full check of a large repository takes what it is given: three at about 4 cores each ran past the 30-minute step bound on 2026-10-26). The capacity above is an upper bound: on each node the module runs at most floor(the cores of its build slice's CPU quota / this) steps at once, the quota read live from its cgroup (issue 1806), so a lowered build CPU share never starves its steps. Above 0.", "minimum": 0.1, "default": 6 + }, + "again-delays": { + "type": "string", + "description": "The delays a step ask whose fetch or publish met its holder away is given again after: after its first run, then after its second, as durations such as 15s,60s (design 47 H2 §799); a third run met so ends the step failed with why fetch: or publish:.", + "trusted": false, + "runtime": true, + "default": "15s,60s" } }, "tools": [ diff --git a/modules/mesh-build-image/cmd/mesh-build-image/compose_test.go b/modules/mesh-build-image/cmd/mesh-build-image/compose_test.go index e06d84c2d..5dd0fc2f3 100644 --- a/modules/mesh-build-image/cmd/mesh-build-image/compose_test.go +++ b/modules/mesh-build-image/cmd/mesh-build-image/compose_test.go @@ -24,7 +24,7 @@ func TestTheModuleInvokesTheImageSeatAndHoldsNoRegistry(t *testing.T) { t.Fatal(err) } if want := []string{"seat:mesh-hand-acts.hand-act", "seat:container-image-registry.publish", "seat:container-image-registry.publish-part", - "seat:container-image-registry.mirror", "seat:mesh-module-assigner.assignments", "seat:mesh-catalog.module", "seat:mesh-catalog.seat-definitions"}; !slices.Equal(m.Invokes, want) { + "seat:container-image-registry.mirror", "seat:mesh-catalog.module", "seat:mesh-catalog.seat-definitions"}; !slices.Equal(m.Invokes, want) { t.Errorf("invokes = %v, want %v", m.Invokes, want) } // The seat's outcomes a copy waits on; the version comes in the ask, so version.numbered is not heard (design 47 G §413). diff --git a/modules/mesh-build-image/cmd/mesh-build-image/copy.go b/modules/mesh-build-image/cmd/mesh-build-image/copy.go index 9a09ca637..48a4da86f 100644 --- a/modules/mesh-build-image/cmd/mesh-build-image/copy.go +++ b/modules/mesh-build-image/cmd/mesh-build-image/copy.go @@ -112,8 +112,8 @@ func (m *Module) copy(ctx context.Context, args map[string]any) (any, error) { raw, err := m.call()("seat:"+steprun.ImageSeat+"."+reg.VerbMirror, reg.MirrorInput{Name: module + "/" + name, Source: from, Digest: digest}) if err != nil { said.Refused(err) - // A seat verb's "no responders" with a holder named is a transport fault (ADR 0664 §1): the seat's tool runner is replaced. - err = steprun.HolderSilentOr(m.call(), steprun.ImageSeat, err) + // A seat verb's "no responders" is a transport fault with no seat call to learn why (design 47 H2 §799, ADR 0664 §1). + err = steprun.PublishFault(err) return nil, fmt.Errorf("%s%s: %s's mirror: %w", builds.NotStored, name, steprun.ImageSeat, err) } var answer reg.MirrorAnswer diff --git a/modules/mesh-build-image/cmd/mesh-build-image/kit_test.go b/modules/mesh-build-image/cmd/mesh-build-image/kit_test.go index 1dea213b0..d151dcf28 100644 --- a/modules/mesh-build-image/cmd/mesh-build-image/kit_test.go +++ b/modules/mesh-build-image/cmd/mesh-build-image/kit_test.go @@ -7,6 +7,8 @@ import ( sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/steprun" "git.novox.be/novox/mesh-sdk/go/forgekey" + "os" + "path/filepath" "strings" "testing" "time" @@ -27,6 +29,30 @@ func TestTheStartSweepEndsWhatAPreviousProcessLeft(t *testing.T) { // ADR 0664 §1 (issue 1748): the module's own paths from a failed fetch or image build to its step's error say a network that // did not answer as a transport fault, and a failure of the source (a RUN line that fails) as none. func TestTheStepsSayAFailedNetworkAsATransportFault(t *testing.T) { + // A step whose fetch or publish met its holder away is given again after the module's again-delays, 15 s after its + // first run and 60 s after its second by default (design 47 H2 §799; H-A-390). + t.Run("again-delays is declared and read", func(t *testing.T) { + raw, err := os.ReadFile(filepath.Join("..", "..", "module.json")) + if err != nil { + t.Fatal(err) + } + var m struct { + Config map[string]json.RawMessage `json:"config"` + } + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatal(err) + } + var c struct { + Default string `json:"default"` + Runtime bool `json:"runtime"` + } + decl, ok := m.Config[steprun.AgainDelaysSetting] + _ = json.Unmarshal(decl, &c) + delay, err := steprun.AgainDelays(c.Default) + if !ok || !c.Runtime || err != nil || delay(1) != 15*time.Second || delay(2) != 60*time.Second { + t.Fatalf("again-delays declared %v as %+v: %v", ok, c, err) + } + }) conformance.Passing(t, []string{conformance.GitFetch, conformance.BuildahBud}, func(runner, words string) error { switch runner { case conformance.GitFetch: diff --git a/modules/mesh-build-image/cmd/mesh-build-image/main.go b/modules/mesh-build-image/cmd/mesh-build-image/main.go index 47a0bfd23..61b05a9c8 100644 --- a/modules/mesh-build-image/cmd/mesh-build-image/main.go +++ b/modules/mesh-build-image/cmd/mesh-build-image/main.go @@ -71,6 +71,14 @@ func main() { steps := m.StepTools() booted := time.Now() puller := m.Puller(steps, nil) + // A step whose fetch or publish met its holder away is given again after again-delays (design 47 H2 §799; H-A-390). + againSetting, _ := stdio.Setting(steprun.AgainDelaysSetting) + againDelay, againErr := steprun.AgainDelays(againSetting) + if againErr != nil { + logf("the setting %s: %v", steprun.AgainDelaysSetting, againErr) + os.Exit(1) + } + puller.Delay = againDelay m.Held = puller if m.Node != "" { // A step a previous process left running is ended (issue 1890): one that names another owner, or none and diff --git a/modules/mesh-build-image/module.json b/modules/mesh-build-image/module.json index 9493c8927..6f99328dc 100644 --- a/modules/mesh-build-image/module.json +++ b/modules/mesh-build-image/module.json @@ -42,6 +42,13 @@ "description": "How many processors one image step of this module needs (buildah, whose RUN lines are mostly fetches). The capacity above is an upper bound: on each node the module runs at most floor(the cores of its build slice's CPU quota / this) steps at once, the quota read live from its cgroup (issue 1806), so a lowered build CPU share never starves its steps. Above 0.", "minimum": 0.1, "default": 2 + }, + "again-delays": { + "type": "string", + "description": "The delays a step ask whose fetch or publish met its holder away is given again after: after its first run, then after its second, as durations such as 15s,60s (design 47 H2 §799); a third run met so ends the step failed with why fetch: or publish:.", + "trusted": false, + "runtime": true, + "default": "15s,60s" } }, "tools": [ @@ -67,7 +74,6 @@ "seat:container-image-registry.publish", "seat:container-image-registry.publish-part", "seat:container-image-registry.mirror", - "seat:mesh-module-assigner.assignments", "seat:mesh-catalog.module", "seat:mesh-catalog.seat-definitions" ], diff --git a/modules/mesh-build-node/cmd/mesh-build-node/kit_test.go b/modules/mesh-build-node/cmd/mesh-build-node/kit_test.go index a9dc94efb..68d5e34a3 100644 --- a/modules/mesh-build-node/cmd/mesh-build-node/kit_test.go +++ b/modules/mesh-build-node/cmd/mesh-build-node/kit_test.go @@ -4,6 +4,8 @@ import ( "context" "encoding/json" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/steprun" + "os" + "path/filepath" "strings" "testing" "time" @@ -37,6 +39,30 @@ func TestTheStartSweepEndsWhatAPreviousProcessLeft(t *testing.T) { // ADR 0664 §1 (issue 1748): the module's own paths from a failed fetch or install to its step's error say a network that did // not answer as a transport fault, and a failure of the source (a package the registry lacks) as none. func TestTheStepsSayAFailedNetworkAsATransportFault(t *testing.T) { + // A step whose fetch or publish met its holder away is given again after the module's again-delays, 15 s after its + // first run and 60 s after its second by default (design 47 H2 §799; H-A-390). + t.Run("again-delays is declared and read", func(t *testing.T) { + raw, err := os.ReadFile(filepath.Join("..", "..", "module.json")) + if err != nil { + t.Fatal(err) + } + var m struct { + Config map[string]json.RawMessage `json:"config"` + } + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatal(err) + } + var c struct { + Default string `json:"default"` + Runtime bool `json:"runtime"` + } + decl, ok := m.Config[steprun.AgainDelaysSetting] + _ = json.Unmarshal(decl, &c) + delay, err := steprun.AgainDelays(c.Default) + if !ok || !c.Runtime || err != nil || delay(1) != 15*time.Second || delay(2) != 60*time.Second { + t.Fatalf("again-delays declared %v as %+v: %v", ok, c, err) + } + }) conformance.Passing(t, []string{conformance.GitFetch, conformance.NpmCI}, func(runner, words string) error { switch runner { case conformance.GitFetch: diff --git a/modules/mesh-build-node/cmd/mesh-build-node/main.go b/modules/mesh-build-node/cmd/mesh-build-node/main.go index 18752e4d7..92a9ca911 100644 --- a/modules/mesh-build-node/cmd/mesh-build-node/main.go +++ b/modules/mesh-build-node/cmd/mesh-build-node/main.go @@ -83,6 +83,14 @@ func main() { steps := m.StepTools() booted := time.Now() puller := m.Puller(steps, nil) + // A step whose fetch or publish met its holder away is given again after again-delays (design 47 H2 §799; H-A-390). + againSetting, _ := stdio.Setting(steprun.AgainDelaysSetting) + againDelay, againErr := steprun.AgainDelays(againSetting) + if againErr != nil { + logf("the setting %s: %v", steprun.AgainDelaysSetting, againErr) + os.Exit(1) + } + puller.Delay = againDelay m.Held = puller if m.Node != "" { // A step a previous process left running is ended (issue 1890): one that names another owner, or none and diff --git a/modules/mesh-build-node/cmd/mesh-build-node/push_test.go b/modules/mesh-build-node/cmd/mesh-build-node/push_test.go index 5e21c8905..fa5b44c01 100644 --- a/modules/mesh-build-node/cmd/mesh-build-node/push_test.go +++ b/modules/mesh-build-node/cmd/mesh-build-node/push_test.go @@ -137,17 +137,9 @@ func TestARefusedPushFailsNotStoredAndNoPackedFileStays(t *testing.T) { if c.words != "" && (err == nil || !strings.HasPrefix(err.Error(), builds.NotStored+"tools: ") || !strings.Contains(err.Error(), c.words)) { t.Errorf("a refused publish answered %v", err) } - if c.noSeat { - // ADR 0664 §1: with a holder named by the assigner the same words are a transport fault, the bus gives the ask again. - m.Call = func(key string, _ any) (json.RawMessage, error) { - if key == steprun.AssignerHolders { - return json.RawMessage(`{"seats":[{"seat":"oci-artifact-registry","holders":[{"module":"mesh-registry","node":"novox"}]}],"holds":1}`), nil - } - return nil, errors.New("nats: no responders available for request") - } - if _, err := m.build(context.Background(), buildArgs()); err == nil || !sdkstep.IsPassing(err) || !strings.Contains(err.Error(), "the holder is silent") { - t.Errorf("no responders with a holder named answered %v, want a transport fault", err) - } + if c.noSeat && (err == nil || !sdkstep.IsPassing(err) || !strings.Contains(err.Error(), "publish: ")) { + // A publish met by no responders is a transport fault with no seat call to learn why (design 47 H2 §799). + t.Errorf("no responders answered %v, want a transport fault opening publish:", err) } var archives []string filepath.WalkDir(work, func(p string, d os.DirEntry, err error) error { diff --git a/modules/mesh-build-node/module.json b/modules/mesh-build-node/module.json index 4f6a44a3f..d14d35aab 100644 --- a/modules/mesh-build-node/module.json +++ b/modules/mesh-build-node/module.json @@ -46,6 +46,13 @@ "description": "How many processors one Node step of this module needs (tsc, esbuild and a test runner; a median step takes 15 s). The capacity above is an upper bound: on each node the module runs at most floor(the cores of its build slice's CPU quota / this) steps at once, the quota read live from its cgroup (issue 1806), so a lowered build CPU share never starves its steps. Above 0.", "minimum": 0.1, "default": 2 + }, + "again-delays": { + "type": "string", + "description": "The delays a step ask whose fetch or publish met its holder away is given again after: after its first run, then after its second, as durations such as 15s,60s (design 47 H2 §799); a third run met so ends the step failed with why fetch: or publish:.", + "trusted": false, + "runtime": true, + "default": "15s,60s" } }, "tools": [ @@ -79,7 +86,6 @@ "seat:oci-artifact-registry.publish-part", "seat:npm-package-registry.publish", "seat:npm-package-registry.publish-part", - "seat:mesh-module-assigner.assignments", "seat:mesh-catalog.module", "seat:mesh-catalog.seat-definitions" ], -- 2.54.0 From 607a1d76eac996a1e3b2c788aa5a82ad5e083fbb Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 22 Nov 2026 00:30:48 +0100 Subject: [PATCH 15/54] GT2: a delivery record keeps its delivery time; gate_times judges it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The head entry keeps merge-called from its opening and the pipeline's phases from its end; a record whose every mover runs ends with delivery-time (from merge-called to the last running of its movers) and class (core when a mover's catalog entry, read once at its entry's opening, says core). gate_times answers each class's budget (budget-core 10m, budget-leaf 3m), the last seven days' median and worst, and every delivery over its budget with its longest phase. Builds: design 47 H2 §810, §785; H-A-394 --- .../cmd/mesh-delivery-gate/bus_test.go | 3 + .../cmd/mesh-delivery-gate/main.go | 3 +- .../cmd/mesh-delivery-gate/manifest_test.go | 2 +- .../cmd/mesh-delivery-gate/movers.go | 33 ++- .../cmd/mesh-delivery-gate/record.go | 22 +- .../cmd/mesh-delivery-gate/record_test.go | 52 ++++- .../cmd/mesh-delivery-gate/recordevents.go | 14 +- .../cmd/mesh-delivery-gate/recordpipeline.go | 7 +- .../cmd/mesh-delivery-gate/recordtools.go | 10 + .../cmd/mesh-delivery-gate/sourcestart.go | 24 +++ .../cmd/mesh-delivery-gate/times.go | 196 ++++++++++++++++++ modules/mesh-delivery-gate/module.json | 49 ++++- 12 files changed, 391 insertions(+), 24 deletions(-) create mode 100644 modules/mesh-delivery-gate/cmd/mesh-delivery-gate/times.go diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/bus_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/bus_test.go index 2cbb06ac0..84e5ac42c 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/bus_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/bus_test.go @@ -376,6 +376,9 @@ func (r *runtime) answer(m rpcMessage) { return toolAnswer(map[string]any{"repository": repository, "branch": branch, "head": head}), nil case "seat:mesh-hand-acts.hand-act": return toolAnswer(map[string]any{"id": "hand-act-1"}), nil + case "seat:" + meshcatalog.Seat + "." + meshcatalog.VerbEntry: + // A mover's catalog entry, read once for its class (design 47 H2 §810): this test's modules are leaves. + return toolAnswer(map[string]any{"entry": map[string]any{"manifest": map[string]any{}}}), nil case "gitea.gitea_note_append", "gitea.gitea_delivery_view", "gitea.gitea_commit_status", "gitea.gitea_merge_pull_request", "gitea.gitea_add_comment": return toolAnswer(map[string]any{}), nil diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/main.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/main.go index 509b4e256..7075dd84a 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/main.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/main.go @@ -60,7 +60,8 @@ func main() { } h.init() h.SaySeat = stdio.SeatPublish - h.Records = &Records{Ledger: sdkLedger{}, Now: h.Now, Retention: h.recordRetention, Say: stdio.SeatPublish, Logf: logf, TooLarge: h.recordTooLarge} + h.Records = &Records{Ledger: sdkLedger{}, Now: h.Now, Retention: h.recordRetention, Say: stdio.SeatPublish, Logf: logf, TooLarge: h.recordTooLarge, + Class: seatCatalog{ask: stdio.Ask}.Class} h.Records.Changed = h.recordChanged h.Forge = seatGit{ask: stdio.Ask} h.Catalog = seatCatalog{ask: stdio.Ask} diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/manifest_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/manifest_test.go index d238d801d..304fee87e 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/manifest_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/manifest_test.go @@ -186,7 +186,7 @@ func TestItClaimsTheDeliverySeatAndFollowsWhatItHandles(t *testing.T) { } // It calls no controller verb and no forge verb: the forge's reads and writes are the bridge's (ADR 0437 §8; design 47 §592). // It uses no operator-channel: its stalled conditions went, the watchman's record-open the one signal (H-A-48). - want := []string{"seat:git.branch", "seat:mesh-catalog.registrations", "seat:mesh-hand-acts.hand-act", WantsEntryVerb, EngineCompositionVerb} + want := []string{"seat:git.branch", "seat:mesh-catalog.entry", "seat:mesh-catalog.registrations", "seat:mesh-hand-acts.hand-act", WantsEntryVerb, EngineCompositionVerb} if len(m.Uses) != 0 { t.Fatalf("uses %v", m.Uses) } diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/movers.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/movers.go index 8b3fdbf4c..4fc43b265 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/movers.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/movers.go @@ -16,6 +16,7 @@ import ( "time" "git.novox.be/novox/mesh-sdk/go/buscalls" + catalogue "git.novox.be/novox/mesh-sdk/go/catalogue" ) // TypeMover is the ledger type of a mover's entry. @@ -26,7 +27,7 @@ const MoversIndex = "movers" // The operations of a mover's entry; the manifest's type mover declares exactly these (record_test.go). const ( - OpMover = "mover" // set-fields: its record's key, its module, its pipeline and its version + OpMover = "mover" // set-fields: its record's key, its module, its class, its pipeline and its version // OpEntry and OpOutcome write its entries and outcomes as they did on the whole record. ) @@ -49,10 +50,12 @@ type Mover struct { Sequence uint64 `json:"-"` // Record is its record's key, `.`, the index's field. - Record string `json:"record"` - Module string `json:"module"` - Pipeline string `json:"pipeline,omitempty"` - Version string `json:"version,omitempty"` + Record string `json:"record"` + Module string `json:"module"` + Pipeline string `json:"pipeline,omitempty"` + Version string `json:"version,omitempty"` + // Class is the module's class as its catalog entry said it when the entry opened (core or leaf, H-A-394 (2)). + Class string `json:"class,omitempty"` Entries map[string]RecordEntry `json:"entries,omitempty"` Outcomes map[string]RecordOutcome `json:"outcomes,omitempty"` } @@ -140,7 +143,7 @@ func (r *Record) join(ms []*Mover) { func (r *Record) Movers() map[string]*Mover { return r.movers } // mover is the record's entry of module, opened in the ledger when it has none, and counted on the head entry. -func (rs *Records) mover(r *Record, module string) (*Mover, error) { +func (rs *Records) mover(r *Record, module, fingerprint string) (*Mover, error) { if m := r.movers[module]; m != nil { return m, nil } @@ -148,8 +151,16 @@ func (rs *Records) mover(r *Record, module string) (*Mover, error) { if err != nil { return nil, err } - m := &Mover{Instance: inst, Record: r.Milestone, Module: module} - seq, err := rs.Ledger.Apply(inst, OpMover, map[string]any{"record": r.Milestone, "module": module}) + class := catalogue.ClassLeaf + if rs.Class != nil && fingerprint != "" { + c, err := rs.Class(fingerprint) + if err != nil { + return nil, fmt.Errorf("the class of %s at %s cannot be read from its catalog entry, so its entry is not opened: %w", module, fingerprint, err) + } + class = firstOf(c, catalogue.ClassLeaf) + } + m := &Mover{Instance: inst, Record: r.Milestone, Module: module, Class: class} + seq, err := rs.Ledger.Apply(inst, OpMover, map[string]any{"record": r.Milestone, "module": module, "class": class}) if err != nil { return nil, err } @@ -167,8 +178,8 @@ func (rs *Records) mover(r *Record, module string) (*Mover, error) { } // SetMover writes a mover's pipeline and version, unless its entry holds them already. -func (rs *Records) SetMover(r *Record, module, pipeline, version string) error { - m, err := rs.mover(r, module) +func (rs *Records) SetMover(r *Record, module, fingerprint, pipeline, version string) error { + m, err := rs.mover(r, module, fingerprint) if err != nil { return err } @@ -180,7 +191,7 @@ func (rs *Records) SetMover(r *Record, module, pipeline, version string) error { if err := rs.fits(m.Instance, next); err != nil { return err } - seq, err := rs.Ledger.Apply(m.Instance, OpMover, map[string]any{"record": r.Milestone, "module": module, + seq, err := rs.Ledger.Apply(m.Instance, OpMover, map[string]any{"record": r.Milestone, "module": module, "class": m.Class, "pipeline": next.Pipeline, "version": next.Version}) if err != nil { return err diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record.go index 6c095e89c..26ed54871 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record.go @@ -22,6 +22,7 @@ import ( "time" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery" + runner "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-sdk/go/buscalls" ) @@ -58,7 +59,7 @@ const ( ) // RecordOperations are a record's operations, in the order the manifest's type allows them. -var RecordOperations = []string{OpOpen, OpGroup, OpTrace, OpMovers, OpAway, OpBack, OpAct, OpEnd} +var RecordOperations = []string{OpOpen, OpGroup, OpTrace, OpMovers, OpPhases, OpAway, OpBack, OpAct, OpEnd} // HeldOperations are the month's list's operations. var HeldOperations = []string{OpHold, OpDrop} @@ -102,6 +103,12 @@ type Record struct { // MoverCount is the count of its movers, each with an entry of its own (design 47 H2 §809); Entries and Outcomes above // are joined from those entries when read. MoverCount int `json:"movers,omitempty"` + // MergeCalled is the merge-called event's own time, from the record's opening; DeliveryTime (a duration) and Class are + // written at its end, Phases at its pipeline's end (design 47 H2 §810, §785; H-A-394). + MergeCalled time.Time `json:"merge-called,omitzero"` + DeliveryTime string `json:"delivery-time,omitempty"` + Class string `json:"class,omitempty"` + Phases *runner.PipelineEndedPhases `json:"phases,omitempty"` movers map[string]*Mover } @@ -368,6 +375,9 @@ type Records struct { // Changed is told of every change of a record that owes the forge an act (A-39): a step's outcome applied, the record // ended (owedacts.go); nil owes none. Changed func(r *Record, c RecordChange) + // Class reads a mover's class from its catalog entry by its fingerprint (design 47 H2 §810; H-A-394 (2)), once, when its + // entry opens; nil reads none, every mover a leaf. + Class func(fingerprint string) (string, error) // TooLarge raises (raised) or clears record-too-large for an entry (§809); nil raises none. TooLarge func(instance string, bytes int, raised bool) @@ -533,8 +543,12 @@ func (rs *Records) Open(o RecordOpening) (r *Record, opened bool, err error) { if err != nil { return nil, false, err } + called := o.Trace.At.UTC() + if o.Trace.At.IsZero() { + called = now + } body := map[string]any{"milestone": milestone, "branch": o.Branch, "source": o.Source, "call": o.Call, "heads": o.Heads, - "opened": now, "form": RecordForm} + "opened": now, "form": RecordForm, "merge-called": called} if _, err := rs.Ledger.Apply(inst, OpOpen, body); err != nil { return nil, false, err } @@ -724,7 +738,7 @@ func (rs *Records) Outcome(r *Record, o RecordOutcome) (applied bool, err error) if had, ok := r.Outcomes[o.Key]; ok && (had.Event != "" && had.Event == o.Event || sameOutcome(had, o)) { return false, nil } - m, err := rs.mover(r, o.Module) + m, err := rs.mover(r, o.Module, o.Fingerprint) if err != nil { return false, err } @@ -755,7 +769,7 @@ func (rs *Records) AddEntry(r *Record, e RecordEntry) error { if had, ok := r.Entries[e.Key]; ok && had == e { return nil } - m, err := rs.mover(r, e.Module) + m, err := rs.mover(r, e.Module, e.Fingerprint) if err != nil { return err } diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record_test.go index da028b6e5..0f583d24f 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record_test.go @@ -17,6 +17,8 @@ import ( stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-sdk/go/buscalls" "git.novox.be/novox/mesh-sdk/go/catalogue" + + runner "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" ) // ledgerFake plays the tool runner's ledger for the gate's `records`, from the types the manifest declares: each operation @@ -214,7 +216,7 @@ func TestTheDeliveryRecordIsALedgerInstanceKeyedByItsSourceRevision(t *testing.T if err := rs.AddEntry(r, RecordEntry{Module: module, Fingerprint: "src2:f" + module}); err != nil { t.Fatal(err) } - if err := rs.SetMover(r, module, "main.90", "0.1.7"); err != nil { + if err := rs.SetMover(r, module, "src2:f"+module, "main.90", "0.1.7"); err != nil { t.Fatal(err) } for _, node := range []string{"ace", "g14", "novox", "shanks"} { @@ -251,6 +253,52 @@ func TestTheDeliveryRecordIsALedgerInstanceKeyedByItsSourceRevision(t *testing.T } }) + // A record keeps its delivery time and class; gate_times judges them against the budgets (design 47 H2 §810; H-A-394). + t.Run("a core record ended 11 minutes after merge-called is over its budget", func(t *testing.T) { + now := at + rs := &Records{Ledger: newLedgerFake(t), Now: func() time.Time { return now }, Class: func(fp string) (string, error) { + if fp == "src2:core" { + return catalogue.ClassCore, nil + } + return "", nil + }} + h := &Holder{Records: rs, Now: func() time.Time { return now }} + for i, c := range []struct { + fp string + took time.Duration + }{{"src2:core", 11 * time.Minute}, {"src2:leaf", 2 * time.Minute}, {"src2:core", 4 * time.Minute}} { + o := opening("main", int64(200+i), heads) + o.Trace.At = now + r, _, err := rs.Open(o) + if err != nil { + t.Fatal(err) + } + if err := rs.AddEntry(r, RecordEntry{Module: "m" + c.fp, Fingerprint: c.fp}); err != nil { + t.Fatal(err) + } + p := now.Add(time.Minute) + if err := rs.SetPhases(r, &runner.PipelineEndedPhases{Taken: &now, Placed: &p, Build: &runner.PipelineEndedPhasesBuild{Start: &p, End: ptrTime(p.Add(7 * time.Minute))}}); err != nil { + t.Fatal(err) + } + if _, err := rs.Outcome(r, RecordOutcome{Module: "m" + c.fp, Fingerprint: c.fp, Node: "ace", Step: "running", Outcome: OutcomeRunning, At: now.Add(c.took)}); err != nil { + t.Fatal(err) + } + now = now.Add(c.took) + if err := rs.End(r, WhyAllRun, ""); err != nil { + t.Fatal(err) + } + if r.DeliveryTime != c.took.String() { + t.Fatalf("record %d keeps %q, want %s", i, r.DeliveryTime, c.took) + } + } + times, err := h.Times() + core, leaf := times[catalogue.ClassCore], times[catalogue.ClassLeaf] + if err != nil || core.Budget != "10m0s" || core.Deliveries != 2 || len(core.Over) != 1 || core.Over[0].DeliveryTime != "11m0s" || + core.Over[0].Longest != "build" || core.Worst != "11m0s" || leaf.Deliveries != 1 || len(leaf.Over) != 0 { + t.Fatalf("gate_times %+v %v", times, err) + } + }) + t.Run("an instance per source revision, a path the tool runner takes", func(t *testing.T) { a, err := RecordInstance(recordMilestone("feat/x", 7)) if err != nil { @@ -1007,3 +1055,5 @@ func (o *ownersFake) Running(node, module string) (string, string, error) { func opening(branch string, source int64, heads map[string]string) RecordOpening { return RecordOpening{Branch: branch, Source: source, Call: fmt.Sprintf("%016x", source), Heads: heads} } + +func ptrTime(t time.Time) *time.Time { return &t } diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/recordevents.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/recordevents.go index 7b90a3d5a..86e785113 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/recordevents.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/recordevents.go @@ -132,11 +132,23 @@ func (rs *Records) End(r *Record, why, by string) error { func (rs *Records) end(r *Record, why, by string) error { e := RecordEnded{At: rs.now(), Why: why, By: by, Away: awayList(r)} - seq, err := rs.Ledger.Apply(r.Instance, OpEnd, map[string]any{"ended": e}) + body := map[string]any{"ended": e} + var took time.Duration + var class string + if why == WhyAllRun { + // Every mover runs: the record keeps its delivery time and class (design 47 H2 §810). + if took, class = timesAtEnd(r); took > 0 { + body["delivery-time"], body["class"] = took.String(), class + } + } + seq, err := rs.Ledger.Apply(r.Instance, OpEnd, body) if err != nil { return err } r.Ended, r.Sequence = &e, seq + if took > 0 { + r.DeliveryTime, r.Class = took.String(), class + } rs.clearTooLarge(r) rs.keep(r) if rs.Changed != nil { diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/recordpipeline.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/recordpipeline.go index af495c710..aec852ae0 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/recordpipeline.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/recordpipeline.go @@ -186,7 +186,7 @@ func (h *Holder) pipelineStarted(p runner.PipelineStartedEvent) error { if err := h.Records.AddEntry(r, RecordEntry{Module: m.Module, Fingerprint: m.Fingerprint}); err != nil { return err } - if err := h.Records.SetMover(r, m.Module, p.Pipeline, m.Version); err != nil { + if err := h.Records.SetMover(r, m.Module, m.Fingerprint, p.Pipeline, m.Version); err != nil { return err } } @@ -209,7 +209,7 @@ func (h *Holder) pipelineEnded(p runner.PipelineEndedEvent) error { if err := h.Records.AddEntry(r, RecordEntry{Module: m.Module, Fingerprint: m.Fingerprint}); err != nil { return err } - if err := h.Records.SetMover(r, m.Module, p.Pipeline, m.Version); err != nil { + if err := h.Records.SetMover(r, m.Module, m.Fingerprint, p.Pipeline, m.Version); err != nil { return err } for _, s := range m.Steps { @@ -226,6 +226,9 @@ func (h *Holder) pipelineEnded(p runner.PipelineEndedEvent) error { } } } + if err := h.Records.SetPhases(r, p.Phases); err != nil { + return err + } if p.Outcome == runner.PipelineEndedOutcomePassed { return nil } diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/recordtools.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/recordtools.go index 6b39b2cbc..330a09dd6 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/recordtools.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/recordtools.go @@ -195,6 +195,16 @@ func recordTools(h *Holder, acts handActs) []stdio.Tool { } return h.Records.RecordOf(strArg(a, "instance"), strArg(a, "milestone")) }}, + {Name: "gate_times", + Description: "Delivery times against their budgets (design 47 H2 §810; ADR 0282 §6): for the classes core and leaf, the " + + "budget (the settings budget-core and budget-leaf), the last seven days' median and worst delivery time from " + + "merge-called to the last running of its movers, and every delivery over its budget, newest first, with its longest phase.", + Run: func(map[string]any) (any, error) { + if h.Records == nil { + return none() + } + return h.Times() + }}, {Name: "gate_records", Description: "Every delivery record kept, oldest first: its milestone, when it opened and ended, its entries, " + "outcomes, nodes away and pins, its size against the bus's value limit, and while open what it waits for; and " + diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/sourcestart.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/sourcestart.go index ccefaaf20..a2d44dcc9 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/sourcestart.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/sourcestart.go @@ -30,6 +30,30 @@ type Registered interface { // seatCatalog reads the seat mesh-catalog through the tool runner. type seatCatalog struct{ ask asking } +// Class is the class of the module at fingerprint, as its catalog entry's manifest says it (design 47 H2 §810; +// H-A-394 (2)): the seat mesh-catalog's entry; a manifest with no class is a leaf. +func (s seatCatalog) Class(fingerprint string) (string, error) { + raw, err := s.ask("seat:"+meshcatalog.Seat+"."+meshcatalog.VerbEntry, meshcatalog.EntryInput{Fingerprint: fingerprint}) + if err != nil { + return "", err + } + answer, said, ok := answerOf(raw) + if !ok { + return "", fmt.Errorf("entry refused: %s", lastLine(said)) + } + var a struct { + Entry struct { + Manifest struct { + Class string `json:"class"` + } `json:"manifest"` + } `json:"entry"` + } + if err := json.Unmarshal(answer, &a); err != nil { + return "", fmt.Errorf("the answer of entry is not readable: %w", err) + } + return a.Entry.Manifest.Class, nil +} + // Registrations reads every registration, a module's and a source-only one alike (design 47 H2 §815), page by page from // the cursor each page answers (§779, §806), so a registration past the first 768 KB is watched too. func (s seatCatalog) Registrations() (meshcatalog.RegistrationsAnswer, error) { diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/times.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/times.go new file mode 100644 index 000000000..8d9a56cc5 --- /dev/null +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/times.go @@ -0,0 +1,196 @@ +package main + +// A delivery record keeps its delivery time (design 47 H2 §810; ADR 0282 §1, §6, §7; ADR 0304; H-A-394): the head entry +// keeps `merge-called` from its opening, the pipeline's `phases` (§785) from its end, and at its end the record's +// `delivery-time`, from `merge-called` to the last `running` of its movers, and its `class`: core when it moves a module +// whose manifest's class (the catalog entry the gate read at the mover's opening) says core, else leaf. gate_times +// answers each class's budget, the last seven days' median and worst, and every delivery over its budget with its +// longest phase; the self-check's probe judges them, the gate raises nothing. + +import ( + "fmt" + "sort" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" + catalogue "git.novox.be/novox/mesh-sdk/go/catalogue" + + runner "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" +) + +// OpPhases writes the pipeline's phases on the head entry (set-fields phases). +const OpPhases = "phases" + +// The budgets' settings and their defaults (ADR 0282, ADR 0304). +const ( + BudgetCoreSetting = "budget-core" + BudgetLeafSetting = "budget-leaf" + DefaultBudgetCore = 10 * time.Minute + DefaultBudgetLeaf = 3 * time.Minute +) + +// timesWindow is how far back gate_times reads (§810: the last seven days). +const timesWindow = 7 * 24 * time.Hour + +// Budgets are the two classes' budgets, read with the one settings reader; a value that is no duration is said and the +// default kept. +func (h *Holder) Budgets() map[string]time.Duration { + out := map[string]time.Duration{} + for class, s := range map[string]struct { + key string + def time.Duration + }{catalogue.ClassCore: {BudgetCoreSetting, DefaultBudgetCore}, catalogue.ClassLeaf: {BudgetLeafSetting, DefaultBudgetLeaf}} { + d, err := stdio.SettingDuration(s.key, s.def) + if err != nil { + h.logf("[mesh-delivery] the setting %s: %v; %s is kept", s.key, err, s.def) + d = s.def + } + out[class] = d + } + return out +} + +// SetPhases writes the pipeline's phases on the record's head entry, unless it holds them already. +func (rs *Records) SetPhases(r *Record, p *runner.PipelineEndedPhases) error { + if p == nil { + return nil + } + next := *r + next.Phases = p + if err := rs.fits(r.Instance, next.head()); err != nil { + return err + } + seq, err := rs.Ledger.Apply(r.Instance, OpPhases, map[string]any{"phases": p}) + if err != nil { + return err + } + r.Phases, r.Sequence = p, seq + rs.fitted(r.Instance) + return nil +} + +// timesAtEnd are the delivery time and the class of a record whose every mover runs: from merge-called to the last running +// of its movers; core when one of its movers is core. No delivery time without merge-called or a running. +func timesAtEnd(r *Record) (time.Duration, string) { + class := catalogue.ClassLeaf + var last time.Time + for _, m := range r.movers { + if m.Class == catalogue.ClassCore { + class = catalogue.ClassCore + } + for _, at := range m.Running() { + if at.After(last) { + last = at + } + } + } + if r.MergeCalled.IsZero() || last.IsZero() || last.Before(r.MergeCalled) { + return 0, class + } + return last.Sub(r.MergeCalled), class +} + +// longestPhase is the record's longest phase of §785 and its length: the take-in, the placing, each step kind's span. +func longestPhase(p *runner.PipelineEndedPhases, mergeCalled time.Time) (string, time.Duration) { + if p == nil { + return "", 0 + } + type span struct { + name string + start, end *time.Time + } + mc := mergeCalled + spans := []span{{"take-in", &mc, p.Taken}, {"placing", p.Taken, p.Placed}} + add := func(name string, start, end *time.Time) { spans = append(spans, span{name, start, end}) } + if b := p.Build; b != nil { + add("build", b.Start, b.End) + } + if b := p.Lint; b != nil { + add("lint", b.Start, b.End) + } + if b := p.Unit; b != nil { + add("unit", b.Start, b.End) + } + if b := p.Integration; b != nil { + add("integration", b.Start, b.End) + } + if b := p.Validate; b != nil { + add("validate", b.Start, b.End) + } + if b := p.Composition; b != nil { + add("composition", b.Start, b.End) + } + name, longest := "", time.Duration(0) + for _, s := range spans { + if s.start == nil || s.end == nil || s.start.IsZero() { + continue + } + if d := s.end.Sub(*s.start); d > longest { + name, longest = s.name, d + } + } + return name, longest +} + +// TimesOver is one delivery over its class's budget, with its longest phase. +type TimesOver struct { + Record string `json:"record"` + Milestone string `json:"milestone"` + Ended string `json:"ended"` + DeliveryTime string `json:"delivery-time"` + Longest string `json:"longest-phase,omitempty"` + LongestTime string `json:"longest-phase-time,omitempty"` +} + +// ClassTimes are one class's times over the last seven days. +type ClassTimes struct { + Budget string `json:"budget"` + Deliveries int `json:"deliveries"` + Median string `json:"median,omitempty"` + Worst string `json:"worst,omitempty"` + Over []TimesOver `json:"over"` +} + +// Times is gate_times: each class's budget, the last seven days' median and worst, and every delivery over its budget, +// newest first, with its longest phase (§810; ADR 0282 §6). +func (h *Holder) Times() (map[string]ClassTimes, error) { + kept, err := h.keptRecords() + if err != nil { + return nil, err + } + budgets, now := h.Budgets(), h.Now().UTC() + by := map[string][]*Record{} + for _, r := range kept { + if r.Ended == nil || r.DeliveryTime == "" || now.Sub(r.Ended.At) > timesWindow { + continue + } + by[firstOf(r.Class, catalogue.ClassLeaf)] = append(by[firstOf(r.Class, catalogue.ClassLeaf)], r) + } + out := map[string]ClassTimes{} + for class, budget := range budgets { + rs := by[class] + ct := ClassTimes{Budget: budget.String(), Deliveries: len(rs), Over: []TimesOver{}} + var ds []time.Duration + sort.Slice(rs, func(i, j int) bool { return rs[i].Ended.At.After(rs[j].Ended.At) }) + for _, r := range rs { + d, err := time.ParseDuration(r.DeliveryTime) + if err != nil { + return nil, fmt.Errorf("the record %s keeps a delivery time %q that is no duration", r.Instance, r.DeliveryTime) + } + ds = append(ds, d) + if d > budget { + o := TimesOver{Record: r.Instance, Milestone: r.Milestone, Ended: r.Ended.At.Format(time.RFC3339), DeliveryTime: d.String()} + if name, l := longestPhase(r.Phases, r.MergeCalled); name != "" { + o.Longest, o.LongestTime = name, l.String() + } + ct.Over = append(ct.Over, o) + } + } + if len(ds) > 0 { + sort.Slice(ds, func(i, j int) bool { return ds[i] < ds[j] }) + ct.Median, ct.Worst = ds[len(ds)/2].String(), ds[len(ds)-1].String() + } + out[class] = ct + } + return out, nil +} diff --git a/modules/mesh-delivery-gate/module.json b/modules/mesh-delivery-gate/module.json index b702a5e67..199323090 100644 --- a/modules/mesh-delivery-gate/module.json +++ b/modules/mesh-delivery-gate/module.json @@ -128,6 +128,18 @@ }, "movers": { "type": "integer" + }, + "merge-called": { + "type": "string" + }, + "delivery-time": { + "type": "string" + }, + "class": { + "type": "string" + }, + "phases": { + "type": "object" } }, "operations": [ @@ -141,7 +153,8 @@ "call", "heads", "opened", - "form" + "form", + "merge-called" ], "sequence": true }, @@ -170,6 +183,14 @@ ], "sequence": true }, + { + "name": "phases", + "effect": "set-fields", + "fields": [ + "phases" + ], + "sequence": true + }, { "name": "away", "effect": "set-entry", @@ -208,7 +229,9 @@ "name": "end", "effect": "set-fields", "fields": [ - "ended" + "ended", + "delivery-time", + "class" ], "sequence": true } @@ -252,6 +275,9 @@ }, "event": { "type": "string" + }, + "class": { + "type": "string" } }, "operations": [ @@ -261,6 +287,7 @@ "fields": [ "record", "module", + "class", "pipeline", "version" ], @@ -369,6 +396,7 @@ }, "invokes": [ "seat:git.branch", + "seat:mesh-catalog.entry", "seat:mesh-catalog.registrations", "seat:mesh-hand-acts.hand-act", "seat:mesh-wants.entry", @@ -396,7 +424,8 @@ "gate_source", "gate_source_again", "gate_source_seed", - "gate_source_start" + "gate_source_start", + "gate_times" ], "config": { "record-retention": { @@ -413,6 +442,20 @@ "minimum": 1, "trusted": false, "default": 10 + }, + "budget-core": { + "type": "string", + "description": "The budget of a delivery that moves a core module, from merge-called to the last running of its movers, as a duration such as 10m (ADR 0282; design 47 H2 §810): gate_times names every delivery over it, and the self-check raises over-budget for it.", + "trusted": false, + "runtime": true, + "default": "10m" + }, + "budget-leaf": { + "type": "string", + "description": "The budget of a delivery that moves only leaf modules, as a duration such as 3m (ADR 0304; design 47 H2 §810).", + "trusted": false, + "runtime": true, + "default": "3m" } }, "resources": [ -- 2.54.0 From 872a02b378dce0989e1aad91b3b083da12a73d0a Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 22 Nov 2026 00:33:39 +0100 Subject: [PATCH 16/54] RG1: a repeated publish ask reads the store's held version in its new form MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The merge of main's H-A-387/H-A-393 fix gave Store.Held a reference and the digest another publish holds; a repeat answers ended when the reference is set. Builds: design 47 H2 §775; H-A-381 --- modules/distribution/internal/holder/transfer.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/distribution/internal/holder/transfer.go b/modules/distribution/internal/holder/transfer.go index ec0b888c2..706f8e989 100644 --- a/modules/distribution/internal/holder/transfer.go +++ b/modules/distribution/internal/holder/transfer.go @@ -415,14 +415,14 @@ func (h *Holder) Repeat(seat, caller string, r PublishRequest) (PublishAnswer, e return PublishAnswer{}, err } if r.Version != "" { - var ok bool + var ref string err := h.timed("the store's read of what it holds (the program's registry)", func() (err error) { - _, ok, err = h.Store.Held(r.Name, r.Version, r.Digest) + ref, _, err = h.Store.Held(r.Name, r.Version, r.Digest) return err }) if err != nil { return PublishAnswer{}, fmt.Errorf("publish: the store cannot say whether it holds %s@%s: %w", r.Name, r.Version, err) - } else if ok { + } else if ref != "" { return PublishAnswer{Transfer: r.Transfer, Ended: true}, nil } } -- 2.54.0 From 5d2d0bade84095f0222a70ce013d1bd9a81accdc Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 22 Nov 2026 00:37:03 +0100 Subject: [PATCH 17/54] SC1: the self-check raises over-budget from the gate's times MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit gate_times answers per class its newest delivery (last) with whether it was over its budget; the self-check's probe D-budget reads gate_times and raises over-budget with the class as its id, naming the source revision and its longest phase, and a newest delivery within its budget clears it. Builds: design 47 H2 §810; H-A-396 --- .../cmd/mesh-delivery-gate/record_test.go | 3 +- .../cmd/mesh-delivery-gate/recordtools.go | 3 +- .../cmd/mesh-delivery-gate/times.go | 19 ++++- .../cmd/mesh-self-check/probes.go | 4 +- .../cmd/mesh-self-check/probes_budget.go | 73 +++++++++++++++++++ .../cmd/mesh-self-check/probes_test.go | 12 ++- modules/mesh-self-check/module.json | 3 +- 7 files changed, 108 insertions(+), 9 deletions(-) create mode 100644 modules/mesh-self-check/cmd/mesh-self-check/probes_budget.go diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record_test.go index 0f583d24f..e0262dc92 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record_test.go @@ -294,7 +294,8 @@ func TestTheDeliveryRecordIsALedgerInstanceKeyedByItsSourceRevision(t *testing.T times, err := h.Times() core, leaf := times[catalogue.ClassCore], times[catalogue.ClassLeaf] if err != nil || core.Budget != "10m0s" || core.Deliveries != 2 || len(core.Over) != 1 || core.Over[0].DeliveryTime != "11m0s" || - core.Over[0].Longest != "build" || core.Worst != "11m0s" || leaf.Deliveries != 1 || len(leaf.Over) != 0 { + core.Over[0].Longest != "build" || core.Worst != "11m0s" || leaf.Deliveries != 1 || len(leaf.Over) != 0 || + core.Last == nil || core.Last.Over || core.Last.DeliveryTime != "4m0s" || leaf.Last == nil || leaf.Last.Over { t.Fatalf("gate_times %+v %v", times, err) } }) diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/recordtools.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/recordtools.go index 330a09dd6..c8ab68e22 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/recordtools.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/recordtools.go @@ -198,7 +198,8 @@ func recordTools(h *Holder, acts handActs) []stdio.Tool { {Name: "gate_times", Description: "Delivery times against their budgets (design 47 H2 §810; ADR 0282 §6): for the classes core and leaf, the " + "budget (the settings budget-core and budget-leaf), the last seven days' median and worst delivery time from " + - "merge-called to the last running of its movers, and every delivery over its budget, newest first, with its longest phase.", + "merge-called to the last running of its movers, every delivery over its budget, newest first, with its longest phase, " + + "and the newest delivery (last) with whether it was over.", Run: func(map[string]any) (any, error) { if h.Records == nil { return none() diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/times.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/times.go index 8d9a56cc5..da2bde271 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/times.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/times.go @@ -142,6 +142,13 @@ type TimesOver struct { LongestTime string `json:"longest-phase-time,omitempty"` } +// TimesLast is a class's newest ended delivery: whether it was over its budget, which the self-check's probe judges +// (design 47 H2 §810; H-A-396). +type TimesLast struct { + TimesOver + Over bool `json:"over"` +} + // ClassTimes are one class's times over the last seven days. type ClassTimes struct { Budget string `json:"budget"` @@ -149,6 +156,7 @@ type ClassTimes struct { Median string `json:"median,omitempty"` Worst string `json:"worst,omitempty"` Over []TimesOver `json:"over"` + Last *TimesLast `json:"last,omitempty"` } // Times is gate_times: each class's budget, the last seven days' median and worst, and every delivery over its budget, @@ -178,11 +186,14 @@ func (h *Holder) Times() (map[string]ClassTimes, error) { return nil, fmt.Errorf("the record %s keeps a delivery time %q that is no duration", r.Instance, r.DeliveryTime) } ds = append(ds, d) + o := TimesOver{Record: r.Instance, Milestone: r.Milestone, Ended: r.Ended.At.Format(time.RFC3339), DeliveryTime: d.String()} + if name, l := longestPhase(r.Phases, r.MergeCalled); name != "" { + o.Longest, o.LongestTime = name, l.String() + } + if ct.Last == nil { + ct.Last = &TimesLast{TimesOver: o, Over: d > budget} + } if d > budget { - o := TimesOver{Record: r.Instance, Milestone: r.Milestone, Ended: r.Ended.At.Format(time.RFC3339), DeliveryTime: d.String()} - if name, l := longestPhase(r.Phases, r.MergeCalled); name != "" { - o.Longest, o.LongestTime = name, l.String() - } ct.Over = append(ct.Over, o) } } diff --git a/modules/mesh-self-check/cmd/mesh-self-check/probes.go b/modules/mesh-self-check/cmd/mesh-self-check/probes.go index 6e944eea3..49fc64641 100644 --- a/modules/mesh-self-check/cmd/mesh-self-check/probes.go +++ b/modules/mesh-self-check/cmd/mesh-self-check/probes.go @@ -35,7 +35,7 @@ const ( // `reads`, which its test holds to these. var ( Invokes = []string{StreamsTool, ClosedTool, SignalsVerb, BannedVerb, AssignmentsTool, SeatHoldersTool, - RevisionVerb, RegistrationsVerb, SourceVerb, HandActVerb} + RevisionVerb, RegistrationsVerb, SourceVerb, HandActVerb, TimesTool} Reads = []string{PresenceRead, HealthRead, ReadingsRead, GoodRead} // Consumes are the views families the probes follow (ADR 0615): the indexer's and the assigner's states are asked // once, by their owners' read tools, and then followed by their events, never read. @@ -104,6 +104,8 @@ func Registry() []Probe { Deferred: "the grants of restricted streams with what each machine composed for them: " + storeOnly}, {ID: "DW", Asserts: "the watchdogs answer: the watchman holds the seat mesh-watchdogs and answers signals", From: "ADR 0227 rule 6: the watchers are watched", Kind: "watchdogs-silent", Run: probeWatchdogs}, + {ID: "D-budget", Asserts: "every class's newest delivery ran on every machine within its budget from its merge call: " + + "the gate's gate_times", From: "ADR 0282 §7, design 47 H2 §810", Kind: kindOverBudget, Run: probeBudget}, {ID: "H-bus", Asserts: "every stream the mesh defines is on the bus, and a request crosses it " + "to the bus's machine and back", From: "ADR 0236, to-be 45 §8", Kind: "core-unhealthy", Run: probeBusHealth}, } diff --git a/modules/mesh-self-check/cmd/mesh-self-check/probes_budget.go b/modules/mesh-self-check/cmd/mesh-self-check/probes_budget.go new file mode 100644 index 000000000..b2e56db1b --- /dev/null +++ b/modules/mesh-self-check/cmd/mesh-self-check/probes_budget.go @@ -0,0 +1,73 @@ +package main + +// The delivery budget's probe (design 47 H2 §810; ADR 0282 §7; H-A-396): it reads the gate's times (gate_times) and raises +// over-budget with the class as its id when that class's newest delivery took longer than its budget, naming the source +// revision and its longest phase; a newest delivery within its budget clears it. It reads no records itself, and the gate +// raises nothing. + +import ( + "context" + "encoding/json" + "fmt" + "sort" + + "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" +) + +// TimesTool is the gate's read of its delivery times against their budgets. +const TimesTool = "mesh-delivery-gate.gate_times" + +// scopeDelivery is the scope of the delivery's findings, the gate's own (A-42). +const scopeDelivery = "delivery" + +// kindOverBudget is the finding of a class whose newest delivery took longer than its budget. +const kindOverBudget = "over-budget" + +// classTimes is one class as gate_times answers it: its budget and its newest delivery. +type classTimes struct { + Budget string `json:"budget"` + Last *struct { + Record string `json:"record"` + Milestone string `json:"milestone"` + Ended string `json:"ended"` + DeliveryTime string `json:"delivery-time"` + Longest string `json:"longest-phase"` + LongestTime string `json:"longest-phase-time"` + Over bool `json:"over"` + } `json:"last"` +} + +func probeBudget(_ context.Context, c *Checker) ([]Finding, error) { + raw, err := c.Ask(TimesTool, map[string]any{}) + if err != nil { + return nil, fmt.Errorf("the gate did not answer %s: %w", TimesTool, err) + } + var times map[string]classTimes + if err := json.Unmarshal(raw, ×); err != nil { + return nil, fmt.Errorf("the gate answered %s with something unreadable: %w", TimesTool, err) + } + classes := make([]string, 0, len(times)) + for class := range times { + classes = append(classes, class) + } + sort.Strings(classes) + var out []Finding + for _, class := range classes { + t := times[class] + if t.Last == nil || !t.Last.Over { + continue + } + longest := "no phase recorded" + if t.Last.Longest != "" { + longest = fmt.Sprintf("its longest phase %s took %s", t.Last.Longest, t.Last.LongestTime) + } + out = append(out, finding(scopeDelivery, class, "", "", Finding{Kind: kindOverBudget, Severity: meshconditionsclient.Warning, + Summary: fmt.Sprintf("the newest %s delivery, %s, took %s from its merge call to running everywhere, over its budget of %s; %s", + class, t.Last.Milestone, t.Last.DeliveryTime, t.Budget, longest), + Evidence: said(fmt.Sprintf("record %s ended %s", t.Last.Record, t.Last.Ended)), + Headline: capital(class) + " deliveries are over their budget", + Explanation: capital(fmt.Sprintf("a delivery moving a %s module should run on every machine within %s of its merge "+ + "call; the newest took %s. The next %s delivery within its budget clears this.", class, t.Budget, t.Last.DeliveryTime, class))})) + } + return sorted(out), nil +} diff --git a/modules/mesh-self-check/cmd/mesh-self-check/probes_test.go b/modules/mesh-self-check/cmd/mesh-self-check/probes_test.go index 7a369328d..9a7739169 100644 --- a/modules/mesh-self-check/cmd/mesh-self-check/probes_test.go +++ b/modules/mesh-self-check/cmd/mesh-self-check/probes_test.go @@ -34,6 +34,8 @@ type bus struct { banned map[string]string // fills are rows nats_streams answers beside the mesh's streams: a module's bucket or log with its size and cap. fills []fillRow + // times is what the gate's gate_times answers. + times string } func (b bus) rows() map[string]*streamRow { @@ -87,6 +89,8 @@ func (b bus) ask(key string, body any) (json.RawMessage, error) { return nil, errors.New("no stream called " + stream) } return json.Marshal(map[string]any{"detail": detail}) + case TimesTool: + return json.RawMessage(b.times), nil case ClosedTool: if b.closed == "" { return json.RawMessage(`{"hours":1,"reaches":true,"users":[]}`), nil @@ -133,6 +137,12 @@ func TestTheBusProbes(t *testing.T) { {"user":"g14.node-tools","closed":30,"dropped":20,"names":[{"name":"node-tools","dropped":20,"reasons":{"Stale Connection":20}}]}, {"user":"ace.records","closed":5,"dropped":3},{"user":"(unauthenticated)","closed":90,"dropped":90}]}`}, []string{"bus.g14=2enode-tools.bus-reconnects"}, "", nil}, + // design 47 H2 §810: a core delivery 11 minutes after its merge call is over its 10 minutes; the next within clears it. + {"D-budget a core delivery over", probeBudget, bus{times: `{"core":{"budget":"10m0s","last":{"record":"record/main_7","milestone":"main.7", + "ended":"2026-11-22T00:11:00Z","delivery-time":"11m0s","longest-phase":"build","longest-phase-time":"7m0s","over":true}}, + "leaf":{"budget":"3m0s"}}`}, []string{"delivery.core.over-budget"}, "", nil}, + {"D-budget the next core delivery within", probeBudget, bus{times: `{"core":{"budget":"10m0s","last":{"record":"record/main_8", + "milestone":"main.8","delivery-time":"4m0s","over":false}},"leaf":{"budget":"3m0s"}}`}, nil, "", nil}, {"DW answers", probeWatchdogs, bus{}, nil, "", nil}, {"DW silent", probeWatchdogs, bus{signals: errors.New("no responders")}, []string{"core.watchdogs.watchdogs-silent"}, "", nil}, {"H-bus healthy", probeBusHealth, bus{}, nil, "", nil}, @@ -338,7 +348,7 @@ func agents(verdicts map[string]string) *memStore { // it lacks; the probes that end with the controller or moved to the watchman are not in it (issue 1952 SC4). func TestTheRegistryIsADR0564sTable(t *testing.T) { moved := []string{"D2", "D3", "D4", "D7", "D8", "D11", "D12", "D13", "D15", "DA", "D-root", "D-fill", "D-heard", - "DW", "H-bus"} + "DW", "H-bus", "D-budget"} var ids []string for _, p := range Registry() { ids = append(ids, p.ID) diff --git a/modules/mesh-self-check/module.json b/modules/mesh-self-check/module.json index fdff2735c..9b07d05fe 100644 --- a/modules/mesh-self-check/module.json +++ b/modules/mesh-self-check/module.json @@ -26,7 +26,8 @@ "seat:mesh-catalog.revision", "seat:mesh-catalog.registrations", "seat:mesh-delivery.source", - "seat:mesh-hand-acts.hand-act" + "seat:mesh-hand-acts.hand-act", + "mesh-delivery-gate.gate_times" ], "reads": [ "mesh-delivery-watchman.presence", -- 2.54.0 From 4e381c323b7b8a5b337a85e6f09fb12c0e8332a8 Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 22 Nov 2026 00:42:37 +0100 Subject: [PATCH 18/54] C3: the seat git's tags name each tag's commit, at the seat's form 2 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit tags answers items {tag, commit} read from the forge's own tag list, a tag the forge names no commit for being an error, never a tag without one; tag writes /vMAJOR.MINOR.PATCH, the form the Go toolchain's direct fallback reads, go/v among them. Builds: design 47 H2 §796, §780; H-A-350 DMC-3; mesh-delivery D5 d2d3ebbf --- modules/gitea/client.ts | 10 ++++++++-- modules/gitea/seat.ts | 16 +++++++++------- modules/gitea/test/guard.test.ts | 8 ++++++-- modules/gitea/test/paging.test.ts | 17 +++++++++++++++++ 4 files changed, 40 insertions(+), 11 deletions(-) diff --git a/modules/gitea/client.ts b/modules/gitea/client.ts index 1b761a8b3..c9cc13f42 100644 --- a/modules/gitea/client.ts +++ b/modules/gitea/client.ts @@ -499,11 +499,17 @@ export class GiteaClient { } /** One page of a repository's tags, newest first, by name: what the seat git's `tags` answers. */ - async tags(owner: string, repo: string, page: number, limit: number): Promise { + async tags(owner: string, repo: string, page: number, limit: number): Promise<{ tag: string; commit: string }[]> { if (!Number.isInteger(page) || page < 1) throw new Error(`${page} is not a page, from 1`); const list = await this.request(`${repoPath(owner, repo)}/tags?page=${page}&limit=${limit}`); if (!Array.isArray(list)) throw new Error(`the forge answered no list of the tags of ${owner}/${repo}`); - return list.map((t) => String(t?.name ?? "")).filter((n) => n !== ""); + // Each tag with the commit it names (design 47 H2 §796): a tag the forge names no commit for is no answer, never a + // tag without its commit. + return list.filter((t) => String(t?.name ?? "") !== "").map((t) => { + const tag = String(t.name), commit = String(t?.commit?.sha ?? ""); + if (!/^[0-9a-f]{40}$/.test(commit)) throw new Error(`the forge named no commit for ${owner}/${repo}'s tag ${tag}`); + return { tag, commit }; + }); } /** The commit a tag names, or null when the repository has no such tag (the forge's 404 only). */ diff --git a/modules/gitea/seat.ts b/modules/gitea/seat.ts index 4af16387d..a9c1804cf 100644 --- a/modules/gitea/seat.ts +++ b/modules/gitea/seat.ts @@ -80,8 +80,10 @@ export function repositoryOf(raw: unknown): { owner: string; repo: string; full: } const SHA = /^[0-9a-f]{40}$/; -/** The git tag of a released Go version (design 51; design 47 §370). */ -const GO_TAG = /^go\/v\d+\.\d+\.\d+$/; +/** The git tag of a released version, /vMAJOR.MINOR.PATCH (design 47 H2 §796; design 47 §370). */ +const VERSION_TAG = /^(?:[A-Za-z0-9_][A-Za-z0-9._-]*\/)+v\d+\.\d+\.\d+$/; +/** The form of the seat git's values and answers this holder answers in (design 47 H2 §780, §796). */ +const GIT_FORM = 2; const STATES = new Set(["pending", "success", "failure", "error", "warning"]); function commitOf(name: string, raw: unknown): string { const sha = String(raw ?? "").trim(); @@ -178,7 +180,7 @@ export function seatTools(gitea: GiteaClient, deps: SeatDeps): ToolDefinition[] })); return { pulls }; }), - verb(GIT_VERBS.tags, "The seat git's tags: a repository's tags, a page at a time, newest first; more when another page exists.", + verb(GIT_VERBS.tags, "The seat git's tags: a repository's tags, each with the commit it names, a page at a time, newest first; more when another page exists.", async (args): Promise => { const { owner, repo, full } = repositoryOf(args.repository); const page = args.page === undefined ? 1 : Number(args.page); @@ -186,8 +188,8 @@ export function seatTools(gitea: GiteaClient, deps: SeatDeps): ToolDefinition[] // One more than the page is asked, so whether another page exists is the forge's answer, not a guess. const got = await gitea.tags(owner, repo, page, TAGS_PAGE + 1); const more = got.length > TAGS_PAGE; - const tags = got.slice(0, TAGS_PAGE).filter((t) => t.startsWith(prefix)); - return { repository: full, page, bound: TAGS_PAGE, tags, ...(more ? { more } : {}) }; + const tags = got.slice(0, TAGS_PAGE).filter((t) => t.tag.startsWith(prefix)); + return { form: GIT_FORM, repository: full, page, bound: TAGS_PAGE, tags, ...(more ? { more } : {}) }; }), verb(GIT_VERBS.trees, "The seat git's trees: the trees of one commit, with files and paths when asked; a listing the forge cut is refused.", async (args): Promise => { @@ -267,11 +269,11 @@ export function seatTools(gitea: GiteaClient, deps: SeatDeps): ToolDefinition[] }), verb(GIT_VERBS.mergePull, "The seat git's merge-pull: one merge call, a delivery's start and its group: the pull requests named (pulls, each {repository, number, head}) and by (who@node); every member is judged before any is merged (still its head, its base a watched branch, no required step failed at its fingerprint), none merged when one is refused, then each merged in the order named; it answers the call and each member's outcome.", async (args): Promise => mergeCall(gitea, args, deps)), - verb(GIT_VERBS.tag, "The seat git's tag: the git tag go/vMAJOR.MINOR.PATCH of a released version, written at the commit its build was made from; a tag that is at that commit is performed.", + verb(GIT_VERBS.tag, "The seat git's tag: the git tag /vMAJOR.MINOR.PATCH of a released version, written at the commit its build was made from; a tag that is at that commit is performed.", async (args): Promise => { const { owner, repo } = repositoryOf(args.repository); const tag = String(args.tag ?? ""); - if (!GO_TAG.test(tag)) throw new Refused(`${JSON.stringify(tag)} is not go/vMAJOR.MINOR.PATCH`); + if (!VERSION_TAG.test(tag)) throw new Refused(`${JSON.stringify(tag)} is not /vMAJOR.MINOR.PATCH`); const commit = commitOf("commit", args.commit); const at = await gitea.tagCommit(owner, repo, tag); if (at === commit) return { tag, performed: true }; diff --git a/modules/gitea/test/guard.test.ts b/modules/gitea/test/guard.test.ts index ae7348ab7..89215abcb 100644 --- a/modules/gitea/test/guard.test.ts +++ b/modules/gitea/test/guard.test.ts @@ -483,7 +483,7 @@ test("every read and act of the seat git answers its contract, and an error is a appliedProtection: async () => ({ rule_name: "main", enable_status_check: true, status_check_contexts: ["mesh/check"] }), getRepo: async () => ({ writes: true }), openPulls: async () => [{ owner: "novox", repo: "app", number: 9, base: "main", head: "feat/x", head_sha: SHA, required: ["mesh/check"], checks: {} }], - tags: async () => ["go/v0.1.1", "go/v0.1.0", "v1"], + tags: async () => [{ tag: "go/v0.1.1", commit: SHA }, { tag: "go/v0.1.0", commit: OTHER }, { tag: "v1", commit: SHA }], commitTrees: async () => ({ root: SHA, trees: { "": SHA } }), getFile: async (_o: string, _r: string, path: string) => (path === "gone" ? (() => { throw new Error("Gitea API GET /repos/novox/app/contents/gone: 404 Not Found"); })() : path === "down" ? down() : { path, sha: OTHER, size: 3, content: "abc" }), compare: async () => ({ total_commits: 2, commits: [{ sha: OTHER }, { sha: SHA }] }), @@ -509,7 +509,8 @@ test("every read and act of the seat git answers its contract, and an error is a ["branch", "no such branch", { repository: "novox/app", branch: "nope" }, { branchHead: gone }, (a) => [a.head, a["error-kind"]], [undefined, "not-found"]], ["open-pulls", "one", {}, {}, (a) => [a.pulls.length, a.pulls[0].required], [1, ["mesh/check"]]], ["open-pulls", "forge down", {}, { openPulls: down }, (a) => [a.pulls, a["error-kind"]], [undefined, "failed"]], - ["tags", "a page", { repository: "novox/app", prefix: "go/" }, {}, (a) => [a.tags, a.page, a.bound, a.more], [["go/v0.1.1", "go/v0.1.0"], 1, 50, undefined]], + // 47 H2 §796: each tag names its commit, at the seat's form 2. + ["tags", "a page, each tag with its commit", { repository: "novox/app", prefix: "go/" }, {}, (a) => [a.tags, a.page, a.bound, a.more, a.form], [[{ tag: "go/v0.1.1", commit: SHA }, { tag: "go/v0.1.0", commit: OTHER }], 1, 50, undefined, 2]], ["tags", "forge down", { repository: "novox/app" }, { tags: down }, (a) => [a.tags, a["error-kind"]], [undefined, "failed"]], ["trees", "root", { repository: "novox/app", commit: SHA }, {}, (a) => [a.commit, a.root], [SHA, SHA]], ["trees", "forge down", { repository: "novox/app", commit: SHA }, { commitTrees: down }, (a) => [a.root, a["error-kind"]], [undefined, "failed"]], @@ -523,6 +524,9 @@ test("every read and act of the seat git answers its contract, and an error is a ["tag", "written", { repository: "novox/app", tag: "go/v1.2.3", commit: SHA }, { tagCommit: async () => null, createTag: async (_o: string, _r: string, tag: string, c: string) => { tagged.push(`${tag}@${c}`); } }, (a) => [a.performed, tagged.splice(0)], [true, [`go/v1.2.3@${SHA}`]]], ["tag", "already at that commit", { repository: "novox/app", tag: "go/v1.2.3", commit: SHA }, { tagCommit: async () => SHA, createTag: async () => { throw new Error("written again"); } }, (a) => [a.performed, a.error], [true, undefined]], ["tag", "at another commit is never moved", { repository: "novox/app", tag: "go/v1.2.3", commit: SHA }, { tagCommit: async () => OTHER }, (a) => [a.performed, a["error-kind"]], [undefined, "refused"]], + // 47 H2 §796: a Go package's tag is /v. + ["tag", "a package directory's version", { repository: "novox/app", tag: "modules/x/go/v1.2.3", commit: SHA }, { tagCommit: async () => null, createTag: async (_o: string, _r: string, tag: string, c: string) => { tagged.push(`${tag}@${c}`); } }, (a) => [a.performed, tagged.splice(0)], [true, [`modules/x/go/v1.2.3@${SHA}`]]], + ["tag", "no directory", { repository: "novox/app", tag: "v1.2.3", commit: SHA }, {}, (a) => [a.performed, a["error-kind"]], [undefined, "refused"]], ["tag", "no Go version", { repository: "novox/app", tag: "v1", commit: SHA }, {}, (a) => [a.performed, a["error-kind"]], [undefined, "refused"]], ["status", "refused state", { id: "s1", repository: "novox/app", commit: SHA, context: "c", state: "bogus" }, {}, (a) => [a.performed, a["error-kind"]], [undefined, "refused"]], ["comment", "first time", { id: "c1", repository: "novox/app", number: 9, text: "hi" }, {}, (a) => [a.performed, a.held, added], [true, undefined, 1]], diff --git a/modules/gitea/test/paging.test.ts b/modules/gitea/test/paging.test.ts index 8bf0bcca9..c9e7d7845 100644 --- a/modules/gitea/test/paging.test.ts +++ b/modules/gitea/test/paging.test.ts @@ -108,3 +108,20 @@ test("a file over 512 KiB is refused from its directory listing, and its body is await new Promise((r) => server.close(() => r())); } }); + +// 47 H2 §796: a tag names its commit; a tag the forge names no commit for is an error, never a tag without one. +test("the forge's tags are read with the commit each names", async () => { + const { GiteaClient } = await import("../client.ts"); + let body: unknown[] = []; + const server = createServer((_req, res) => res.end(JSON.stringify(body))); + await new Promise((r) => server.listen(0, r)); + try { + const client = new GiteaClient(`http://127.0.0.1:${(server.address() as any).port}`, "t"); + body = [{ name: "go/v0.1.1", commit: { sha: "a".repeat(40) } }, { name: "go/v0.1.0", commit: { sha: "b".repeat(40) } }]; + assert.deepEqual(await client.tags("novox", "hq", 1, 51), [{ tag: "go/v0.1.1", commit: "a".repeat(40) }, { tag: "go/v0.1.0", commit: "b".repeat(40) }]); + body = [{ name: "go/v0.1.2" }]; + await assert.rejects(client.tags("novox", "hq", 1, 51), /named no commit for novox\/hq's tag go\/v0.1.2/); + } finally { + server.close(); + } +}); -- 2.54.0 From cb5de6fd31b523e70f7ae43a800952bd024f336d Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 22 Nov 2026 00:43:38 +0100 Subject: [PATCH 19/54] MR1: the recovery module's whole test waits its start by the setup bound MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The module's start is the test's setup: its two waits on the bus take testbus.SetupBound (half of MESH_TEST_LEVEL_BOUND) and say a starved setup in the one line testbus.Starved writes; its 30 s values go. Builds: design 47 H2 §800; H-A-327, H-A-368 --- .../mesh-recovery/whole_integration_test.go | 22 ++++++++++++++----- 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/modules/mesh-recovery/cmd/mesh-recovery/whole_integration_test.go b/modules/mesh-recovery/cmd/mesh-recovery/whole_integration_test.go index 0aad2a95a..9e7904dba 100644 --- a/modules/mesh-recovery/cmd/mesh-recovery/whole_integration_test.go +++ b/modules/mesh-recovery/cmd/mesh-recovery/whole_integration_test.go @@ -15,6 +15,7 @@ import ( "crypto/sha256" "encoding/hex" "encoding/json" + "fmt" "os" "path/filepath" "strings" @@ -258,7 +259,8 @@ func TestRecoveryWhole(t *testing.T) { events <- heard{m.Subject, b} }) var kept []heard - waitEvent := func(t *testing.T, what string, ok func(heard) bool) map[string]any { + // saidWithin waits for an event ok takes, within bound, and ends the test by fail when none came. + saidWithin := func(t *testing.T, what string, bound time.Duration, fail func(string), ok func(heard) bool) map[string]any { t.Helper() for i, h := range kept { if ok(h) { @@ -266,7 +268,7 @@ func TestRecoveryWhole(t *testing.T) { return h.body } } - deadline := time.After(30 * time.Second) + deadline := time.After(bound) for { select { case h := <-events: @@ -275,10 +277,15 @@ func TestRecoveryWhole(t *testing.T) { } kept = append(kept, h) case <-deadline: - t.Fatalf("not said within 30s: %s", what) + fail(fmt.Sprintf("not said within %s: %s", bound, what)) + return nil } } } + waitEvent := func(t *testing.T, what string, ok func(heard) bool) map[string]any { + t.Helper() + return saidWithin(t, what, 30*time.Second, func(words string) { t.Helper(); t.Fatal(words) }, ok) + } endOf := func(t *testing.T, event, act string) map[string]any { t.Helper() return waitEvent(t, event+" of "+act, func(h heard) bool { @@ -347,13 +354,16 @@ func TestRecoveryWhole(t *testing.T) { done := make(chan error, 1) go func() { done <- run(runCtx, func(k string) string { return env[k] }, testLog{t}) }() defer func() { stop(); <-done }() - waitEvent(t, "the start's removal of points past their keep days", func(h heard) bool { + // The module's start is the test's setup: its waits on the bus take the setup bound of the level's bound and say a starved + // setup in its one line, never a value of the test's own (design 47 H2 §800, H-A-327, H-A-368). + setup := testbus.SetupBound(t) + saidWithin(t, "the start's removal of points past their keep days", setup, func(words string) { testbus.Starved(t, words) }, func(h heard) bool { return h.subject == "mesh.seat.mesh-recovery.event."+seat.EventExpiryEnded }) select { case <-listed: - case <-time.After(30 * time.Second): - t.Fatal("the module did not read its seat's definition from the seat list within 30s") + case <-time.After(setup): + testbus.Starved(t, fmt.Sprintf("the module did not read its seat's definition from the seat list within %s", setup)) } var first string -- 2.54.0 From 54fe0dd7cc12a61f51b4f4f89331f3d41cb3b00e Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 22 Nov 2026 00:45:38 +0100 Subject: [PATCH 20/54] C1: the forge module says away when the forge or its store is away MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A 401, 502, 503 or connection fault is judged by the forge's own unauthenticated version read: the read failing, or naming the store unavailable, answers the kind away with part and words, which every caller reads as a transport fault; the read passing, a 401 (the fresh token refused too) is the forge refusing the mesh's admin account, never worded as an account to create. Builds: design 47 H2 §776; H-A-350 DMC-2, H-A-354 --- modules/gitea/client.ts | 25 ++++++++++++ .../seatgit/whole_integration_test.go | 28 +++++++++++++ modules/gitea/seat.ts | 40 ++++++++++++++++++- modules/gitea/test/guard.test.ts | 8 ++++ modules/gitea/test/paging.test.ts | 28 +++++++++++++ modules/gitea/test/token.test.ts | 6 +-- modules/gitea/token.ts | 10 ++--- 7 files changed, 133 insertions(+), 12 deletions(-) diff --git a/modules/gitea/client.ts b/modules/gitea/client.ts index c9cc13f42..54c39d5d4 100644 --- a/modules/gitea/client.ts +++ b/modules/gitea/client.ts @@ -192,6 +192,9 @@ export class DirectoryNotInTree extends Error { } /** One request to the forge, cut at the limit and failing loud naming the request. */ +/** What a forge's answer says when its store does not answer (design 47 H2 §776). */ +const STORE_UNAVAILABLE = /database|\bdb\b|store/i; + async function timedFetch(url: string, init: RequestInit, label: string, timeoutMs: number): Promise { try { return await fetch(url, { ...init, signal: init.signal ? AbortSignal.any([init.signal, AbortSignal.timeout(timeoutMs)]) : AbortSignal.timeout(timeoutMs) }); @@ -276,6 +279,28 @@ export class GiteaClient { return this.request(path, options); } + /** + * The forge's own version read, unauthenticated (GET /api/v1/version): whether the forge answers at all, and whether + * it names its store unavailable (design 47 H2 §776). What the seat git reads before it says a 401, a 502, a 503 or a + * connection fault is the forge's refusal or the forge away; never throws. + */ + async forgeVersion(): Promise<{ ok: true; version: string } | { ok: false; words: string; store: boolean }> { + const label = "GET /version"; + try { + const res = await timedFetch(`${this.baseUrl}/api/v1/version`, { headers: { Accept: "application/json" } }, label, this.requestTimeoutMs); + const text = await res.text(); + const store = STORE_UNAVAILABLE.test(text); + if (!res.ok) return { ok: false, words: `the forge's version read answered ${res.status} ${text}`.trim(), store }; + let version = ""; + try { version = String(JSON.parse(text)?.version ?? ""); } catch { /* no version said */ } + if (store) return { ok: false, words: `the forge's version read names its store unavailable: ${text}`, store }; + if (!version) return { ok: false, words: `the forge's version read named no version: ${text}`, store }; + return { ok: true, version }; + } catch (err) { + return { ok: false, words: `the forge's version read: ${err instanceof Error ? err.message : String(err)}`, store: false }; + } + } + // ---- Repositories ---- /** Every repository this token can see, one page. `/user/repos` is only what the token's own diff --git a/modules/gitea/internal/seatgit/whole_integration_test.go b/modules/gitea/internal/seatgit/whole_integration_test.go index 2109e7eb2..19ed27d79 100644 --- a/modules/gitea/internal/seatgit/whole_integration_test.go +++ b/modules/gitea/internal/seatgit/whole_integration_test.go @@ -12,6 +12,7 @@ package seatgit // - A runner's step-ended of a failed required step at a pull request's head and fingerprint is kept, and a merge call // naming that head merges nothing and names the step (§735, test 114). // - A call naming two pull requests in two repositories merges both, in order, and says one merge-called (§734, §736). +// - With the forge's store stopped a merge call answers away, and called again once it is back it lands (H2 §776). // - A forge that refuses the second merge gives one merge-called with the first member and `refused`, and merge-partial // raised; a later call landing the refused member clears it (§767, test 115). @@ -54,12 +55,26 @@ type forge struct { pulls map[string]*pull merges []string paths []string + // away is the forge's store stopped (§776): the API answers 503 and the version read names the store unavailable. + away bool } func (f *forge) serve(w http.ResponseWriter, r *http.Request) { f.mu.Lock() defer f.mu.Unlock() f.paths = append(f.paths, r.Method+" "+r.URL.Path) + if r.URL.Path == "/api/v1/version" { + if f.away { + http.Error(w, `{"message":"the database is unavailable"}`, http.StatusInternalServerError) + return + } + _, _ = w.Write([]byte(`{"version":"1.27.3"}`)) + return + } + if f.away { + http.Error(w, "Service Unavailable", http.StatusServiceUnavailable) + return + } m := regexp.MustCompile(`^/api/v1/repos/([^/]+/[^/]+)/pulls/(\d+)(/merge)?$`).FindStringSubmatch(r.URL.Path) if m != nil { p := f.pulls[m[1]+"#"+m[2]] @@ -456,6 +471,19 @@ func TestTheSeatGitWholeOverARealBus(t *testing.T) { t.Fatalf("merged %v", got) } + // With the forge's store stopped, a merge call answers away, merging nothing (H2 §776); called again once the store is + // back, it lands (below). + f.mu.Lock() + f.away = true + f.mu.Unlock() + away := rt.mergePull(map[string]any{"repository": "novox/app", "number": 7, "head": A}, map[string]any{"repository": "novox/lib", "number": 10, "head": B}) + if away["error-kind"] != "away" || away["part"] != "the forge's store" || len(f.merged()) != 0 { + t.Fatalf("a merge call with the forge's store stopped answered %v, merged %v", away, f.merged()) + } + f.mu.Lock() + f.away = false + f.mu.Unlock() + // Two pull requests in two repositories merge, in order, and say one merge-called (§734, §736). both := rt.mergePull(map[string]any{"repository": "novox/app", "number": 7, "head": A}, map[string]any{"repository": "novox/lib", "number": 10, "head": B}) call, _ := both["call"].(string) diff --git a/modules/gitea/seat.ts b/modules/gitea/seat.ts index a9c1804cf..a8854e36a 100644 --- a/modules/gitea/seat.ts +++ b/modules/gitea/seat.ts @@ -7,7 +7,8 @@ // // **A failed read answers an error, never an empty answer**: the forge's 404 is the kind `not-found`, its refusal (another // 4xx, or a refusal of this holder) the kind `refused`, a forge that failed or did not answer the kind `failed`; an empty list or an absent file is answered -// only when the forge said so. +// only when the forge said so. A 401, 502, 503 or connection fault whose version read fails, or names the forge's store +// unavailable, is the kind `away` (design 47 H2 §776; failureOf). // // **The acts carry the caller's id** (status, comment, view, note): an id the pull request or the note already holds is // answered `held` and nothing is done again. `merge-pull` is one merge call of one or more pull requests, every member @@ -24,6 +25,8 @@ import type { CommitStatus, GiteaClient, GiteaComment, GiteaPull } from "./clien import { DirectoryNotInTree, MAX_FILE_BYTES } from "./client.js"; import { containedFrom, newestByContext } from "./checks.js"; import { Refused, checkName } from "./guard.js"; +import { AdminRefused } from "./token.js"; +import { awayAnswer } from "@novox/mesh-sdk/callers"; import { appendNote, viewBody, viewComment, type Runner } from "./delivery.js"; import type { Judged } from "./failedsteps.js"; import { mergeCall } from "./mergecall.js"; @@ -71,6 +74,39 @@ export function failure(err: unknown): { error: string; "error-kind": ErrorKind return { error: text, "error-kind": refused ? "refused" : "failed" }; } +/** The forge's own version read, which tells the forge away from its refusal (client.ts forgeVersion). */ +export type VersionRead = Pick; + +/** + * Which failure may mean the forge or its store is away (design 47 H2 §776): a 401 (or the admin account refused at the + * mint the 401 asked), a 502 or 503, or a connection fault (no answer, or an answer cut at its limit); else null. + */ +function awayCause(err: unknown): "unauthorized" | "unavailable" | null { + if (err instanceof AdminRefused) return "unauthorized"; + const text = err instanceof Error ? err.message : String(err); + if (/^Gitea [^:]*: 401 /.test(text)) return "unauthorized"; + if (/^Gitea [^:]*: 50[23] /.test(text)) return "unavailable"; + if (/^Gitea API [^:]*: no answer in /.test(text)) return "unavailable"; + if (err instanceof TypeError && /fetch failed/.test(text)) return "unavailable"; + return null; +} + +/** + * A failure as the contract says it, the forge's own version read deciding a 401, a 502, a 503 or a connection fault + * (design 47 H2 §776): the read failing, or naming the forge's store unavailable, answers `away`, which every caller reads + * as a transport fault; the read passing, a 401 is the forge refusing the mesh's admin account (the client minted a + * fresh token on the first 401 and was refused again, token.ts), never worded as an account to create; else as failure. + */ +export async function failureOf(forge: VersionRead, err: unknown): Promise<{ error: string; "error-kind": ErrorKind; part?: string; words?: string }> { + const cause = awayCause(err); + if (cause === null) return failure(err); + const text = err instanceof Error ? err.message : String(err); + const read = await forge.forgeVersion(); + if (read.ok === false) return awayAnswer(read.store ? "the forge's store" : "the forge", `${text}; ${read.words}`); + if (cause === "unauthorized") return { error: `the forge refuses the mesh's admin account: ${text}`, "error-kind": "refused" }; + return failure(err); +} + /** owner/name of a `repository` argument, each checked as a name (guard.ts). */ export function repositoryOf(raw: unknown): { owner: string; repo: string; full: string } { const parts = String(raw ?? "").split("/"); @@ -143,7 +179,7 @@ export function seatTools(gitea: GiteaClient, deps: SeatDeps): ToolDefinition[] try { return await act(args); } catch (err) { - return failure(err); + return failureOf(gitea, err); } }, }); diff --git a/modules/gitea/test/guard.test.ts b/modules/gitea/test/guard.test.ts index 89215abcb..397877407 100644 --- a/modules/gitea/test/guard.test.ts +++ b/modules/gitea/test/guard.test.ts @@ -474,6 +474,7 @@ test("every read and act of the seat git answers its contract, and an error is a const SHA = "a".repeat(40), OTHER = "b".repeat(40); const gone = () => { throw new Error("Gitea API GET /repos/novox/app/pulls/9: 404 Not Found"); }; const down = () => { throw new Error("Gitea API GET /repos/novox/app/x: 502 Bad Gateway"); }; + const unauthorized = () => { throw new Error("Gitea API /repos/novox/app/pulls/9: 401 Unauthorized"); }; const pr = { number: 9, title: "t", state: "open", merged: false, head: "feat/x", head_sha: SHA, base: "main", user: "jochen", html_url: "http://f/9", mergeable: true }; const tagged: string[] = []; let comments: { id: number; body: string }[] = [], noteShown = "", added = 0, noteWritten = 0; @@ -483,6 +484,7 @@ test("every read and act of the seat git answers its contract, and an error is a appliedProtection: async () => ({ rule_name: "main", enable_status_check: true, status_check_contexts: ["mesh/check"] }), getRepo: async () => ({ writes: true }), openPulls: async () => [{ owner: "novox", repo: "app", number: 9, base: "main", head: "feat/x", head_sha: SHA, required: ["mesh/check"], checks: {} }], + forgeVersion: async () => ({ ok: true, version: "1.27.3" }), tags: async () => [{ tag: "go/v0.1.1", commit: SHA }, { tag: "go/v0.1.0", commit: OTHER }, { tag: "v1", commit: SHA }], commitTrees: async () => ({ root: SHA, trees: { "": SHA } }), getFile: async (_o: string, _r: string, path: string) => (path === "gone" ? (() => { throw new Error("Gitea API GET /repos/novox/app/contents/gone: 404 Not Found"); })() : path === "down" ? down() : { path, sha: OTHER, size: 3, content: "abc" }), @@ -505,6 +507,12 @@ test("every read and act of the seat git answers its contract, and an error is a ["pull", "by merge commit", { repository: "novox/app", "merge-commit": OTHER }, {}, (a) => [a.pull.state, a.pull["merge-commit"], a.merging], ["merged", OTHER, undefined]], ["pull", "unknown", { repository: "novox/app", number: 404 }, {}, (a) => [a.pull, a["error-kind"]], [undefined, "not-found"]], ["pull", "forge down", { repository: "novox/app", number: 9 }, { getPullRequest: down }, (a) => [a.pull, a["error-kind"]], [undefined, "failed"]], + // 47 H2 §776: a 401, 502, 503 or connection fault is away when the forge's version read fails or names its store + // unavailable; with the read passing a 401 (the fresh token refused too) is the forge refusing the mesh's admin account. + ["pull", "502 and the version read failing is away", { repository: "novox/app", number: 9 }, { getPullRequest: down, forgeVersion: async () => ({ ok: false, words: "no answer", store: false }) }, (a) => [a.pull, a["error-kind"], a.part], [undefined, "away", "the forge"]], + ["pull", "401 and the store named unavailable is away", { repository: "novox/app", number: 9 }, { getPullRequest: unauthorized, forgeVersion: async () => ({ ok: false, words: "database is down", store: true }) }, (a) => [a["error-kind"], a.part, /401/.test(a.words)], ["away", "the forge's store", true]], + ["pull", "a connection fault and the version read failing is away", { repository: "novox/app", number: 9 }, { getPullRequest: async () => { throw new TypeError("fetch failed"); }, forgeVersion: async () => ({ ok: false, words: "fetch failed", store: false }) }, (a) => [a["error-kind"]], ["away"]], + ["pull", "401 with the version read passing is refused", { repository: "novox/app", number: 9 }, { getPullRequest: unauthorized }, (a) => [a["error-kind"], /refuses the mesh's admin account/.test(a.error), /[Cc]reate/.test(a.error)], ["refused", true, false]], ["branch", "a branch's head, no protection read (ADR 0682 §13)", { repository: "novox/app", branch: "main" }, { branchHead: async () => SHA, appliedProtection: async () => { throw new Error("read"); } }, (a) => [a.head, a.protected, a.error], [SHA, undefined, undefined]], ["branch", "no such branch", { repository: "novox/app", branch: "nope" }, { branchHead: gone }, (a) => [a.head, a["error-kind"]], [undefined, "not-found"]], ["open-pulls", "one", {}, {}, (a) => [a.pulls.length, a.pulls[0].required], [1, ["mesh/check"]]], diff --git a/modules/gitea/test/paging.test.ts b/modules/gitea/test/paging.test.ts index c9e7d7845..f5d3010c4 100644 --- a/modules/gitea/test/paging.test.ts +++ b/modules/gitea/test/paging.test.ts @@ -125,3 +125,31 @@ test("the forge's tags are read with the commit each names", async () => { server.close(); } }); + +// 47 H2 §776: the forge's own version read, unauthenticated, tells the forge away from its refusal. +test("the forge's version read says whether the forge answers and whether its store is away", async () => { + const { GiteaClient } = await import("../client.ts"); + let status = 200, body = `{"version":"1.27.3"}`; + const auth: (string | undefined)[] = []; + const server = createServer((req, res) => { auth.push(req.headers.authorization); res.statusCode = status; res.end(body); }); + await new Promise((r) => server.listen(0, r)); + const port = (server.address() as any).port; + try { + const client = new GiteaClient(`http://127.0.0.1:${port}`, "t"); + const rows: [number, string, unknown][] = [ + [200, `{"version":"1.27.3"}`, { ok: true, version: "1.27.3" }], + [500, "database is locked", { ok: false, store: true }], + [503, "Service Unavailable", { ok: false, store: false }], + ]; + for (const [s, b, want] of rows) { + status = s; body = b; + const got: any = await client.forgeVersion(); + assert.deepEqual(got.ok ? got : { ok: got.ok, store: got.store }, want, `${s} ${b}`); + } + assert.deepEqual(auth.filter(Boolean), [], "the version read is unauthenticated"); + } finally { + server.close(); + } + const gone: any = await new GiteaClient(`http://127.0.0.1:${port}`, "t").forgeVersion(); + assert.deepEqual([gone.ok, gone.store], [false, false]); +}); diff --git a/modules/gitea/test/token.test.ts b/modules/gitea/test/token.test.ts index a33477b82..289020c57 100644 --- a/modules/gitea/test/token.test.ts +++ b/modules/gitea/test/token.test.ts @@ -456,9 +456,9 @@ test("the admin account is refused: said plainly, nothing kept, and the next cal await assert.rejects(client.listRepos(), (err: unknown) => { assert.ok(err instanceof AdminRefused, String(err)); - assert.match(err.message, /refused the admin account "mesh-admin" \(401\)/); - assert.match(err.message, /admin-bootstrap step creates it/); - assert.match(err.message, /came from a predecessor/); + // 47 H2 §776: the forge's refusal, never worded as an account to create. + assert.match(err.message, /the forge refuses the mesh's admin account "mesh-admin" \(401\)/); + assert.doesNotMatch(err.message, /[Cc]reate/); assert.ok(!err.message.includes(PASSWORD)); return true; }); diff --git a/modules/gitea/token.ts b/modules/gitea/token.ts index 2c02a65d7..5e079e8bf 100644 --- a/modules/gitea/token.ts +++ b/modules/gitea/token.ts @@ -91,15 +91,11 @@ export class ConfiguredToken implements TokenSource { } } -/** The forge would not take the admin account: it is missing, or its password is not the one the mesh holds. */ +/** The forge would not take the admin account: it refused the login the mesh holds (design 47 H2 §776). Said as the + * forge's refusal, never as an account to create: whether the forge is away is the seat's to judge (seat.ts). */ export class AdminRefused extends Error { constructor(admin: string, status: number) { - super( - `the forge refused the admin account "${admin}" (${status}) — it does not exist there, or its password is not ` + - `the one the vault delivered. The admin-bootstrap step creates it on a forge the mesh raised; a forge whose data ` + - `came from a predecessor does not have it. Create "${admin}" on the forge with the delivered password and the ` + - `token is minted on the next call — the tools stay registered and the watcher keeps trying`, - ); + super(`the forge refuses the mesh's admin account "${admin}" (${status})`); this.name = "AdminRefused"; } } -- 2.54.0 From 681de09f043936a6d560cf6636a66fa77538965b Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 22 Nov 2026 00:49:42 +0100 Subject: [PATCH 21/54] RG2: the registry's list pages by module and cursor; the sweep reads it page by page MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit oci-artifact-registry's list filters by name or module while it reads and answers at most 768 KB a page with capped, total and the cursor next, in order of the names; container-image-registry's answers whole as its form says. The registry takes a removal for retention whoever asks it: the refusal "not the sweep's" and its read of the sweep seat's holder go. The sweep reads the list from its cursor until no page is left. Builds: design 47 H2 §813 (7), §811, §779; H-A-350 DMC-10, H-A-399 --- .../holder_integration_test.go | 95 +++++++++++++++---- .../cmd/artifact-store-tools/listed.go | 42 ++++++-- .../cmd/artifact-store-tools/store_test.go | 15 ++- .../cmd/distribution-tools/main.go | 52 +++++++--- modules/distribution/go.mod | 2 +- .../internal/holder/holder_test.go | 22 +---- modules/distribution/internal/holder/list.go | 90 +++++++++++++++--- .../distribution/internal/holder/list_test.go | 39 +++++++- modules/distribution/internal/holder/seats.go | 3 - modules/distribution/internal/holder/verbs.go | 14 +-- 10 files changed, 282 insertions(+), 92 deletions(-) diff --git a/modules/artifact-store-tools/cmd/artifact-store-tools/holder_integration_test.go b/modules/artifact-store-tools/cmd/artifact-store-tools/holder_integration_test.go index 4e81af3d0..9c2c64958 100644 --- a/modules/artifact-store-tools/cmd/artifact-store-tools/holder_integration_test.go +++ b/modules/artifact-store-tools/cmd/artifact-store-tools/holder_integration_test.go @@ -11,6 +11,7 @@ package main import ( "context" "encoding/json" + "fmt" "strings" "sync" "testing" @@ -244,26 +245,82 @@ func TestTheSeatSweepsAtABuildsEndOnARealBusWithNoLease(t *testing.T) { } time.Sleep(20 * time.Millisecond) } - mu.Lock() - defer mu.Unlock() - if len(acts) != 3 || !strings.Contains(acts[0], archiveRef("letta", 2)) || !strings.Contains(acts[1], imageRef("letta", 2)) || - !strings.Contains(acts[2], mirror(1)) { - t.Fatalf("the hand acts recorded are %v; the store was asked %v", acts, door.asked) - } - if len(gone) != 1 || !strings.Contains(gone[0], `"fingerprint":"`+fp(2)+`"`) { - t.Fatalf("artifacts-gone said %v", gone) - } + func() { + mu.Lock() + defer mu.Unlock() + if len(acts) != 3 || !strings.Contains(acts[0], archiveRef("letta", 2)) || !strings.Contains(acts[1], imageRef("letta", 2)) || + !strings.Contains(acts[2], mirror(1)) { + t.Fatalf("the hand acts recorded are %v; the store was asked %v", acts, door.asked) + } + if len(gone) != 1 || !strings.Contains(gone[0], `"fingerprint":"`+fp(2)+`"`) { + t.Fatalf("artifacts-gone said %v", gone) + } - // S25 through the real findings bucket: letta's wanted image missing from the store is a finding there. - view, _ := mem.Listed(ctx) - repo, digest, _ := splitReference(imageRef("letta", 1)) - delete(view.L.Repos[repo].Revisions, digest) - if err := h.Observe(ctx); err != nil { - t.Fatal(err) - } - if keys, _ := bus.Keys("artifact-store-tools." + FindingsState); len(keys) != 1 || keys[0] != "module.letta.ace.image-missing" { - t.Fatalf("the findings are %v", keys) - } + // S25 through the real findings bucket: letta's wanted image missing from the store is a finding there. + view, _ := mem.Listed(ctx) + repo, digest, _ := splitReference(imageRef("letta", 1)) + delete(view.L.Repos[repo].Revisions, digest) + if err := h.Observe(ctx); err != nil { + t.Fatal(err) + } + if keys, _ := bus.Keys("artifact-store-tools." + FindingsState); len(keys) != 1 || keys[0] != "module.letta.ace.image-missing" { + t.Fatalf("the findings are %v", keys) + } + }() + + // Design 47 H2 §813 (7): a pass reads oci-artifact-registry's list page by page, each at most 768 KB, over 20,000 + // artifacts, so its view holds every one. + t.Run("a pass reads 20,000 artifacts page by page", func(t *testing.T) { + const many, pageBytes = 20000, 768 << 10 + repos := make([]map[string]any, many) + for i := range repos { + repos[i] = map[string]any{"name": fmt.Sprintf("m%05d/bundle", i), "manifests": []map[string]any{{ + "digest": fmt.Sprintf("sha256:%064x", i), "artifact": fmt.Sprintf("sha256:%064x", many+i), "size": 100}}} + } + pages := 0 + sub, err := nc.Subscribe("mesh.seat.oci-artifact-registry.tool.list.paged", func(m *nats.Msg) { + var in struct{ Next string } + _ = json.Unmarshal(m.Data, &in) + page, size, next := []map[string]any{}, 0, "" + for _, r := range repos { + if r["name"].(string) <= in.Next { + continue + } + raw, _ := json.Marshal(r) + if size+len(raw)+1 > pageBytes-1024 { // its comma, and room for the answer's other fields + next = page[len(page)-1]["name"].(string) + break + } + size += len(raw) + 1 + page = append(page, r) + } + pages++ + out, _ := json.Marshal(map[string]any{"repositories": page, "next": next, "capped": next != "", "total": many}) + _ = m.Respond(out) + }) + if err != nil { + t.Fatal(err) + } + defer func() { _ = sub.Unsubscribe() }() + store := &Store{Ask: func(key string, body any) (json.RawMessage, error) { + if key != "seat:oci-artifact-registry.list" { + return ask(key, body) + } + raw, _ := json.Marshal(body) + m, err := nc.Request("mesh.seat.oci-artifact-registry.tool.list.paged", raw, 5*time.Second) + if err != nil { + return nil, err + } + if len(m.Data) > pageBytes { + t.Errorf("a page of %d bytes", len(m.Data)) + } + return m.Data, nil + }} + got, err := store.ListOf("oci-artifact-registry", "") + if err != nil || len(got) != many || got[many-1].Name != "m19999/bundle" || pages < 2 { + t.Fatalf("read %d repositories in %d pages: %v", len(got), pages, err) + } + }) } // findingsAt reads and writes the holder's own findings in its bucket, artifact-store-tools_findings. diff --git a/modules/artifact-store-tools/cmd/artifact-store-tools/listed.go b/modules/artifact-store-tools/cmd/artifact-store-tools/listed.go index e83e881ea..61e7a59ac 100644 --- a/modules/artifact-store-tools/cmd/artifact-store-tools/listed.go +++ b/modules/artifact-store-tools/cmd/artifact-store-tools/listed.go @@ -79,20 +79,46 @@ type Store struct { Ask func(key string, body any) (json.RawMessage, error) } -// ListOf asks seat's list, of one repository or, with name empty, of every one. +// ListOf asks seat's list, of one repository or, with name empty, of every one. oci-artifact-registry's list is read page by +// page from its cursor until it answers none (design 47 H2 §813 (7), §779); container-image-registry's answers whole. func (s *Store) ListOf(seat, name string) ([]reg.Repository, error) { if s == nil || s.Ask == nil { return nil, fmt.Errorf("this process reaches no registry seat") } - raw, err := s.Ask("seat:"+seat+"."+reg.VerbList, reg.ListInput{Name: name}) - if err != nil { - return nil, fmt.Errorf("%s's list: %w", seat, err) + if seat != ociseat.Seat { + raw, err := s.Ask("seat:"+seat+"."+reg.VerbList, reg.ListInput{Name: name}) + if err != nil { + return nil, fmt.Errorf("%s's list: %w", seat, err) + } + var a reg.ListAnswer + if err := json.Unmarshal(raw, &a); err != nil { + return nil, fmt.Errorf("%s's answer to list: %w", seat, err) + } + return a.Repositories, nil } - var a reg.ListAnswer - if err := json.Unmarshal(raw, &a); err != nil { - return nil, fmt.Errorf("%s's answer to list: %w", seat, err) + var out []reg.Repository + seen := map[string]bool{} + for in := (ociseat.ListInput{Name: name}); ; { + raw, err := s.Ask("seat:"+seat+"."+reg.VerbList, in) + if err != nil { + return nil, fmt.Errorf("%s's list: %w", seat, err) + } + var a struct { + Repositories []reg.Repository `json:"repositories"` + Next string `json:"next"` + } + if err := json.Unmarshal(raw, &a); err != nil { + return nil, fmt.Errorf("%s's answer to list: %w", seat, err) + } + out = append(out, a.Repositories...) + if a.Next == "" { + return out, nil + } + if seen[a.Next] { + return nil, fmt.Errorf("%s's list answered the cursor %q twice", seat, a.Next) + } + seen[a.Next], in.Next = true, a.Next } - return a.Repositories, nil } // View reads the store now, through both seats. diff --git a/modules/artifact-store-tools/cmd/artifact-store-tools/store_test.go b/modules/artifact-store-tools/cmd/artifact-store-tools/store_test.go index 3af7f16e7..5943e7e10 100644 --- a/modules/artifact-store-tools/cmd/artifact-store-tools/store_test.go +++ b/modules/artifact-store-tools/cmd/artifact-store-tools/store_test.go @@ -92,7 +92,20 @@ func (f *fakeStore) ask(t *testing.T) func(string, any) (json.RawMessage, error) repos = append(repos, reg.Repository{Name: name, Manifests: ms}) } } - return json.Marshal(reg.ListAnswer{Repositories: repos}) + if seat != "oci-artifact-registry" { + return json.Marshal(reg.ListAnswer{Repositories: repos}) + } + // oci-artifact-registry pages (design 47 H2 §813 (7)): one repository a page, after the one its cursor names. + var page struct{ Next string } + _ = json.Unmarshal(raw, &page) + for len(repos) > 0 && repos[0].Name <= page.Next { + repos = repos[1:] + } + next := "" + if len(repos) > 1 { + repos, next = repos[:1], repos[0].Name + } + return json.Marshal(map[string]any{"repositories": repos, "next": next, "capped": next != ""}) } } diff --git a/modules/distribution/cmd/distribution-tools/main.go b/modules/distribution/cmd/distribution-tools/main.go index 2e5fb3b45..330c3c944 100644 --- a/modules/distribution/cmd/distribution-tools/main.go +++ b/modules/distribution/cmd/distribution-tools/main.go @@ -22,10 +22,10 @@ import ( "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient" meshconditions "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" reg "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/containerimageregistry" + oci "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/ociartifactregistry" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/parts" stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-sdk/go/busstate" - "git.novox.be/novox/mesh-sdk/go/seatlist" "git.novox.be/novox/mesh-sdk/go/views" "distribution/internal/holder" @@ -314,15 +314,7 @@ func tools(v *holder.Verbs, windows kept[holder.Window], taken *asks) []stdio.To } return reg.RemoveAnswer{Removed: got.Removed, Kept: got.Kept, Reason: got.Reason, Tags: got.Tags, Artifact: ®.Artifact{Name: in.Name, Version: in.Version, Tag: in.Tag, Digest: in.Digest}}, nil }}, - stdio.Tool{Name: seat + "." + holder.VerbList, Description: "What the store holds of one repository, or of every repository the seat holds: each manifest's digest, whether it is an index and the manifests it names, the tags naming it, its size and whether it is held (ADR 0632 §3). Read from the store's own files; changes nothing.", - Input: obj(map[string]any{"name": s("/; every repository when not given")}), - Run: func(args map[string]any) (any, error) { - repos, err := v.List(context.Background(), seat, stdio.CallerOf(args), str(args, "name")) - if err != nil { - return nil, err - } - return reg.ListAnswer{Repositories: repos}, nil - }}, + listTool(v, seat), stdio.Tool{Name: seat + "." + holder.VerbHold, Description: "Keep one untagged bundle, archive or copied base from retention (the artifact sweep alone); says held.", Input: obj(map[string]any{"name": s("name"), "digest": s("sha256:…"), "reason": s("which of the sweep's rules keeps it")}, "name", "digest", "reason"), Run: func(args map[string]any) (any, error) { @@ -495,7 +487,7 @@ func main() { Transfers: boundedTransfers{parts.BusTransfers[holder.Opening](holder.TransfersState)}, Cap: boundedCap(parts.BusCap(holder.IncomingObjects)), Record: recordAct, Idle: boundedArm(parts.BusArm(holder.IdleState))} files := &holder.Files{Container: or(os.Getenv("MESH_REGISTRY_CONTAINER"), "mesh-registry"), Run: holder.ExecRunner, UID: os.Getuid()} v := &holder.Verbs{Holder: h, Catalog: store, Holds: kept[holder.Hold]{state: "holds", key: holder.Hold.Key}, Mirrors: mirrors{}, - Files: files, SweepHolder: func() (string, error) { return seatlist.HolderOf(stdio.Ask, holder.SweepSeat) }} + Files: files} go followMirrors(v) go parts.FollowIdle(holder.IdleState, parts.IDs(h.Transfers), h.Expired, func(id string, dropped bool, err error) { if err != nil { @@ -523,3 +515,41 @@ func or(v, d string) string { } return v } + +// listTool is the seat's verb list. On oci-artifact-registry it filters by name or module while it reads and pages at 768 KB +// by the cursor next (design 47 H2 §779, §811, §813 (7)); on container-image-registry, whose form takes no cursor, it answers +// whole. +func listTool(v *holder.Verbs, seat string) stdio.Tool { + s := func(d string) map[string]any { return map[string]any{"type": "string", "description": d} } + description := "What the store holds of one repository, or of every repository the seat holds: each manifest's digest, whether it is an index and the manifests it names, the tags naming it, its size and whether it is held (ADR 0632 §3). Read from the store's own files; changes nothing." + props := map[string]any{"name": s("/; every repository when not given")} + if seat == holder.ArtifactSeat { + description += " Filtered by name or module while it reads and paged by the cursor next, at most 768 KB a page, capped saying the rest is read from next." + props["module"] = s("only the repositories of this module, /*") + props["next"] = s("the cursor the previous page answered; absent for the first page") + } + return stdio.Tool{Name: seat + "." + holder.VerbList, Description: description, Input: map[string]any{"type": "object", "properties": props}, + Run: func(args map[string]any) (any, error) { + if seat != holder.ArtifactSeat { + page, err := v.List(context.Background(), seat, stdio.CallerOf(args), holder.ListQuery{Name: str(args, "name")}) + if err != nil { + return nil, err + } + return reg.ListAnswer{Repositories: page.Repositories}, nil + } + in, err := arg[oci.ListInput](args) + if err != nil { + return nil, err + } + page, err := v.List(context.Background(), seat, stdio.CallerOf(args), holder.ListQuery{Name: in.Name, Module: in.Module, Next: in.Next, PageBytes: holder.ListPageBytes}) + if err != nil { + return nil, err + } + out := oci.ListAnswer{Form: oci.Form, Repositories: []oci.Repository{}, Capped: page.Capped, Next: page.Next, Total: int64(page.Total)} + raw, _ := json.Marshal(page.Repositories) + if err := json.Unmarshal(raw, &out.Repositories); err != nil { + return nil, err + } + return out, nil + }} +} diff --git a/modules/distribution/go.mod b/modules/distribution/go.mod index a16a3916a..4379a5784 100644 --- a/modules/distribution/go.mod +++ b/modules/distribution/go.mod @@ -10,6 +10,7 @@ require ( git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/containerimageregistry v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/ociartifactregistry v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-sdk/go v0.0.0-00010101000000-000000000000 github.com/nats-io/nats.go v1.54.0 @@ -31,7 +32,6 @@ require ( require ( git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshhandacts v0.0.0-00010101000000-000000000000 // indirect - git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/ociartifactregistry v0.0.0-00010101000000-000000000000 // indirect ) require git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery v0.0.0-00010101000000-000000000000 // indirect diff --git a/modules/distribution/internal/holder/holder_test.go b/modules/distribution/internal/holder/holder_test.go index 127318716..069f08745 100644 --- a/modules/distribution/internal/holder/holder_test.go +++ b/modules/distribution/internal/holder/holder_test.go @@ -216,8 +216,7 @@ func newBed(builderOK bool) *bed { return nil }, Finding: func(kind, subject, _ string, raised bool) { b.findings[kind+" "+subject] = raised }} - b.v = &Verbs{Holder: b.h, Catalog: b.cat, Holds: memHolds{}, Mirrors: memMirrors{}, - SweepHolder: func() (string, error) { return "artifact-store-tools", nil }} + b.v = &Verbs{Holder: b.h, Catalog: b.cat, Holds: memHolds{}, Mirrors: memMirrors{}} return b } @@ -366,10 +365,6 @@ func TestTheHolderOfTheTwoImageSeats(t *testing.T) { _, err := b.v.Remove(ArtifactSeat, "novox.mesh-delivery-builder", RemoveRequest{Name: "m/program", Version: "0.1.4", Reason: "branch ended"}) return err }, "has no tag"}, - {"a retention removal from another caller than the sweep", func(b *bed) error { - _, err := b.v.Remove(ArtifactSeat, "novox.mesh-delivery-builder", RemoveRequest{Name: "m/program", Digest: part.Digest, Reason: ReasonRetention}) - return err - }, "artifact sweep's alone"}, {"a retention removal of a held archive", func(b *bed) error { _ = b.v.HoldOne(ArtifactSeat, "novox.artifact-store-tools", HoldRequest{Name: "m/a", Kind: KindArchive, Digest: part.Digest}) _, err := b.v.Remove(ArtifactSeat, "novox.artifact-store-tools", RemoveRequest{Name: "m/a", Digest: part.Digest, Reason: ReasonRetention}) @@ -378,16 +373,6 @@ func TestTheHolderOfTheTwoImageSeats(t *testing.T) { {"a hold of an image", func(b *bed) error { return b.v.HoldOne(ImageSeat, "novox.artifact-store-tools", HoldRequest{Name: "m/i", Kind: KindImage, Digest: part.Digest}) }, "never held"}, - {"a retention removal by the module that held the sweep seat before", func(b *bed) error { - b.v.SweepHolder = func() (string, error) { return "artifact-sweeper", nil } // the seat's holder as the seat says it now - _, err := b.v.Remove(ArtifactSeat, "novox.artifact-store-tools", RemoveRequest{Name: "m/program", Digest: part.Digest, Reason: ReasonRetention}) - return err - }, "artifact sweep's alone"}, - {"a retention removal while the sweep's holder cannot be read", func(b *bed) error { - b.v.SweepHolder = func() (string, error) { return "", errors.New("the seat's holders are not answered") } - _, err := b.v.Remove(ArtifactSeat, "novox.artifact-store-tools", RemoveRequest{Name: "m/program", Digest: part.Digest, Reason: ReasonRetention}) - return err - }, "cannot be read"}, {"a release in distribution's own name", func(b *bed) error { _, err := b.v.Release(ArtifactSeat, "novox.distribution", HoldRequest{Name: "m/a", Digest: part.Digest}) return err @@ -490,7 +475,8 @@ func TestTheHolderOfTheTwoImageSeats(t *testing.T) { t.Run("the sweep's removal, hold and release", func(t *testing.T) { b := newBed(true) sweep := "novox.artifact-store-tools" - if _, err := b.v.Remove(ArtifactSeat, sweep, RemoveRequest{Name: "m/a", Digest: part.Digest, Kind: KindArchive, Reason: ReasonRetention}); err != nil { + // A removal for retention is taken whoever asks it; the seat's holders are not read (design 47 H2 §813 (7)). + if _, err := b.v.Remove(ArtifactSeat, "novox.artifact-sweeper", RemoveRequest{Name: "m/a", Digest: part.Digest, Kind: KindArchive, Reason: ReasonRetention}); err != nil { t.Fatal(err) } if err := b.v.HoldOne(ArtifactSeat, sweep, HoldRequest{Name: "m/b", Kind: KindArchive, Digest: part.Digest}); err != nil { @@ -507,7 +493,7 @@ func TestTheHolderOfTheTwoImageSeats(t *testing.T) { for _, s := range b.said { events = append(events, s.event+" "+s.body.By) } - if want := []string{"removed " + sweep, "held " + sweep, "released person.jochen"}; !reflect.DeepEqual(events, want) { + if want := []string{"removed novox.artifact-sweeper", "held " + sweep, "released person.jochen"}; !reflect.DeepEqual(events, want) { t.Errorf("said %v, want %v", events, want) } b.cat.tags["m/p:0.1.5-feat-x.1"] = part.Digest diff --git a/modules/distribution/internal/holder/list.go b/modules/distribution/internal/holder/list.go index 022387c4b..4ae7e7ca7 100644 --- a/modules/distribution/internal/holder/list.go +++ b/modules/distribution/internal/holder/list.go @@ -9,9 +9,12 @@ package holder import ( "context" + "encoding/base64" + "encoding/json" "errors" "fmt" "sort" + "strings" reg "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/containerimageregistry" ) @@ -25,27 +28,52 @@ type StoreFiles interface { Manifests(ctx context.Context, l *Listing) (map[string]*Manifest, map[string]string) } -// List is `list` on seat: the repositories it holds, or name alone; a repository the seat holds nothing of is not named, -// and name not held is an empty answer. A manifest whose content cannot be read fails the answer, by name: a listing -// missing one would read as the store not holding it. -func (v *Verbs) List(ctx context.Context, seat, caller, name string) ([]reg.Repository, error) { +// ListPageBytes is the most a page of oci-artifact-registry's list answers (design 47 H2 §811): 768 KB. +const ListPageBytes = 768 << 10 + +// ListQuery is what a list is asked: Name or Module (`/*`) filter the repositories while the files are read, Next +// is the cursor a page answered, and PageBytes the most a page answers, 0 for the whole answer at once (the seat +// container-image-registry's list, whose form takes no cursor). +type ListQuery struct { + Name, Module, Next string + PageBytes int +} + +// ListPage is one page of a list: its repositories in order of their names, Total the repositories the filters hold in +// all, and Next the cursor of the next page, empty at the end; Capped says the page stopped at PageBytes. +type ListPage struct { + Repositories []reg.Repository + Capped bool + Total int + Next string +} + +// List is `list` on seat: the repositories it holds, or those q's filters name; a repository the seat holds nothing of is +// not named, and a name not held is an empty answer. A paged list continues after the repository its cursor names, in +// order of the names (design 47 H2 §779, §811, §813 (7)). A manifest whose content cannot be read fails the answer, by +// name: a listing missing one would read as the store not holding it. +func (v *Verbs) List(ctx context.Context, seat, caller string, q ListQuery) (ListPage, error) { if seat != ImageSeat && seat != ArtifactSeat { - return nil, fmt.Errorf("list: %s is no seat of distribution's", seat) + return ListPage{}, fmt.Errorf("list: %s is no seat of distribution's", seat) } if v.Files == nil { - return nil, errors.New("list: this process reads no store files") + return ListPage{}, errors.New("list: this process reads no store files") + } + after, err := listCursor(q.Next) + if err != nil { + return ListPage{}, err } l, err := v.Files.List(ctx) if err != nil { - return nil, fmt.Errorf("list: the store's files cannot be listed: %w", err) + return ListPage{}, fmt.Errorf("list: the store's files cannot be listed: %w", err) } - if name != "" { - r, held := l.Repos[name] - l = &Listing{Blobs: l.Blobs, Repos: map[string]*Repo{}} - if held { - l.Repos[name] = r + kept := &Listing{Blobs: l.Blobs, Repos: map[string]*Repo{}} + for name, r := range l.Repos { + if (q.Name == "" || name == q.Name) && (q.Module == "" || strings.HasPrefix(name, q.Module+"/")) { + kept.Repos[name] = r } } + l = kept docs, unread := v.Files.Manifests(ctx, l) if len(unread) > 0 { keys := make([]string, 0, len(unread)) @@ -53,11 +81,11 @@ func (v *Verbs) List(ctx context.Context, seat, caller, name string) ([]reg.Repo keys = append(keys, k) } sort.Strings(keys) - return nil, fmt.Errorf("list: %d manifests cannot be read, the first %s: %s", len(keys), keys[0], unread[keys[0]]) + return ListPage{}, fmt.Errorf("list: %d manifests cannot be read, the first %s: %s", len(keys), keys[0], unread[keys[0]]) } holds, err := v.Holds.List() if err != nil { - return nil, fmt.Errorf("list: the holds cannot be read: %w", err) + return ListPage{}, fmt.Errorf("list: the holds cannot be read: %w", err) } held := map[string]bool{} for _, h := range holds { @@ -102,5 +130,37 @@ func (v *Verbs) List(ctx context.Context, seat, caller, name string) ([]reg.Repo out = append(out, reg.Repository{Name: repo, Manifests: manifests}) } } - return out, nil + page := ListPage{Repositories: []reg.Repository{}, Total: len(out)} + size := 0 + for _, r := range out { + if r.Name <= after { + continue + } + if q.PageBytes > 0 { + raw, _ := json.Marshal(r) + if size+len(raw) > q.PageBytes { + if len(page.Repositories) == 0 { + return ListPage{}, fmt.Errorf("list: the repository %s alone is over a page's %d bytes (%d)", r.Name, q.PageBytes, len(raw)) + } + page.Capped = true + page.Next = base64.RawURLEncoding.EncodeToString([]byte(page.Repositories[len(page.Repositories)-1].Name)) + break + } + size += len(raw) + } + page.Repositories = append(page.Repositories, r) + } + return page, nil +} + +// listCursor is the repository a page's cursor continues after, "" for the first page; a cursor no page answered is refused. +func listCursor(next string) (string, error) { + if next == "" { + return "", nil + } + name, err := base64.RawURLEncoding.DecodeString(next) + if err != nil || len(name) == 0 { + return "", fmt.Errorf("list: %q is no cursor a page of list answered", next) + } + return string(name), nil } diff --git a/modules/distribution/internal/holder/list_test.go b/modules/distribution/internal/holder/list_test.go index b49cd7dc7..3f1562738 100644 --- a/modules/distribution/internal/holder/list_test.go +++ b/modules/distribution/internal/holder/list_test.go @@ -2,6 +2,7 @@ package holder import ( "context" + "encoding/json" "fmt" "reflect" "strings" @@ -132,7 +133,8 @@ func TestListAnswersTheStoreThroughTheSeat(t *testing.T) { t.Fatal(err) } ctx := context.Background() - images, err := b.v.List(ctx, ImageSeat, "mesh-anyone", "") + page, err := b.v.List(ctx, ImageSeat, "mesh-anyone", ListQuery{}) + images := page.Repositories if err != nil { t.Fatal(err) } @@ -155,7 +157,8 @@ func TestListAnswersTheStoreThroughTheSeat(t *testing.T) { if !reflect.DeepEqual(images, want) { t.Errorf("the image seat lists %+v\nwant %+v", images, want) } - artifacts, err := b.v.List(ctx, ArtifactSeat, "mesh-anyone", "") + page, err = b.v.List(ctx, ArtifactSeat, "mesh-anyone", ListQuery{}) + artifacts := page.Repositories if err != nil { t.Fatal(err) } @@ -166,16 +169,42 @@ func TestListAnswersTheStoreThroughTheSeat(t *testing.T) { if !reflect.DeepEqual(artifacts, wantA) { t.Errorf("the artifact seat lists %+v\nwant %+v", artifacts, wantA) } - if one, err := b.v.List(ctx, ImageSeat, "mesh-anyone", "app/tools"); err != nil || len(one) != 0 { + if one, err := b.v.List(ctx, ImageSeat, "mesh-anyone", ListQuery{Name: "app/tools"}); err != nil || len(one.Repositories) != 0 { t.Errorf("the image seat's list of a bundle's repository: %+v, %v", one, err) } - if one, err := b.v.List(ctx, ArtifactSeat, "mesh-anyone", "mesh/facts"); err != nil || len(one) != 1 || one[0].Name != "mesh/facts" { + if one, err := b.v.List(ctx, ArtifactSeat, "mesh-anyone", ListQuery{Name: "mesh/facts"}); err != nil || len(one.Repositories) != 1 || one.Repositories[0].Name != "mesh/facts" { t.Errorf("one repository: %+v, %v", one, err) } + // Design 47 H2 §813 (7), §811, §779: the artifact seat's list filters by module while it reads, and pages by its cursor in + // order of the names, each page within its bytes, saying capped and the total the filters hold. + if mod, err := b.v.List(ctx, ArtifactSeat, "mesh-anyone", ListQuery{Module: "app", PageBytes: ListPageBytes}); err != nil || + !reflect.DeepEqual(mod, ListPage{Repositories: wantA[:1], Total: 1}) { + t.Errorf("the module app's repositories: %+v, %v", mod, err) + } + first, _ := json.Marshal(wantA[0]) + second, _ := json.Marshal(wantA[1]) + var paged []reg.Repository + q := ListQuery{PageBytes: max(len(first), len(second))} // one repository a page + for pages := 0; ; pages++ { + p, err := b.v.List(ctx, ArtifactSeat, "mesh-anyone", q) + if err != nil || p.Total != 2 || p.Capped != (p.Next != "") || pages > 2 { + t.Fatalf("page %d: %+v, %v", pages, p, err) + } + paged = append(paged, p.Repositories...) + if q.Next = p.Next; q.Next == "" { + break + } + } + if !reflect.DeepEqual(paged, wantA) { + t.Errorf("the pages hold %+v, want %+v", paged, wantA) + } + if _, err := b.v.List(ctx, ArtifactSeat, "mesh-anyone", ListQuery{Next: "!"}); err == nil { + t.Error("a cursor no page answered was taken") + } // A manifest whose content cannot be read fails the answer: a listing without it would read as the store not holding it. delete(f.manifests, dg("m-old")) b.v.Files = &Files{Container: "mesh-registry", Run: f.run(t)} // what was read is kept: a fresh reader reads again - if _, err := b.v.List(ctx, ImageSeat, "mesh-anyone", ""); err == nil || !strings.Contains(err.Error(), "cannot be read") { + if _, err := b.v.List(ctx, ImageSeat, "mesh-anyone", ListQuery{}); err == nil || !strings.Contains(err.Error(), "cannot be read") { t.Errorf("an unreadable manifest: %v", err) } // The files' listing: a repository with slashes read, a cut or stray one refused. diff --git a/modules/distribution/internal/holder/seats.go b/modules/distribution/internal/holder/seats.go index 3acec39c8..7df52723c 100644 --- a/modules/distribution/internal/holder/seats.go +++ b/modules/distribution/internal/holder/seats.go @@ -19,9 +19,6 @@ const ( OldSeat = "mesh-artifact-store" ) -// SweepSeat is the seat of the artifact sweep, whose holder alone removes an untagged version for retention. -const SweepSeat = "mesh-artifact-sweep" - // The verbs of the seats (design 26 *A seat's definition*; design 51 *One contract*). const ( VerbPublish = "publish" diff --git a/modules/distribution/internal/holder/verbs.go b/modules/distribution/internal/holder/verbs.go index 5451e1382..13493bbbb 100644 --- a/modules/distribution/internal/holder/verbs.go +++ b/modules/distribution/internal/holder/verbs.go @@ -88,9 +88,6 @@ type Verbs struct { Mirrors Mirrors // Files are the store's own files, which list reads (ADR 0632 §3). Files StoreFiles - // SweepHolder is the module holding the seat `mesh-artifact-sweep`, as the seat's holders say it now (design 47 G §7, §428); - // a removal for retention is that module's alone. - SweepHolder func() (string, error) } // VersionsAnswer is `versions`: the versions the seat holds of name. @@ -150,7 +147,7 @@ type Removal struct { const ReasonSharedDigest = reg.RemoveAnswerReasonSharedDigest // Remove is `remove`: the builder removes a tagged version; the artifact sweep's holder an untagged bundle, archive or -// copied base with the reason retention, which a hold keeps. Every removal says `removed` with its reason. +// copied base with the reason retention, which a hold keeps, its caller not judged (design 47 H2 §813 (7)). Every removal says `removed` with its reason. func (v *Verbs) Remove(seat, caller string, r RemoveRequest) (Removal, error) { if strings.TrimSpace(r.Name) == "" || strings.TrimSpace(r.Reason) == "" { return Removal{}, errors.New("remove: a name and a reason are required") @@ -158,13 +155,8 @@ func (v *Verbs) Remove(seat, caller string, r RemoveRequest) (Removal, error) { ev := Event{Name: r.Name, Version: r.Version, Kind: r.Kind, Digest: r.Digest, Holder: Self, Reason: r.Reason, By: caller} switch { case r.Reason == ReasonRetention: - sweep, err := v.SweepHolder() - if err != nil { - return Removal{}, fmt.Errorf("remove: the holder of %s cannot be read, so nothing is removed for retention: %w", SweepSeat, err) - } - if CallerModule(caller) != sweep { - return Removal{}, fmt.Errorf("remove: refused for %s: a removal for retention is the artifact sweep's alone", caller) - } + // The removal for retention the holder of mesh-artifact-sweep asks; its caller is not judged, as every account may call + // every verb (design 47 H2 §813 (7)). if r.Digest == "" { return Removal{}, errors.New("remove: a removal for retention names the digest") } -- 2.54.0 From b93253e6987e50af6d858522021d88b5b3748831 Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 22 Nov 2026 00:51:01 +0100 Subject: [PATCH 22/54] C2: the forge module takes an act of the seat git once by its ask id MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Each act (status, comment, view, note, merge-pull, tag) carrying an ask id is taken in the state taken, key git., for the setting ask-taken-bound (10m, read from MESH_CONFIG_ASK_TAKEN_BOUND): a repeat while the act runs joins it; after it ended, the repeat answers what the forge holds of the act, read afresh (a merge call's pull requests merged with their merge commits), and the act is never done twice. An act that performed nothing drops its key, so the kept call's repeat acts. Builds: design 47 H2 §775, §820; H-A-381, H-A-400, H-A-359a --- modules/gitea/delivery.ts | 9 +++ modules/gitea/index.ts | 9 ++- modules/gitea/kept.ts | 1 + modules/gitea/module.json | 12 ++++ modules/gitea/seat.ts | 110 ++++++++++++++++++++++++++----- modules/gitea/taken.ts | 107 ++++++++++++++++++++++++++++++ modules/gitea/test/kept.test.ts | 4 +- modules/gitea/test/taken.test.ts | 63 ++++++++++++++++++ 8 files changed, 295 insertions(+), 20 deletions(-) create mode 100644 modules/gitea/taken.ts create mode 100644 modules/gitea/test/taken.test.ts diff --git a/modules/gitea/delivery.ts b/modules/gitea/delivery.ts index ed65da6f4..6548954e0 100644 --- a/modules/gitea/delivery.ts +++ b/modules/gitea/delivery.ts @@ -68,6 +68,15 @@ export async function appendNote(runner: Runner, container: string, owner: strin return { added: true, lines: lines.length + 1 }; } +/** Whether a commit's note holds a line carrying the act's id, read without writing (design 47 H2 §775: a repeat of a + * taken ask answers what its act did). */ +export async function noteHolds(runner: Runner, container: string, owner: string, repo: string, commit: string, + notesRef: string, id: string): Promise { + const shown = await runner("docker", noteArgs(container, owner, repo, commit, notesRef)); + if (shown.code !== 0 && shown.code !== 1) throw new Error(`the note on ${commit.slice(0, 8)} could not be read (git exited ${shown.code})`); + return shown.code === 0 && shown.stdout.split("\n").some((l) => l.includes(`[${id}]`)); +} + /** The marker that makes one comment of a pull request the delivery's view. */ export const VIEW_MARKER = ""; diff --git a/modules/gitea/index.ts b/modules/gitea/index.ts index 6418924f7..858d3bcdb 100644 --- a/modules/gitea/index.ts +++ b/modules/gitea/index.ts @@ -32,7 +32,7 @@ import { emit } from "@novox/mesh-sdk/events"; import { registerModuleTools } from "@novox/mesh-sdk/tools"; import { statesAnnouncement } from "./kept.js"; -import { state } from "@novox/mesh-sdk/state"; +import { state, TAKEN_STATE } from "@novox/mesh-sdk/state"; import { on } from "@novox/mesh-sdk/events"; import { broker } from "@novox/mesh-sdk/messaging"; import { STATUS_STATE, keepStatus, statusTools, viewWaitsFinding, type Follower, type Status } from "@novox/mesh-sdk/views"; @@ -45,6 +45,7 @@ import { REGISTRATION_CHANGED_EVENT, Registrations } from "./registrations.js"; import { FailedSteps, Heads, readHeads } from "./failedsteps.js"; import { failedStepsTools } from "./tools/failedsteps.js"; import { Calls, callRetentionDays, sayUnsaid } from "./calls.js"; +import { Once, askTakenBoundMs, takenState } from "./taken.js"; import { Partials } from "./partials.js"; import { callTools } from "./tools/calls.js"; import { seatPublish } from "@novox/mesh-sdk/runtime"; @@ -282,6 +283,10 @@ const heldHeads = new Heads(); let reloadHeads: () => Promise = async () => {}; // The merge calls, each written before its first merge (calls.ts; design 47 H §766). const calls = new Calls(state("calls"), () => callRetentionDays(), (line) => console.log(`[gitea] ${line}`)); +// Each act of the seat git is taken once by its ask id, the ids kept in the state taken for ask-taken-bound (taken.ts; +// design 47 H2 §775). +const once = new Once(takenState(state(TAKEN_STATE), askTakenBoundMs(process.env, (line) => console.log(`[gitea] ${line}`))), + (line) => console.log(`[gitea] ${line}`)); // At the start, every call whose merge-called is not said says it for what landed (§766). const partials = new Partials(state("findings"), calls, (line) => console.log(`[gitea] ${line}`)); void sayUnsaid(calls, seatPublish, (line) => console.log(`[gitea] ${line}`)).catch((err) => @@ -297,7 +302,7 @@ void failedSteps.expireSeen().catch((err) => console.log(`[gitea] the seen keys registerModuleTools("gitea", () => [eventsTool(pace), ...readAllTools(readAll), ...writeTools(writes), ...statusTools("gitea", "gitea_first_views", "gitea_first_view_again", followers, firstViews, async (act) => { await broker().request("seat:mesh-hand-acts.hand-act", act); }), ...(gitea ? seatTools(gitea, { run, container: FORGE_CONTAINER, registered: () => registrations.written, judge: (head) => failedSteps.judge(head), - watched: () => registrations.watched, calls, say: seatPublish, partials }) : []), + watched: () => registrations.watched, calls, say: seatPublish, partials, once }) : []), ...callTools(calls, partials, seatPublish, async (act) => { await broker().request("seat:mesh-hand-acts.hand-act", act); }), ...failedStepsTools(failedSteps, async (act) => { await broker().request("seat:mesh-hand-acts.hand-act", act); })]); diff --git a/modules/gitea/kept.ts b/modules/gitea/kept.ts index 790780182..368cdc388 100644 --- a/modules/gitea/kept.ts +++ b/modules/gitea/kept.ts @@ -12,6 +12,7 @@ export const KEPT: readonly { readonly name: string; readonly readable?: boolean { name: "failed-steps" }, { name: "first-views" }, { name: "idle" }, + { name: "taken" }, { name: "transfers" }, ]; diff --git a/modules/gitea/module.json b/modules/gitea/module.json index 40de8e851..a6ace6e09 100644 --- a/modules/gitea/module.json +++ b/modules/gitea/module.json @@ -237,6 +237,13 @@ "default": [ "jschoubben" ] + }, + "ask-taken-bound": { + "type": "string", + "description": "How long the forge module keeps the id of an act of the seat git it took, as a duration such as 10m (design 47 H2 §775): an act given again with that id within it acts not again but answers what the forge holds of it. The largest caller's bound, the pipeline runner's ask-unanswered-bound, by default.", + "trusted": false, + "runtime": true, + "default": "10m" } }, "facts": { @@ -635,6 +642,11 @@ { "name": "calls", "history": 1 + }, + { + "name": "taken", + "key-ttl": true, + "max-mib": 8 } ], "reads": [ diff --git a/modules/gitea/seat.ts b/modules/gitea/seat.ts index a8854e36a..ffefb69fe 100644 --- a/modules/gitea/seat.ts +++ b/modules/gitea/seat.ts @@ -27,11 +27,13 @@ import { containedFrom, newestByContext } from "./checks.js"; import { Refused, checkName } from "./guard.js"; import { AdminRefused } from "./token.js"; import { awayAnswer } from "@novox/mesh-sdk/callers"; -import { appendNote, viewBody, viewComment, type Runner } from "./delivery.js"; +import { appendNote, noteHolds, viewBody, viewComment, type Runner } from "./delivery.js"; import type { Judged } from "./failedsteps.js"; -import { mergeCall } from "./mergecall.js"; +import { mergeCall, membersOf } from "./mergecall.js"; import type { Calls, Say } from "./calls.js"; import type { Partials } from "./partials.js"; +import type { Once } from "./taken.js"; +import { askIdOf } from "@novox/mesh-sdk/callers"; /** The verbs of the seat, in its definition's order. */ export const VERBS: readonly string[] = Object.values(GIT_VERBS); @@ -60,6 +62,8 @@ export interface SeatDeps { say?: Say; /** The findings merge-partial (partials.ts; design 47 H §767). */ partials?: Partials; + /** Takes each act once by its ask id (taken.ts; design 47 H2 §775): absent, an act acts at every call. */ + once?: Once; } /** A failure as the contract says it: the forge's or this holder's refusal, or a forge that failed. */ @@ -171,18 +175,24 @@ async function mergingOf(gitea: GiteaClient, owner: string, repo: string, p: Git /** The seat's verbs as tools `git.`, each answering in the seat's contract. */ export function seatTools(gitea: GiteaClient, deps: SeatDeps): ToolDefinition[] { - const verb = (name: string, description: string, act: (args: Record) => Promise): ToolDefinition => ({ + const answered = async (args: Record, act: (args: Record) => Promise): Promise => { + try { + return await act(args); + } catch (err) { + return failureOf(gitea, err); + } + }; + // An act given `standing`, what it did read afresh, is taken once by its ask id (§775, H-A-381, H-A-400). + const verb = (name: string, description: string, act: (args: Record) => Promise, + standing?: (args: Record) => Promise): ToolDefinition => ({ name: `${GIT}.${name}`, description, input: {}, - run: async (args: Record) => { - try { - return await act(args); - } catch (err) { - return failureOf(gitea, err); - } - }, + run: async (args: Record) => standing && deps.once + ? deps.once.run(askIdOf(args), () => answered(args, act), () => answered(args, standing)) + : answered(args, act), }); + const s = standingActs(gitea, deps); return [ verb(GIT_VERBS.pull, "The seat git's pull: one pull request by its repository and number or by its merge commit, and for an open one what the forge would merge now.", @@ -273,7 +283,7 @@ export function seatTools(gitea: GiteaClient, deps: SeatDeps): ToolDefinition[] await gitea.setCommitStatus(owner, repo, commit, { state: state as CommitStatus["state"], context: String(args.context ?? ""), description: String(args.description ?? ""), ...(args.target ? { target_url: String(args.target) } : {}) }); return { id, performed: true }; - }), + }, s.status), verb(GIT_VERBS.comment, "The seat git's comment: an act of the forge bridge, a comment on a pull request carrying the act's id; an id it already holds is held.", async (args): Promise => { const id = idOf(args.id); @@ -282,7 +292,7 @@ export function seatTools(gitea: GiteaClient, deps: SeatDeps): ToolDefinition[] if (holds(await gitea.listComments(owner, repo, number), id)) return { id, held: true, performed: true }; await gitea.addComment(owner, repo, number, `${String(args.text ?? "")}\n\n${actMarker(id)}`); return { id, performed: true }; - }), + }, s.comment), verb(GIT_VERBS.view, "The seat git's view: an act of the forge bridge, the delivery's one view comment kept on a pull request; an id it already holds is held.", async (args): Promise => { const id = idOf(args.id); @@ -294,7 +304,7 @@ export function seatTools(gitea: GiteaClient, deps: SeatDeps): ToolDefinition[] if (existing) await gitea.editComment(owner, repo, existing.id, body); else await gitea.addComment(owner, repo, number, body); return { id, performed: true }; - }), + }, s.view), verb(GIT_VERBS.note, "The seat git's note: an act of the forge bridge, one line on a commit's note under refs/notes/mesh-plan carrying the act's id; an id it already holds is held.", async (args): Promise => { const id = idOf(args.id); @@ -302,9 +312,9 @@ export function seatTools(gitea: GiteaClient, deps: SeatDeps): ToolDefinition[] const commit = commitOf("commit", args.commit); const said = await appendNote(deps.run, deps.container, owner, repo, commit, "mesh-plan", `${String(args.line ?? "")} [${id}]`, id); return said.added ? { id, performed: true } : { id, held: true, performed: true }; - }), + }, s.note), verb(GIT_VERBS.mergePull, "The seat git's merge-pull: one merge call, a delivery's start and its group: the pull requests named (pulls, each {repository, number, head}) and by (who@node); every member is judged before any is merged (still its head, its base a watched branch, no required step failed at its fingerprint), none merged when one is refused, then each merged in the order named; it answers the call and each member's outcome.", - async (args): Promise => mergeCall(gitea, args, deps)), + async (args): Promise => mergeCall(gitea, args, deps), s.mergePull), verb(GIT_VERBS.tag, "The seat git's tag: the git tag /vMAJOR.MINOR.PATCH of a released version, written at the commit its build was made from; a tag that is at that commit is performed.", async (args): Promise => { const { owner, repo } = repositoryOf(args.repository); @@ -316,10 +326,78 @@ export function seatTools(gitea: GiteaClient, deps: SeatDeps): ToolDefinition[] if (at !== null) throw new Refused(`${tag} is at ${at.slice(0, 12)} already, not at ${commit.slice(0, 12)}; a tag is never moved`); await gitea.createTag(owner, repo, tag, commit); return { tag, performed: true }; - }), + }, s.tag), ]; } function holds(comments: GiteaComment[], id: string): boolean { return comments.some((c) => String(c.body ?? "").includes(actMarker(id))); } + +/** + * What each act did, read afresh from the forge, for an ask given again after its act ended (design 47 H2 §775; H-A-381 + * (1)): the act is never performed twice, and what the forge holds is what the act answers when done. A repeat for which + * the forge holds nothing of the act is a refusal naming it. + */ +function standingActs(gitea: GiteaClient, deps: SeatDeps) { + const never = (ask: string, what: string) => new Refused(`the ask ${ask} was taken and its act acted, but ${what}: answered as it stands, never acted again`); + const askOf = (args: Record) => askIdOf(args); + return { + status: async (args: Record): Promise => { + const id = idOf(args.id); + const { owner, repo } = repositoryOf(args.repository); + const commit = commitOf("commit", args.commit); + const context = String(args.context ?? ""), state = String(args.state ?? ""); + const now = newestByContext((await gitea.combinedStatus(owner, repo, commit)).statuses).find((x) => x.context === context); + if (!now || now.state !== state) throw never(askOf(args), `${commit.slice(0, 12)} carries no ${context} ${state}`); + return { id, held: true, performed: true }; + }, + comment: async (args: Record): Promise => { + const id = idOf(args.id); + const { owner, repo, full } = repositoryOf(args.repository); + const number = numberOf(args.number); + if (!holds(await gitea.listComments(owner, repo, number), id)) throw never(askOf(args), `${full}#${number} holds no comment of the act ${id}`); + return { id, held: true, performed: true }; + }, + view: async (args: Record): Promise => { + const id = idOf(args.id); + const { owner, repo, full } = repositoryOf(args.repository); + const number = numberOf(args.number); + const existing = viewComment(await gitea.listComments(owner, repo, number)); + if (!existing?.body.includes(actMarker(id))) throw never(askOf(args), `${full}#${number}'s view is not the act ${id}'s`); + return { id, held: true, performed: true }; + }, + note: async (args: Record): Promise => { + const id = idOf(args.id); + const { owner, repo } = repositoryOf(args.repository); + const commit = commitOf("commit", args.commit); + if (!(await noteHolds(deps.run, deps.container, owner, repo, commit, "mesh-plan", id))) { + throw never(askOf(args), `the note on ${commit.slice(0, 12)} holds no line of the act ${id}`); + } + return { id, held: true, performed: true }; + }, + mergePull: async (args: Record): Promise => { + // Each member as the forge holds it: merged, with its merge commit; the call that merged them, when one is kept. + const members = membersOf(args.pulls); + const out = await Promise.all(members.map(async (m) => { + const p = await gitea.getPullRequest(m.owner, m.repo, m.number); + return p.merged && p.merge_commit_sha + ? { repository: m.repository, number: m.number, state: "merged" as const, commit: p.merge_commit_sha } + : { repository: m.repository, number: m.number, state: "refused" as const, why: `not merged: the forge holds it ${p.state === "open" ? "open" : "closed"}` }; + })); + if (!out.some((m) => m.state === "merged")) throw never(askOf(args), "the forge merged none of its pull requests"); + const named = (c: { members: { repository: string; number: number; state: string }[] }) => + out.every((m) => m.state !== "merged" || c.members.some((k) => k.repository === m.repository && k.number === m.number && k.state === "merged")); + const kept = (await deps.calls?.all())?.find(named); + return { ...(kept ? { call: kept.call } : {}), members: out }; + }, + tag: async (args: Record): Promise => { + const { owner, repo } = repositoryOf(args.repository); + const tag = String(args.tag ?? ""); + const commit = commitOf("commit", args.commit); + const at = await gitea.tagCommit(owner, repo, tag); + if (at !== commit) throw never(askOf(args), at === null ? `the forge holds no tag ${tag}` : `${tag} is at ${at.slice(0, 12)}, not ${commit.slice(0, 12)}`); + return { tag, performed: true }; + }, + }; +} diff --git a/modules/gitea/taken.ts b/modules/gitea/taken.ts new file mode 100644 index 000000000..90d73e676 --- /dev/null +++ b/modules/gitea/taken.ts @@ -0,0 +1,107 @@ +// **The forge module takes an act of the seat git once by its ask id** (novox/hq design 47 H2 §775; H-A-381, H-A-400): a +// kept call carries its ask id (the runtime names it in the call's `_meta`, mesh-sdk askIdOf), and the forge module keeps +// each id it took in its state `taken`, key `git.`, for `ask-taken-bound` (a setting, default 10 minutes, the +// largest caller's bound). An ask given again while its act runs joins that act and gets its answer; given again after +// the act ended, it answers the act's state as it stands, read afresh from the forge (a merge call answers each pull +// request as it is: merged, with its merge commit), and the act is never performed twice. No answer is kept in `taken`. +// The key is kept only for an act that acted: an answer that performed nothing (away, a transport fault, a refusal +// before any act) drops it, so the kept call's repeat under the same id acts (H-A-400 (1)). A repeat for which the +// forge's state says the act that acted never happened is a refusal naming it. + +import { takeOnce, takenKey, TAKEN_STATE, type State } from "@novox/mesh-sdk/state"; + +/** The seat the forge module takes asks of. */ +export const SEAT = "git"; + +/** ask-taken-bound's default (H-A-381 (2)). */ +export const ASK_TAKEN_BOUND_DEFAULT = "10m"; + +/** A duration as the mesh writes one ("10m", "1h30m", "45s", "500ms"), in milliseconds; null when it is none. */ +export function durationMs(text: string): number | null { + const s = text.trim(); + if (!/^(\d+(\.\d+)?(ms|h|m|s))+$/.test(s)) return null; + let ms = 0; + for (const [, n, , unit] of s.matchAll(/(\d+(\.\d+)?)(ms|h|m|s)/g)) { + ms += Number(n) * ({ ms: 1, s: 1_000, m: 60_000, h: 3_600_000 } as Record)[unit]; + } + return ms > 0 ? ms : null; +} + +/** ask-taken-bound in milliseconds, from the word the node-engine composes for it (H-A-370, H-A-400 (2)); the default when + * it is unset. A value that is no duration is said and the default taken. */ +export function askTakenBoundMs(env: NodeJS.ProcessEnv = process.env, log: (line: string) => void = () => {}): number { + const raw = env.MESH_CONFIG_ASK_TAKEN_BOUND; + const fallback = durationMs(ASK_TAKEN_BOUND_DEFAULT)!; + if (raw === undefined || raw.trim() === "") return fallback; + const ms = durationMs(raw); + if (ms === null) { + log(`the setting ask-taken-bound is ${JSON.stringify(raw)}, no duration: the default ${ASK_TAKEN_BOUND_DEFAULT} holds`); + return fallback; + } + return ms; +} + +/** The state taken, as the forge module reaches it: take an ask once, drop a key whose act performed nothing. */ +export interface Taken { + /** Whether this holder takes the ask now (mesh-sdk takeOnce: created, refused when the key exists). */ + take(ask: string): Promise; + /** Drops the key of an ask whose act performed nothing, so its repeat acts. */ + drop(ask: string): Promise; +} + +/** The module's state taken over the runtime, its keys living ask-taken-bound. */ +export function takenState(st: Pick, "delete">, boundMs: number): Taken { + return { + take: (ask) => takeOnce(SEAT, ask, boundMs, TAKEN_STATE), + drop: (ask) => st.delete(takenKey(SEAT, ask)), + }; +} + +/** Whether an act's answer performed nothing: an error answer, or a merge call that merged no member. */ +export function performedNothing(answer: object): boolean { + const a = answer as Record; + if (a["error-kind"] !== undefined) return true; + if (Array.isArray(a.members)) return !a.members.some((m: any) => m?.state === "merged"); + return false; +} + +/** + * Takes an act once by its ask id: `act` runs for an ask this holder takes now; an ask given while its act runs joins it; + * an ask taken before whose act ended answers `standing`, the act's state read afresh. An act with no ask id acts as + * before. The taken key is dropped when the act performed nothing. + */ +export class Once { + private readonly running = new Map>(); + + constructor(private readonly taken: Taken, private readonly log: (line: string) => void = () => {}) {} + + async run(ask: string, act: () => Promise, standing: () => Promise): Promise { + if (!ask) return act(); + const joined = this.running.get(ask); + if (joined) return joined; + if (!(await this.taken.take(ask))) { + // Taken before, and its act ended (or another process runs it): what it did, read afresh, never done again. + return this.running.get(ask) ?? standing(); + } + const acting = (async () => { + let answer: object; + try { + answer = await act(); + } catch (err) { + await this.taken.drop(ask).catch(() => {}); + throw err; + } + if (performedNothing(answer)) { + await this.taken.drop(ask).catch((err) => + this.log(`the ask ${ask} performed nothing and its taken key could not be dropped, so its repeat is answered as it stands — ${err instanceof Error ? err.message : err}`)); + } + return answer; + })(); + this.running.set(ask, acting); + try { + return await acting; + } finally { + this.running.delete(ask); + } + } +} diff --git a/modules/gitea/test/kept.test.ts b/modules/gitea/test/kept.test.ts index d4a1cb604..f7d7639de 100644 --- a/modules/gitea/test/kept.test.ts +++ b/modules/gitea/test/kept.test.ts @@ -16,8 +16,8 @@ test("the states the module announces are the states its manifest declares", asy test("the announcement has the layout mesh-sdk's go/announce reads: version 1, the module, the node, the states sorted by name", async () => { const { statesAnnouncement } = await import("../kept.ts"); - assert.deepEqual(statesAnnouncement("gitea", "ace"), { v: 1, module: "gitea", node: "ace", states: [{ name: "announced" }, { name: "bound", readable: true }, { name: "calls" }, { name: "failed-steps" }, { name: "findings", readable: true }, { name: "first-views" }, { name: "idle" }, { name: "transfers" }] }); - assert.deepEqual(statesAnnouncement("gitea", undefined), { v: 1, module: "gitea", states: [{ name: "announced" }, { name: "bound", readable: true }, { name: "calls" }, { name: "failed-steps" }, { name: "findings", readable: true }, { name: "first-views" }, { name: "idle" }, { name: "transfers" }] }); + assert.deepEqual(statesAnnouncement("gitea", "ace"), { v: 1, module: "gitea", node: "ace", states: [{ name: "announced" }, { name: "bound", readable: true }, { name: "calls" }, { name: "failed-steps" }, { name: "findings", readable: true }, { name: "first-views" }, { name: "idle" }, { name: "taken" }, { name: "transfers" }] }); + assert.deepEqual(statesAnnouncement("gitea", undefined), { v: 1, module: "gitea", states: [{ name: "announced" }, { name: "bound", readable: true }, { name: "calls" }, { name: "failed-steps" }, { name: "findings", readable: true }, { name: "first-views" }, { name: "idle" }, { name: "taken" }, { name: "transfers" }] }); }); test("index.ts says the announcement as the event `states` before the tools are served", () => { diff --git a/modules/gitea/test/taken.test.ts b/modules/gitea/test/taken.test.ts new file mode 100644 index 000000000..c053ae422 --- /dev/null +++ b/modules/gitea/test/taken.test.ts @@ -0,0 +1,63 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; + +// 47 H2 §775, H-A-381, H-A-400: the forge module takes an act of the seat git once by its ask id. Given the same ask id +// twice it acts once: a repeat while the act runs joins it, a repeat after it ended answers what the forge holds of the +// act; an act that performed nothing (away, a refusal before any merge) drops its key, so the repeat acts. +test("an act of the seat git given the same ask id twice acts once", async () => { + const { seatTools } = await import("../dist/seat.js"); + const { Once, durationMs, askTakenBoundMs } = await import("../dist/taken.js"); + const { runAs } = await import("@novox/mesh-sdk/callers"); + const SHA = "a".repeat(40), MERGE = "c".repeat(40); + const keys = new Set(); + const taken = { take: async (ask: string) => (keys.has(ask) ? false : (keys.add(ask), true)), drop: async (ask: string) => { keys.delete(ask); } }; + let merges = 0, comments: { id: number; body: string }[] = [], added = 0, away = false; + const pr = () => ({ number: 9, state: merges ? "closed" : "open", merged: merges > 0, head: "feat/x", head_sha: SHA, base: "main", ...(merges ? { merge_commit_sha: MERGE } : {}) }); + let release: () => void = () => {}, entered: () => void = () => {}; + const merging = new Promise((r) => { entered = r; }); + const forge: Record = { + forgeVersion: async () => (away ? { ok: false, words: "no answer", store: true } : { ok: true, version: "1.27.3" }), + getPullRequest: async () => { if (away) throw new Error("Gitea API /repos/novox/app/pulls/9: 503 Service Unavailable"); return pr(); }, + mergePullRequest: async () => { const held = new Promise((r) => { release = r; }); entered(); await held; merges++; }, + commitParents: async () => [SHA, SHA], + listComments: async () => comments, + addComment: async (_o: string, _r: string, _n: number, body: string) => { added++; comments.push({ id: added, body }); }, + }; + const seat = seatTools(forge as any, { run: (async () => ({ stdout: "", code: 1 })) as any, container: "gitea", registered: () => true, + judge: async () => null, watched: () => new Map([["novox/app", new Set(["main"])]]), once: new Once(taken) }); + const call = (verb: string, ask: string, args: Record) => + runAs({ "mesh/caller": "ace.x", "mesh/ask": ask }, args, () => seat.find((x: any) => x.name === `git.${verb}`)!.run(args)) as Promise; + const merge = { by: "lead@shanks", pulls: [{ repository: "novox/app", number: 9, head: SHA }] }; + + // Away first: nothing performed, the key dropped, so the kept call's repeat under the same id acts (H-A-400 (1)). + away = true; + assert.equal((await call("merge-pull", "ask1", merge))["error-kind"], "away"); + assert.equal(keys.has("ask1"), false, "an act that performed nothing keeps no taken key"); + away = false; + + // The repeat while the first runs joins it: one merge, both answered the same. + const first = call("merge-pull", "ask1", merge); + await merging; + const joined = call("merge-pull", "ask1", merge); + release(); + const [a, b] = await Promise.all([first, joined]); + assert.equal(merges, 1, "a repeat while the act runs joins it"); + assert.deepEqual(b, a); + assert.equal(a.members[0].state, "merged"); + + // After it ended, the repeat answers the pull request as the forge holds it, merged with its merge commit, and merges nothing. + const again = await call("merge-pull", "ask1", merge); + assert.deepEqual([merges, again.members], [1, [{ repository: "novox/app", number: 9, state: "merged", commit: MERGE }]]); + + // A comment given its ask again is not written twice; a repeat whose comment the forge does not hold is refused, naming it. + const comment = { id: "c1", repository: "novox/app", number: 9, text: "hi" }; + assert.equal((await call("comment", "ask2", comment)).performed, true); + assert.deepEqual([(await call("comment", "ask2", comment)).held, added], [true, 1]); + comments = []; + const gone = await call("comment", "ask2", comment); + assert.deepEqual([gone["error-kind"], /ask2 was taken/.test(gone.error), added], ["refused", true, 1]); + + // ask-taken-bound is read from the word the node-engine composes (H-A-400 (2)), 10 minutes by default. + assert.deepEqual([askTakenBoundMs({}), askTakenBoundMs({ MESH_CONFIG_ASK_TAKEN_BOUND: "90s" }), askTakenBoundMs({ MESH_CONFIG_ASK_TAKEN_BOUND: "soon" }), durationMs("1h30m")], + [600_000, 90_000, 600_000, 5_400_000]); +}); -- 2.54.0 From 6b17965d98cc0e7f3c5724445e7d137f18d15de1 Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 22 Nov 2026 00:52:01 +0100 Subject: [PATCH 23/54] C5: forge_merged lists a repository's merged pull requests by number MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit From..to inclusive, each read from the forge one by one, with its merge commit, when it was merged and the merge call that made it when a kept call names it merged; at most 768 KB, capped with the next number as the cursor. Builds: design 47 H2 §812, §811; H-A-402 --- modules/gitea/index.ts | 3 +- modules/gitea/test/calls.test.ts | 24 ++++++++++++++ modules/gitea/tools/calls.ts | 54 ++++++++++++++++++++++++++++++++ 3 files changed, 80 insertions(+), 1 deletion(-) diff --git a/modules/gitea/index.ts b/modules/gitea/index.ts index 858d3bcdb..6ba5c79f6 100644 --- a/modules/gitea/index.ts +++ b/modules/gitea/index.ts @@ -47,7 +47,7 @@ import { failedStepsTools } from "./tools/failedsteps.js"; import { Calls, callRetentionDays, sayUnsaid } from "./calls.js"; import { Once, askTakenBoundMs, takenState } from "./taken.js"; import { Partials } from "./partials.js"; -import { callTools } from "./tools/calls.js"; +import { callTools, mergedTool } from "./tools/calls.js"; import { seatPublish } from "@novox/mesh-sdk/runtime"; import { seat as GIT, events as GIT_EVENTS, type BranchMovedEvent, type PullClosedEvent, type PullMergedEvent, type PullUpdatedEvent } from "@novox/mesh-delivery-contracts-git"; import { seat as RUNNER, events as RUNNER_EVENTS, type StepEndedEvent } from "@novox/mesh-delivery-contracts-mesh-pipeline-runner"; @@ -304,6 +304,7 @@ registerModuleTools("gitea", () => [eventsTool(pace), ...readAllTools(readAll), ...(gitea ? seatTools(gitea, { run, container: FORGE_CONTAINER, registered: () => registrations.written, judge: (head) => failedSteps.judge(head), watched: () => registrations.watched, calls, say: seatPublish, partials, once }) : []), ...callTools(calls, partials, seatPublish, async (act) => { await broker().request("seat:mesh-hand-acts.hand-act", act); }), + ...(gitea ? [mergedTool(gitea, calls)] : []), ...failedStepsTools(failedSteps, async (act) => { await broker().request("seat:mesh-hand-acts.hand-act", act); })]); /** What the forge's event asks of the announcer at once (issue 729, 1422): see watch.ts. */ diff --git a/modules/gitea/test/calls.test.ts b/modules/gitea/test/calls.test.ts index fddb0574a..be2bc43eb 100644 --- a/modules/gitea/test/calls.test.ts +++ b/modules/gitea/test/calls.test.ts @@ -117,3 +117,27 @@ test("forge_calls reads the calls newest first; forge_call_say_again says one ag const none: any = await tool("forge_call_say_again").run({ call: "00000000000000bb", why: "x" }); assert.equal(none.said, false, "a call that landed none says none"); }); + +// 47 H2 §812, §811, H-A-402: forge_merged lists a repository's merged pull requests by number, from..to, each with its merge +// commit, when, and the merge call that made it; a number the forge does not hold or one not merged is left out; capped at +// 768 KB with the next number as the cursor. +test("forge_merged lists merged pull requests by number with the merge call that made each", async () => { + const { mergedTool, ANSWER_CAP_BYTES } = await import("../dist/tools/calls.js"); + const sha = (n: number) => String(n % 10).repeat(40); + const forge = { getPullRequest: async (_o: string, _r: string, n: number) => { + if (n === 3) throw new Error("Gitea API /repos/novox/app/pulls/3: 404 Not Found"); + return n === 4 ? { number: n, state: "open", merged: false } : { number: n, state: "closed", merged: true, merge_commit_sha: sha(n), merged_at: `2026-11-2${n}T10:00:00Z` }; + } }; + const calls = { all: async () => [{ call: "00000000000000aa", members: [{ repository: "novox/app", number: 2, state: "merged" }, { repository: "novox/lib", number: 1, state: "merged" }] }] }; + const got: any = await mergedTool(forge as any, calls as any).run({ repository: "novox/app", from: 1, to: 5 }); + assert.deepEqual(got, { repository: "novox/app", from: 1, to: 5, total: 5, merged: [ + { number: 1, "merge-commit": sha(1), "merged-at": "2026-11-21T10:00:00Z" }, + { number: 2, "merge-commit": sha(2), "merged-at": "2026-11-22T10:00:00Z", call: "00000000000000aa" }, + { number: 5, "merge-commit": sha(5), "merged-at": "2026-11-25T10:00:00Z" }] }); + await assert.rejects(mergedTool(forge as any, calls as any).run({ repository: "novox/app", from: 5, to: 4 }), /to not below from/); + // Past 768 KB it is capped, the cursor the next number to read from. + const many: any = await mergedTool(forge as any, { all: async () => [] } as any).run({ repository: "novox/app", from: 5, to: 20_000 }); + assert.equal(many.capped, true); + assert.ok(JSON.stringify(many.merged).length <= ANSWER_CAP_BYTES); + assert.equal(many.cursor, many.merged.at(-1).number + 1); +}); diff --git a/modules/gitea/tools/calls.ts b/modules/gitea/tools/calls.ts index 5737a80eb..06f0789fb 100644 --- a/modules/gitea/tools/calls.ts +++ b/modules/gitea/tools/calls.ts @@ -7,6 +7,8 @@ import { callerOf } from "@novox/mesh-sdk/callers"; import type { HandAct } from "@novox/mesh-sdk/views"; import type { Partials } from "../partials.js"; import { sayCall, type Calls, type Say } from "../calls.js"; +import type { GiteaClient } from "../client.js"; +import { repositoryOf } from "../seat.js"; const word = (v: unknown): string => (typeof v === "string" ? v.trim() : ""); @@ -70,3 +72,55 @@ export function callTools(calls: Calls, partials: Partials, say: Say, acts: (act }, ]; } + +/** The most a read tool answers (design 47 H2 §811). */ +export const ANSWER_CAP_BYTES = 768 * 1024; + +/** + * forge_merged (design 47 H2 §812, §811; H-A-402): a repository's merged pull requests by number, from `from` to `to` + * inclusive, each read from the forge one by one (the forge lists none by number), with its merge commit, when it was + * merged, and the merge call that made it when a kept call names it merged. A number the forge does not hold, or a pull + * request not merged, is left out. At most 768 KB: past it `capped`, with `cursor` the next number to give as `from`. + */ +export function mergedTool(forge: Pick, calls: Pick): ToolDefinition { + return { + name: "forge_merged", + description: "A repository's merged pull requests by number, from..to inclusive: each with its merge commit, when it was merged, and " + + "the merge call that made it (none when it was merged outside a merge call). At most 768 KB: capped, with the cursor to give as from " + + "(design 47 H2 §812).", + input: { type: "object", required: ["repository", "from", "to"], properties: { + repository: { type: "string", description: "the repository, /" }, + from: { type: "integer", description: "the first pull request number" }, + to: { type: "integer", description: "the last pull request number" } } }, + run: async (args: Record) => { + const { owner, repo, full } = repositoryOf(args.repository); + const from = Number(args.from), to = Number(args.to); + if (!Number.isInteger(from) || from < 1 || !Number.isInteger(to) || to < from) { + throw new Error("forge_merged names from and to, pull request numbers from 1, to not below from"); + } + const madeBy = new Map(); + for (const c of await calls.all()) { + for (const m of c.members) if (m.repository === full && m.state === "merged") madeBy.set(m.number, c.call); + } + const merged: { number: number; "merge-commit": string; "merged-at"?: string; call?: string }[] = []; + let size = 0; + for (let n = from; n <= to; n++) { + let p; + try { + p = await forge.getPullRequest(owner, repo, n); + } catch (err) { + if (/: 404 /.test(err instanceof Error ? err.message : "")) continue; + throw err; + } + if (!p.merged || !p.merge_commit_sha) continue; + const row = { number: n, "merge-commit": p.merge_commit_sha, ...(p.merged_at ? { "merged-at": p.merged_at } : {}), + ...(madeBy.has(n) ? { call: madeBy.get(n) } : {}) }; + const rowSize = JSON.stringify(row).length + 1; + if (size + rowSize > ANSWER_CAP_BYTES) return { repository: full, from, to, total: to - from + 1, merged, capped: true, cursor: n }; + size += rowSize; + merged.push(row); + } + return { repository: full, from, to, total: to - from + 1, merged }; + }, + }; +} -- 2.54.0 From ee5e7f232f5312ba3d96202cf05e1bb7f23ee46c Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 22 Nov 2026 00:55:14 +0100 Subject: [PATCH 24/54] P1: the packet filter holder reads a port's scope from its derived rule's comment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A derived accept rule's one comment says derived / scope , its module list cut with a count past 128 bytes; allow without a source is refused for a port whose derived rules name no scope any, read from the comments, never from a rule's shape. Builds: design 47 H2 §813 (4); H-A-357, H-A-384 (1), H-A-349 DQ-11 --- modules/nftables/allowances.ts | 44 +++++++++++++----------- modules/nftables/test/allowances.test.ts | 21 ++++++++--- 2 files changed, 40 insertions(+), 25 deletions(-) diff --git a/modules/nftables/allowances.ts b/modules/nftables/allowances.ts index ce60ceb39..69826a2c6 100644 --- a/modules/nftables/allowances.ts +++ b/modules/nftables/allowances.ts @@ -324,8 +324,8 @@ export class Allowances { const until = untilOf(args.until, now); const why = whyOf(args.why); if (!source && privateOnly(await this.asks.meshTable(), protocol, port)) { - refuse("port", `${protocol} ${port} is admitted by a derived rule of the loaded mesh table from given sources only (the private network's, such as the store's ` + - "port or the broker's management port); an allowance from any source would widen it: give a source"); + refuse("port", `${protocol} ${port} is admitted by a derived rule of the loaded mesh table from given sources only (its scope private-network or ` + + "local-network, such as the store's port or the broker's management port); an allowance from any source would widen it: give a source"); } const most = await this.asks.most(); const kept = await this.ownKeys(); @@ -442,23 +442,27 @@ export function markLoaded(allowances: Allowance[], meshTable: string): { }; } -/** Whether the loaded mesh table's input chain accepts `protocol` `port` only with a source match (`ip saddr` or `ip6 - * saddr`) and never from any source: a port a derived rule admits from the private network only (§3, H-A-315 (7), - * H-A-319). */ -export function privateOnly(meshTable: string, protocol: Protocol, port: number): boolean { - const input = /chain input \{([\s\S]*?)\n\t?\}/.exec(meshTable)?.[1] ?? ""; - let withSource = false; - for (const line of input.split("\n")) { - if (!/\baccept\b/.test(line) || !line.includes(`${protocol} dport `)) continue; - const ports = /dport (\{[^}]*\}|\S+)/.exec(line)?.[1] ?? ""; - const names = ports.replace(/[{}]/g, "").split(",").map((p) => p.trim()).filter(Boolean); - const named = names.some((p) => { - const [lo, hi] = p.split("-").map(Number); - return hi === undefined ? lo === port : lo <= port && port <= hi; - }); - if (!named) continue; - if (!/\bip6? saddr\b/.test(line)) return false; - withSource = true; +/** The scopes the node-engine's render writes on a derived accept rule (design 47 H2 §813 (4), H-A-357). */ +export type Scope = "any" | "private-network" | "local-network"; + +/** + * The scopes the loaded mesh table's derived accept rules give `protocol` `port`, read from each rule's one comment + * `derived / scope ` (design 47 H2 §813 (4), + * H-A-357), the module list cut with a count past 128 bytes (`derived a,b,+4 tcp/443 scope any`, H-A-384 (1)); never + * from the rule's shape. A port is one port or a range `lo-hi`. + */ +export function portScopes(meshTable: string, protocol: Protocol, port: number): Scope[] { + const out: Scope[] = []; + for (const m of meshTable.matchAll(/comment "derived \S+ (tcp|udp)\/(\d+)(?:-(\d+))? scope (any|private-network|local-network)"/g)) { + const lo = Number(m[2]), hi = m[3] === undefined ? lo : Number(m[3]); + if (m[1] === protocol && lo <= port && port <= hi) out.push(m[4] as Scope); } - return withSource; + return out; +} + +/** Whether a derived rule admits `protocol` `port` from given sources only (the private network or the local network) + * and none from any source: its scope read from the rules' comments (§813 (4), H-A-315 (7), H-A-319). */ +export function privateOnly(meshTable: string, protocol: Protocol, port: number): boolean { + const scopes = portScopes(meshTable, protocol, port); + return scopes.length > 0 && !scopes.includes("any"); } diff --git a/modules/nftables/test/allowances.test.ts b/modules/nftables/test/allowances.test.ts index f905db821..44b0342a5 100644 --- a/modules/nftables/test/allowances.test.ts +++ b/modules/nftables/test/allowances.test.ts @@ -10,10 +10,18 @@ import { interface Timer { ms: number; fire: () => void; cancelled: boolean } -// The loaded mesh table: the store's port from the private network only, 80 from anywhere and 8443 from both. +// The loaded mesh table: the store's port from the private network only, 80 from anywhere and 8443 from both; each derived +// accept rule carries the one comment the node-engine's render writes, `derived / scope ` +// (47 H2 §813 (4), H-A-357), its module list cut with a count past 128 bytes (H-A-384 (1)). const TABLE = 'table inet mesh {\n\tchain input {\n\t\ttype filter hook input priority filter; policy drop;\n' + - '\t\tip saddr { 10.77.0.1, 10.77.0.2 } tcp dport 5432 accept\n\t\tip6 saddr { fd77::1 } tcp dport 5432 accept\n' + - '\t\ttcp dport 80 accept\n\t\tip saddr { 10.77.0.1 } tcp dport { 8443, 9500-9600 } accept\n\t\ttcp dport 8443 accept\n\t}\n}\n'; + '\t\tip saddr { 10.77.0.1, 10.77.0.2 } tcp dport 5432 accept comment "derived postgres tcp/5432 scope private-network"\n' + + '\t\tip6 saddr { fd77::1 } tcp dport 5432 accept comment "derived postgres tcp/5432 scope private-network"\n' + + '\t\ttcp dport 80 accept comment "derived route-proxy tcp/80 scope any"\n' + + '\t\tip saddr { 10.77.0.1 } tcp dport 8443 accept comment "derived a,b,+4 tcp/8443 scope private-network"\n' + + '\t\ttcp dport 8443 accept comment "derived c tcp/8443 scope any"\n' + + '\t\tip saddr { 192.168.1.0/24 } tcp dport 9500-9600 accept comment "derived nats tcp/9500-9600 scope local-network"\n' + + '\t\ttcp dport 7000 accept comment "derived x,+2 tcp/7000 scope private-network"\n' + + '\t\tip saddr { 10.77.0.1 } tcp dport 7100 accept\n\t}\n}\n'; // gate, when given, holds every delete until it opens, and every get waits a turn of the event loop: a slow store. function fake(at = "2026-11-21T20:00:00Z", node = "shanks", kept = new Map(), gate?: Promise) { @@ -189,9 +197,12 @@ test("allowed asks converged again while none answered, and refuses when it does assert.equal(asks.asked, asked + 2, "an answer held is not asked again"); }); -test("privateOnly: a port accepted only with a source match, never from any source", () => { +test("privateOnly: a port a derived rule's comment scopes to the private or the local network, none to any", () => { assert.equal(privateOnly(TABLE, "tcp", 5432), true); - assert.equal(privateOnly(TABLE, "tcp", 9550), true, "a range of a set"); + assert.equal(privateOnly(TABLE, "tcp", 9550), true, "a range, local-network"); + // 47 H2 §813 (4): the scope is read from the rule's comment, never from its shape. + assert.equal(privateOnly(TABLE, "tcp", 7000), true, "a comment cut with a count says private-network, whatever the shape"); + assert.equal(privateOnly(TABLE, "tcp", 7100), false, "a rule with a source match and no derived comment names no scope"); assert.equal(privateOnly(TABLE, "tcp", 80), false); assert.equal(privateOnly(TABLE, "tcp", 8443), false, "accepted from anywhere too"); assert.equal(privateOnly(TABLE, "udp", 5432), false); -- 2.54.0 From 0cb4090096e79b5cc5bbe085b0825f473410c97f Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 22 Nov 2026 00:58:29 +0100 Subject: [PATCH 25/54] C2: the environment check takes a declared setting's word as the node-engine's MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The forge module reads ask-taken-bound from MESH_CONFIG_ASK_TAKEN_BOUND, the word the node-engine composes for every setting a manifest declares; the check takes MESH_CONFIG_ of each declared config key as set, and an undeclared one as set by nothing. Builds: design 47 H2 §775, §781; H-A-370, H-A-400 (2) --- checks/environment/main.go | 12 ++++++++++++ checks/environment/main_test.go | 24 +++++++++++++----------- 2 files changed, 25 insertions(+), 11 deletions(-) diff --git a/checks/environment/main.go b/checks/environment/main.go index e8cd65a69..2ffb4b89a 100644 --- a/checks/environment/main.go +++ b/checks/environment/main.go @@ -77,6 +77,7 @@ func Check(root string) (map[string]map[string][]string, error) { } set := map[string]bool{} setBy(manifest, set) + settingWords(manifest, set) reads := map[string]map[string]bool{} consts := map[string]map[string]string{} // directory -> constant -> word var goFiles []string @@ -180,6 +181,17 @@ func setBy(v any, set map[string]bool) { } } +// settingWords collects the words the node-engine composes for every setting the manifest's `config` declares, +// MESH_CONFIG_, the key upper-cased with '-' and '.' as '_', into the module's processes and its tool runner slice +// alike (design 47 H2 §781; H-A-370, H-A-400 (2)). +func settingWords(v any, set map[string]bool) { + m, _ := v.(map[string]any) + config, _ := m["config"].(map[string]any) + for key := range config { + set["MESH_CONFIG_"+strings.NewReplacer("-", "_", ".", "_").Replace(strings.ToUpper(key))] = true + } +} + // Listed is the list's ` ` lines; a line starting with # and a blank line are not entries. func Listed(path string) (map[string]bool, error) { f, err := os.Open(path) diff --git a/checks/environment/main_test.go b/checks/environment/main_test.go index f79842cd1..fd69078e7 100644 --- a/checks/environment/main_test.go +++ b/checks/environment/main_test.go @@ -32,13 +32,14 @@ func module(t *testing.T, root, name, manifest string, files map[string]string) // not; tests, node_modules and dist are not read. func TestAWordReadThatNothingSetsIsFound(t *testing.T) { root := t.TempDir() - module(t, root, "reader", `{"module":"reader","build":{"artifacts":[{"name":"tools","env":{"MESH_SET":"x"}}]},`+ + module(t, root, "reader", `{"module":"reader","config":{"ask-taken-bound":{"type":"string"}},"build":{"artifacts":[{"name":"tools","env":{"MESH_SET":"x"}}]},`+ `"resources":[{"id":"p","type":"process","environment":{"variables":{"MESH_VARIABLE":"y"}}}]}`, map[string]string{ - "cmd/a/main.go": `package main; const reach = "MESH_REACH"; var _ = os.Getenv(reach); var _ = os.Getenv("MESH_SET")`, - "cmd/a/other.go": `package main; var _, _ = os.LookupEnv("MESH_LITERAL"); var _ = os.Getenv("MESH_MODULE")`, - "cmd/a/write.go": `package main; var _ = []string{"MESH_WRITTEN_FOR_A_CHILD=1"}`, - "cmd/a/a_test.go": `package main; var _ = os.Getenv("MESH_IN_A_TEST")`, - "client.ts": `const a = process.env.MESH_TS ?? "d"; const b = env["MESH_BRACKET"]; const c = env.MESH_VARIABLE;`, + "cmd/a/main.go": `package main; const reach = "MESH_REACH"; var _ = os.Getenv(reach); var _ = os.Getenv("MESH_SET")`, + "cmd/a/other.go": `package main; var _, _ = os.LookupEnv("MESH_LITERAL"); var _ = os.Getenv("MESH_MODULE")`, + "cmd/a/write.go": `package main; var _ = []string{"MESH_WRITTEN_FOR_A_CHILD=1"}`, + "cmd/a/a_test.go": `package main; var _ = os.Getenv("MESH_IN_A_TEST")`, + // 47 H2 §781, H-A-370: a declared setting's word, which the node-engine composes; one undeclared is set by nothing. + "client.ts": `const a = process.env.MESH_TS ?? "d"; const b = env["MESH_BRACKET"]; const c = env.MESH_VARIABLE; const d = env.MESH_CONFIG_ASK_TAKEN_BOUND; const e = env.MESH_CONFIG_UNDECLARED;`, "files/env": "MESH_FROM_FILE=${config:x}\n", "index.ts": `const f = env.MESH_FROM_FILE;`, "test/x.test.ts": `env.MESH_IN_A_TEST_DIR`, @@ -50,16 +51,17 @@ func TestAWordReadThatNothingSetsIsFound(t *testing.T) { t.Fatal(err) } want := map[string]map[string][]string{"reader": { - "MESH_REACH": {"cmd/a/main.go"}, - "MESH_LITERAL": {"cmd/a/other.go"}, - "MESH_TS": {"client.ts"}, - "MESH_BRACKET": {"client.ts"}, + "MESH_REACH": {"cmd/a/main.go"}, + "MESH_LITERAL": {"cmd/a/other.go"}, + "MESH_TS": {"client.ts"}, + "MESH_BRACKET": {"client.ts"}, + "MESH_CONFIG_UNDECLARED": {"client.ts"}, }} if !reflect.DeepEqual(got, want) { t.Fatalf("found %v, want %v", got, want) } refused, stale := Judge(got, map[string]bool{"reader MESH_REACH": true, "reader MESH_GONE": true}) - if !reflect.DeepEqual(refused, []string{"reader MESH_BRACKET", "reader MESH_LITERAL", "reader MESH_TS"}) || + if !reflect.DeepEqual(refused, []string{"reader MESH_BRACKET", "reader MESH_CONFIG_UNDECLARED", "reader MESH_LITERAL", "reader MESH_TS"}) || !reflect.DeepEqual(stale, []string{"reader MESH_GONE"}) { t.Errorf("refused %v, stale %v", refused, stale) } -- 2.54.0 From ac4ed46359d88d77070e41d7b9e3d90e635d4160 Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 22 Nov 2026 01:01:26 +0100 Subject: [PATCH 26/54] M1: every manifest says its class; the delivery modules' stores name their caps MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A module that claims a seat is core, one that claims none a leaf (141 manifests; the build modules and the registry are the other developer's). The forge module's, the gate's and the sweep's states each name 8 MiB, and their findings stream 8 MiB for 7 days. Builds: design 47 H2 §810, §812, §783, §820; H-A-394 (1), H-A-411 (1), (2), H-A-378 --- modules/adwaita/module.json | 1 + modules/artifact-store-tools/module.json | 8 +++-- modules/asus-zephyrus-g14/module.json | 1 + modules/audit-logger/module.json | 1 + modules/avahi/module.json | 1 + modules/baserow/module.json | 1 + modules/blueman/module.json | 1 + modules/bluetooth/module.json | 1 + modules/brother/module.json | 1 + modules/buildah/module.json | 1 + modules/ca-trust/module.json | 1 + modules/claude-code/module.json | 1 + modules/claude-licence-manager/module.json | 1 + modules/clipmenu/module.json | 1 + modules/confluence/module.json | 1 + modules/cups/module.json | 1 + modules/dbus/module.json | 1 + modules/desk-channel/module.json | 1 + modules/dev-tools/module.json | 1 + modules/disk-load/module.json | 1 + modules/dmenu/module.json | 1 + modules/dnsmasq/module.json | 1 + modules/docker-compose/module.json | 1 + modules/docker/module.json | 1 + modules/dunst/module.json | 1 + modules/fail2ban/module.json | 1 + modules/feh/module.json | 1 + modules/flatpak/module.json | 1 + modules/fonts/module.json | 1 + modules/forticlient/module.json | 1 + .../cmd/package-registry/retention_test.go | 7 ++-- modules/gitea/module.json | 35 ++++++++++++++----- modules/gitlab/module.json | 1 + modules/gnome-keyring/module.json | 1 + modules/go/module.json | 1 + modules/grafana/module.json | 1 + modules/hello-web/module.json | 1 + modules/home-assistant/module.json | 1 + modules/hostname/module.json | 1 + modules/i3/module.json | 1 + modules/i3status-rust/module.json | 1 + modules/icecast/module.json | 1 + modules/influxdb/module.json | 1 + modules/invoicing/module.json | 1 + modules/jira/module.json | 1 + modules/kernel/module.json | 1 + modules/keycloak/module.json | 1 + modules/lab/module.json | 1 + modules/lemurs/module.json | 1 + modules/letta/module.json | 1 + modules/local-model-consumer/module.json | 1 + modules/localization/module.json | 1 + modules/logrotate/module.json | 1 + modules/mailu/module.json | 1 + modules/marrytts/module.json | 1 + modules/matrix/module.json | 1 + modules/memory-pressure/module.json | 1 + modules/mesh-catalog/module.json | 1 + modules/mesh-cli/module.json | 1 + modules/mesh-conditions/module.json | 1 + modules/mesh-config/module.json | 1 + modules/mesh-delivery-gate/module.json | 16 ++++++--- modules/mesh-files/module.json | 1 + modules/mesh-hand-acts/module.json | 1 + modules/mesh-healers/module.json | 1 + modules/mesh-issues/module.json | 1 + modules/mesh-knowledge/module.json | 1 + modules/mesh-mail-notifier/module.json | 1 + modules/mesh-mail-reader/module.json | 1 + modules/mesh-mail-web/module.json | 1 + modules/mesh-mail/module.json | 1 + modules/mesh-provisioner-client/module.json | 1 + modules/mesh-provisioner/module.json | 1 + modules/mesh-reads/module.json | 1 + modules/mesh-recovery/module.json | 1 + modules/mesh-self-check/module.json | 1 + modules/mesh-vault/module.json | 1 + modules/mesh-watcher/module.json | 1 + modules/messenger/module.json | 1 + modules/minio/module.json | 1 + modules/model-usage/module.json | 1 + modules/mongodb/module.json | 1 + modules/mosquitto/module.json | 1 + modules/mounts/module.json | 1 + modules/mssql/module.json | 1 + modules/n8n/module.json | 1 + modules/nats/module.json | 1 + modules/netcheck/module.json | 1 + modules/networkmanager/module.json | 1 + modules/nextcloud-client/module.json | 1 + modules/nextcloud/module.json | 1 + modules/nfs-server/module.json | 1 + modules/nftables/module.json | 1 + modules/nm-applet/module.json | 1 + modules/node-env/module.json | 1 + modules/nodejs/module.json | 1 + modules/nodered/module.json | 1 + modules/ollama/module.json | 1 + modules/only-office/module.json | 1 + modules/openai-consumer/module.json | 1 + modules/openrazer/module.json | 1 + modules/pacman/module.json | 1 + modules/photos-eef/module.json | 1 + modules/photos-filip/module.json | 1 + modules/picom/module.json | 1 + modules/polychromatic/module.json | 1 + modules/postgres/module.json | 1 + modules/power/module.json | 1 + modules/powerlevel10k/module.json | 1 + modules/records/module.json | 1 + modules/redis/module.json | 1 + modules/restic/module.json | 1 + modules/rofi/module.json | 1 + modules/route-adapter/module.json | 1 + modules/screen-lock/module.json | 1 + modules/searxng/module.json | 1 + modules/sensors/module.json | 1 + modules/showcase/module.json | 1 + modules/slack/module.json | 1 + modules/snapd/module.json | 1 + modules/ssh-client/module.json | 1 + modules/sshd/module.json | 1 + modules/step-ca/module.json | 1 + modules/sudo/module.json | 1 + modules/supabase/module.json | 1 + modules/systemd-networkd/module.json | 1 + modules/systemd-resolved/module.json | 1 + modules/systemd/module.json | 1 + modules/telegram/module.json | 1 + modules/time-sync/module.json | 1 + modules/triggerhappy/module.json | 1 + modules/umami/module.json | 1 + modules/unifi/module.json | 1 + modules/website/module.json | 1 + modules/xclip/module.json | 1 + modules/xdg/module.json | 1 + modules/xorg/module.json | 1 + modules/xterm/module.json | 1 + modules/zfs/module.json | 1 + modules/zsh-autosuggestions/module.json | 1 + modules/zsh-syntax-highlighting/module.json | 1 + modules/zsh/module.json | 1 + 142 files changed, 186 insertions(+), 18 deletions(-) diff --git a/modules/adwaita/module.json b/modules/adwaita/module.json index b47fbf37e..44bbe78dd 100644 --- a/modules/adwaita/module.json +++ b/modules/adwaita/module.json @@ -1,6 +1,7 @@ { "module": "adwaita", "version": "0.1", + "class": "leaf", "moved": true, "capabilities": [ "package-manager" diff --git a/modules/artifact-store-tools/module.json b/modules/artifact-store-tools/module.json index 5ed0d1b66..26622f54e 100644 --- a/modules/artifact-store-tools/module.json +++ b/modules/artifact-store-tools/module.json @@ -1,6 +1,7 @@ { "module": "artifact-store-tools", "version": "0.1", + "class": "core", "packages": { "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds": "0.1" }, @@ -111,7 +112,10 @@ { "name": "findings", "storage": "key-value", - "template": "state" + "template": "state", + "options": { + "max-mib": 8 + } }, { "name": "finding-events", @@ -124,7 +128,7 @@ "findings-stated.*" ], "max-age-days": 7, - "max-mib": 16 + "max-mib": 8 } } ] diff --git a/modules/asus-zephyrus-g14/module.json b/modules/asus-zephyrus-g14/module.json index b5152b6d7..4055b81cd 100644 --- a/modules/asus-zephyrus-g14/module.json +++ b/modules/asus-zephyrus-g14/module.json @@ -1,6 +1,7 @@ { "module": "asus-zephyrus-g14", "version": "0.1", + "class": "leaf", "moved": true, "templates": "templates", "capabilities": [ diff --git a/modules/audit-logger/module.json b/modules/audit-logger/module.json index 2ed839dce..4314b5af2 100644 --- a/modules/audit-logger/module.json +++ b/modules/audit-logger/module.json @@ -1,6 +1,7 @@ { "module": "audit-logger", "version": "0.1", + "class": "leaf", "slug": "audit", "consumes": [ "**" diff --git a/modules/avahi/module.json b/modules/avahi/module.json index 01731ee6e..b73cbcca7 100644 --- a/modules/avahi/module.json +++ b/modules/avahi/module.json @@ -1,6 +1,7 @@ { "module": "avahi", "version": "0.1", + "class": "leaf", "capabilities": [ "package-manager", "service-manager" diff --git a/modules/baserow/module.json b/modules/baserow/module.json index f92a50402..ef7de33b6 100644 --- a/modules/baserow/module.json +++ b/modules/baserow/module.json @@ -1,6 +1,7 @@ { "module": "baserow", "version": "0.1", + "class": "leaf", "capabilities": [ "container-runtime" ], diff --git a/modules/blueman/module.json b/modules/blueman/module.json index 8a79ba2d6..c896e7afb 100644 --- a/modules/blueman/module.json +++ b/modules/blueman/module.json @@ -1,6 +1,7 @@ { "module": "blueman", "version": "0.1", + "class": "leaf", "capabilities": [ "package-manager" ], diff --git a/modules/bluetooth/module.json b/modules/bluetooth/module.json index 6ee44cb77..8d908e6b5 100644 --- a/modules/bluetooth/module.json +++ b/modules/bluetooth/module.json @@ -1,6 +1,7 @@ { "module": "bluetooth", "version": "0.1", + "class": "leaf", "capabilities": [ "package-manager", "service-manager" diff --git a/modules/brother/module.json b/modules/brother/module.json index e1c1f7ae1..158ea6898 100644 --- a/modules/brother/module.json +++ b/modules/brother/module.json @@ -1,6 +1,7 @@ { "module": "brother", "version": "0.1", + "class": "core", "requires": [ { "name": "s3-bucket", diff --git a/modules/buildah/module.json b/modules/buildah/module.json index 4e97d7cdd..4f2cdad59 100644 --- a/modules/buildah/module.json +++ b/modules/buildah/module.json @@ -1,6 +1,7 @@ { "module": "buildah", "version": "0.1", + "class": "leaf", "capabilities": [ "package-manager", "user-namespaces" diff --git a/modules/ca-trust/module.json b/modules/ca-trust/module.json index 88602b90a..eed1faf32 100644 --- a/modules/ca-trust/module.json +++ b/modules/ca-trust/module.json @@ -1,6 +1,7 @@ { "module": "ca-trust", "version": "0.1", + "class": "core", "slug": "catrust", "capabilities": [ "service-manager" diff --git a/modules/claude-code/module.json b/modules/claude-code/module.json index 73ae6b48b..cbc6cab7d 100644 --- a/modules/claude-code/module.json +++ b/modules/claude-code/module.json @@ -1,6 +1,7 @@ { "module": "claude-code", "version": "0.1", + "class": "leaf", "moved": true, "slug": "agent", "capabilities": [ diff --git a/modules/claude-licence-manager/module.json b/modules/claude-licence-manager/module.json index 62145c08b..5b6c70668 100644 --- a/modules/claude-licence-manager/module.json +++ b/modules/claude-licence-manager/module.json @@ -1,6 +1,7 @@ { "module": "claude-licence-manager", "version": "0.1", + "class": "core", "slug": "licmgr", "runs-as": "claude-licence-manager", "requires": [ diff --git a/modules/clipmenu/module.json b/modules/clipmenu/module.json index 1c59e061b..168f925de 100644 --- a/modules/clipmenu/module.json +++ b/modules/clipmenu/module.json @@ -1,6 +1,7 @@ { "module": "clipmenu", "version": "0.1", + "class": "core", "moved": true, "capabilities": [ "package-manager" diff --git a/modules/confluence/module.json b/modules/confluence/module.json index 3580e2d1f..26dbb3333 100644 --- a/modules/confluence/module.json +++ b/modules/confluence/module.json @@ -1,6 +1,7 @@ { "module": "confluence", "version": "0.1", + "class": "leaf", "slug": "confl", "own-secrets": { "token": "${dir:state}/token" diff --git a/modules/cups/module.json b/modules/cups/module.json index a11903c28..2156e3bbc 100644 --- a/modules/cups/module.json +++ b/modules/cups/module.json @@ -1,6 +1,7 @@ { "module": "cups", "version": "0.1", + "class": "leaf", "capabilities": [ "package-manager", "service-manager" diff --git a/modules/dbus/module.json b/modules/dbus/module.json index e71f5100a..9bdd57ba8 100644 --- a/modules/dbus/module.json +++ b/modules/dbus/module.json @@ -1,6 +1,7 @@ { "module": "dbus", "version": "0.1", + "class": "core", "capabilities": [ "package-manager", "service-manager" diff --git a/modules/desk-channel/module.json b/modules/desk-channel/module.json index 562945690..bf8b4487a 100644 --- a/modules/desk-channel/module.json +++ b/modules/desk-channel/module.json @@ -1,6 +1,7 @@ { "module": "desk-channel", "version": "0.1", + "class": "core", "slug": "desk", "runs-as": "desk-channel", "claims": [ diff --git a/modules/dev-tools/module.json b/modules/dev-tools/module.json index 0505c8666..754963765 100644 --- a/modules/dev-tools/module.json +++ b/modules/dev-tools/module.json @@ -1,6 +1,7 @@ { "module": "dev-tools", "version": "0.1", + "class": "leaf", "capabilities": [ "package-manager" ], diff --git a/modules/disk-load/module.json b/modules/disk-load/module.json index a2798476a..f19a46849 100644 --- a/modules/disk-load/module.json +++ b/modules/disk-load/module.json @@ -1,6 +1,7 @@ { "module": "disk-load", "version": "0.1", + "class": "leaf", "tools": [ "disk_load", "disk_top", diff --git a/modules/dmenu/module.json b/modules/dmenu/module.json index 61986d1e2..71d0c4172 100644 --- a/modules/dmenu/module.json +++ b/modules/dmenu/module.json @@ -1,6 +1,7 @@ { "module": "dmenu", "version": "0.1", + "class": "leaf", "capabilities": [ "package-manager" ], diff --git a/modules/dnsmasq/module.json b/modules/dnsmasq/module.json index 2318baa09..8516bb620 100644 --- a/modules/dnsmasq/module.json +++ b/modules/dnsmasq/module.json @@ -1,6 +1,7 @@ { "module": "dnsmasq", "version": "0.1", + "class": "core", "upgrade": { "policy": "record", "why": "the mesh's resolver: a build that breaks it can stop a machine resolving the bus, and then neither the gate's rollback nor a push reaches it (hq ADR 0236, issue 260)" diff --git a/modules/docker-compose/module.json b/modules/docker-compose/module.json index 7ace4b8ce..92caf5835 100644 --- a/modules/docker-compose/module.json +++ b/modules/docker-compose/module.json @@ -1,6 +1,7 @@ { "module": "docker-compose", "version": "0.1", + "class": "leaf", "capabilities": [ "package-manager" ], diff --git a/modules/docker/module.json b/modules/docker/module.json index a51d21c9d..c4be0284b 100644 --- a/modules/docker/module.json +++ b/modules/docker/module.json @@ -1,6 +1,7 @@ { "module": "docker", "version": "0.1", + "class": "core", "capabilities": [ "package-manager", "service-manager", diff --git a/modules/dunst/module.json b/modules/dunst/module.json index 7321e05e7..7821e3f9d 100644 --- a/modules/dunst/module.json +++ b/modules/dunst/module.json @@ -1,6 +1,7 @@ { "module": "dunst", "version": "0.1", + "class": "core", "moved": true, "capabilities": [ "package-manager" diff --git a/modules/fail2ban/module.json b/modules/fail2ban/module.json index 7e5008ab3..1fe831870 100644 --- a/modules/fail2ban/module.json +++ b/modules/fail2ban/module.json @@ -1,6 +1,7 @@ { "module": "fail2ban", "version": "0.1", + "class": "core", "capabilities": [ "firewall" ], diff --git a/modules/feh/module.json b/modules/feh/module.json index 14de69d04..49dada45b 100644 --- a/modules/feh/module.json +++ b/modules/feh/module.json @@ -1,6 +1,7 @@ { "module": "feh", "version": "0.1", + "class": "leaf", "moved": true, "capabilities": [ "package-manager" diff --git a/modules/flatpak/module.json b/modules/flatpak/module.json index d7fc53c52..da3d6ae04 100644 --- a/modules/flatpak/module.json +++ b/modules/flatpak/module.json @@ -1,6 +1,7 @@ { "module": "flatpak", "version": "0.1", + "class": "leaf", "capabilities": [ "package-manager" ], diff --git a/modules/fonts/module.json b/modules/fonts/module.json index cac0dc2d9..14b7463c2 100644 --- a/modules/fonts/module.json +++ b/modules/fonts/module.json @@ -1,6 +1,7 @@ { "module": "fonts", "version": "0.1", + "class": "leaf", "capabilities": [ "package-manager" ], diff --git a/modules/forticlient/module.json b/modules/forticlient/module.json index 24ce2f444..c849b0fe4 100644 --- a/modules/forticlient/module.json +++ b/modules/forticlient/module.json @@ -1,6 +1,7 @@ { "module": "forticlient", "version": "0.1", + "class": "leaf", "upgrade": { "policy": "record", "why": "its adapter routes the company's domains on the machine a person works on: a build that breaks it cuts the person off from work names until a person pushes the next (hq ADR 0236, ADR 0247)" diff --git a/modules/gitea/cmd/package-registry/retention_test.go b/modules/gitea/cmd/package-registry/retention_test.go index 4eb926efb..11baa150d 100644 --- a/modules/gitea/cmd/package-registry/retention_test.go +++ b/modules/gitea/cmd/package-registry/retention_test.go @@ -480,15 +480,16 @@ func TestToolsAreTheOnesTheReadmeNamesAndSayWhatTheyDo(t *testing.T) { Objects any `json:"objects"` State any `json:"state"` }{m.Objects, m.State}) - for _, want := range []string{`"name":"incoming"`, `"transfers"`, `"key-ttl":true,"name":"idle"`} { + // Each with its cap (design 47 H2 §812, H-A-411 (2)). + for _, want := range []string{`"name":"incoming"`, `"name":"transfers"`, `"key-ttl":true,"max-mib":8,"name":"idle"`} { if !strings.Contains(string(req), want) { t.Errorf("the manifest does not keep %s in its old sections (%s)", want, req) } } // findings is one local name in one form (novox/hq ADR 0595 §11): the old section state, which the TypeScript bundle // announces readable for the condition keeper (kept.ts), never also under requires. - if strings.Contains(string(req), `"name":"findings"`) { - t.Errorf("findings is required as well as kept in state: %s", req) + if required, _ := json.Marshal(m.Requires); strings.Contains(string(required), `"name":"findings"`) { + t.Errorf("findings is required as well as kept in state: %s", required) } // transfers keeps for good: a lifetime of its own would drop a transfer's key and leave its chunks in incoming. if strings.Contains(string(req), `"name":"transfers","options"`) { diff --git a/modules/gitea/module.json b/modules/gitea/module.json index a6ace6e09..6d490aedc 100644 --- a/modules/gitea/module.json +++ b/modules/gitea/module.json @@ -1,6 +1,7 @@ { "module": "gitea", "version": "0.1", + "class": "core", "moved": true, "requires": [ "postgres-database", @@ -623,25 +624,41 @@ } ], "state": [ - "bound", - "announced", - "findings", + { + "name": "bound", + "max-mib": 8 + }, + { + "name": "announced", + "max-mib": 8 + }, + { + "name": "findings", + "max-mib": 8 + }, { "name": "first-views", - "history": 1 + "history": 1, + "max-mib": 8 + }, + { + "name": "transfers", + "max-mib": 8 }, - "transfers", { "name": "idle", - "key-ttl": true + "key-ttl": true, + "max-mib": 8 }, { "name": "failed-steps", - "history": 1 + "history": 1, + "max-mib": 8 }, { "name": "calls", - "history": 1 + "history": 1, + "max-mib": 8 }, { "name": "taken", @@ -682,7 +699,7 @@ "findings-stated.*" ], "max-age-days": 7, - "max-mib": 16 + "max-mib": 8 } ], "objects": [ diff --git a/modules/gitlab/module.json b/modules/gitlab/module.json index 960832669..7bded31ca 100644 --- a/modules/gitlab/module.json +++ b/modules/gitlab/module.json @@ -1,6 +1,7 @@ { "module": "gitlab", "version": "0.1", + "class": "leaf", "own-secrets": { "token": "${dir:state}/token" }, diff --git a/modules/gnome-keyring/module.json b/modules/gnome-keyring/module.json index 76ca633c5..a7b663deb 100644 --- a/modules/gnome-keyring/module.json +++ b/modules/gnome-keyring/module.json @@ -1,6 +1,7 @@ { "module": "gnome-keyring", "version": "0.1", + "class": "core", "moved": true, "capabilities": [ "package-manager" diff --git a/modules/go/module.json b/modules/go/module.json index ca0264bd8..10cb723e5 100644 --- a/modules/go/module.json +++ b/modules/go/module.json @@ -1,6 +1,7 @@ { "module": "go", "version": "0.1", + "class": "leaf", "capabilities": [ "package-manager" ], diff --git a/modules/grafana/module.json b/modules/grafana/module.json index ce84cc904..8acd7b622 100644 --- a/modules/grafana/module.json +++ b/modules/grafana/module.json @@ -1,6 +1,7 @@ { "module": "grafana", "version": "0.1", + "class": "leaf", "emits": [ "alert.firing" ], diff --git a/modules/hello-web/module.json b/modules/hello-web/module.json index bbe7fd9af..d6cf28b03 100644 --- a/modules/hello-web/module.json +++ b/modules/hello-web/module.json @@ -2,6 +2,7 @@ "module": "hello-web", "slug": "hello", "version": "0.1", + "class": "leaf", "capabilities": [ "container-runtime" ], diff --git a/modules/home-assistant/module.json b/modules/home-assistant/module.json index f43735649..fad6eec97 100644 --- a/modules/home-assistant/module.json +++ b/modules/home-assistant/module.json @@ -1,6 +1,7 @@ { "module": "home-assistant", "version": "0.1", + "class": "leaf", "slug": "hass", "capabilities": [ "container-runtime" diff --git a/modules/hostname/module.json b/modules/hostname/module.json index 94ec44788..dc9a84014 100644 --- a/modules/hostname/module.json +++ b/modules/hostname/module.json @@ -1,6 +1,7 @@ { "module": "hostname", "version": "0.1", + "class": "core", "claims": [ { "name": "node-hostname", diff --git a/modules/i3/module.json b/modules/i3/module.json index 04a5810b4..315b77deb 100644 --- a/modules/i3/module.json +++ b/modules/i3/module.json @@ -1,6 +1,7 @@ { "module": "i3", "version": "0.1", + "class": "core", "moved": true, "capabilities": [ "package-manager" diff --git a/modules/i3status-rust/module.json b/modules/i3status-rust/module.json index 8e052bf67..4e85779f9 100644 --- a/modules/i3status-rust/module.json +++ b/modules/i3status-rust/module.json @@ -1,6 +1,7 @@ { "module": "i3status-rust", "version": "0.1", + "class": "core", "moved": true, "capabilities": [ "package-manager" diff --git a/modules/icecast/module.json b/modules/icecast/module.json index 052147018..ca6de7876 100644 --- a/modules/icecast/module.json +++ b/modules/icecast/module.json @@ -1,6 +1,7 @@ { "module": "icecast", "version": "0.1", + "class": "leaf", "requires": [ "route", "secret" diff --git a/modules/influxdb/module.json b/modules/influxdb/module.json index 23e0e4b25..80d2f225a 100644 --- a/modules/influxdb/module.json +++ b/modules/influxdb/module.json @@ -1,6 +1,7 @@ { "module": "influxdb", "version": "0.1", + "class": "leaf", "provides": [ { "name": "influxdb-api", diff --git a/modules/invoicing/module.json b/modules/invoicing/module.json index 89ace98a3..f56e75359 100644 --- a/modules/invoicing/module.json +++ b/modules/invoicing/module.json @@ -1,6 +1,7 @@ { "module": "invoicing", "version": "0.1", + "class": "leaf", "slug": "invoice", "capabilities": [ "container-runtime" diff --git a/modules/jira/module.json b/modules/jira/module.json index 81b7e845d..3d48f9f63 100644 --- a/modules/jira/module.json +++ b/modules/jira/module.json @@ -1,6 +1,7 @@ { "module": "jira", "version": "0.1", + "class": "leaf", "own-secrets": { "token": "${dir:state}/token" }, diff --git a/modules/kernel/module.json b/modules/kernel/module.json index d5fd22bc9..97a21caed 100644 --- a/modules/kernel/module.json +++ b/modules/kernel/module.json @@ -1,6 +1,7 @@ { "module": "kernel", "version": "0.1", + "class": "leaf", "capabilities": [ "service-manager" ], diff --git a/modules/keycloak/module.json b/modules/keycloak/module.json index 38f9b0610..cfad61b60 100644 --- a/modules/keycloak/module.json +++ b/modules/keycloak/module.json @@ -1,6 +1,7 @@ { "module": "keycloak", "version": "0.1", + "class": "leaf", "moved": true, "upgrade": { "policy": "record", diff --git a/modules/lab/module.json b/modules/lab/module.json index 4bddf1725..0d2f574e2 100644 --- a/modules/lab/module.json +++ b/modules/lab/module.json @@ -1,6 +1,7 @@ { "module": "lab", "version": "0.1", + "class": "leaf", "moved": true, "capabilities": [ "container-runtime", diff --git a/modules/lemurs/module.json b/modules/lemurs/module.json index 668e94a1f..8c143702d 100644 --- a/modules/lemurs/module.json +++ b/modules/lemurs/module.json @@ -1,6 +1,7 @@ { "module": "lemurs", "version": "0.1", + "class": "core", "capabilities": [ "package-manager", "service-manager", diff --git a/modules/letta/module.json b/modules/letta/module.json index d97ba80cb..fffb0e5ce 100644 --- a/modules/letta/module.json +++ b/modules/letta/module.json @@ -1,6 +1,7 @@ { "module": "letta", "version": "0.1", + "class": "leaf", "capabilities": [ "container-runtime" ], diff --git a/modules/local-model-consumer/module.json b/modules/local-model-consumer/module.json index 47403651f..cb35bdcad 100644 --- a/modules/local-model-consumer/module.json +++ b/modules/local-model-consumer/module.json @@ -1,6 +1,7 @@ { "module": "local-model-consumer", "version": "0.1", + "class": "leaf", "slug": "local", "requires": [ "model-access" diff --git a/modules/localization/module.json b/modules/localization/module.json index 14d5845e8..dad52ffbf 100644 --- a/modules/localization/module.json +++ b/modules/localization/module.json @@ -1,6 +1,7 @@ { "module": "localization", "version": "0.1", + "class": "leaf", "capabilities": [ "service-manager" ], diff --git a/modules/logrotate/module.json b/modules/logrotate/module.json index 1c41ac39b..4ec39c136 100644 --- a/modules/logrotate/module.json +++ b/modules/logrotate/module.json @@ -1,6 +1,7 @@ { "module": "logrotate", "version": "0.1", + "class": "leaf", "capabilities": [ "package-manager", "service-manager" diff --git a/modules/mailu/module.json b/modules/mailu/module.json index c1dd44617..f88907ddd 100644 --- a/modules/mailu/module.json +++ b/modules/mailu/module.json @@ -1,6 +1,7 @@ { "module": "mailu", "version": "0.1", + "class": "leaf", "upgrade": { "policy": "record", "why": "people's mail: any change to how its containers are declared recreates them together in one send, and the mail is down for everyone until they are up again — a person chooses the moment (hq ADR 0242, issue 295)" diff --git a/modules/marrytts/module.json b/modules/marrytts/module.json index ddadf075c..916c106f2 100644 --- a/modules/marrytts/module.json +++ b/modules/marrytts/module.json @@ -1,6 +1,7 @@ { "module": "marrytts", "version": "0.1", + "class": "leaf", "capabilities": [ "container-runtime" ], diff --git a/modules/matrix/module.json b/modules/matrix/module.json index 3c6282545..38ff40b06 100644 --- a/modules/matrix/module.json +++ b/modules/matrix/module.json @@ -1,6 +1,7 @@ { "module": "matrix", "version": "0.1", + "class": "leaf", "capabilities": [ "container-runtime" ], diff --git a/modules/memory-pressure/module.json b/modules/memory-pressure/module.json index 6fe32ff11..c4fc5ffec 100644 --- a/modules/memory-pressure/module.json +++ b/modules/memory-pressure/module.json @@ -1,6 +1,7 @@ { "module": "memory-pressure", "version": "0.1", + "class": "leaf", "capabilities": [ "package-manager", "service-manager" diff --git a/modules/mesh-catalog/module.json b/modules/mesh-catalog/module.json index f3353220c..421804df1 100644 --- a/modules/mesh-catalog/module.json +++ b/modules/mesh-catalog/module.json @@ -2,6 +2,7 @@ "module": "mesh-catalog", "slug": "catalog", "version": "0.1", + "class": "leaf", "capabilities": [ "container-runtime" ], diff --git a/modules/mesh-cli/module.json b/modules/mesh-cli/module.json index d07f47d85..86325ec23 100644 --- a/modules/mesh-cli/module.json +++ b/modules/mesh-cli/module.json @@ -1,6 +1,7 @@ { "module": "mesh-cli", "version": "0.1", + "class": "leaf", "moved": true, "requires": [ "mcp-endpoint" diff --git a/modules/mesh-conditions/module.json b/modules/mesh-conditions/module.json index f8b9ca440..4bfcc3fe1 100644 --- a/modules/mesh-conditions/module.json +++ b/modules/mesh-conditions/module.json @@ -1,6 +1,7 @@ { "module": "mesh-conditions", "version": "0.1", + "class": "core", "runs-as": "mesh-conditions", "claims": [ { diff --git a/modules/mesh-config/module.json b/modules/mesh-config/module.json index 58d0d9540..e6ac9ed63 100644 --- a/modules/mesh-config/module.json +++ b/modules/mesh-config/module.json @@ -1,6 +1,7 @@ { "module": "mesh-config", "version": "0.1", + "class": "core", "runs-as": "mesh-config", "claims": [ { diff --git a/modules/mesh-delivery-gate/module.json b/modules/mesh-delivery-gate/module.json index 199323090..7a8fe136f 100644 --- a/modules/mesh-delivery-gate/module.json +++ b/modules/mesh-delivery-gate/module.json @@ -1,6 +1,7 @@ { "module": "mesh-delivery-gate", "version": "0.1", + "class": "core", "slug": "gate", "claims": [ { @@ -368,12 +369,18 @@ { "name": "heads", "storage": "key-value", - "template": "state" + "template": "state", + "options": { + "max-mib": 8 + } }, { "name": "findings", "storage": "key-value", - "template": "state" + "template": "state", + "options": { + "max-mib": 8 + } }, { "name": "finding-events", @@ -386,7 +393,7 @@ "findings-stated.*" ], "max-age-days": 7, - "max-mib": 16 + "max-mib": 8 } } ], @@ -405,7 +412,8 @@ "state": [ { "name": "source", - "history": 64 + "history": 64, + "max-mib": 8 } ], "capabilities": [ diff --git a/modules/mesh-files/module.json b/modules/mesh-files/module.json index c43c13d49..4b75b5f33 100644 --- a/modules/mesh-files/module.json +++ b/modules/mesh-files/module.json @@ -1,6 +1,7 @@ { "module": "mesh-files", "version": "0.1", + "class": "core", "claims": [ { "name": "node-files", diff --git a/modules/mesh-hand-acts/module.json b/modules/mesh-hand-acts/module.json index ef8bb16a9..6131b4e2e 100644 --- a/modules/mesh-hand-acts/module.json +++ b/modules/mesh-hand-acts/module.json @@ -1,6 +1,7 @@ { "module": "mesh-hand-acts", "version": "0.1", + "class": "core", "runs-as": "mesh-hand-acts", "claims": [ { diff --git a/modules/mesh-healers/module.json b/modules/mesh-healers/module.json index ccfb3216d..66bfbe727 100644 --- a/modules/mesh-healers/module.json +++ b/modules/mesh-healers/module.json @@ -1,6 +1,7 @@ { "module": "mesh-healers", "version": "0.1", + "class": "core", "runs-as": "mesh-healers", "claims": [ { diff --git a/modules/mesh-issues/module.json b/modules/mesh-issues/module.json index 7e1f48975..ec6bdc8f8 100644 --- a/modules/mesh-issues/module.json +++ b/modules/mesh-issues/module.json @@ -1,6 +1,7 @@ { "module": "mesh-issues", "version": "0.1", + "class": "core", "slug": "issues", "seats": [ { diff --git a/modules/mesh-knowledge/module.json b/modules/mesh-knowledge/module.json index 73ab8fe61..339872e77 100644 --- a/modules/mesh-knowledge/module.json +++ b/modules/mesh-knowledge/module.json @@ -1,6 +1,7 @@ { "module": "mesh-knowledge", "version": "0.1", + "class": "core", "slug": "knowledge", "requires": [ "postgres-database", diff --git a/modules/mesh-mail-notifier/module.json b/modules/mesh-mail-notifier/module.json index 8197fb30f..290b2b2c2 100644 --- a/modules/mesh-mail-notifier/module.json +++ b/modules/mesh-mail-notifier/module.json @@ -1,6 +1,7 @@ { "module": "mesh-mail-notifier", "version": "0.1", + "class": "core", "runs-as": "mesh-mail-notifier", "claims": [ { diff --git a/modules/mesh-mail-reader/module.json b/modules/mesh-mail-reader/module.json index 5ec31bd2b..211b6dbe5 100644 --- a/modules/mesh-mail-reader/module.json +++ b/modules/mesh-mail-reader/module.json @@ -1,6 +1,7 @@ { "module": "mesh-mail-reader", "version": "0.1", + "class": "core", "runs-as": "mesh-mail-reader", "claims": [ { diff --git a/modules/mesh-mail-web/module.json b/modules/mesh-mail-web/module.json index ddb16db66..f9192aac6 100644 --- a/modules/mesh-mail-web/module.json +++ b/modules/mesh-mail-web/module.json @@ -1,6 +1,7 @@ { "module": "mesh-mail-web", "version": "0.1", + "class": "core", "slug": "mail-web", "runs-as": "mesh-mail-web", "claims": [ diff --git a/modules/mesh-mail/module.json b/modules/mesh-mail/module.json index ce94111a8..42ad7e902 100644 --- a/modules/mesh-mail/module.json +++ b/modules/mesh-mail/module.json @@ -1,6 +1,7 @@ { "module": "mesh-mail", "version": "0.1", + "class": "core", "slug": "mail", "seats": [ { diff --git a/modules/mesh-provisioner-client/module.json b/modules/mesh-provisioner-client/module.json index 53a205b6c..c086a9afc 100644 --- a/modules/mesh-provisioner-client/module.json +++ b/modules/mesh-provisioner-client/module.json @@ -1,6 +1,7 @@ { "module": "mesh-provisioner-client", "version": "0.1", + "class": "core", "claims": [ { "name": "node-provisioner", diff --git a/modules/mesh-provisioner/module.json b/modules/mesh-provisioner/module.json index 15028d356..2d4e79405 100644 --- a/modules/mesh-provisioner/module.json +++ b/modules/mesh-provisioner/module.json @@ -1,6 +1,7 @@ { "module": "mesh-provisioner", "version": "0.1", + "class": "core", "runs-as": "mesh-provisioner", "claims": [ { diff --git a/modules/mesh-reads/module.json b/modules/mesh-reads/module.json index 4dae766f8..1067ef63e 100644 --- a/modules/mesh-reads/module.json +++ b/modules/mesh-reads/module.json @@ -1,6 +1,7 @@ { "module": "mesh-reads", "version": "0.1", + "class": "core", "runs-as": "mesh-reads", "claims": [ { diff --git a/modules/mesh-recovery/module.json b/modules/mesh-recovery/module.json index 9a955a3f8..3e138a482 100644 --- a/modules/mesh-recovery/module.json +++ b/modules/mesh-recovery/module.json @@ -1,6 +1,7 @@ { "module": "mesh-recovery", "version": "0.1", + "class": "core", "runs-as": "mesh-recovery", "claims": [ { diff --git a/modules/mesh-self-check/module.json b/modules/mesh-self-check/module.json index 9b07d05fe..16ee72e74 100644 --- a/modules/mesh-self-check/module.json +++ b/modules/mesh-self-check/module.json @@ -1,6 +1,7 @@ { "module": "mesh-self-check", "version": "0.1", + "class": "core", "packages": { "git.novox.be/novox/mesh-catalog/modules/restic/backupgood": "0.1", "git.novox.be/novox/mesh-catalog/modules/disk-load/diskreadings": "0.1" diff --git a/modules/mesh-vault/module.json b/modules/mesh-vault/module.json index 9332157b0..1e9300a8e 100644 --- a/modules/mesh-vault/module.json +++ b/modules/mesh-vault/module.json @@ -1,6 +1,7 @@ { "module": "mesh-vault", "version": "0.1", + "class": "core", "packages": { "git.novox.be/novox/mesh-delivery/modules/mesh-grants/grantsstate": "0.1" }, diff --git a/modules/mesh-watcher/module.json b/modules/mesh-watcher/module.json index 020e049a6..9ffeaa193 100644 --- a/modules/mesh-watcher/module.json +++ b/modules/mesh-watcher/module.json @@ -1,6 +1,7 @@ { "module": "mesh-watcher", "version": "0.1", + "class": "leaf", "slug": "watch", "consumes": [ "mesh-self-check.checked", diff --git a/modules/messenger/module.json b/modules/messenger/module.json index 020de147d..4f89a31b8 100644 --- a/modules/messenger/module.json +++ b/modules/messenger/module.json @@ -1,6 +1,7 @@ { "module": "messenger", "version": "0.1", + "class": "core", "slug": "msgr", "runs-as": "messenger", "seats": [ diff --git a/modules/minio/module.json b/modules/minio/module.json index 4f829c495..9a7ed812f 100644 --- a/modules/minio/module.json +++ b/modules/minio/module.json @@ -1,6 +1,7 @@ { "module": "minio", "version": "0.1", + "class": "core", "upgrade": { "policy": "record", "why": "holds the photos themselves (irreplaceable, kept by photos) for its consumers: a person takes each build, after a backup (hq ADR 0236)" diff --git a/modules/model-usage/module.json b/modules/model-usage/module.json index 6e5b3c96f..86ef6f202 100644 --- a/modules/model-usage/module.json +++ b/modules/model-usage/module.json @@ -1,6 +1,7 @@ { "module": "model-usage", "version": "0.1", + "class": "leaf", "slug": "usage", "capabilities": [ "container-runtime" diff --git a/modules/mongodb/module.json b/modules/mongodb/module.json index 5e0961211..27ab92027 100644 --- a/modules/mongodb/module.json +++ b/modules/mongodb/module.json @@ -1,6 +1,7 @@ { "module": "mongodb", "version": "0.1", + "class": "leaf", "upgrade": { "policy": "record", "why": "a provider whose restart drops every consumer on its machine, and which holds the photos' albums (irreplaceable, kept by photos): a person takes each build, after a backup (hq ADR 0236)" diff --git a/modules/mosquitto/module.json b/modules/mosquitto/module.json index e1e0bf0c6..a8bb15153 100644 --- a/modules/mosquitto/module.json +++ b/modules/mosquitto/module.json @@ -1,6 +1,7 @@ { "module": "mosquitto", "version": "0.1", + "class": "leaf", "slug": "mosq", "provides": [ { diff --git a/modules/mounts/module.json b/modules/mounts/module.json index cd89f3659..2da577468 100644 --- a/modules/mounts/module.json +++ b/modules/mounts/module.json @@ -1,6 +1,7 @@ { "module": "mounts", "version": "0.1", + "class": "core", "packages": { "git.novox.be/novox/mesh-catalog/modules/disk-load/diskreadings": "0.1" }, diff --git a/modules/mssql/module.json b/modules/mssql/module.json index a34bcea2e..52f62daa9 100644 --- a/modules/mssql/module.json +++ b/modules/mssql/module.json @@ -1,6 +1,7 @@ { "module": "mssql", "version": "0.1", + "class": "leaf", "upgrade": { "policy": "record", "why": "a provider whose restart drops every consumer on its machine, and whose new version may upgrade its databases in place: a person takes each build, after a backup (hq ADR 0236)" diff --git a/modules/n8n/module.json b/modules/n8n/module.json index bdbfbd24d..d67bc7a60 100644 --- a/modules/n8n/module.json +++ b/modules/n8n/module.json @@ -1,6 +1,7 @@ { "module": "n8n", "version": "0.1", + "class": "leaf", "capabilities": [ "container-runtime" ], diff --git a/modules/nats/module.json b/modules/nats/module.json index 105d9a30d..bcc65be90 100644 --- a/modules/nats/module.json +++ b/modules/nats/module.json @@ -1,6 +1,7 @@ { "module": "nats", "version": "0.1", + "class": "core", "packages": { "git.novox.be/novox/mesh-delivery/modules/mesh-grants/busgrants": "0.1" }, diff --git a/modules/netcheck/module.json b/modules/netcheck/module.json index bb9b92f93..ba762203c 100644 --- a/modules/netcheck/module.json +++ b/modules/netcheck/module.json @@ -1,6 +1,7 @@ { "module": "netcheck", "version": "0.1", + "class": "leaf", "emits": [ "tunnel.endpoint-changed", "tunnel.handshake-stale", diff --git a/modules/networkmanager/module.json b/modules/networkmanager/module.json index 89d4ecc40..3433f40ba 100644 --- a/modules/networkmanager/module.json +++ b/modules/networkmanager/module.json @@ -1,6 +1,7 @@ { "module": "networkmanager", "version": "0.1", + "class": "core", "upgrade": { "policy": "record", "why": "the machine's network: a build that breaks it can cut the machine off from the bus, and then neither the gate's rollback nor a push reaches it (hq ADR 0236)" diff --git a/modules/nextcloud-client/module.json b/modules/nextcloud-client/module.json index 9b6c1c0c5..5b603e465 100644 --- a/modules/nextcloud-client/module.json +++ b/modules/nextcloud-client/module.json @@ -1,6 +1,7 @@ { "module": "nextcloud-client", "version": "0.1", + "class": "leaf", "capabilities": [ "package-manager" ], diff --git a/modules/nextcloud/module.json b/modules/nextcloud/module.json index 4560c106a..c4efbe8a4 100644 --- a/modules/nextcloud/module.json +++ b/modules/nextcloud/module.json @@ -1,6 +1,7 @@ { "module": "nextcloud", "version": "0.1", + "class": "leaf", "slug": "ncloud", "requires": [ "postgres-database", diff --git a/modules/nfs-server/module.json b/modules/nfs-server/module.json index 1bb18465f..650a0aa40 100644 --- a/modules/nfs-server/module.json +++ b/modules/nfs-server/module.json @@ -1,6 +1,7 @@ { "module": "nfs-server", "version": "0.1", + "class": "core", "packages": { "git.novox.be/novox/mesh-catalog/modules/disk-load/diskreadings": "0.1" }, diff --git a/modules/nftables/module.json b/modules/nftables/module.json index 68964f4fa..33c275f66 100644 --- a/modules/nftables/module.json +++ b/modules/nftables/module.json @@ -1,6 +1,7 @@ { "module": "nftables", "version": "0.1", + "class": "core", "upgrade": { "policy": "record", "why": "the machine's packet filter: a build that breaks it can cut the machine off from the bus, and then neither the gate's rollback nor a push reaches it (hq ADR 0236)" diff --git a/modules/nm-applet/module.json b/modules/nm-applet/module.json index 7552731c7..22f916059 100644 --- a/modules/nm-applet/module.json +++ b/modules/nm-applet/module.json @@ -1,6 +1,7 @@ { "module": "nm-applet", "version": "0.1", + "class": "leaf", "capabilities": [ "package-manager" ], diff --git a/modules/node-env/module.json b/modules/node-env/module.json index 83437459e..db48bb5f4 100644 --- a/modules/node-env/module.json +++ b/modules/node-env/module.json @@ -1,6 +1,7 @@ { "module": "node-env", "version": "0.1", + "class": "core", "moved": true, "claims": [ { diff --git a/modules/nodejs/module.json b/modules/nodejs/module.json index d34a6d884..0762ab3ac 100644 --- a/modules/nodejs/module.json +++ b/modules/nodejs/module.json @@ -1,6 +1,7 @@ { "module": "nodejs", "version": "0.1", + "class": "leaf", "capabilities": [ "package-manager" ], diff --git a/modules/nodered/module.json b/modules/nodered/module.json index a633c7c0a..672625046 100644 --- a/modules/nodered/module.json +++ b/modules/nodered/module.json @@ -1,6 +1,7 @@ { "module": "nodered", "version": "0.1", + "class": "leaf", "emits": [ "flows.deployed" ], diff --git a/modules/ollama/module.json b/modules/ollama/module.json index d22f5f721..690f62adc 100644 --- a/modules/ollama/module.json +++ b/modules/ollama/module.json @@ -1,6 +1,7 @@ { "module": "ollama", "version": "0.1", + "class": "leaf", "capabilities": [ "container-runtime" ], diff --git a/modules/only-office/module.json b/modules/only-office/module.json index e54ee8b06..98f530cf5 100644 --- a/modules/only-office/module.json +++ b/modules/only-office/module.json @@ -1,6 +1,7 @@ { "module": "only-office", "version": "0.1", + "class": "core", "slug": "office", "provides": [ { diff --git a/modules/openai-consumer/module.json b/modules/openai-consumer/module.json index 1a6d7408b..20d633cfa 100644 --- a/modules/openai-consumer/module.json +++ b/modules/openai-consumer/module.json @@ -1,6 +1,7 @@ { "module": "openai-consumer", "version": "0.1", + "class": "leaf", "slug": "openai", "capabilities": [ "container-runtime" diff --git a/modules/openrazer/module.json b/modules/openrazer/module.json index 7337e9e17..6e30f9513 100644 --- a/modules/openrazer/module.json +++ b/modules/openrazer/module.json @@ -1,6 +1,7 @@ { "module": "openrazer", "version": "0.1", + "class": "leaf", "capabilities": [ "package-manager" ], diff --git a/modules/pacman/module.json b/modules/pacman/module.json index fff50da70..580aaaa54 100644 --- a/modules/pacman/module.json +++ b/modules/pacman/module.json @@ -1,6 +1,7 @@ { "module": "pacman", "version": "0.1", + "class": "core", "capabilities": [ "package-manager", "service-manager" diff --git a/modules/photos-eef/module.json b/modules/photos-eef/module.json index 3bb9940f3..6f9989e33 100644 --- a/modules/photos-eef/module.json +++ b/modules/photos-eef/module.json @@ -1,6 +1,7 @@ { "module": "photos-eef", "version": "0.1", + "class": "leaf", "slug": "eef", "capabilities": [ "container-runtime" diff --git a/modules/photos-filip/module.json b/modules/photos-filip/module.json index 8287b4a1f..c526f3818 100644 --- a/modules/photos-filip/module.json +++ b/modules/photos-filip/module.json @@ -1,6 +1,7 @@ { "module": "photos-filip", "version": "0.1", + "class": "leaf", "slug": "filip", "capabilities": [ "container-runtime" diff --git a/modules/picom/module.json b/modules/picom/module.json index 31d53c43e..bc61080d1 100644 --- a/modules/picom/module.json +++ b/modules/picom/module.json @@ -1,6 +1,7 @@ { "module": "picom", "version": "0.1", + "class": "core", "capabilities": [ "package-manager" ], diff --git a/modules/polychromatic/module.json b/modules/polychromatic/module.json index b02d2a30f..8930b1469 100644 --- a/modules/polychromatic/module.json +++ b/modules/polychromatic/module.json @@ -1,6 +1,7 @@ { "module": "polychromatic", "version": "0.1", + "class": "leaf", "moved": true, "requires": [ "x11-display" diff --git a/modules/postgres/module.json b/modules/postgres/module.json index c441d7181..e41fab60c 100644 --- a/modules/postgres/module.json +++ b/modules/postgres/module.json @@ -1,6 +1,7 @@ { "module": "postgres", "version": "0.1", + "class": "core", "moved": true, "upgrade": { "policy": "record", diff --git a/modules/power/module.json b/modules/power/module.json index cf5b061bc..e4b54f8f4 100644 --- a/modules/power/module.json +++ b/modules/power/module.json @@ -1,6 +1,7 @@ { "module": "power", "version": "0.1", + "class": "core", "moved": true, "templates": "templates", "capabilities": [ diff --git a/modules/powerlevel10k/module.json b/modules/powerlevel10k/module.json index 72c4e0243..d719efb31 100644 --- a/modules/powerlevel10k/module.json +++ b/modules/powerlevel10k/module.json @@ -1,6 +1,7 @@ { "module": "powerlevel10k", "version": "0.1", + "class": "leaf", "moved": true, "resources": [ { diff --git a/modules/records/module.json b/modules/records/module.json index 3e23fe06d..a41fe6961 100644 --- a/modules/records/module.json +++ b/modules/records/module.json @@ -1,6 +1,7 @@ { "module": "records", "version": "0.1", + "class": "leaf", "slug": "records", "requires": [ "git" diff --git a/modules/redis/module.json b/modules/redis/module.json index 4a7c95998..357accec8 100644 --- a/modules/redis/module.json +++ b/modules/redis/module.json @@ -1,6 +1,7 @@ { "module": "redis", "version": "0.1", + "class": "leaf", "provides": [ { "name": "redis-cache", diff --git a/modules/restic/module.json b/modules/restic/module.json index 313d3e7de..54ac3c376 100644 --- a/modules/restic/module.json +++ b/modules/restic/module.json @@ -1,6 +1,7 @@ { "module": "restic", "version": "0.1", + "class": "core", "packages": { "git.novox.be/novox/mesh-catalog/modules/disk-load/diskreadings": "0.1" }, diff --git a/modules/rofi/module.json b/modules/rofi/module.json index 69e7a5f70..1bba3a54f 100644 --- a/modules/rofi/module.json +++ b/modules/rofi/module.json @@ -1,6 +1,7 @@ { "module": "rofi", "version": "0.1", + "class": "core", "moved": true, "capabilities": [ "package-manager" diff --git a/modules/route-adapter/module.json b/modules/route-adapter/module.json index 28bbfa869..ee7e27f74 100644 --- a/modules/route-adapter/module.json +++ b/modules/route-adapter/module.json @@ -1,6 +1,7 @@ { "module": "route-adapter", "version": "0.1", + "class": "leaf", "moved": true, "slug": "radapt", "capabilities": [ diff --git a/modules/screen-lock/module.json b/modules/screen-lock/module.json index e73e79230..5f184f289 100644 --- a/modules/screen-lock/module.json +++ b/modules/screen-lock/module.json @@ -1,6 +1,7 @@ { "module": "screen-lock", "version": "0.1", + "class": "core", "moved": true, "capabilities": [ "package-manager" diff --git a/modules/searxng/module.json b/modules/searxng/module.json index d066a5cfb..14add9c82 100644 --- a/modules/searxng/module.json +++ b/modules/searxng/module.json @@ -1,6 +1,7 @@ { "module": "searxng", "version": "0.1", + "class": "leaf", "capabilities": [ "container-runtime" ], diff --git a/modules/sensors/module.json b/modules/sensors/module.json index d029e350d..79a476d6f 100644 --- a/modules/sensors/module.json +++ b/modules/sensors/module.json @@ -1,6 +1,7 @@ { "module": "sensors", "version": "0.1", + "class": "leaf", "capabilities": [ "package-manager", "service-manager" diff --git a/modules/showcase/module.json b/modules/showcase/module.json index 7152479c7..06fea3efc 100644 --- a/modules/showcase/module.json +++ b/modules/showcase/module.json @@ -1,6 +1,7 @@ { "module": "showcase", "version": "0.1", + "class": "core", "templates": "templates", "slug": "show", "capabilities": [ diff --git a/modules/slack/module.json b/modules/slack/module.json index 7736b3a1e..f791e2451 100644 --- a/modules/slack/module.json +++ b/modules/slack/module.json @@ -1,6 +1,7 @@ { "module": "slack", "version": "0.1", + "class": "leaf", "moved": true, "requires": [ "x11-display" diff --git a/modules/snapd/module.json b/modules/snapd/module.json index e0049a7fc..cd5ecd1d3 100644 --- a/modules/snapd/module.json +++ b/modules/snapd/module.json @@ -1,6 +1,7 @@ { "module": "snapd", "version": "0.1", + "class": "leaf", "tools": [ "snapd_status", "snapd_list", diff --git a/modules/ssh-client/module.json b/modules/ssh-client/module.json index 933b64acb..c4f8c3a91 100644 --- a/modules/ssh-client/module.json +++ b/modules/ssh-client/module.json @@ -1,6 +1,7 @@ { "module": "ssh-client", "version": "0.1", + "class": "leaf", "moved": true, "tools": [ "ssh_client_hosts", diff --git a/modules/sshd/module.json b/modules/sshd/module.json index c8bd15edd..b799d5a87 100644 --- a/modules/sshd/module.json +++ b/modules/sshd/module.json @@ -1,6 +1,7 @@ { "module": "sshd", "version": "0.1", + "class": "leaf", "upgrade": { "policy": "record", "why": "the operator's way into the machine when the mesh cannot reach it: a build that breaks it is found only when that way is needed, which no gate sees (hq ADR 0236)" diff --git a/modules/step-ca/module.json b/modules/step-ca/module.json index 0656bdfa8..887945e1f 100644 --- a/modules/step-ca/module.json +++ b/modules/step-ca/module.json @@ -1,6 +1,7 @@ { "module": "step-ca", "version": "0.1", + "class": "leaf", "capabilities": [ "container-runtime" ], diff --git a/modules/sudo/module.json b/modules/sudo/module.json index 5b84c2a6e..019d9f992 100644 --- a/modules/sudo/module.json +++ b/modules/sudo/module.json @@ -1,6 +1,7 @@ { "module": "sudo", "version": "0.1", + "class": "leaf", "capabilities": [ "package-manager" ], diff --git a/modules/supabase/module.json b/modules/supabase/module.json index 64141d342..e67fad8e2 100644 --- a/modules/supabase/module.json +++ b/modules/supabase/module.json @@ -1,6 +1,7 @@ { "module": "supabase", "version": "0.1", + "class": "leaf", "capabilities": [ "container-runtime" ], diff --git a/modules/systemd-networkd/module.json b/modules/systemd-networkd/module.json index e906eee0a..5c0e60f68 100644 --- a/modules/systemd-networkd/module.json +++ b/modules/systemd-networkd/module.json @@ -1,6 +1,7 @@ { "module": "systemd-networkd", "version": "0.1", + "class": "core", "upgrade": { "policy": "record", "why": "the machine's network: a build that breaks it can cut the machine off from the bus, and then neither the gate's rollback nor a push reaches it (hq ADR 0236)" diff --git a/modules/systemd-resolved/module.json b/modules/systemd-resolved/module.json index 270010bb6..c62dbd7b4 100644 --- a/modules/systemd-resolved/module.json +++ b/modules/systemd-resolved/module.json @@ -1,6 +1,7 @@ { "module": "systemd-resolved", "version": "0.1", + "class": "core", "upgrade": { "policy": "record", "why": "the machine's names: a build that breaks this resolver stops every name resolving on a machine a person works on, the bus's included, and then neither the gate's rollback nor a push reaches it (hq ADR 0236, ADR 0247)" diff --git a/modules/systemd/module.json b/modules/systemd/module.json index a2c492a17..2f3cff730 100644 --- a/modules/systemd/module.json +++ b/modules/systemd/module.json @@ -1,6 +1,7 @@ { "module": "systemd", "version": "0.1", + "class": "core", "capabilities": [ "service-manager", "package-manager" diff --git a/modules/telegram/module.json b/modules/telegram/module.json index a333cfce8..22304d74c 100644 --- a/modules/telegram/module.json +++ b/modules/telegram/module.json @@ -1,6 +1,7 @@ { "module": "telegram", "version": "0.1", + "class": "core", "slug": "tgrm", "runs-as": "telegram", "claims": [ diff --git a/modules/time-sync/module.json b/modules/time-sync/module.json index ba4f1cf53..de09c9e59 100644 --- a/modules/time-sync/module.json +++ b/modules/time-sync/module.json @@ -1,6 +1,7 @@ { "module": "time-sync", "version": "0.1", + "class": "leaf", "capabilities": [ "package-manager", "service-manager" diff --git a/modules/triggerhappy/module.json b/modules/triggerhappy/module.json index bbc84be9e..babb95e7b 100644 --- a/modules/triggerhappy/module.json +++ b/modules/triggerhappy/module.json @@ -1,6 +1,7 @@ { "module": "triggerhappy", "version": "0.1", + "class": "core", "moved": true, "capabilities": [ "service-manager" diff --git a/modules/umami/module.json b/modules/umami/module.json index eb8df76e3..d1c618a88 100644 --- a/modules/umami/module.json +++ b/modules/umami/module.json @@ -1,6 +1,7 @@ { "module": "umami", "version": "0.1", + "class": "leaf", "capabilities": [ "container-runtime" ], diff --git a/modules/unifi/module.json b/modules/unifi/module.json index 67348a912..c9f09c7e5 100644 --- a/modules/unifi/module.json +++ b/modules/unifi/module.json @@ -1,6 +1,7 @@ { "module": "unifi", "version": "0.1", + "class": "leaf", "capabilities": [ "container-runtime" ], diff --git a/modules/website/module.json b/modules/website/module.json index b65fb187a..07970c19e 100644 --- a/modules/website/module.json +++ b/modules/website/module.json @@ -1,6 +1,7 @@ { "module": "website", "version": "0.1", + "class": "leaf", "capabilities": [ "container-runtime" ], diff --git a/modules/xclip/module.json b/modules/xclip/module.json index 9c0829670..a094b45ed 100644 --- a/modules/xclip/module.json +++ b/modules/xclip/module.json @@ -1,6 +1,7 @@ { "module": "xclip", "version": "0.1", + "class": "leaf", "capabilities": [ "package-manager" ], diff --git a/modules/xdg/module.json b/modules/xdg/module.json index f9afd0214..72b0650a2 100644 --- a/modules/xdg/module.json +++ b/modules/xdg/module.json @@ -1,6 +1,7 @@ { "module": "xdg", "version": "0.1", + "class": "leaf", "capabilities": [ "package-manager" ], diff --git a/modules/xorg/module.json b/modules/xorg/module.json index 72ea858d5..26550fec3 100644 --- a/modules/xorg/module.json +++ b/modules/xorg/module.json @@ -1,6 +1,7 @@ { "module": "xorg", "version": "0.1", + "class": "core", "moved": true, "capabilities": [ "package-manager", diff --git a/modules/xterm/module.json b/modules/xterm/module.json index def76a72d..195510057 100644 --- a/modules/xterm/module.json +++ b/modules/xterm/module.json @@ -1,6 +1,7 @@ { "module": "xterm", "version": "0.1", + "class": "core", "moved": true, "capabilities": [ "package-manager" diff --git a/modules/zfs/module.json b/modules/zfs/module.json index 072a3f285..0594e54c8 100644 --- a/modules/zfs/module.json +++ b/modules/zfs/module.json @@ -1,6 +1,7 @@ { "module": "zfs", "version": "0.1", + "class": "leaf", "capabilities": [ "bus-state-calls", "ledger-calls", diff --git a/modules/zsh-autosuggestions/module.json b/modules/zsh-autosuggestions/module.json index b4702085c..16415c596 100644 --- a/modules/zsh-autosuggestions/module.json +++ b/modules/zsh-autosuggestions/module.json @@ -1,6 +1,7 @@ { "module": "zsh-autosuggestions", "version": "0.1", + "class": "leaf", "moved": true, "capabilities": [ "package-manager" diff --git a/modules/zsh-syntax-highlighting/module.json b/modules/zsh-syntax-highlighting/module.json index 4e4ea24a0..8fa4c1cb4 100644 --- a/modules/zsh-syntax-highlighting/module.json +++ b/modules/zsh-syntax-highlighting/module.json @@ -1,6 +1,7 @@ { "module": "zsh-syntax-highlighting", "version": "0.1", + "class": "leaf", "moved": true, "capabilities": [ "package-manager" diff --git a/modules/zsh/module.json b/modules/zsh/module.json index 028f2213d..c4bbfb570 100644 --- a/modules/zsh/module.json +++ b/modules/zsh/module.json @@ -1,6 +1,7 @@ { "module": "zsh", "version": "0.1", + "class": "core", "moved": true, "capabilities": [ "package-manager" -- 2.54.0 From 74dd8af6992562302c8205f751556d1e418aef91 Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 22 Nov 2026 01:03:06 +0100 Subject: [PATCH 27/54] M1: the gate prunes its state heads toward its cap at its own writes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit heads is written through the SDK's pruned state, which judges the fill each put answers and past prune-mark (70 %) prunes the oldest keys to prune-target (50 %); the gate serves mesh-delivery-gate_state_prune_now and mesh-delivery-gate_state_keep, each written in the hand-act log. Builds: design 47 H2 §812; H-A-380, H-A-386, H-A-411 (3) --- .../cmd/mesh-delivery-gate/heads.go | 23 ++++++++++++++++++- .../cmd/mesh-delivery-gate/main.go | 2 ++ modules/mesh-delivery-gate/module.json | 22 +++++++++++++++++- 3 files changed, 45 insertions(+), 2 deletions(-) diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/heads.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/heads.go index ea4d1d7bd..c2b80d4dd 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/heads.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/heads.go @@ -12,7 +12,9 @@ import ( "encoding/json" "errors" "fmt" + "os" "sort" + "sync" "time" "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" @@ -20,6 +22,8 @@ import ( "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-sdk/go/keys" + "git.novox.be/novox/mesh-sdk/go/stateobjects" + "git.novox.be/novox/mesh-sdk/go/views" ) // HeadUntaken is the finding of a head no merge call delivered; warning, the operator its resolver (H-A-81). @@ -61,10 +65,27 @@ func (kvHeads) Get(key string) (*HeldHead, error) { } func (kvHeads) Put(key string, v HeldHead) error { - _, err := stdio.State(seat.StateHeads).Put(key, v) + _, err := prunedHeads().Put(key, v) return err } +// prunedHeads is the state heads, pruned toward its cap at the gate's own writes (design 47 H2 §812; H-A-411 (3)): past +// prune-mark of its 8 MiB the oldest keys go down to prune-target, in one journal line. A setting that is no percentage +// is said, and the defaults hold. +var prunedHeads = sync.OnceValue(func() *stdio.PrunedState { + p, err := stdio.PruneSettings() + if err != nil { + fmt.Fprintf(os.Stderr, "the prune of heads keeps its defaults (%d %% to %d %%): %v\n", stdio.PruneMarkDefault, stdio.PruneTargetDefault, err) + p = stdio.Prune{Mark: stdio.PruneMarkDefault, Target: stdio.PruneTargetDefault} + } + return stdio.State(seat.StateHeads).Pruned(p) +}) + +// headsPruneTools are the gate's two prune tools over heads, each written in the hand-act log (H-A-380 (3)). +func headsPruneTools(acts views.HandActs) []stdio.Tool { + return stateobjects.PruneTools(Module, []stateobjects.Pruning{{Name: seat.StateHeads, State: prunedHeads()}}, acts) +} + func (kvHeads) Delete(key string) error { return stdio.State(seat.StateHeads).Delete(key) } func (s kvHeads) All() (map[string]HeldHead, error) { diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/main.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/main.go index 7075dd84a..42dadf5a4 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/main.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/main.go @@ -263,6 +263,8 @@ func tools(h *Holder, l *listening) []stdio.Tool { list = append(list, recordTools(h, meshhandactsclient.Recorder{Ask: stdio.Ask})...) list = append(list, sourceTools(h, meshhandactsclient.Recorder{Ask: stdio.Ask})...) list = append(list, headTools(h, meshhandactsclient.Recorder{Ask: stdio.Ask})...) + // The prune of heads by hand, now or kept from it (design 47 H2 §812, H-A-380 (3)). + list = append(list, headsPruneTools(meshhandactsclient.Recorder{Ask: stdio.Ask})...) // A verb that changes a delivery asks for a look at what it makes possible (react.go). for i := range list { run := list[i].Run diff --git a/modules/mesh-delivery-gate/module.json b/modules/mesh-delivery-gate/module.json index 7a8fe136f..37f784c26 100644 --- a/modules/mesh-delivery-gate/module.json +++ b/modules/mesh-delivery-gate/module.json @@ -433,7 +433,9 @@ "gate_source_again", "gate_source_seed", "gate_source_start", - "gate_times" + "gate_times", + "mesh-delivery-gate_state_keep", + "mesh-delivery-gate_state_prune_now" ], "config": { "record-retention": { @@ -464,6 +466,24 @@ "trusted": false, "runtime": true, "default": "3m" + }, + "prune-mark": { + "type": "number", + "description": "The fill of the gate's state heads, in percent of its cap, past which the gate prunes it at its own write, oldest first (design 47 H2 §812): below the bus module's raise mark of 75 %.", + "trusted": false, + "runtime": true, + "minimum": 1, + "maximum": 100, + "default": 70 + }, + "prune-target": { + "type": "number", + "description": "The fill, in percent of the cap, a prune of the gate's state heads takes it down to (design 47 H2 §812); below prune-mark.", + "trusted": false, + "runtime": true, + "minimum": 1, + "maximum": 100, + "default": 50 } }, "resources": [ -- 2.54.0 From bff5e98b308c799da710178e20a7adf0c5b83383 Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 22 Nov 2026 01:05:16 +0100 Subject: [PATCH 28/54] C6: merge-pull refuses a source-only head whose check failed, naming the step MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The runner's step-ended of a source-only check names the registration's key source.. as its module and the commit's root tree hash as its fingerprint, so its subject holds more parts than a module's: the forge module hears every subject under step-ended and judges it as a module's. The seat's whole test refuses an hq pull request whose unit step failed. Builds: design 47 H2 §815; H-A-408 --- modules/gitea/index.ts | 6 +++-- .../seatgit/whole_integration_test.go | 24 ++++++++++++++++--- modules/gitea/module.json | 2 +- 3 files changed, 26 insertions(+), 6 deletions(-) diff --git a/modules/gitea/index.ts b/modules/gitea/index.ts index 6ba5c79f6..9d80727cd 100644 --- a/modules/gitea/index.ts +++ b/modules/gitea/index.ts @@ -459,8 +459,10 @@ if (gitea) { const read = await readHeads(forge, registrations.watched, (line) => console.log(`[gitea] ${line}`)); await failedSteps.lost(heldHeads.replace(read.names, read.branches, read.pulls)); }; - // The runner's step-ended is said on step-ended... (the seat's definition). - const stepEnded = `${RUNNER}.${RUNNER_EVENTS.stepEnded}.*.*.*`; + // The runner's step-ended is said on step-ended... (the seat's definition); a source-only + // check's names the registration's key source.. as its module (design 47 H2 §815, H-A-408), so every + // subject under step-ended is heard. + const stepEnded = `${RUNNER}.${RUNNER_EVENTS.stepEnded}.#`; void bindUntilHeld(stepEnded, () => on(stepEnded, (event) => failedSteps.stepEnded(event.body))); // A branch's head is read from the forge, never taken from the event (the seat's definition of branch-moved). void bindUntilHeld(`${GIT}.${GIT_EVENTS.branchMoved}`, () => on(`${GIT}.${GIT_EVENTS.branchMoved}`, async (event) => { diff --git a/modules/gitea/internal/seatgit/whole_integration_test.go b/modules/gitea/internal/seatgit/whole_integration_test.go index 19ed27d79..f29ed9d47 100644 --- a/modules/gitea/internal/seatgit/whole_integration_test.go +++ b/modules/gitea/internal/seatgit/whole_integration_test.go @@ -12,6 +12,7 @@ package seatgit // - A runner's step-ended of a failed required step at a pull request's head and fingerprint is kept, and a merge call // naming that head merges nothing and names the step (§735, test 114). // - A call naming two pull requests in two repositories merges both, in order, and says one merge-called (§734, §736). +// - An hq pull request, of a source-only registration, whose Go checks failed is refused naming the step (H2 §815). // - With the forge's store stopped a merge call answers away, and called again once it is back it lands (H2 §776). // - A forge that refuses the second merge gives one merge-called with the first member and `refused`, and merge-partial // raised; a later call landing the refused member clears it (§767, test 115). @@ -405,7 +406,7 @@ func TestTheSeatGitWholeOverARealBus(t *testing.T) { } kvs[name] = kv } - A, B, X := strings.Repeat("a", 40), strings.Repeat("b", 40), strings.Repeat("e", 40) + A, B, X, H := strings.Repeat("a", 40), strings.Repeat("b", 40), strings.Repeat("e", 40), strings.Repeat("f", 40) // A call stopped after the second of three merges, kept and not said (§766). stopped, _ := json.Marshal(map[string]any{"by": "lead@shanks", "account": "shanks.claude-code", "at": "2026-11-20T03:00:00Z", "said": false, "members": []any{ @@ -422,6 +423,7 @@ func TestTheSeatGitWholeOverARealBus(t *testing.T) { "novox/app#7": {repo: "novox/app", base: "main", head: A, number: 7}, "novox/lib#9": {repo: "novox/lib", base: "main", head: B, number: 9, refuse: true}, "novox/lib#10": {repo: "novox/lib", base: "main", head: B, number: 10}, + "novox/hq#11": {repo: "novox/hq", base: "main", head: H, number: 11}, }} srv := httptest.NewServer(http.HandlerFunc(f.serve)) t.Cleanup(srv.Close) @@ -441,7 +443,10 @@ func TestTheSeatGitWholeOverARealBus(t *testing.T) { t.Cleanup(func() { _ = in.Close(); _ = cmd.Process.Kill(); _ = cmd.Wait() }) rt := &runtime{t: t, in: in, js: js, kvs: kvs, wait: map[string]chan json.RawMessage{}, regs: []map[string]any{ {"module": "app", "origin": "novox/app", "branches": []string{"main"}}, - {"module": "lib", "origin": "novox/lib", "branches": []string{"main"}}}} + {"module": "lib", "origin": "novox/lib", "branches": []string{"main"}}, + // A source-only registration (H2 §815): no module, one watched branch, its check a build seat's step. + {"origin": "novox/hq", "branches": []string{"main"}, "policy": map[string]any{"steps": map[string]any{"unit": "required"}}, + "check": map[string]any{"seat": "node-build-go", "step": "unit"}}}} go rt.read(out) // At the start the stopped call says one merge-called naming the two that landed (§766). @@ -457,7 +462,7 @@ func TestTheSeatGitWholeOverARealBus(t *testing.T) { until(t, "the watched heads read after the registrations", func() bool { return f.asked("GET /api/v1/repos/novox/app/branches") }) // A failed required step at a head's fingerprint refuses that head (§735, test 114), heard on the module's consumer. - until(t, "the runner's step-ended followed", func() bool { return rt.subscribed("mesh-pipeline-runner.step-ended.*.*.*") }) + until(t, "the runner's step-ended followed", func() bool { return rt.subscribed("mesh-pipeline-runner.step-ended.#") }) rt.ask("mesh/event", map[string]any{"envelope": map[string]any{"key": "mesh-pipeline-runner.step-ended.app.main." + X, "node": "test", "body": map[string]any{"v": 1, "pipeline": "main.4", "module": "app", "branch": "main", "commit": X, "fingerprint": "fp-x", "at": "2026-11-20T03:00:00Z", "result": map[string]any{"step": "unit", "status": "required", "outcome": "failed"}}}}) @@ -471,6 +476,19 @@ func TestTheSeatGitWholeOverARealBus(t *testing.T) { t.Fatalf("merged %v", got) } + // An hq pull request whose Go checks fail is refused by merge-pull naming the failed step (H2 §815): the runner's + // step-ended of a source-only check names the registration's key as its module and the commit's root tree hash as its + // fingerprint (H-A-408), judged as a module's. + rt.ask("mesh/event", map[string]any{"envelope": map[string]any{"key": "mesh-pipeline-runner.step-ended.source.novox.hq.main." + H, "node": "test", + "body": map[string]any{"v": 1, "pipeline": "main.5", "module": "source.novox.hq", "branch": "main", "commit": H, "fingerprint": strings.Repeat("7", 40), + "at": "2026-11-22T01:00:00Z", "result": map[string]any{"step": "unit", "status": "required", "outcome": "failed"}}}}) + hq := rt.mergePull(map[string]any{"repository": "novox/hq", "number": 11, "head": H}) + hqm, _ := hq["members"].([]any) + if len(hqm) != 1 || hqm[0].(map[string]any)["state"] != "refused" || + !strings.Contains(fmt.Sprint(hqm[0].(map[string]any)["why"]), "the unit step of source.novox.hq failed in run main.5") { + t.Fatalf("an hq head at a failed check answered %v", hq) + } + // With the forge's store stopped, a merge call answers away, merging nothing (H2 §776); called again once the store is // back, it lands (below). f.mu.Lock() diff --git a/modules/gitea/module.json b/modules/gitea/module.json index 6d490aedc..722686476 100644 --- a/modules/gitea/module.json +++ b/modules/gitea/module.json @@ -683,7 +683,7 @@ "consumes": [ "mesh-catalog.catalog-revision", "mesh-catalog.registration-changed", - "mesh-pipeline-runner.step-ended.*.*.*", + "mesh-pipeline-runner.step-ended.#", "git.branch-moved", "git.pull-updated", "git.pull-closed", -- 2.54.0 From a6132b32f6439d27c2727ede77ccb57ddae2ac22 Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 22 Nov 2026 01:07:28 +0100 Subject: [PATCH 29/54] C6: a source-only check's step-ended is heard on its escaped subject part MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The manifest validator takes no # in consumes, and a key part escapes its dots (mesh-sdk keys.EscapeName), so the forge module keeps hearing step-ended.*.*.*: a source-only check's module source.. is one escaped subject part, and its body names it whole. The whole test says it so. Builds: design 47 H2 §815; H-A-408 --- modules/gitea/index.ts | 6 +++--- modules/gitea/internal/seatgit/whole_integration_test.go | 6 +++--- modules/gitea/module.json | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/modules/gitea/index.ts b/modules/gitea/index.ts index 9d80727cd..61792f30f 100644 --- a/modules/gitea/index.ts +++ b/modules/gitea/index.ts @@ -460,9 +460,9 @@ if (gitea) { await failedSteps.lost(heldHeads.replace(read.names, read.branches, read.pulls)); }; // The runner's step-ended is said on step-ended... (the seat's definition); a source-only - // check's names the registration's key source.. as its module (design 47 H2 §815, H-A-408), so every - // subject under step-ended is heard. - const stepEnded = `${RUNNER}.${RUNNER_EVENTS.stepEnded}.#`; + // check's module is the registration's key source.. (design 47 H2 §815, H-A-408), one subject part as + // a key part escapes it (mesh-sdk keys.EscapeName), and its body names it whole. + const stepEnded = `${RUNNER}.${RUNNER_EVENTS.stepEnded}.*.*.*`; void bindUntilHeld(stepEnded, () => on(stepEnded, (event) => failedSteps.stepEnded(event.body))); // A branch's head is read from the forge, never taken from the event (the seat's definition of branch-moved). void bindUntilHeld(`${GIT}.${GIT_EVENTS.branchMoved}`, () => on(`${GIT}.${GIT_EVENTS.branchMoved}`, async (event) => { diff --git a/modules/gitea/internal/seatgit/whole_integration_test.go b/modules/gitea/internal/seatgit/whole_integration_test.go index f29ed9d47..eaa20face 100644 --- a/modules/gitea/internal/seatgit/whole_integration_test.go +++ b/modules/gitea/internal/seatgit/whole_integration_test.go @@ -462,7 +462,7 @@ func TestTheSeatGitWholeOverARealBus(t *testing.T) { until(t, "the watched heads read after the registrations", func() bool { return f.asked("GET /api/v1/repos/novox/app/branches") }) // A failed required step at a head's fingerprint refuses that head (§735, test 114), heard on the module's consumer. - until(t, "the runner's step-ended followed", func() bool { return rt.subscribed("mesh-pipeline-runner.step-ended.#") }) + until(t, "the runner's step-ended followed", func() bool { return rt.subscribed("mesh-pipeline-runner.step-ended.*.*.*") }) rt.ask("mesh/event", map[string]any{"envelope": map[string]any{"key": "mesh-pipeline-runner.step-ended.app.main." + X, "node": "test", "body": map[string]any{"v": 1, "pipeline": "main.4", "module": "app", "branch": "main", "commit": X, "fingerprint": "fp-x", "at": "2026-11-20T03:00:00Z", "result": map[string]any{"step": "unit", "status": "required", "outcome": "failed"}}}}) @@ -478,8 +478,8 @@ func TestTheSeatGitWholeOverARealBus(t *testing.T) { // An hq pull request whose Go checks fail is refused by merge-pull naming the failed step (H2 §815): the runner's // step-ended of a source-only check names the registration's key as its module and the commit's root tree hash as its - // fingerprint (H-A-408), judged as a module's. - rt.ask("mesh/event", map[string]any{"envelope": map[string]any{"key": "mesh-pipeline-runner.step-ended.source.novox.hq.main." + H, "node": "test", + // fingerprint (H-A-408), escaped as one part of its subject, judged as a module's. + rt.ask("mesh/event", map[string]any{"envelope": map[string]any{"key": "mesh-pipeline-runner.step-ended.source=2enovox=2ehq.main." + H, "node": "test", "body": map[string]any{"v": 1, "pipeline": "main.5", "module": "source.novox.hq", "branch": "main", "commit": H, "fingerprint": strings.Repeat("7", 40), "at": "2026-11-22T01:00:00Z", "result": map[string]any{"step": "unit", "status": "required", "outcome": "failed"}}}}) hq := rt.mergePull(map[string]any{"repository": "novox/hq", "number": 11, "head": H}) diff --git a/modules/gitea/module.json b/modules/gitea/module.json index 722686476..6d490aedc 100644 --- a/modules/gitea/module.json +++ b/modules/gitea/module.json @@ -683,7 +683,7 @@ "consumes": [ "mesh-catalog.catalog-revision", "mesh-catalog.registration-changed", - "mesh-pipeline-runner.step-ended.#", + "mesh-pipeline-runner.step-ended.*.*.*", "git.branch-moved", "git.pull-updated", "git.pull-closed", -- 2.54.0 From c63b4ae31d65594988b81468fe2e73c4bb04fbe0 Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 22 Nov 2026 01:07:45 +0100 Subject: [PATCH 30/54] BM2: a test that cannot run says NOT RUN; the level's bound and the install archive reach the tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Go build module's level result carries not-run (the SDK reader's NOT RUN skips), its tests get MESH_TEST_LEVEL_BOUND, the level's bound, from which mesh-sdk's helpers take the setup bound; a test call naming an install archive reads it from the artifact store by its reference and unpacks it into the workspace's tree before go test. check-needs is read from the module's manifest, on the Go artifact whose path holds the package, and the marker file goes: the Node build module's manifest declares it on its bundle. Builds: design 47 H2 §800; H-A-294, H-A-350 DMC-6, H-A-368, H-A-399 (1) (2) (3) --- .../mesh-build-go/cmd/mesh-build-go/builds.go | 67 +++++++++++++++++++ .../cmd/mesh-build-go/builds_test.go | 27 ++++++++ .../mesh-build-go/cmd/mesh-build-go/digest.go | 67 ++++++++++++------- .../cmd/mesh-build-go/digest_test.go | 9 ++- .../mesh-build-go/cmd/mesh-build-go/levels.go | 5 ++ .../mesh-build-go/cmd/mesh-build-go/test.go | 24 ++++++- .../cmd/mesh-build-go/test_test.go | 13 +++- .../mesh-build-go/testdata/captured/README | 3 + .../testdata/captured/notrun.json | 13 ++++ .../testdata/repos/notrun/go.mod | 3 + .../testdata/repos/notrun/npm/npm.go | 5 ++ .../testdata/repos/notrun/npm/npm_test.go | 13 ++++ .../cmd/mesh-build-node/check-needs | 3 - modules/mesh-build-node/module.json | 3 + 14 files changed, 220 insertions(+), 35 deletions(-) create mode 100644 modules/mesh-build-go/cmd/mesh-build-go/testdata/captured/notrun.json create mode 100644 modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/notrun/go.mod create mode 100644 modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/notrun/npm/npm.go create mode 100644 modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/notrun/npm/npm_test.go delete mode 100644 modules/mesh-build-node/cmd/mesh-build-node/check-needs diff --git a/modules/mesh-build-go/cmd/mesh-build-go/builds.go b/modules/mesh-build-go/cmd/mesh-build-go/builds.go index 200b01c75..588ac2c1f 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/builds.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/builds.go @@ -10,13 +10,18 @@ import ( "archive/zip" "bytes" "context" + "crypto/sha256" + "encoding/hex" "fmt" "io" + "net/http" "os" "path" "path/filepath" "strings" + artifactref "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/containerimageregistryclient/reference" + imageseat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/containerimageregistry" sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/steprun" ) @@ -108,3 +113,65 @@ func unzipBuild(data []byte, dir string) error { } return nil } + +// **A step's install archive, unpacked into the workspace** (design 47 H2 §800; H-A-399 (2)): an integration step whose +// module's Go artifact needs node-build-node names the archive the Node build module's install published to the artifact +// store, as its reference (`oci-artifact-registry//@`); the level's tests run with its tree +// laid into the workspace's tree. It is read from the store by its digest, as a pinned archive is fetched by its pin: +// bytes that are not the digest's are refused, and a store that does not answer is a transport fault (§799). +var installForms = artifactref.Seats{Image: imageseat.Seat, Object: ArtifactSeat} + +// layInstall fetches the archive reference names from the artifact store and unpacks it into tree. +func (m *Module) layInstall(ctx context.Context, workspace, tree, reference string) error { + where, kept := installForms.InArtifactStore(reference) + _, digest, _ := strings.Cut(where, "/blobs/") + if !kept || !strings.HasPrefix(digest, "sha256:") { + return fmt.Errorf("the install archive %q is no reference of an archive in the artifact store", reference) + } + base, err := steprun.StoreURL(m.ArtifactStoreFile) + if err != nil { + return fmt.Errorf("the install archive %s is read from the artifact store: %w", reference, err) + } + request, err := http.NewRequestWithContext(ctx, http.MethodGet, strings.TrimRight(base, "/")+"/v2/"+where, nil) + if err != nil { + return err + } + response, err := m.httpClient().Do(request) + if err != nil { + return steprun.Transport(fmt.Errorf("cannot fetch the install archive %s: %w", reference, err)) + } + defer response.Body.Close() + if response.StatusCode != http.StatusOK { + return fmt.Errorf("the artifact store answered %s for the install archive %s", response.Status, reference) + } + dir := filepath.Join(m.Work, workspace, "upstream") + if err := os.MkdirAll(dir, 0o755); err != nil { + return err + } + part, err := os.CreateTemp(dir, "install.*.part") + if err != nil { + return err + } + defer os.Remove(part.Name()) + hash := sha256.New() + n, err := io.Copy(io.MultiWriter(part, hash), io.LimitReader(response.Body, maxUpstream+1)) + if cerr := part.Close(); err == nil { + err = cerr + } + if err != nil { + return steprun.Transport(fmt.Errorf("fetching the install archive %s: %w", reference, err)) + } + if n > maxUpstream { + return fmt.Errorf("the install archive %s is larger than %d bytes, more than a machine unpacks", reference, maxUpstream) + } + if got := "sha256:" + hex.EncodeToString(hash.Sum(nil)); got != digest { + return fmt.Errorf("the install archive %s read as %s, not its digest", reference, got) + } + if err := unpackUpstream(part.Name(), tree, ""); err != nil { + return fmt.Errorf("the install archive %s: %w", reference, err) + } + if m.Say != nil { + m.Say("laid the install archive %s into the workspace %s", reference, workspace) + } + return nil +} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/builds_test.go b/modules/mesh-build-go/cmd/mesh-build-go/builds_test.go index bfc578415..3e09bd1ba 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/builds_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/builds_test.go @@ -14,6 +14,7 @@ import ( "testing" sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/steprun" ) // Design 47 G §465, §373 (BG7): a fetch naming builds reads each member from the artifact store by its digest, checks the @@ -28,9 +29,21 @@ func TestAFetchLaysAGroupMembersBuildFromTheArtifactStore(t *testing.T) { zw.Close() sum := sha256.Sum256(zbuf.Bytes()) digest := "sha256:" + hex.EncodeToString(sum[:]) + // Design 47 H2 §800: an install archive, as the Node build module packs it, its paths from the workspace's tree. + npm := t.TempDir() + os.MkdirAll(filepath.Join(npm, "modules", "gitea", "node_modules", "x"), 0o755) + os.WriteFile(filepath.Join(npm, "modules", "gitea", "node_modules", "x", "index.js"), []byte("module.exports = 1\n"), 0o644) + install, installDigest, err := steprun.Pack(npm) + if err != nil { + t.Fatal(err) + } var asked []string store := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) { asked = append(asked, r.URL.Path) + if r.URL.Path == "/v2/gitea/node-modules/blobs/"+installDigest { + rw.Write(install) + return + } if r.URL.Path != "/v2/lib/pkg/blobs/"+digest { http.NotFound(rw, r) return @@ -68,7 +81,21 @@ func TestAFetchLaysAGroupMembersBuildFromTheArtifactStore(t *testing.T) { t.Errorf("member %+v: error %v", c, err) } } + // Design 47 H2 §800: the step's install archive is read from the store by its reference and unpacked into the tree; a + // reference whose bytes are not its digest is refused, and a store that does not answer is a transport fault (§799). + if err := m.layInstall(context.Background(), "ws1", filepath.Join(root, "tree"), "oci-artifact-registry/gitea/node-modules@"+installDigest); err != nil { + t.Fatal(err) + } + if b, err := os.ReadFile(filepath.Join(root, "tree", "modules", "gitea", "node_modules", "x", "index.js")); err != nil || string(b) != "module.exports = 1\n" { + t.Errorf("the install archive was not unpacked into the tree: %q, %v", b, err) + } + if err := m.layInstall(context.Background(), "ws1", filepath.Join(root, "tree"), "oci-artifact-registry/gitea/node-modules@"+digest); err == nil { + t.Error("an install archive whose bytes are not its digest was laid") + } store.Close() + if err := m.layInstall(context.Background(), "ws1", filepath.Join(root, "tree"), "oci-artifact-registry/gitea/node-modules@"+installDigest); !sdkstep.IsPassing(err) { + t.Errorf("an install archive from a store that does not answer: %v, want a transport fault", err) + } if err := m.layBuilds(context.Background(), root, []string{"."}, []sdkstep.FetchBuild{member}); err == nil || !strings.Contains(err.Error(), "example.test/lib") { t.Errorf("a store that does not answer: %v", err) } diff --git a/modules/mesh-build-go/cmd/mesh-build-go/digest.go b/modules/mesh-build-go/cmd/mesh-build-go/digest.go index 9aa9c4616..ed77b326d 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/digest.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/digest.go @@ -18,12 +18,14 @@ import ( "os" "path" "path/filepath" + "slices" "sort" "strconv" "strings" sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep" trunk "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/trunk" + "git.novox.be/novox/mesh-sdk/go/catalogue" ) // DigestVersion is the form of the digest's text: changed whenever what the digest hashes changes in kind. @@ -56,31 +58,44 @@ const namesVariable = "MESH_NODE_NAMES" // from the Go test log with each verdict, replaces the lists (issue 1936). const ReadsOutsideMarker = "reads-outside" -// NeedsMarker is the name of a file in a package's directory that names, one per line, what its integration tests need -// of the node they run on (issue 890; mesh-delivery-sdk steprun.AskNeeds): `node-build-node` for tests that read the Node build -// module's installed toolchain. The digest reports it (DigestResult.Needs); a check ask leaves such a package untested on -// a node that lacks it, and the checker asks it again of a node that has it. A marker beside the tests, as -// ReadsOutsideMarker is, and not a manifest field: it is a property of the package's tests, read only where they are -// run, and the manifest describes the module on the machines. The unit level is never given anything of the node (ADR -// 0533 §1), so a marker binds the integration level alone. Blank lines and lines starting with # are not read. -const NeedsMarker = "check-needs" - -// readNeeds is what the marker in dir names; nil when there is none. -func readNeeds(dir string) ([]string, error) { - raw, err := os.ReadFile(filepath.Join(dir, NeedsMarker)) - if errors.Is(err, fs.ErrNotExist) { - return nil, nil - } - if err != nil { - return nil, err - } - var needs []string - for _, l := range strings.Split(string(raw), "\n") { - if l = strings.TrimSpace(l); l != "" && !strings.HasPrefix(l, "#") { - needs = append(needs, l) +// readNeeds is what a package's integration tests need of the node they run on (issue 890; mesh-delivery-sdk +// steprun.AskNeeds): the seats its module's manifest names in `check-needs` on each Go artifact whose path holds the +// package's directory (design 47 H2 §800; H-A-350 DMC-6, H-A-399 (3)), `node-build-node` for tests that read the Node +// build module's installed toolchain. The manifest is the nearest module.json in dir or above it within the tree; nil +// when there is none, or none of its Go artifacts holds dir. The digest reports it (DigestResult.Needs); a check ask +// leaves such a package untested on a node that lacks it, and the checker asks it again of a node that has it. The unit +// level is never given anything of the node (ADR 0533 §1), so the field binds the integration level alone. +func readNeeds(tree, dir string) ([]string, error) { + for at := dir; within(tree, at); at = filepath.Dir(at) { + raw, err := os.ReadFile(filepath.Join(at, "module.json")) + if errors.Is(err, fs.ErrNotExist) { + if at == tree { + break + } + continue } + if err != nil { + return nil, err + } + manifest, err := catalogue.Decode(raw) + if err != nil { + return nil, fmt.Errorf("the manifest %s: %w", filepath.Join(at, "module.json"), err) + } + var needs []string + if manifest.Build != nil { + for _, a := range manifest.Build.Artifacts { + if a.Language == "go" && len(a.CheckNeeds) > 0 && within(filepath.Join(at, filepath.FromSlash(a.From)), dir) { + for _, n := range a.CheckNeeds { + if !slices.Contains(needs, n) { + needs = append(needs, n) + } + } + } + } + } + return needs, nil } - return needs, nil + return nil, nil } // integrationTags are the build tags of the integration level, its run and its digest alike: integration, and isolation @@ -149,7 +164,7 @@ type DigestResult struct { // of the files markers name, which several packages share (the catalogue's manifests). neverKept map[string]string hashedFiles map[string]string - // Needs are the packages whose tests at this level need something of the node, as their NeedsMarker names it. + // Needs are the packages whose tests at this level need something of the node, as their module's manifest names them (check-needs). Needs []sdkstep.PackageNeeds `json:"needs,omitempty"` // JudgedBy is the program that ran the digest (ADR 0535 §5): "installed", or "candidate " under the // runner workspace. @@ -340,9 +355,9 @@ func digestModule(res *DigestResult, pkgs map[string]*goPackage, tree string, he return fmt.Errorf("go list gave no test binary for %s", path) } if needs { - n, err := readNeeds(root.Dir) + n, err := readNeeds(tree, root.Dir) if err != nil { - return fmt.Errorf("%s: its %s: %w", path, NeedsMarker, err) + return fmt.Errorf("%s: its check-needs: %w", path, err) } if len(n) > 0 { res.Needs = append(res.Needs, sdkstep.PackageNeeds{Package: path, Needs: n}) diff --git a/modules/mesh-build-go/cmd/mesh-build-go/digest_test.go b/modules/mesh-build-go/cmd/mesh-build-go/digest_test.go index df2fbb19d..ecca63a39 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/digest_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/digest_test.go @@ -353,9 +353,12 @@ func TestAListedReadsOutsideMarkerKeepsTheDigestOfOnlyWhatItLists(t *testing.T) before = after } - // Issue 890/AC3: a package whose directory carries the needs marker is named with its needs at the integration level, - // comments and blank lines not read, and at the unit level not at all. - edit(t, tree, "a/"+NeedsMarker, "# its integration tests copy the installed Node toolchain\n\nnode-build-node\n") + // Issue 890/AC3, design 47 H2 §800 (H-A-350 DMC-6): a package a Go artifact of its module's manifest holds is named + // with the artifact's check-needs at the integration level, a package no such artifact holds (c) not, and at the + // unit level not at all. + edit(t, tree, "module.json", `{"module":"digest","build":{"artifacts":[`+ + `{"name":"tools","kind":"bundle","language":"go","from":"a","check-needs":["node-build-node"]},`+ + `{"name":"other","kind":"bundle","language":"go","from":"c"}]}}`) if n := digests(t, m, LevelIntegration).Needs; len(n) != 1 || n[0].Package != a || strings.Join(n[0].Needs, ",") != "node-build-node" { t.Errorf("the integration level's needs = %+v, want %s needing node-build-node", n, a) } diff --git a/modules/mesh-build-go/cmd/mesh-build-go/levels.go b/modules/mesh-build-go/cmd/mesh-build-go/levels.go index 1233b5d0c..2d702ec0d 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/levels.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/levels.go @@ -59,6 +59,10 @@ type TestResult struct { SetupBound float64 `json:"setup_bound,omitempty"` Slowest []TestTime `json:"slowest,omitempty"` Failed []Failure `json:"failed,omitempty"` + // NotRun are the tests that said they did not run (`t.Skip("NOT RUN: ")`), each with its reason (design 47 H2 + // §800, H-A-294): counted apart from passed and failed, they leave the level passing, listed here, where the delivery + // SDK's test step reads them into the step's end. + NotRun []sdkstep.NotRun `json:"not-run,omitempty"` // Store and Bus are what the integration level ran against: the postgres major version and the nats-server // version (ADR 0470 §4). // OverBound says a bound was kept no longer, and OverWhich which: setup, run or both. The level's verdict, Passed, is @@ -86,6 +90,7 @@ func readGoTest(res *TestResult, dir, out string) (noise []string, packages int) res.Tests += read.Tests res.Skipped += read.Skipped res.Failed = append(res.Failed, read.Failed...) + res.NotRun = append(res.NotRun, read.NotRun...) res.Packages = append(res.Packages, read.Packages...) res.Slowest = append(res.Slowest, read.Times...) res.Elapsed = max(res.Elapsed, read.Elapsed) diff --git a/modules/mesh-build-go/cmd/mesh-build-go/test.go b/modules/mesh-build-go/cmd/mesh-build-go/test.go index e76b9b2c5..0b7a0e0fc 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/test.go @@ -42,7 +42,8 @@ func (m *Module) testTool() StepTool { "run": map[string]any{"type": "string", "description": "optional go test -run pattern: only the tests it matches run (a try's tests)"}, sdkstep.NodeNamesArg: map[string]any{"type": "array", "items": map[string]any{"type": "string"}, "description": "optional, the names of the mesh's nodes: the tests get them comma-separated in " + sdkstep.NodeNamesEnv + " (a check that keeps node names out of the repository reads them there)"}, - "runner": runnerProperty, + "runner": runnerProperty, + "archive": map[string]any{"type": "string", "description": "optional, at the integration level: the reference of the install archive in the artifact store (oci-artifact-registry//@), unpacked into the workspace's tree before the tests run (a package whose module's Go artifact names check-needs)"}, }, "required": []string{"workspace", "level"}, }, @@ -85,6 +86,12 @@ func (m *Module) test(ctx context.Context, args map[string]any) (any, error) { if err != nil { return nil, err } + if archive, _ := args["archive"].(string); archive != "" { + // Design 47 H2 §800: the step's install archive is laid into the tree before go test. + if err := m.layInstall(ctx, workspace, tree, archive); err != nil { + return nil, err + } + } var res *TestResult switch { case runner == RunnerWorkspace && (level == LevelUnit || level == LevelIntegration): @@ -312,7 +319,7 @@ func (m *Module) runLevel(ctx context.Context, tree string, o levelOpts) (*TestR args = append(args, byDir[d]...) dr := &runs[i] dr.res = &TestResult{} - dr.r = run(ctx, append(m.goEnvFor(filepath.Join(tree, d)), o.env...), "go", args...) + dr.r = run(ctx, append(append(m.goEnvFor(filepath.Join(tree, d)), o.env...), levelBoundEnv(o.bounds)...), "go", args...) dr.noise, dr.packages = readGoTest(dr.res, d, dr.r.Stdout) }() } @@ -329,6 +336,7 @@ func (m *Module) runLevel(ctx context.Context, tree string, o levelOpts) (*TestR res.Tests += dr.res.Tests res.Skipped += dr.res.Skipped res.Failed = append(res.Failed, dr.res.Failed...) + res.NotRun = append(res.NotRun, dr.res.NotRun...) res.Packages = append(res.Packages, dr.res.Packages...) res.Slowest = append(res.Slowest, dr.res.Slowest...) if dr.res.Elapsed > res.Elapsed { @@ -342,6 +350,18 @@ func (m *Module) runLevel(ctx context.Context, tree string, o levelOpts) (*TestR return res, nil } +// LevelBoundEnv is the word each level's tests get its bound in, as a duration ("10m0s"; design 47 H2 §800): mesh-sdk's +// test helpers take a setup bound of half of it, never a value in the test (testbus.SetupBound). +const LevelBoundEnv = "MESH_TEST_LEVEL_BOUND" + +// levelBoundEnv is the level's run bound as LevelBoundEnv, nothing when the level has none. +func levelBoundEnv(b Bounds) []string { + if b.Run <= 0 { + return nil + } + return []string{LevelBoundEnv + "=" + b.Run.String()} +} + // runAnswer is what a step runner answers. type runAnswer = forgekey.Ran diff --git a/modules/mesh-build-go/cmd/mesh-build-go/test_test.go b/modules/mesh-build-go/cmd/mesh-build-go/test_test.go index 779b4dba5..afbded703 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/test_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/test_test.go @@ -5,6 +5,7 @@ import ( "encoding/json" "os" "path/filepath" + "slices" "strings" "testing" "time" @@ -101,6 +102,14 @@ func TestUnitLevelNamesTheFailingTestAndFailsTheLevel(t *testing.T) { if !good || res.Tests != 2 { t.Errorf("packages = %+v, tests = %d; want the good package passed with its test, 2 tests", res.Packages, res.Tests) } + + // Design 47 H2 §800: a test that says NOT RUN leaves its level passing, listed in the result's not-run with its reason. + m, states, ws, _ = workspaceFor(t, "notrun") + res, st = runUnit(t, m, states, ws) + want := []sdkstep.NotRun{{Test: "fixture/notrun/npm TestWithNpm", Reason: "npm is not on this node"}} + if st.State != sdkstep.StepPassed || !res.Passed || !slices.Equal(res.NotRun, want) { + t.Errorf("step %s, passed %v, not run %+v; want the level passing with not run %+v", st.State, res.Passed, res.NotRun, want) + } } // Issue 1921: a fixture with a failing test in each of two packages: one level's run names both, each with its own @@ -271,7 +280,8 @@ func TestUnitLevelRunsGoTestJsonInTheModulesDirectoryWithTheStepsEnvironment(t * if line != want { t.Errorf("command = %q, want %q", line, want) } - for _, w := range []string{"GOTOOLCHAIN=local", "GOFLAGS=-mod=mod", "GOMODCACHE=" + filepath.Join(m.Cache, "gomod")} { + // Design 47 H2 §800: the level's bound reaches the tests as MESH_TEST_LEVEL_BOUND, a duration. + for _, w := range []string{"GOTOOLCHAIN=local", "GOFLAGS=-mod=mod", "GOMODCACHE=" + filepath.Join(m.Cache, "gomod"), "MESH_TEST_LEVEL_BOUND=" + m.unitBound().String()} { if !strings.Contains(env, w) { t.Errorf("environment %s lacks %s", env, w) } @@ -310,6 +320,7 @@ func TestATestOfAnUnknownWorkspaceOrAnotherLevelIsRefused(t *testing.T) { {"workspace": "ws1", "level": "stress"}, {"workspace": "ws1", "level": "unit", "packages": []any{"-run=x"}}, {"workspace": "ws1", "level": "unit", "packages": []any{"../x"}}, + {"workspace": "ws1", "level": "integration", "archive": "gitea/node-modules"}, // §800: an install archive the store does not name } { run(args) if st := states.waitEnd(t); st.State != sdkstep.StepFailed { diff --git a/modules/mesh-build-go/cmd/mesh-build-go/testdata/captured/README b/modules/mesh-build-go/cmd/mesh-build-go/testdata/captured/README index c2bebf153..f7331934c 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/testdata/captured/README +++ b/modules/mesh-build-go/cmd/mesh-build-go/testdata/captured/README @@ -29,3 +29,6 @@ other digests; the tests therefore compare the stream always and the digest only The go-test-json-*.jsonl captures of issue 1901 (2026-10-23 and 2026-10-25) moved to mesh-sdk's conformance kit (go/buildstep/conformance/captured/go, with their description) on 2026-10-26, with the reader they test: the build module reads go test -json through buildstep.ReadGoTest and runs the kit (kit_test.go). + +notrun.json was captured the same way on 2026-11-22 (go1.27.3) for design 47 H2 §800 in repos/notrun: a passing test +beside one that skips with "NOT RUN: ". diff --git a/modules/mesh-build-go/cmd/mesh-build-go/testdata/captured/notrun.json b/modules/mesh-build-go/cmd/mesh-build-go/testdata/captured/notrun.json new file mode 100644 index 000000000..fcf6e370e --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/testdata/captured/notrun.json @@ -0,0 +1,13 @@ +{"Time":"2026-11-22T01:01:16.828741416+01:00","Action":"start","Package":"fixture/notrun/npm"} +{"Time":"2026-11-22T01:01:16.830715274+01:00","Action":"run","Package":"fixture/notrun/npm","Test":"TestTwo"} +{"Time":"2026-11-22T01:01:16.830773544+01:00","Action":"output","Package":"fixture/notrun/npm","Test":"TestTwo","Output":"=== RUN TestTwo\n","OutputType":"frame"} +{"Time":"2026-11-22T01:01:16.830797205+01:00","Action":"output","Package":"fixture/notrun/npm","Test":"TestTwo","Output":"--- PASS: TestTwo (0.00s)\n","OutputType":"frame"} +{"Time":"2026-11-22T01:01:16.830801905+01:00","Action":"pass","Package":"fixture/notrun/npm","Test":"TestTwo","Elapsed":0} +{"Time":"2026-11-22T01:01:16.830807995+01:00","Action":"run","Package":"fixture/notrun/npm","Test":"TestWithNpm"} +{"Time":"2026-11-22T01:01:16.830811005+01:00","Action":"output","Package":"fixture/notrun/npm","Test":"TestWithNpm","Output":"=== RUN TestWithNpm\n","OutputType":"frame"} +{"Time":"2026-11-22T01:01:16.830814835+01:00","Action":"output","Package":"fixture/notrun/npm","Test":"TestWithNpm","Output":" npm_test.go:12: NOT RUN: npm is not on this node\n"} +{"Time":"2026-11-22T01:01:16.830822885+01:00","Action":"output","Package":"fixture/notrun/npm","Test":"TestWithNpm","Output":"--- SKIP: TestWithNpm (0.00s)\n","OutputType":"frame"} +{"Time":"2026-11-22T01:01:16.830826595+01:00","Action":"skip","Package":"fixture/notrun/npm","Test":"TestWithNpm","Elapsed":0} +{"Time":"2026-11-22T01:01:16.830830225+01:00","Action":"output","Package":"fixture/notrun/npm","Output":"PASS\n","OutputType":"frame"} +{"Time":"2026-11-22T01:01:16.831057928+01:00","Action":"output","Package":"fixture/notrun/npm","Output":"ok \tfixture/notrun/npm\t0.002s\n"} +{"Time":"2026-11-22T01:01:16.831077789+01:00","Action":"pass","Package":"fixture/notrun/npm","Elapsed":0.002} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/notrun/go.mod b/modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/notrun/go.mod new file mode 100644 index 000000000..d8e47ef9a --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/notrun/go.mod @@ -0,0 +1,3 @@ +module fixture/notrun + +go 1.25.0 diff --git a/modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/notrun/npm/npm.go b/modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/notrun/npm/npm.go new file mode 100644 index 000000000..187f97ac0 --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/notrun/npm/npm.go @@ -0,0 +1,5 @@ +// Package npm is a fixture whose one test lacks the tooling its step gives. +package npm + +// Two is two. +func Two() int { return 2 } diff --git a/modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/notrun/npm/npm_test.go b/modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/notrun/npm/npm_test.go new file mode 100644 index 000000000..faef1086e --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/notrun/npm/npm_test.go @@ -0,0 +1,13 @@ +package npm + +import "testing" + +func TestTwo(t *testing.T) { + if Two() != 2 { + t.Errorf("Two() = %d, want 2", Two()) + } +} + +func TestWithNpm(t *testing.T) { + t.Skip("NOT RUN: npm is not on this node") +} diff --git a/modules/mesh-build-node/cmd/mesh-build-node/check-needs b/modules/mesh-build-node/cmd/mesh-build-node/check-needs deleted file mode 100644 index 7d69db718..000000000 --- a/modules/mesh-build-node/cmd/mesh-build-node/check-needs +++ /dev/null @@ -1,3 +0,0 @@ -# Its integration tests copy the Node toolchain the Node build module installed on this node and run node and npm -# (realMachine, build_integration_test.go): they run only where the node holds node-build-node (issue 890). -node-build-node diff --git a/modules/mesh-build-node/module.json b/modules/mesh-build-node/module.json index d14d35aab..994095fa2 100644 --- a/modules/mesh-build-node/module.json +++ b/modules/mesh-build-node/module.json @@ -211,6 +211,9 @@ "binary": "mesh-build-node", "loads": [ "mesh-build-node" + ], + "check-needs": [ + "node-build-node" ] }, { -- 2.54.0 From 751919186161518b7462105d97b3729581b036db Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 22 Nov 2026 01:10:54 +0100 Subject: [PATCH 31/54] BM5: the steps read tools list an ask's steps under its asker, so they are never empty while asks run MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Each build module's steps read tool listed only `.` keys and so skipped every ask's step, kept as `..`: it now reads every key of its node, newest id first. The Go build module's test of capacity 0 also holds the seat's current at form 2: no ask taken, why not capacity 0, then paused. Builds: design 47 H2 §801; H-A-399 (1) --- .../cmd/mesh-build-go/asks_test.go | 23 +++++++++++++++++++ .../mesh-build-go/cmd/mesh-build-go/read.go | 12 +++++----- .../cmd/mesh-build-go/read_test.go | 17 ++++++++++---- .../cmd/mesh-build-image/read.go | 12 +++++----- .../cmd/mesh-build-image/read_test.go | 18 +++++++++++---- .../cmd/mesh-build-node/read.go | 12 +++++----- .../cmd/mesh-build-node/read_test.go | 17 ++++++++++---- 7 files changed, 80 insertions(+), 31 deletions(-) diff --git a/modules/mesh-build-go/cmd/mesh-build-go/asks_test.go b/modules/mesh-build-go/cmd/mesh-build-go/asks_test.go index 485f0b1df..88542ab6b 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/asks_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/asks_test.go @@ -100,6 +100,29 @@ func TestAtCapacityZeroNothingIsPulledAndTheToolsAnswer(t *testing.T) { if src.Waiting(sdkstep.AskDelivery) != 1 || len(calls) != 0 { t.Fatalf("a node at capacity 0 pulled: waiting %d, calls %v", src.Waiting(sdkstep.AskDelivery), calls) } + // Design 47 H2 §801: the seat's current says the holder takes no ask and why: capacity 0, and once paused, paused. + current := func() string { + for _, tool := range m.SeatTools(p) { + if tool.Name == Seat+".current" { + out, err := tool.Run(map[string]any{}) + if err != nil { + t.Fatal(err) + } + return string(mustJSON(out)) + } + } + t.Fatal("no current verb") + return "" + } + if got := current(); !strings.Contains(got, `"takes":false`) || !strings.Contains(got, `"why-not":"capacity 0"`) { + t.Errorf("current at capacity 0 = %s", got) + } + if err := p.Pause("test@node"); err != nil { + t.Fatal(err) + } + if got := current(); !strings.Contains(got, `"why-not":"paused"`) { + t.Errorf("current paused = %s", got) + } for _, tool := range m.Tools(steps) { if tool.Name == "build_go_fetch" { out, err := tool.Run(map[string]any{"repository": "novox/x", "commit": askCommit}) diff --git a/modules/mesh-build-go/cmd/mesh-build-go/read.go b/modules/mesh-build-go/cmd/mesh-build-go/read.go index fc53441a6..b036c8b3b 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/read.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/read.go @@ -147,15 +147,15 @@ func (m *Module) RecentSteps(limit int) (lines []StepLine, cut bool, err error) if err != nil { return nil, false, err } - own := keys[:0:0] + // Every key of this node's steps, an ask's under its asker (`..`) as well as `.` (design 47 + // H2 §801): the tool is never empty while asks run. Ordered by the id, newest first. + own, ids := keys[:0:0], map[string]string{} for _, k := range keys { - if strings.HasPrefix(k, m.Node+".") { - if _, node, _, err := sdkstep.ParseStepKey(k); err == nil && node == m.Node { - own = append(own, k) - } + if _, node, id, err := sdkstep.ParseStepKey(k); err == nil && node == m.Node { + own, ids[k] = append(own, k), id } } - sort.Sort(sort.Reverse(sort.StringSlice(own))) + sort.SliceStable(own, func(i, j int) bool { return ids[own[i]] > ids[own[j]] }) if n := min(max(limit*4, 100), MaxStateReads); len(own) > n { own, cut = own[:n], true } diff --git a/modules/mesh-build-go/cmd/mesh-build-go/read_test.go b/modules/mesh-build-go/cmd/mesh-build-go/read_test.go index 880fa8962..8b5d2776a 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/read_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/read_test.go @@ -157,10 +157,12 @@ func TestRecentStepsReadsOnlyTheNodesKeysInParallelAndBounded(t *testing.T) { reads int64 steps int cut bool + asked bool // design 47 H2 §801: the newest step is an ask's, keyed under its asker }{ - {20, 0, 20, 10, false}, - {600, 0, 100, 10, true}, - {1500, 300, MaxStateReads, 300, true}, + {20, 0, 20, 10, false, false}, + {600, 0, 100, 10, true, false}, + {1500, 300, MaxStateReads, 300, true, false}, + {19, 0, 20, 10, false, true}, } { var reads, inFlight, maxFlight atomic.Int64 stored := map[string][]byte{} @@ -170,6 +172,10 @@ func TestRecentStepsReadsOnlyTheNodesKeysInParallelAndBounded(t *testing.T) { stored[n+".s"+fmt.Sprintf("%04d", i)] = b } } + if row.asked { + b, _ := sdkstep.EncodeStep(sdkstep.Step{ID: "s9999", Tool: "t", Node: "shanks", State: sdkstep.StepRunning, Started: t0.Add(time.Hour)}) + stored["mesh-pipeline-runner.shanks.s9999"] = b + } m := &Module{Node: "shanks"} m.ReadSteps = StateRead{ Keys: func() ([]string, error) { @@ -186,7 +192,7 @@ func TestRecentStepsReadsOnlyTheNodesKeysInParallelAndBounded(t *testing.T) { } time.Sleep(time.Millisecond) inFlight.Add(-1) - if !strings.HasPrefix(key, "shanks.") { + if _, node, _, err := sdkstep.ParseStepKey(key); err != nil || node != "shanks" { t.Errorf("read %s, a key of another node", key) } return stored[key], nil @@ -200,6 +206,9 @@ func TestRecentStepsReadsOnlyTheNodesKeysInParallelAndBounded(t *testing.T) { if row.cut && len(lines) > 0 && lines[0].ID != fmt.Sprintf("s%04d", row.perNode-1) { t.Errorf("%d a node: newest answered %s", row.perNode, lines[0].ID) } + if row.asked && (len(lines) == 0 || lines[0].ID != "s9999") { + t.Errorf("an ask's step under its asker is not the newest line: %+v", lines) + } if maxFlight.Load() < 2 { t.Errorf("%d a node: values read one at a time", row.perNode) } diff --git a/modules/mesh-build-image/cmd/mesh-build-image/read.go b/modules/mesh-build-image/cmd/mesh-build-image/read.go index d37522613..bd3402064 100644 --- a/modules/mesh-build-image/cmd/mesh-build-image/read.go +++ b/modules/mesh-build-image/cmd/mesh-build-image/read.go @@ -146,15 +146,15 @@ func (m *Module) RecentSteps(limit int) (lines []StepLine, cut bool, err error) if err != nil { return nil, false, err } - own := keys[:0:0] + // Every key of this node's steps, an ask's under its asker (`..`) as well as `.` (design 47 + // H2 §801): the tool is never empty while asks run. Ordered by the id, newest first. + own, ids := keys[:0:0], map[string]string{} for _, k := range keys { - if strings.HasPrefix(k, m.Node+".") { - if _, node, _, err := sdkstep.ParseStepKey(k); err == nil && node == m.Node { - own = append(own, k) - } + if _, node, id, err := sdkstep.ParseStepKey(k); err == nil && node == m.Node { + own, ids[k] = append(own, k), id } } - sort.Sort(sort.Reverse(sort.StringSlice(own))) + sort.SliceStable(own, func(i, j int) bool { return ids[own[i]] > ids[own[j]] }) if n := min(max(limit*4, 100), MaxStateReads); len(own) > n { own, cut = own[:n], true } diff --git a/modules/mesh-build-image/cmd/mesh-build-image/read_test.go b/modules/mesh-build-image/cmd/mesh-build-image/read_test.go index 9776771ce..338259e68 100644 --- a/modules/mesh-build-image/cmd/mesh-build-image/read_test.go +++ b/modules/mesh-build-image/cmd/mesh-build-image/read_test.go @@ -3,7 +3,6 @@ package main import ( "encoding/json" "fmt" - "strings" "sync/atomic" "testing" "time" @@ -58,10 +57,12 @@ func TestRecentStepsReadsOnlyTheNodesKeysInParallelAndBounded(t *testing.T) { reads int64 steps int cut bool + asked bool // design 47 H2 §801: the newest step is an ask's, keyed under its asker }{ - {20, 0, 20, 10, false}, - {600, 0, 100, 10, true}, - {1500, 300, MaxStateReads, 300, true}, + {20, 0, 20, 10, false, false}, + {600, 0, 100, 10, true, false}, + {1500, 300, MaxStateReads, 300, true, false}, + {19, 0, 20, 10, false, true}, } { var reads, inFlight, maxFlight atomic.Int64 stored := map[string][]byte{} @@ -71,6 +72,10 @@ func TestRecentStepsReadsOnlyTheNodesKeysInParallelAndBounded(t *testing.T) { stored[n+".s"+fmt.Sprintf("%04d", i)] = b } } + if row.asked { + b, _ := sdkstep.EncodeStep(sdkstep.Step{ID: "s9999", Tool: "t", Node: "shanks", State: sdkstep.StepRunning, Started: t0.Add(time.Hour)}) + stored["mesh-pipeline-runner.shanks.s9999"] = b + } m := &Module{Node: "shanks"} m.ReadSteps = StateRead{ Keys: func() ([]string, error) { @@ -87,7 +92,7 @@ func TestRecentStepsReadsOnlyTheNodesKeysInParallelAndBounded(t *testing.T) { } time.Sleep(time.Millisecond) inFlight.Add(-1) - if !strings.HasPrefix(key, "shanks.") { + if _, node, _, err := sdkstep.ParseStepKey(key); err != nil || node != "shanks" { t.Errorf("read %s, a key of another node", key) } return stored[key], nil @@ -101,6 +106,9 @@ func TestRecentStepsReadsOnlyTheNodesKeysInParallelAndBounded(t *testing.T) { if row.cut && len(lines) > 0 && lines[0].ID != fmt.Sprintf("s%04d", row.perNode-1) { t.Errorf("%d a node: newest answered %s", row.perNode, lines[0].ID) } + if row.asked && (len(lines) == 0 || lines[0].ID != "s9999") { + t.Errorf("an ask's step under its asker is not the newest line: %+v", lines) + } if maxFlight.Load() < 2 { t.Errorf("%d a node: values read one at a time", row.perNode) } diff --git a/modules/mesh-build-node/cmd/mesh-build-node/read.go b/modules/mesh-build-node/cmd/mesh-build-node/read.go index ec7b2d2a8..6b6a29dac 100644 --- a/modules/mesh-build-node/cmd/mesh-build-node/read.go +++ b/modules/mesh-build-node/cmd/mesh-build-node/read.go @@ -147,15 +147,15 @@ func (m *Module) RecentSteps(limit int) (lines []StepLine, cut bool, err error) if err != nil { return nil, false, err } - own := keys[:0:0] + // Every key of this node's steps, an ask's under its asker (`..`) as well as `.` (design 47 + // H2 §801): the tool is never empty while asks run. Ordered by the id, newest first. + own, ids := keys[:0:0], map[string]string{} for _, k := range keys { - if strings.HasPrefix(k, m.Node+".") { - if _, node, _, err := sdkstep.ParseStepKey(k); err == nil && node == m.Node { - own = append(own, k) - } + if _, node, id, err := sdkstep.ParseStepKey(k); err == nil && node == m.Node { + own, ids[k] = append(own, k), id } } - sort.Sort(sort.Reverse(sort.StringSlice(own))) + sort.SliceStable(own, func(i, j int) bool { return ids[own[i]] > ids[own[j]] }) if n := min(max(limit*4, 100), MaxStateReads); len(own) > n { own, cut = own[:n], true } diff --git a/modules/mesh-build-node/cmd/mesh-build-node/read_test.go b/modules/mesh-build-node/cmd/mesh-build-node/read_test.go index 70be8af51..ae61e13c0 100644 --- a/modules/mesh-build-node/cmd/mesh-build-node/read_test.go +++ b/modules/mesh-build-node/cmd/mesh-build-node/read_test.go @@ -157,10 +157,12 @@ func TestRecentStepsReadsOnlyTheNodesKeysInParallelAndBounded(t *testing.T) { reads int64 steps int cut bool + asked bool // design 47 H2 §801: the newest step is an ask's, keyed under its asker }{ - {20, 0, 20, 10, false}, - {600, 0, 100, 10, true}, - {1500, 300, MaxStateReads, 300, true}, + {20, 0, 20, 10, false, false}, + {600, 0, 100, 10, true, false}, + {1500, 300, MaxStateReads, 300, true, false}, + {19, 0, 20, 10, false, true}, } { var reads, inFlight, maxFlight atomic.Int64 stored := map[string][]byte{} @@ -170,6 +172,10 @@ func TestRecentStepsReadsOnlyTheNodesKeysInParallelAndBounded(t *testing.T) { stored[n+".s"+fmt.Sprintf("%04d", i)] = b } } + if row.asked { + b, _ := sdkstep.EncodeStep(sdkstep.Step{ID: "s9999", Tool: "t", Node: "shanks", State: sdkstep.StepRunning, Started: t0.Add(time.Hour)}) + stored["mesh-pipeline-runner.shanks.s9999"] = b + } m := &Module{Node: "shanks"} m.ReadSteps = StateRead{ Keys: func() ([]string, error) { @@ -186,7 +192,7 @@ func TestRecentStepsReadsOnlyTheNodesKeysInParallelAndBounded(t *testing.T) { } time.Sleep(time.Millisecond) inFlight.Add(-1) - if !strings.HasPrefix(key, "shanks.") { + if _, node, _, err := sdkstep.ParseStepKey(key); err != nil || node != "shanks" { t.Errorf("read %s, a key of another node", key) } return stored[key], nil @@ -200,6 +206,9 @@ func TestRecentStepsReadsOnlyTheNodesKeysInParallelAndBounded(t *testing.T) { if row.cut && len(lines) > 0 && lines[0].ID != fmt.Sprintf("s%04d", row.perNode-1) { t.Errorf("%d a node: newest answered %s", row.perNode, lines[0].ID) } + if row.asked && (len(lines) == 0 || lines[0].ID != "s9999") { + t.Errorf("an ask's step under its asker is not the newest line: %+v", lines) + } if maxFlight.Load() < 2 { t.Errorf("%d a node: values read one at a time", row.perNode) } -- 2.54.0 From 9afbd3575950e2fd0ce576d937e0350fb5890704 Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 22 Nov 2026 01:18:06 +0100 Subject: [PATCH 32/54] NP1: the NFS server and package manager modules' tests fold into one test per file, within 50 as the check counts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The check counts every test the integration level runs, untagged ones in: the NFS server module held 51 and the package manager module 121, over the budget of 50. Each folded file's tests run in order as rows of one test that keeps the first one's name and logs each case before it runs; the root helper's test sockets are numbered per call, as one test now holds several listeners, and the case that skips without pacman-conf runs last. Counts now 48 and 42. Builds: design 47 H2 §819; H-A-349 DQ-14 --- .../cmd/nfs-server/reconcile_test.go | 25 +++++++-- .../cmd/pacman-tools/aftercheck_test.go | 28 ++++++++-- modules/pacman/cmd/pacman-tools/dkms_test.go | 28 ++++++++-- .../cmd/pacman-tools/keeper_manifest_test.go | 31 ++++++++-- .../pacman/cmd/pacman-tools/keeper_test.go | 34 ++++++++--- .../pacman/cmd/pacman-tools/manifest_test.go | 32 +++++++++-- .../pacman/cmd/pacman-tools/pacman_test.go | 52 ++++++++++++----- .../pacman/cmd/pacman-tools/rollback_test.go | 28 ++++++++-- modules/pacman/cmd/pacman-tools/root_test.go | 56 ++++++++++++++----- .../pacman/cmd/pacman-tools/rounds_test.go | 31 ++++++++-- modules/pacman/cmd/pacman-tools/runs_test.go | 31 ++++++++-- .../cmd/pacman-tools/transaction_test.go | 40 ++++++++++--- .../cmd/pacman-tools/upgrade_health_test.go | 37 +++++++++--- modules/pacman/cmd/pacman-tools/view_test.go | 28 ++++++++-- 14 files changed, 382 insertions(+), 99 deletions(-) diff --git a/modules/nfs-server/cmd/nfs-server/reconcile_test.go b/modules/nfs-server/cmd/nfs-server/reconcile_test.go index d6be1ada7..94194c704 100644 --- a/modules/nfs-server/cmd/nfs-server/reconcile_test.go +++ b/modules/nfs-server/cmd/nfs-server/reconcile_test.go @@ -21,7 +21,7 @@ func countExportfs(ran []string) int { // and the kernel still exported the refused folder while the module said it was healthy. A failed reload // is tried again on the next pass, and while the kernel exports anything of the module's that the file no // longer carries, the module is unhealthy and says what. -func TestARefusalWhoseReloadFailedIsReloadedAgainAndIsUnhealthyUntilItIs(t *testing.T) { +func testARefusalWhoseReloadFailedIsReloadedAgainAndIsUnhealthyUntilItIs(t *testing.T) { f := newFake(t) f.mountinfo("/", "/srv/pool") f.dir("/srv/pool") @@ -72,7 +72,7 @@ func TestARefusalWhoseReloadFailedIsReloadedAgainAndIsUnhealthyUntilItIs(t *test // that never happened — is reconciled: the first pass always reloads, a later pass reloads when the kernel // exports one of the module's folders to a client the file no longer has, and the module is unhealthy // while it cannot. -func TestAWithdrawnGrantIsTakenFromTheKernel(t *testing.T) { +func testAWithdrawnGrantIsTakenFromTheKernel(t *testing.T) { f := newFake(t) f.mountinfo("/", "/srv/library") f.dir("/srv/library") @@ -122,7 +122,7 @@ func TestAWithdrawnGrantIsTakenFromTheKernel(t *testing.T) { // The review of #142: SIGTERM in the middle of a pass on a plain restart (the config is still there) ends // the pass, and the exports stay: the export file is unchanged and the kernel's table keeps the share. -func TestSIGTERMDuringAPassOnARestartLeavesTheExports(t *testing.T) { +func testSIGTERMDuringAPassOnARestartLeavesTheExports(t *testing.T) { f := newFake(t) f.mountinfo("/", "/srv/library") f.dir("/srv/library") @@ -173,7 +173,7 @@ func TestSIGTERMDuringAPassOnARestartLeavesTheExports(t *testing.T) { // unassignment then stops it with the config still there. The node-engine disables a process's unit before // it stops it (`systemctl disable --now`), and a restart or a shutdown leaves it enabled, so a process // whose own unit is disabled when it is stopped is being taken away, whatever the config says. -func TestAProcessWhoseUnitIsDisabledAtStopCleansUpWithTheConfigStillThere(t *testing.T) { +func testAProcessWhoseUnitIsDisabledAtStopCleansUpWithTheConfigStillThere(t *testing.T) { f := newFake(t) f.mountinfo("/", "/srv/library") f.dir("/srv/library") @@ -199,3 +199,20 @@ func TestAProcessWhoseUnitIsDisabledAtStopCleansUpWithTheConfigStillThere(t *tes t.Errorf("the kernel still exports the library after an unassignment with the config in place: %q", f.read(Etab)) } } + +// TestARefusalWhoseReloadFailedIsReloadedAgainAndIsUnhealthyUntilItIs runs this file's cases in order as one test, each a row: the check counts every top-level test against the +// module's budget of 50 (design 47 H2 §819). A failing case is named by the line logged before it. +func TestARefusalWhoseReloadFailedIsReloadedAgainAndIsUnhealthyUntilItIs(t *testing.T) { + for _, c := range []struct { + name string + run func(*testing.T) + }{ + {"TestARefusalWhoseReloadFailedIsReloadedAgainAndIsUnhealthyUntilItIs", testARefusalWhoseReloadFailedIsReloadedAgainAndIsUnhealthyUntilItIs}, + {"TestAWithdrawnGrantIsTakenFromTheKernel", testAWithdrawnGrantIsTakenFromTheKernel}, + {"TestSIGTERMDuringAPassOnARestartLeavesTheExports", testSIGTERMDuringAPassOnARestartLeavesTheExports}, + {"TestAProcessWhoseUnitIsDisabledAtStopCleansUpWithTheConfigStillThere", testAProcessWhoseUnitIsDisabledAtStopCleansUpWithTheConfigStillThere}, + } { + t.Logf("case %s", c.name) + c.run(t) + } +} diff --git a/modules/pacman/cmd/pacman-tools/aftercheck_test.go b/modules/pacman/cmd/pacman-tools/aftercheck_test.go index 5410f8bb6..9b4d80f3d 100644 --- a/modules/pacman/cmd/pacman-tools/aftercheck_test.go +++ b/modules/pacman/cmd/pacman-tools/aftercheck_test.go @@ -74,7 +74,7 @@ func checkUpgrades(t *testing.T, m *Machine) *Upgrades { return &Upgrades{Node: "node-a", Machine: m, Runs: runs} } -func TestTheCheckNamesTheFilesTheTransactionLeftAndTheNewsSinceThePreviousUpgrade(t *testing.T) { +func testTheCheckNamesTheFilesTheTransactionLeftAndTheNewsSinceThePreviousUpgrade(t *testing.T) { u := checkUpgrades(t, checkMachine("", "7.2.8-arch1-2")) c := u.AfterCheck(checkedRun(StepStarted, StepFinished), &Record{}) if strings.Join(c.Pacnew, " ") != "/etc/pacman.d/mirrorlist.pacnew" || strings.Join(c.Pacsave, " ") != "/etc/lemurs/config.toml.pacsave" { @@ -88,7 +88,7 @@ func TestTheCheckNamesTheFilesTheTransactionLeftAndTheNewsSinceThePreviousUpgrad } } -func TestNewsThatCannotBeReadIsSaidNotShownAsNone(t *testing.T) { +func testNewsThatCannotBeReadIsSaidNotShownAsNone(t *testing.T) { u := checkUpgrades(t, checkMachine("", "7.2.8-arch1-2")) fetch = func(string) ([]byte, error) { return nil, errors.New("no route to host") } c := u.AfterCheck(checkedRun(StepStarted, StepFinished), &Record{}) @@ -97,7 +97,7 @@ func TestNewsThatCannotBeReadIsSaidNotShownAsNone(t *testing.T) { } } -func TestAUnitNewlyFailedFailsTheCheckAndOneFailedBeforeDoesNot(t *testing.T) { +func testAUnitNewlyFailedFailsTheCheckAndOneFailedBeforeDoesNot(t *testing.T) { u := checkUpgrades(t, checkMachine("backup.service loaded failed failed The backup\nweb.service loaded failed failed The web\n", "7.2.8-arch1-2")) c := u.AfterCheck(checkedRun(StepStarted, StepFinished), &Record{FailedUnits: []string{"backup.service"}}) if strings.Join(c.NewlyFailed, " ") != "web.service" || c.Passed { @@ -105,7 +105,7 @@ func TestAUnitNewlyFailedFailsTheCheckAndOneFailedBeforeDoesNot(t *testing.T) { } } -func TestARestartIsNeededWhenTheRunningKernelIsNoLongerTheInstalledOne(t *testing.T) { +func testARestartIsNeededWhenTheRunningKernelIsNoLongerTheInstalledOne(t *testing.T) { u := checkUpgrades(t, checkMachine("", "7.2.9-arch1-1")) c := u.AfterCheck(checkedRun(StepStarted, StepFinished), &Record{}) if !c.RestartNeeded || c.InstalledKernel != "7.2.9-arch1-1" || !c.Passed { @@ -113,9 +113,27 @@ func TestARestartIsNeededWhenTheRunningKernelIsNoLongerTheInstalledOne(t *testin } } -func TestTheCheckPassesOnlyWhenTheTransactionSucceeded(t *testing.T) { +func testTheCheckPassesOnlyWhenTheTransactionSucceeded(t *testing.T) { u := checkUpgrades(t, checkMachine("", "7.2.8-arch1-2")) if c := u.AfterCheck(checkedRun(StepStarted, StepFailed), &Record{}); c.Passed { t.Errorf("%+v", c) } } + +// TestTheCheckNamesTheFilesTheTransactionLeftAndTheNewsSinceThePreviousUpgrade runs this file's cases in order as one test, each a row: the check counts every top-level test against the +// module's budget of 50 (design 47 H2 §819). A failing case is named by the line logged before it. +func TestTheCheckNamesTheFilesTheTransactionLeftAndTheNewsSinceThePreviousUpgrade(t *testing.T) { + for _, c := range []struct { + name string + run func(*testing.T) + }{ + {"TestTheCheckNamesTheFilesTheTransactionLeftAndTheNewsSinceThePreviousUpgrade", testTheCheckNamesTheFilesTheTransactionLeftAndTheNewsSinceThePreviousUpgrade}, + {"TestNewsThatCannotBeReadIsSaidNotShownAsNone", testNewsThatCannotBeReadIsSaidNotShownAsNone}, + {"TestAUnitNewlyFailedFailsTheCheckAndOneFailedBeforeDoesNot", testAUnitNewlyFailedFailsTheCheckAndOneFailedBeforeDoesNot}, + {"TestARestartIsNeededWhenTheRunningKernelIsNoLongerTheInstalledOne", testARestartIsNeededWhenTheRunningKernelIsNoLongerTheInstalledOne}, + {"TestTheCheckPassesOnlyWhenTheTransactionSucceeded", testTheCheckPassesOnlyWhenTheTransactionSucceeded}, + } { + t.Logf("case %s", c.name) + c.run(t) + } +} diff --git a/modules/pacman/cmd/pacman-tools/dkms_test.go b/modules/pacman/cmd/pacman-tools/dkms_test.go index 1c3961381..ca3124570 100644 --- a/modules/pacman/cmd/pacman-tools/dkms_test.go +++ b/modules/pacman/cmd/pacman-tools/dkms_test.go @@ -9,7 +9,7 @@ import ( "testing" ) -func TestTheNewKernelsReleaseIsDerivedAsTheInstalledOnesIs(t *testing.T) { +func testTheNewKernelsReleaseIsDerivedAsTheInstalledOnesIs(t *testing.T) { for _, c := range [][4]string{ {"7.2.8.arch1-2", "7.2.8-arch1-2", "7.3.1.arch1-1", "7.3.1-arch1-1"}, {"6.18.54-2", "6.18.54-2-lts", "6.18.60-1", "6.18.60-1-lts"}, @@ -72,7 +72,7 @@ func gateUpgrades(t *testing.T, m *Machine, linuxTo string, reachable bool) *Upg const zfsStatus = "zfs/2.4.4, 7.2.8-arch1-2, x86_64: installed\nzfs/2.4.4, 6.18.54-2-lts, x86_64: installed\n" -func TestAKernelPastAModulesLinuxMaximumIsHeldWithItsHeaders(t *testing.T) { +func testAKernelPastAModulesLinuxMaximumIsHeldWithItsHeaders(t *testing.T) { u := gateUpgrades(t, gateMachine(t, zfsStatus, capturedZFSSource(t)), "7.3.1.arch1-1", true) held, ignore, why := u.Gate() if why != "" { @@ -86,14 +86,14 @@ func TestAKernelPastAModulesLinuxMaximumIsHeldWithItsHeaders(t *testing.T) { } } -func TestAKernelWithinTheBoundIsNotHeld(t *testing.T) { +func testAKernelWithinTheBoundIsNotHeld(t *testing.T) { u := gateUpgrades(t, gateMachine(t, zfsStatus, capturedZFSSource(t)), "7.2.9.arch1-1", true) if held, ignore, why := u.Gate(); why != "" || len(held) != 0 || len(ignore) != 0 { t.Errorf("%+v %v %q", held, ignore, why) } } -func TestABuildExclusivePatternHoldsTheKernelsItDoesNotMatch(t *testing.T) { +func testABuildExclusivePatternHoldsTheKernelsItDoesNotMatch(t *testing.T) { src := map[string]string{"/usr/src/made-up-driver-1.0/dkms.conf": "PACKAGE_NAME=\"made-up-driver\"\nBUILD_EXCLUSIVE_KERNEL=\"^(6\\.|7\\.[0-2]\\.)\"\n"} u := gateUpgrades(t, gateMachine(t, "made-up-driver/1.0, 7.2.8-arch1-2, x86_64: installed\n", src), "7.3.1.arch1-1", true) held, ignore, why := u.Gate() @@ -103,7 +103,7 @@ func TestABuildExclusivePatternHoldsTheKernelsItDoesNotMatch(t *testing.T) { } } -func TestNoRunWhenWhatTheUpgradeWouldChangeIsNotKnown(t *testing.T) { +func testNoRunWhenWhatTheUpgradeWouldChangeIsNotKnown(t *testing.T) { u := gateUpgrades(t, gateMachine(t, zfsStatus, capturedZFSSource(t)), "7.3.1.arch1-1", false) if _, _, why := u.Gate(); !strings.Contains(why, "could not reach its mirrors") { t.Errorf("unreachable: %q", why) @@ -113,3 +113,21 @@ func TestNoRunWhenWhatTheUpgradeWouldChangeIsNotKnown(t *testing.T) { t.Errorf("no updates key: %q", why) } } + +// TestTheNewKernelsReleaseIsDerivedAsTheInstalledOnesIs runs this file's cases in order as one test, each a row: the check counts every top-level test against the +// module's budget of 50 (design 47 H2 §819). A failing case is named by the line logged before it. +func TestTheNewKernelsReleaseIsDerivedAsTheInstalledOnesIs(t *testing.T) { + for _, c := range []struct { + name string + run func(*testing.T) + }{ + {"TestTheNewKernelsReleaseIsDerivedAsTheInstalledOnesIs", testTheNewKernelsReleaseIsDerivedAsTheInstalledOnesIs}, + {"TestAKernelPastAModulesLinuxMaximumIsHeldWithItsHeaders", testAKernelPastAModulesLinuxMaximumIsHeldWithItsHeaders}, + {"TestAKernelWithinTheBoundIsNotHeld", testAKernelWithinTheBoundIsNotHeld}, + {"TestABuildExclusivePatternHoldsTheKernelsItDoesNotMatch", testABuildExclusivePatternHoldsTheKernelsItDoesNotMatch}, + {"TestNoRunWhenWhatTheUpgradeWouldChangeIsNotKnown", testNoRunWhenWhatTheUpgradeWouldChangeIsNotKnown}, + } { + t.Logf("case %s", c.name) + c.run(t) + } +} diff --git a/modules/pacman/cmd/pacman-tools/keeper_manifest_test.go b/modules/pacman/cmd/pacman-tools/keeper_manifest_test.go index 658c1e7da..7576e9bd2 100644 --- a/modules/pacman/cmd/pacman-tools/keeper_manifest_test.go +++ b/modules/pacman/cmd/pacman-tools/keeper_manifest_test.go @@ -40,7 +40,7 @@ func strs(v any) []string { return out } -func TestTheWeeklyTimerAndItsServiceAreGone(t *testing.T) { +func testTheWeeklyTimerAndItsServiceAreGone(t *testing.T) { m := manifest(t) for _, r := range m.Resources { id, _ := r["id"].(string) @@ -58,7 +58,7 @@ func TestTheWeeklyTimerAndItsServiceAreGone(t *testing.T) { } } -func TestTheKeeperIsAProcessUnderItsOwnAccountWithABusAccountOfItsOwn(t *testing.T) { +func testTheKeeperIsAProcessUnderItsOwnAccountWithABusAccountOfItsOwn(t *testing.T) { m := manifest(t) user := m.resource(t, "keeper-account") if user["type"] != "user" || user["name"] != "pacman-keeper" || user["shell"] != "/usr/bin/nologin" { @@ -95,7 +95,7 @@ func TestTheKeeperIsAProcessUnderItsOwnAccountWithABusAccountOfItsOwn(t *testing } } -func TestTheKeeperDeclaresTheModulesStatesAndEvents(t *testing.T) { +func testTheKeeperDeclaresTheModulesStatesAndEvents(t *testing.T) { raw := rawManifest(t) k := manifest(t).resource(t, "keeper") bus, _ := k["bus"].(map[string]any) @@ -152,7 +152,7 @@ func configValue(t *testing.T, key string) map[string]any { return v } -func TestTheConfigValuesLandInTheKeepersConfigFileWhichRestartsIt(t *testing.T) { +func testTheConfigValuesLandInTheKeepersConfigFileWhichRestartsIt(t *testing.T) { order := configValue(t, "upgrade-order") if order["type"] != "list" || order["trusted"] != true || order["predefined"] != "upgrade-order" || !reflect.DeepEqual(order["default"], []any{}) || order["required"] != nil { @@ -191,7 +191,7 @@ func TestTheConfigValuesLandInTheKeepersConfigFileWhichRestartsIt(t *testing.T) } } -func TestUpgradeOrderIsATrustedListOfTheMesh(t *testing.T) { +func testUpgradeOrderIsATrustedListOfTheMesh(t *testing.T) { raw, err := os.ReadFile(filepath.Join(moduleDir, "..", "..", "predefined.json")) if err != nil { t.Fatal(err) @@ -216,7 +216,7 @@ func TestUpgradeOrderIsATrustedListOfTheMesh(t *testing.T) { // never place "." or a placement by default: for this module both resolve under /var/lib/pacman, the package manager's // own database directory (issue 1841: the node-engine used it as found and reported it unhealthy, and wrote keeper.json // into it). What the keeper and its root helper keep there is still named through ${dir:state}. -func TestTheStateDirectoryIsOutsideThePackageManagersOwn(t *testing.T) { +func testTheStateDirectoryIsOutsideThePackageManagersOwn(t *testing.T) { m := rawManifest(t) resources, _ := m["resources"].([]any) var state map[string]any @@ -245,3 +245,22 @@ func TestTheStateDirectoryIsOutsideThePackageManagersOwn(t *testing.T) { } } } + +// TestTheWeeklyTimerAndItsServiceAreGone runs this file's cases in order as one test, each a row: the check counts every top-level test against the +// module's budget of 50 (design 47 H2 §819). A failing case is named by the line logged before it. +func TestTheWeeklyTimerAndItsServiceAreGone(t *testing.T) { + for _, c := range []struct { + name string + run func(*testing.T) + }{ + {"TestTheWeeklyTimerAndItsServiceAreGone", testTheWeeklyTimerAndItsServiceAreGone}, + {"TestTheKeeperIsAProcessUnderItsOwnAccountWithABusAccountOfItsOwn", testTheKeeperIsAProcessUnderItsOwnAccountWithABusAccountOfItsOwn}, + {"TestTheKeeperDeclaresTheModulesStatesAndEvents", testTheKeeperDeclaresTheModulesStatesAndEvents}, + {"TestTheConfigValuesLandInTheKeepersConfigFileWhichRestartsIt", testTheConfigValuesLandInTheKeepersConfigFileWhichRestartsIt}, + {"TestUpgradeOrderIsATrustedListOfTheMesh", testUpgradeOrderIsATrustedListOfTheMesh}, + {"TestTheStateDirectoryIsOutsideThePackageManagersOwn", testTheStateDirectoryIsOutsideThePackageManagersOwn}, + } { + t.Logf("case %s", c.name) + c.run(t) + } +} diff --git a/modules/pacman/cmd/pacman-tools/keeper_test.go b/modules/pacman/cmd/pacman-tools/keeper_test.go index 21ddf0946..beb9826d2 100644 --- a/modules/pacman/cmd/pacman-tools/keeper_test.go +++ b/modules/pacman/cmd/pacman-tools/keeper_test.go @@ -342,7 +342,7 @@ func installedOf(t *testing.T, s *fakeStore, key string) InstalledValue { // 1840/AC1: at the start every package is written with its version, its reason and its repository or foreign, // and written again only when it differs from the key. -func TestAtItsStartTheKeeperWritesEveryPackageOnlyWhenItDiffersFromTheKey(t *testing.T) { +func testAtItsStartTheKeeperWritesEveryPackageOnlyWhenItDiffersFromTheKey(t *testing.T) { db := &pacmanDB{sl: syncDBs, checkup: Ran{Status: 2}} db.set("glibc 2.42-1\nlinux 6.17.1-1\nyay 12.0-1\nzsh 5.9-1\n", "linux\nyay\nzsh\n") k, states, ev, _, _ := newTestKeeper(db) @@ -375,7 +375,7 @@ func TestAtItsStartTheKeeperWritesEveryPackageOnlyWhenItDiffersFromTheKey(t *tes // 1840/AC2: a transaction that adds, removes and changes packages is one package.changed naming the node and // exactly those; one that changes nothing writes and emits nothing; a lock held again waits for its release. -func TestAReleasedLockSaysExactlyWhatTheTransactionChangedOnce(t *testing.T) { +func testAReleasedLockSaysExactlyWhatTheTransactionChangedOnce(t *testing.T) { db := &pacmanDB{sl: syncDBs, checkup: Ran{Status: 2}} db.set("glibc 2.42-1\nlinux 6.17.1-1\nzsh 5.9-1\n", "linux\nzsh\n") k, states, ev, clock, _ := newTestKeeper(db) @@ -440,7 +440,7 @@ func bigDB(n int) string { // 1840/AC3: a value over the bus's cap is not written: the key keeps its last value and `.not-current` // holds the size and since when, written once; the next write that fits deletes it. -func TestAValueOverTheCapKeepsTheLastAndMarksTheNodeNotCurrent(t *testing.T) { +func testAValueOverTheCapKeepsTheLastAndMarksTheNodeNotCurrent(t *testing.T) { db := &pacmanDB{sl: syncDBs, checkup: Ran{Status: 2}} db.set("glibc 2.42-1\n", "") k, states, ev, clock, _ := newTestKeeper(db) @@ -483,7 +483,7 @@ func TestAValueOverTheCapKeepsTheLastAndMarksTheNodeNotCurrent(t *testing.T) { } // The mark goes too when the node's packages come back to what the key holds: the key is current again. -func TestTheMarkGoesWhenThePackagesComeBackToTheKey(t *testing.T) { +func testTheMarkGoesWhenThePackagesComeBackToTheKey(t *testing.T) { db := &pacmanDB{sl: syncDBs, checkup: Ran{Status: 2}} db.set("glibc 2.42-1\n", "") k, states, _, _, _ := newTestKeeper(db) @@ -501,7 +501,7 @@ func TestTheMarkGoesWhenThePackagesComeBackToTheKey(t *testing.T) { // The loop: a released lock is read, a hook runs after it with what it changed, a watched state's change reaches // its hook, and a wake comes at the time it asked. -func TestTheLoopRunsTheHooksOnItsOneGoroutine(t *testing.T) { +func testTheLoopRunsTheHooksOnItsOneGoroutine(t *testing.T) { db := &pacmanDB{sl: syncDBs, checkup: Ran{Status: 2}} db.set("glibc 2.42-1\n", "") k, states, _, clock, w := newTestKeeper(db) @@ -567,7 +567,7 @@ func waitFor(t *testing.T, mu *sync.Mutex, seen *[]string, want string) { } // The watch ending ends the keeper, loudly. -func TestTheKeeperStopsWhenTheWatchEnds(t *testing.T) { +func testTheKeeperStopsWhenTheWatchEnds(t *testing.T) { db := &pacmanDB{sl: syncDBs, checkup: Ran{Status: 2}} k, _, _, _, w := newTestKeeper(db) close(w.c) @@ -578,7 +578,7 @@ func TestTheKeeperStopsWhenTheWatchEnds(t *testing.T) { // The machine's watch: the lock deleted and the mirror list written or moved in are each a signal; anything else // in those directories is none. -func TestTheWatchSeesTheLockReleasedAndTheMirrorListRewritten(t *testing.T) { +func testTheWatchSeesTheLockReleasedAndTheMirrorListRewritten(t *testing.T) { lockDir, mirrorDir := t.TempDir(), t.TempDir() w, err := WatchMachine(lockDir, LockName, mirrorDir, MirrorName) if err != nil { @@ -628,3 +628,23 @@ func TestTheWatchSeesTheLockReleasedAndTheMirrorListRewritten(t *testing.T) { } } } + +// TestAtItsStartTheKeeperWritesEveryPackageOnlyWhenItDiffersFromTheKey runs this file's cases in order as one test, each a row: the check counts every top-level test against the +// module's budget of 50 (design 47 H2 §819). A failing case is named by the line logged before it. +func TestAtItsStartTheKeeperWritesEveryPackageOnlyWhenItDiffersFromTheKey(t *testing.T) { + for _, c := range []struct { + name string + run func(*testing.T) + }{ + {"TestAtItsStartTheKeeperWritesEveryPackageOnlyWhenItDiffersFromTheKey", testAtItsStartTheKeeperWritesEveryPackageOnlyWhenItDiffersFromTheKey}, + {"TestAReleasedLockSaysExactlyWhatTheTransactionChangedOnce", testAReleasedLockSaysExactlyWhatTheTransactionChangedOnce}, + {"TestAValueOverTheCapKeepsTheLastAndMarksTheNodeNotCurrent", testAValueOverTheCapKeepsTheLastAndMarksTheNodeNotCurrent}, + {"TestTheMarkGoesWhenThePackagesComeBackToTheKey", testTheMarkGoesWhenThePackagesComeBackToTheKey}, + {"TestTheLoopRunsTheHooksOnItsOneGoroutine", testTheLoopRunsTheHooksOnItsOneGoroutine}, + {"TestTheKeeperStopsWhenTheWatchEnds", testTheKeeperStopsWhenTheWatchEnds}, + {"TestTheWatchSeesTheLockReleasedAndTheMirrorListRewritten", testTheWatchSeesTheLockReleasedAndTheMirrorListRewritten}, + } { + t.Logf("case %s", c.name) + c.run(t) + } +} diff --git a/modules/pacman/cmd/pacman-tools/manifest_test.go b/modules/pacman/cmd/pacman-tools/manifest_test.go index 45cd3f68e..2aaafdee5 100644 --- a/modules/pacman/cmd/pacman-tools/manifest_test.go +++ b/modules/pacman/cmd/pacman-tools/manifest_test.go @@ -14,7 +14,7 @@ import ( "testing" ) -func TestItHoldsThePackageManagerSeatAndDeclaresItsPackage(t *testing.T) { +func testItHoldsThePackageManagerSeatAndDeclaresItsPackage(t *testing.T) { m := manifest(t) if len(m.Claims) != 1 || m.Claims[0]["name"] != "node-package-manager" || m.Claims[0]["scope"] != "node" || m.Claims[0]["serves"] != nil { t.Fatalf("claims: %v", m.Claims) @@ -34,7 +34,7 @@ func TestItHoldsThePackageManagerSeatAndDeclaresItsPackage(t *testing.T) { } } -func TestPacmanConfIsWholeTheUnionOfRepositoriesAndTheImprovedOptions(t *testing.T) { +func testPacmanConfIsWholeTheUnionOfRepositoriesAndTheImprovedOptions(t *testing.T) { f := manifest(t).resource(t, "config") content := templateText(t, f) if f["path"] != "/etc/pacman.conf" || f["into"] != nil || !strings.HasPrefix(content, MeshHeader) { @@ -79,7 +79,7 @@ func TestPacmanConfIsWholeTheUnionOfRepositoriesAndTheImprovedOptions(t *testing // Every repository of the shipped file reads the mirror list reflector writes, and the file pacman-conf proves reads // nothing of the machine: the unit level runs shut off (novox/hq ADR 0448), where /etc/pacman.d is not there // (issue 1891). -func TestTheProvenConfReadsNothingOfTheMachine(t *testing.T) { +func testTheProvenConfReadsNothingOfTheMachine(t *testing.T) { content := templateText(t, manifest(t).resource(t, "config")) if n := strings.Count(content, "Include = "+Mirrorlist+"\n"); n != 3 || strings.Count(content, "Include") != n { t.Fatalf("every repository reads %s, and nothing else is included: %d of 3\n%s", Mirrorlist, n, content) @@ -105,7 +105,7 @@ func provenConf(t *testing.T, content, dir string) string { return strings.ReplaceAll(content, "Include = "+Mirrorlist, "Include = "+list) } -func TestReflectorRanksWithTheModulesOptionsAndSavesNoList(t *testing.T) { +func testReflectorRanksWithTheModulesOptionsAndSavesNoList(t *testing.T) { content := templateText(t, manifest(t).resource(t, "mirrors")) opts := map[string]string{} for _, l := range strings.Split(content, "\n") { @@ -124,7 +124,7 @@ func TestReflectorRanksWithTheModulesOptionsAndSavesNoList(t *testing.T) { } } -func TestTheReflectorPackageIsDeclaredBeforeTheFileItShips(t *testing.T) { +func testTheReflectorPackageIsDeclaredBeforeTheFileItShips(t *testing.T) { order := map[string]int{} for i, r := range manifest(t).Resources { order[r["id"].(string)] = i @@ -136,7 +136,7 @@ func TestTheReflectorPackageIsDeclaredBeforeTheFileItShips(t *testing.T) { // TestThePackageManagersConfigIsDeclaredBeforeItsPackage holds design 42 zfs-owns-zfs §1 and A-26: the // apply stops at the first failure, and the package fails while its config file is missing. -func TestThePackageManagersConfigIsDeclaredBeforeItsPackage(t *testing.T) { +func testThePackageManagersConfigIsDeclaredBeforeItsPackage(t *testing.T) { order := map[string]int{} for i, r := range manifest(t).Resources { order[r["id"].(string)] = i @@ -145,3 +145,23 @@ func TestThePackageManagersConfigIsDeclaredBeforeItsPackage(t *testing.T) { t.Fatalf("the config file comes before the package: %v", order) } } + +// TestItHoldsThePackageManagerSeatAndDeclaresItsPackage runs this file's cases in order as one test, each a row: the check counts every top-level test against the +// module's budget of 50 (design 47 H2 §819). A failing case is named by the line logged before it; the case that +// skips where pacman-conf is not installed runs last, so its skip ends no other case. +func TestItHoldsThePackageManagerSeatAndDeclaresItsPackage(t *testing.T) { + for _, c := range []struct { + name string + run func(*testing.T) + }{ + {"TestItHoldsThePackageManagerSeatAndDeclaresItsPackage", testItHoldsThePackageManagerSeatAndDeclaresItsPackage}, + {"TestTheProvenConfReadsNothingOfTheMachine", testTheProvenConfReadsNothingOfTheMachine}, + {"TestReflectorRanksWithTheModulesOptionsAndSavesNoList", testReflectorRanksWithTheModulesOptionsAndSavesNoList}, + {"TestTheReflectorPackageIsDeclaredBeforeTheFileItShips", testTheReflectorPackageIsDeclaredBeforeTheFileItShips}, + {"TestThePackageManagersConfigIsDeclaredBeforeItsPackage", testThePackageManagersConfigIsDeclaredBeforeItsPackage}, + {"TestPacmanConfIsWholeTheUnionOfRepositoriesAndTheImprovedOptions", testPacmanConfIsWholeTheUnionOfRepositoriesAndTheImprovedOptions}, + } { + t.Logf("case %s", c.name) + c.run(t) + } +} diff --git a/modules/pacman/cmd/pacman-tools/pacman_test.go b/modules/pacman/cmd/pacman-tools/pacman_test.go index f980d2b5e..a93360bca 100644 --- a/modules/pacman/cmd/pacman-tools/pacman_test.go +++ b/modules/pacman/cmd/pacman-tools/pacman_test.go @@ -16,7 +16,7 @@ core/base-devel 1-2 (base-devel) Basic tools to build Arch Linux packages ` -func TestSearchReadsHeaderAndDescriptionAndWhatIsInstalled(t *testing.T) { +func testSearchReadsHeaderAndDescriptionAndWhatIsInstalled(t *testing.T) { f := ParseSearch(searchOut) if len(f) != 3 { t.Fatalf("%+v", f) @@ -32,7 +32,7 @@ func TestSearchReadsHeaderAndDescriptionAndWhatIsInstalled(t *testing.T) { } } -func TestASearchThatFindsNothingIsEmptyAndAFailureIsAnError(t *testing.T) { +func testASearchThatFindsNothingIsEmptyAndAFailureIsAnError(t *testing.T) { m := machine(fake(func(c call) Ran { return Ran{Status: 1} }, nil), 1000) r, err := m.Search("nothing", 50) if err != nil || r["count"] != 0 { @@ -54,7 +54,7 @@ Install Reason : Explicitly installed ` -func TestInfoReadsListsAsListsAndFallsBackToTheRepositories(t *testing.T) { +func testInfoReadsListsAsListsAndFallsBackToTheRepositories(t *testing.T) { p := ParseInfo(infoOut) if len(p) != 1 { t.Fatalf("%v", p) @@ -79,7 +79,7 @@ func TestInfoReadsListsAsListsAndFallsBackToTheRepositories(t *testing.T) { } } -func TestReadInstalledSaysWhyAndWhichRepositoryOrForeign(t *testing.T) { +func testReadInstalledSaysWhyAndWhichRepositoryOrForeign(t *testing.T) { m := machine(byLine(map[string]Ran{ "pacman -Q": {Stdout: "zsh 5.9-1\nglibc 2.42-1\nyay 12.0-1\n"}, "pacman -Qeq": {Stdout: "yay\nzsh\n"}, @@ -95,7 +95,7 @@ func TestReadInstalledSaysWhyAndWhichRepositoryOrForeign(t *testing.T) { } } -func TestOwnsAnswersNoOwnerAsAnAnswer(t *testing.T) { +func testOwnsAnswersNoOwnerAsAnAnswer(t *testing.T) { m := machine(byLine(map[string]Ran{ "pacman -Qo -- /usr/bin/zsh": {Stdout: "/usr/bin/zsh is owned by zsh 5.9.2-1\n"}, "pacman -Qo -- /etc/hostname": {Status: 1, Stderr: "error: No package owns /etc/hostname\n"}, @@ -115,7 +115,7 @@ func TestOwnsAnswersNoOwnerAsAnAnswer(t *testing.T) { } } -func TestUpdateListReadsCheckupdatesAndItsNothingToDo(t *testing.T) { +func testUpdateListReadsCheckupdatesAndItsNothingToDo(t *testing.T) { m := machine(byLine(map[string]Ran{"checkupdates": {Stdout: "zsh 5.9-1 -> 5.9-2\nlinux 6.1-1 -> 6.2-1\n"}}, nil), 1000) r, err := m.UpdateList() if err != nil || len(r) != 2 || r[0] != (VersionChange{"linux", "6.1-1", "6.2-1"}) { @@ -136,7 +136,7 @@ func lockless(m *Machine) *Machine { return m } -func TestAnUpgradesUnitIsReadBack(t *testing.T) { +func testAnUpgradesUnitIsReadBack(t *testing.T) { unit := "mesh-pacman-upgrade-1791115200" m := machine(byLine(map[string]Ran{ "systemctl show " + unit + " --no-pager --property=LoadState --property=ActiveState --property=SubState --property=Result --property=ExecMainStatus": {Stdout: "LoadState=loaded\nActiveState=failed\nSubState=failed\nResult=exit-code\nExecMainStatus=1\n"}, @@ -165,7 +165,7 @@ func orphanMachine(calls *[]call) *Machine { }, calls), 1000)) } -func TestRemovingOrphansTakesOnlyOrphansNamedOrAll(t *testing.T) { +func testRemovingOrphansTakesOnlyOrphansNamedOrAll(t *testing.T) { var calls []call m := orphanMachine(&calls) if _, err := m.RemoveOrphans(nil, false); err == nil || !strings.Contains(err.Error(), "name the orphans") { @@ -193,7 +193,7 @@ func TestRemovingOrphansTakesOnlyOrphansNamedOrAll(t *testing.T) { } } -func TestCacheSaysWhatCleaningWouldFreeAndCleansThroughSudo(t *testing.T) { +func testCacheSaysWhatCleaningWouldFreeAndCleansThroughSudo(t *testing.T) { var calls []call m := machine(fake(func(c call) Ran { switch c.String() { @@ -227,7 +227,7 @@ const pacmanLog = `[2026-09-24T17:47:36+0200] [PACMAN] starting full system upgr [2022-01-01 10:00] [ALPM] installed old (1-1) ` -func TestHistoryReadsTheLogSinceADayAndTheLastFullUpgrade(t *testing.T) { +func testHistoryReadsTheLogSinceADayAndTheLastFullUpgrade(t *testing.T) { m := machine(nil, 1000) m.ReadFile = func(p string) ([]byte, error) { return []byte(pacmanLog), nil } r, err := m.History("2026-09-01", "", "", 10) @@ -263,7 +263,7 @@ const feed = ` Old newshttps://example.org/news/b/oldMon, 01 Jun 2026 09:00:00 +0000 ` -func TestNewsSinceTheLastUpgrade(t *testing.T) { +func testNewsSinceTheLastUpgrade(t *testing.T) { items, err := ParseNews([]byte(feed), time.Date(2026, 9, 1, 0, 0, 0, 0, time.UTC)) if err != nil || len(items) != 1 || items[0].Title != "Manual intervention needed" || items[0].Summary != "Do this first ." { t.Fatalf("%+v %v", items, err) @@ -314,7 +314,7 @@ var rankedAt = time.Date(2026, 11, 1, 15, 0, 0, 0, time.UTC) // and never its --save; a ranking that fails or keeps none is no list. pacman_mirrors answers the shared list, its // ranker and age and whether this node's file matches it (the file as found when not); with refresh it asks this node's // keeper to rank again, refused while a round runs, naming it. -func TestTheMeshsMirrorListIsRankedFreshAndAnsweredWithThisNodesFile(t *testing.T) { +func testTheMeshsMirrorListIsRankedFreshAndAnsweredWithThisNodesFile(t *testing.T) { conf := "# comment\n--save /etc/pacman.d/mirrorlist\n--country Belgium,Netherlands\n--sort rate\n" if o := strings.Join(ReflectorOptions(conf), " "); o != "--country Belgium,Netherlands --sort rate" { t.Errorf("the options are %q", o) @@ -381,7 +381,7 @@ func TestTheMeshsMirrorListIsRankedFreshAndAnsweredWithThisNodesFile(t *testing. } } -func TestConfigIsReadAsPacmanConfResolvesIt(t *testing.T) { +func testConfigIsReadAsPacmanConfResolvesIt(t *testing.T) { c := ParseConf("[options]\nHoldPkg = pacman\nHoldPkg = glibc\nCheckSpace\nParallelDownloads = 5\n[core]\nUsage = All\nServer = https://a/core\nServer = https://b/core\n[extra]\nServer = https://a/extra\n") if len(c.Options["HoldPkg"]) != 2 || c.Options["ParallelDownloads"][0] != "5" || len(c.Repositories) != 2 || c.Repositories[0].Servers != 2 || c.Repositories[0].FirstServer != "https://a/core" { t.Fatalf("%+v", c) @@ -430,3 +430,29 @@ func checkFreshest(t *testing.T) { t.Errorf("unknown syncs not detected: %+v", r) } } + +// TestSearchReadsHeaderAndDescriptionAndWhatIsInstalled runs this file's cases in order as one test, each a row: the check counts every top-level test against the +// module's budget of 50 (design 47 H2 §819). A failing case is named by the line logged before it. +func TestSearchReadsHeaderAndDescriptionAndWhatIsInstalled(t *testing.T) { + for _, c := range []struct { + name string + run func(*testing.T) + }{ + {"TestSearchReadsHeaderAndDescriptionAndWhatIsInstalled", testSearchReadsHeaderAndDescriptionAndWhatIsInstalled}, + {"TestASearchThatFindsNothingIsEmptyAndAFailureIsAnError", testASearchThatFindsNothingIsEmptyAndAFailureIsAnError}, + {"TestInfoReadsListsAsListsAndFallsBackToTheRepositories", testInfoReadsListsAsListsAndFallsBackToTheRepositories}, + {"TestReadInstalledSaysWhyAndWhichRepositoryOrForeign", testReadInstalledSaysWhyAndWhichRepositoryOrForeign}, + {"TestOwnsAnswersNoOwnerAsAnAnswer", testOwnsAnswersNoOwnerAsAnAnswer}, + {"TestUpdateListReadsCheckupdatesAndItsNothingToDo", testUpdateListReadsCheckupdatesAndItsNothingToDo}, + {"TestAnUpgradesUnitIsReadBack", testAnUpgradesUnitIsReadBack}, + {"TestRemovingOrphansTakesOnlyOrphansNamedOrAll", testRemovingOrphansTakesOnlyOrphansNamedOrAll}, + {"TestCacheSaysWhatCleaningWouldFreeAndCleansThroughSudo", testCacheSaysWhatCleaningWouldFreeAndCleansThroughSudo}, + {"TestHistoryReadsTheLogSinceADayAndTheLastFullUpgrade", testHistoryReadsTheLogSinceADayAndTheLastFullUpgrade}, + {"TestNewsSinceTheLastUpgrade", testNewsSinceTheLastUpgrade}, + {"TestTheMeshsMirrorListIsRankedFreshAndAnsweredWithThisNodesFile", testTheMeshsMirrorListIsRankedFreshAndAnsweredWithThisNodesFile}, + {"TestConfigIsReadAsPacmanConfResolvesIt", testConfigIsReadAsPacmanConfResolvesIt}, + } { + t.Logf("case %s", c.name) + c.run(t) + } +} diff --git a/modules/pacman/cmd/pacman-tools/rollback_test.go b/modules/pacman/cmd/pacman-tools/rollback_test.go index ff7ac532d..54bdf82fd 100644 --- a/modules/pacman/cmd/pacman-tools/rollback_test.go +++ b/modules/pacman/cmd/pacman-tools/rollback_test.go @@ -45,7 +45,7 @@ func rollbackStates(t *testing.T) (*fakeStates, *Machine, *[]call) { return states, m, &calls } -func TestThePlanSaysEachPackageNowToTheRecordWithItsSourceTheAddedAndNoForeign(t *testing.T) { +func testThePlanSaysEachPackageNowToTheRecordWithItsSourceTheAddedAndNoForeign(t *testing.T) { _, m, calls := rollbackStates(t) p, err := PlanRollback(m, m.States(StateInstalled), m.States(StateRecords), "node-a", rollbackRecord.Run, false, true) if err != nil { @@ -79,7 +79,7 @@ func TestThePlanSaysEachPackageNowToTheRecordWithItsSourceTheAddedAndNoForeign(t } } -func TestAnotherNodesPlanReadsNoCacheAndTakesUnknownArchitecturesFromTheDatedRepository(t *testing.T) { +func testAnotherNodesPlanReadsNoCacheAndTakesUnknownArchitecturesFromTheDatedRepository(t *testing.T) { _, m, calls := rollbackStates(t) m.Node = "node-b" r, err := m.Rollback("node-a", rollbackRecord.Run, false, false) @@ -97,7 +97,7 @@ func TestAnotherNodesPlanReadsNoCacheAndTakesUnknownArchitecturesFromTheDatedRep } } -func TestNoPlanFromPackagesNotCurrentOrWithoutARecord(t *testing.T) { +func testNoPlanFromPackagesNotCurrentOrWithoutARecord(t *testing.T) { states, m, _ := rollbackStates(t) if _, err := m.Rollback("node-a", "20250101T000000Z", false, false); err == nil || !strings.Contains(err.Error(), "no record") { t.Errorf("no record: %v", err) @@ -108,7 +108,7 @@ func TestNoPlanFromPackagesNotCurrentOrWithoutARecord(t *testing.T) { } } -func TestApplyWritesAnAskForTheNodesKeeper(t *testing.T) { +func testApplyWritesAnAskForTheNodesKeeper(t *testing.T) { _, m, _ := rollbackStates(t) asks := newBucket() saved := upgradeState @@ -125,7 +125,7 @@ func TestApplyWritesAnAskForTheNodesKeeper(t *testing.T) { } } -func TestAnAskedRollbackIsOneTransactionWithItsEventsAndTheCheck(t *testing.T) { +func testAnAskedRollbackIsOneTransactionWithItsEventsAndTheCheck(t *testing.T) { k := newKeeperBed(t) k.records.Put(NodeKey("node-a", rollbackRecord.Run), rollbackRecord) k.installed.Put("node-a", rollbackNow) @@ -193,3 +193,21 @@ func (k *keeperBed) takeAsk(t *testing.T, a Ask) error { raw, _ := json.Marshal(a) return k.u.TakeAsk(stdio.StateChange{State: StateAsks, Key: key, Op: "put", Value: raw, Revision: rev}) } + +// TestThePlanSaysEachPackageNowToTheRecordWithItsSourceTheAddedAndNoForeign runs this file's cases in order as one test, each a row: the check counts every top-level test against the +// module's budget of 50 (design 47 H2 §819). A failing case is named by the line logged before it. +func TestThePlanSaysEachPackageNowToTheRecordWithItsSourceTheAddedAndNoForeign(t *testing.T) { + for _, c := range []struct { + name string + run func(*testing.T) + }{ + {"TestThePlanSaysEachPackageNowToTheRecordWithItsSourceTheAddedAndNoForeign", testThePlanSaysEachPackageNowToTheRecordWithItsSourceTheAddedAndNoForeign}, + {"TestAnotherNodesPlanReadsNoCacheAndTakesUnknownArchitecturesFromTheDatedRepository", testAnotherNodesPlanReadsNoCacheAndTakesUnknownArchitecturesFromTheDatedRepository}, + {"TestNoPlanFromPackagesNotCurrentOrWithoutARecord", testNoPlanFromPackagesNotCurrentOrWithoutARecord}, + {"TestApplyWritesAnAskForTheNodesKeeper", testApplyWritesAnAskForTheNodesKeeper}, + {"TestAnAskedRollbackIsOneTransactionWithItsEventsAndTheCheck", testAnAskedRollbackIsOneTransactionWithItsEventsAndTheCheck}, + } { + t.Logf("case %s", c.name) + c.run(t) + } +} diff --git a/modules/pacman/cmd/pacman-tools/root_test.go b/modules/pacman/cmd/pacman-tools/root_test.go index c5303f7e6..b0bf3f231 100644 --- a/modules/pacman/cmd/pacman-tools/root_test.go +++ b/modules/pacman/cmd/pacman-tools/root_test.go @@ -15,6 +15,7 @@ import ( "path/filepath" "reflect" "strings" + "sync/atomic" "testing" ) @@ -64,13 +65,16 @@ func (s stubProc) Readlink(name string) (string, error) { const aRun = "20261024T040000Z" -// testSocket is a socket in the abstract namespace, named for the test: a path under a test's temporary -// directory can pass the 108 bytes a socket's path may hold. +// testSocket is a socket in the abstract namespace, named for the test and numbered per call, as one test runs +// several cases whose listeners live until its end: a path under a test's temporary directory can pass the 108 +// bytes a socket's path may hold. func testSocket(t *testing.T) string { - return fmt.Sprintf("@pacman-root-test-%d-%s", os.Getpid(), t.Name()) + return fmt.Sprintf("@pacman-root-test-%d-%s-%d", os.Getpid(), t.Name(), sockets.Add(1)) } -func TestTheRootHelperStartsAnUpgradeAsTheRunsUnit(t *testing.T) { +var sockets atomic.Int64 + +func testTheRootHelperStartsAnUpgradeAsTheRunsUnit(t *testing.T) { b := newRootBed(t) a := b.h.Handle(RootRequest{Verb: VerbTransaction, Kind: RunUpgrade, Run: aRun, Ignore: []string{"linux", "linux-headers"}}) if a.Error != "" { @@ -83,7 +87,7 @@ func TestTheRootHelperStartsAnUpgradeAsTheRunsUnit(t *testing.T) { } } -func TestTheRootHelperStartsARollbackWithThePlanInTheRequest(t *testing.T) { +func testTheRootHelperStartsARollbackWithThePlanInTheRequest(t *testing.T) { b := newRootBed(t) plan := []RollbackItem{ {Name: "bash", From: "5.3.20-1", To: "5.3.15-1", Source: SourceCache, Location: CacheDir + "/bash-5.3.15-1-x86_64.pkg.tar.zst"}, @@ -115,7 +119,7 @@ func TestTheRootHelperStartsARollbackWithThePlanInTheRequest(t *testing.T) { } // And (ADR 0585 §3) the verb mirrorlist writes the mirror list file whole from the servers it is sent, running nothing. -func TestTheRootHelperReadsTheReplacedFilesMappings(t *testing.T) { +func testTheRootHelperReadsTheReplacedFilesMappings(t *testing.T) { b := newRootBed(t) servers := []string{"https://a.example/$repo/os/$arch", "https://b.example/archlinux/$repo/os/$arch"} if a := b.h.Handle(RootRequest{Verb: VerbMirrorlist, Servers: servers}); a.Error != "" { @@ -131,7 +135,7 @@ func TestTheRootHelperReadsTheReplacedFilesMappings(t *testing.T) { } } -func TestTheRootHelperReadsATransactionUnitsJournal(t *testing.T) { +func testTheRootHelperReadsATransactionUnitsJournal(t *testing.T) { b := newRootBed(t) unit := TransactionUnit(RunUpgrade, aRun) a := b.h.Handle(RootRequest{Verb: VerbJournal, Unit: unit, Lines: 40}) @@ -144,7 +148,7 @@ func TestTheRootHelperReadsATransactionUnitsJournal(t *testing.T) { } } -func TestTheRootHelperRefusesEverythingOffItsList(t *testing.T) { +func testTheRootHelperRefusesEverythingOffItsList(t *testing.T) { good := RollbackItem{Name: "bash", From: "5.3.20-1", To: "5.3.15-1", Source: SourceCache, Location: CacheDir + "/bash-5.3.15-1-x86_64.pkg.tar.zst"} with := func(f func(*RollbackItem)) []RollbackItem { i := good; f(&i); return []RollbackItem{i} } cases := map[string]RootRequest{ @@ -211,14 +215,14 @@ func serveOnce(t *testing.T, peer int, req RootRequest) (RootAnswer, error) { return rootClient{Socket: sock}.ask(req) } -func TestTheRootHelperAnswersTheKeepersAccountOverItsSocket(t *testing.T) { +func testTheRootHelperAnswersTheKeepersAccountOverItsSocket(t *testing.T) { a, err := serveOnce(t, os.Getuid(), RootRequest{Verb: VerbMaps}) if err != nil || a.Error != "" || len(a.Mapped) != 1 { t.Fatalf("the keeper's own account was answered %+v, %v", a, err) } } -func TestTheRootHelperRefusesAnyOtherAccount(t *testing.T) { +func testTheRootHelperRefusesAnyOtherAccount(t *testing.T) { a, err := serveOnce(t, os.Getuid()+1, RootRequest{Verb: VerbMaps}) if err == nil && a.Error == "" { t.Fatalf("another account was answered: %+v", a) @@ -228,7 +232,7 @@ func TestTheRootHelperRefusesAnyOtherAccount(t *testing.T) { } } -func TestTheClientCarriesTheRootActsAndSaysARefusal(t *testing.T) { +func testTheClientCarriesTheRootActsAndSaysARefusal(t *testing.T) { b := newRootBed(t) b.h.Peer = os.Getuid() sock := testSocket(t) @@ -254,7 +258,7 @@ func TestTheClientCarriesTheRootActsAndSaysARefusal(t *testing.T) { } } -func TestTheRootHelperIsRootWithNoBusAndNoSudoRule(t *testing.T) { +func testTheRootHelperIsRootWithNoBusAndNoSudoRule(t *testing.T) { m := manifest(t) r := m.resource(t, "root-helper") if r["type"] != "process" || r["name"] != "pacman-root" || r["artifact"] != "tools" || @@ -283,7 +287,7 @@ func TestTheRootHelperIsRootWithNoBusAndNoSudoRule(t *testing.T) { } } -func TestTheRollbackUnitInstallsThePlanThenRemovesWhatWasAskedInOrder(t *testing.T) { +func testTheRollbackUnitInstallsThePlanThenRemovesWhatWasAskedInOrder(t *testing.T) { dir := t.TempDir() plan := []RollbackItem{ {Name: "bash", To: "5.3.15-1", Source: SourceCache, Location: CacheDir + "/bash-5.3.15-1-x86_64.pkg.tar.zst"}, @@ -314,7 +318,7 @@ func TestTheRollbackUnitInstallsThePlanThenRemovesWhatWasAskedInOrder(t *testing } } -func TestTheRollbackUnitChangesNothingWhenAPackageIsNotFound(t *testing.T) { +func testTheRollbackUnitChangesNothingWhenAPackageIsNotFound(t *testing.T) { file := filepath.Join(t.TempDir(), aRun+".json") raw, _ := json.Marshal([]RollbackItem{{Name: "zlib", To: "1.3.1-2", Source: SourceRepository, Location: ArchiveURL + "/repos/2026/10/04/core/os/$arch"}}) _ = os.WriteFile(file, raw, 0o600) @@ -324,3 +328,27 @@ func TestTheRollbackUnitChangesNothingWhenAPackageIsNotFound(t *testing.T) { t.Fatalf("exit %d, ran %q: a package the archive does not hold fails the unit before any change", code, ran) } } + +// TestTheRootHelperStartsAnUpgradeAsTheRunsUnit runs this file's cases in order as one test, each a row: the check counts every top-level test against the +// module's budget of 50 (design 47 H2 §819). A failing case is named by the line logged before it. +func TestTheRootHelperStartsAnUpgradeAsTheRunsUnit(t *testing.T) { + for _, c := range []struct { + name string + run func(*testing.T) + }{ + {"TestTheRootHelperStartsAnUpgradeAsTheRunsUnit", testTheRootHelperStartsAnUpgradeAsTheRunsUnit}, + {"TestTheRootHelperStartsARollbackWithThePlanInTheRequest", testTheRootHelperStartsARollbackWithThePlanInTheRequest}, + {"TestTheRootHelperReadsTheReplacedFilesMappings", testTheRootHelperReadsTheReplacedFilesMappings}, + {"TestTheRootHelperReadsATransactionUnitsJournal", testTheRootHelperReadsATransactionUnitsJournal}, + {"TestTheRootHelperRefusesEverythingOffItsList", testTheRootHelperRefusesEverythingOffItsList}, + {"TestTheRootHelperAnswersTheKeepersAccountOverItsSocket", testTheRootHelperAnswersTheKeepersAccountOverItsSocket}, + {"TestTheRootHelperRefusesAnyOtherAccount", testTheRootHelperRefusesAnyOtherAccount}, + {"TestTheClientCarriesTheRootActsAndSaysARefusal", testTheClientCarriesTheRootActsAndSaysARefusal}, + {"TestTheRootHelperIsRootWithNoBusAndNoSudoRule", testTheRootHelperIsRootWithNoBusAndNoSudoRule}, + {"TestTheRollbackUnitInstallsThePlanThenRemovesWhatWasAskedInOrder", testTheRollbackUnitInstallsThePlanThenRemovesWhatWasAskedInOrder}, + {"TestTheRollbackUnitChangesNothingWhenAPackageIsNotFound", testTheRollbackUnitChangesNothingWhenAPackageIsNotFound}, + } { + t.Logf("case %s", c.name) + c.run(t) + } +} diff --git a/modules/pacman/cmd/pacman-tools/rounds_test.go b/modules/pacman/cmd/pacman-tools/rounds_test.go index 65463f51b..5a9545e67 100644 --- a/modules/pacman/cmd/pacman-tools/rounds_test.go +++ b/modules/pacman/cmd/pacman-tools/rounds_test.go @@ -85,7 +85,7 @@ func (b *roundBed) state(t *testing.T) map[string]NodeInRound { return out } -func TestTheRoundOpensSaturdayAtFourAndThePassedCheckGivesTheNextItsTurn(t *testing.T) { +func testTheRoundOpensSaturdayAtFourAndThePassedCheckGivesTheNextItsTurn(t *testing.T) { if o := RoundOpening(time.Date(2026, 10, 22, 15, 0, 0, 0, time.UTC)); !o.Equal(time.Date(2026, 10, 24, 4, 0, 0, 0, time.UTC)) { t.Fatalf("the opening of the week of Thursday 22 October: %s", o) } @@ -144,7 +144,7 @@ func TestTheRoundOpensSaturdayAtFourAndThePassedCheckGivesTheNextItsTurn(t *test } } -func TestNoOrderNoRound(t *testing.T) { +func testNoOrderNoRound(t *testing.T) { b := newRoundBed(t) b.now = time.Date(2026, 10, 24, 5, 0, 0, 0, time.UTC) if next := b.step(t, "node-a"); !next.IsZero() { @@ -155,7 +155,7 @@ func TestNoOrderNoRound(t *testing.T) { } } -func TestANodeOffAtItsTurnRunsAtItsNextStartAndAStalledTurnIsSaid(t *testing.T) { +func testANodeOffAtItsTurnRunsAtItsNextStartAndAStalledTurnIsSaid(t *testing.T) { b := newRoundBed(t, "node-a", "node-b", "node-c") b.now = time.Date(2026, 10, 24, 4, 0, 0, 0, time.UTC) b.step(t, "node-a") @@ -183,7 +183,7 @@ func warrant(a oc.Ask, option string, level string, at time.Time) json.RawMessag return raw } -func TestOnlyTheOperatorsApprovePassesANodeAndAPassNeedsAReason(t *testing.T) { +func testOnlyTheOperatorsApprovePassesANodeAndAPassNeedsAReason(t *testing.T) { b := newRoundBed(t, "node-a", "node-b", "node-c") b.now = time.Date(2026, 10, 24, 4, 0, 0, 0, time.UTC) b.step(t, "node-a") @@ -254,7 +254,7 @@ func TestOnlyTheOperatorsApprovePassesANodeAndAPassNeedsAReason(t *testing.T) { } } -func TestAPassTheOperatorKeepsWaitingPassesNothing(t *testing.T) { +func testAPassTheOperatorKeepsWaitingPassesNothing(t *testing.T) { b := newRoundBed(t, "node-a", "node-b") b.now = time.Date(2026, 10, 24, 4, 0, 0, 0, time.UTC) b.step(t, "node-a") // node-a runs and never ends @@ -287,7 +287,7 @@ func TestAPassTheOperatorKeepsWaitingPassesNothing(t *testing.T) { } } -func TestPacmanRoundReadsTheRounds(t *testing.T) { +func testPacmanRoundReadsTheRounds(t *testing.T) { b := newRoundBed(t, "node-a", "node-b") b.now = time.Date(2026, 10, 24, 4, 0, 0, 0, time.UTC) b.step(t, "node-a") @@ -304,3 +304,22 @@ func TestPacmanRoundReadsTheRounds(t *testing.T) { t.Error("a round that is not there") } } + +// TestTheRoundOpensSaturdayAtFourAndThePassedCheckGivesTheNextItsTurn runs this file's cases in order as one test, each a row: the check counts every top-level test against the +// module's budget of 50 (design 47 H2 §819). A failing case is named by the line logged before it. +func TestTheRoundOpensSaturdayAtFourAndThePassedCheckGivesTheNextItsTurn(t *testing.T) { + for _, c := range []struct { + name string + run func(*testing.T) + }{ + {"TestTheRoundOpensSaturdayAtFourAndThePassedCheckGivesTheNextItsTurn", testTheRoundOpensSaturdayAtFourAndThePassedCheckGivesTheNextItsTurn}, + {"TestNoOrderNoRound", testNoOrderNoRound}, + {"TestANodeOffAtItsTurnRunsAtItsNextStartAndAStalledTurnIsSaid", testANodeOffAtItsTurnRunsAtItsNextStartAndAStalledTurnIsSaid}, + {"TestOnlyTheOperatorsApprovePassesANodeAndAPassNeedsAReason", testOnlyTheOperatorsApprovePassesANodeAndAPassNeedsAReason}, + {"TestAPassTheOperatorKeepsWaitingPassesNothing", testAPassTheOperatorKeepsWaitingPassesNothing}, + {"TestPacmanRoundReadsTheRounds", testPacmanRoundReadsTheRounds}, + } { + t.Logf("case %s", c.name) + c.run(t) + } +} diff --git a/modules/pacman/cmd/pacman-tools/runs_test.go b/modules/pacman/cmd/pacman-tools/runs_test.go index d339b695f..98352a4e5 100644 --- a/modules/pacman/cmd/pacman-tools/runs_test.go +++ b/modules/pacman/cmd/pacman-tools/runs_test.go @@ -24,7 +24,7 @@ func capturedLog(t *testing.T) string { // The captured log holds three full upgrades among other commands: one that ended before any transaction, one // whose transaction completed, and a later one that upgraded one package. -func TestEveryFullUpgradeOfTheLogIsARunReadFromIt(t *testing.T) { +func testEveryFullUpgradeOfTheLogIsARunReadFromIt(t *testing.T) { runs := PastUpgrades("node-a", capturedLog(t)) if len(runs) != 3 { t.Fatalf("%d runs, want the three full upgrades and none of the other commands: %+v", len(runs), runs) @@ -61,7 +61,7 @@ func TestEveryFullUpgradeOfTheLogIsARunReadFromIt(t *testing.T) { } } -func TestATransactionWithoutAnEndOrThatFailedIsAFailedRun(t *testing.T) { +func testATransactionWithoutAnEndOrThatFailedIsAFailedRun(t *testing.T) { log := "[2026-10-10T04:00:00+0000] [PACMAN] Running 'pacman -Syu --noconfirm'\n" + "[2026-10-10T04:00:01+0000] [PACMAN] starting full system upgrade\n" + "[2026-10-10T04:00:05+0000] [ALPM] transaction started\n" + @@ -82,7 +82,7 @@ func TestATransactionWithoutAnEndOrThatFailedIsAFailedRun(t *testing.T) { } } -func TestTheLogIsReadIntoRunsOnceAndASecondStartReadsNothing(t *testing.T) { +func testTheLogIsReadIntoRunsOnceAndASecondStartReadsNothing(t *testing.T) { m := machine(nil, 1000) reads := 0 log := capturedLog(t) @@ -116,7 +116,7 @@ func TestTheLogIsReadIntoRunsOnceAndASecondStartReadsNothing(t *testing.T) { // A run's key is created at its first step and then only updated from the revision its writer holds: a step // is written once, and a run another writer changed meanwhile is refused, never overwritten. -func TestAStepIsWrittenOnceAndAnotherWritersChangeIsNeverOverwritten(t *testing.T) { +func testAStepIsWrittenOnceAndAnotherWritersChangeIsNeverOverwritten(t *testing.T) { runs := newBucket() u := &Upgrades{Node: "node-a", Runs: runs} r := &openRun{Run: Run{Node: "node-a", Run: "20261024T040000Z", Kind: RunUpgrade, From: FromAsk}} @@ -142,7 +142,7 @@ func TestAStepIsWrittenOnceAndAnotherWritersChangeIsNeverOverwritten(t *testing. } } -func TestPacmanUpgradesAnswersAllOneNodeDaysAndOneRun(t *testing.T) { +func testPacmanUpgradesAnswersAllOneNodeDaysAndOneRun(t *testing.T) { runs := newBucket() put := func(node, id string, steps ...Step) { r := Run{Node: node, Run: id, Kind: RunUpgrade, From: FromRound, Round: "2026-W43", Steps: steps} @@ -194,7 +194,7 @@ func TestPacmanUpgradesAnswersAllOneNodeDaysAndOneRun(t *testing.T) { } } -func TestTheUpgradesToolReadsTheRunsState(t *testing.T) { +func testTheUpgradesToolReadsTheRunsState(t *testing.T) { runs := newBucket() saved := upgradeState upgradeState = func(name string) Bucket { @@ -215,3 +215,22 @@ func TestTheUpgradesToolReadsTheRunsState(t *testing.T) { } t.Fatal("no pacman_upgrades") } + +// TestEveryFullUpgradeOfTheLogIsARunReadFromIt runs this file's cases in order as one test, each a row: the check counts every top-level test against the +// module's budget of 50 (design 47 H2 §819). A failing case is named by the line logged before it. +func TestEveryFullUpgradeOfTheLogIsARunReadFromIt(t *testing.T) { + for _, c := range []struct { + name string + run func(*testing.T) + }{ + {"TestEveryFullUpgradeOfTheLogIsARunReadFromIt", testEveryFullUpgradeOfTheLogIsARunReadFromIt}, + {"TestATransactionWithoutAnEndOrThatFailedIsAFailedRun", testATransactionWithoutAnEndOrThatFailedIsAFailedRun}, + {"TestTheLogIsReadIntoRunsOnceAndASecondStartReadsNothing", testTheLogIsReadIntoRunsOnceAndASecondStartReadsNothing}, + {"TestAStepIsWrittenOnceAndAnotherWritersChangeIsNeverOverwritten", testAStepIsWrittenOnceAndAnotherWritersChangeIsNeverOverwritten}, + {"TestPacmanUpgradesAnswersAllOneNodeDaysAndOneRun", testPacmanUpgradesAnswersAllOneNodeDaysAndOneRun}, + {"TestTheUpgradesToolReadsTheRunsState", testTheUpgradesToolReadsTheRunsState}, + } { + t.Logf("case %s", c.name) + c.run(t) + } +} diff --git a/modules/pacman/cmd/pacman-tools/transaction_test.go b/modules/pacman/cmd/pacman-tools/transaction_test.go index 1ea5466a2..18306804f 100644 --- a/modules/pacman/cmd/pacman-tools/transaction_test.go +++ b/modules/pacman/cmd/pacman-tools/transaction_test.go @@ -180,7 +180,7 @@ func steps(r Run) string { return strings.Join(s, " ") } -func TestAnAskedUpgradeIsARunWrittenAndEmittedAtEachStep(t *testing.T) { +func testAnAskedUpgradeIsARunWrittenAndEmittedAtEachStep(t *testing.T) { k := newKeeperBed(t) key, err := k.ask(t, "node-a", ActUpgrade) if err != nil { @@ -238,7 +238,7 @@ func TestAnAskedUpgradeIsARunWrittenAndEmittedAtEachStep(t *testing.T) { } } -func TestAFailedTransactionIsWrittenWithItsExitAndLastErrorAndChecked(t *testing.T) { +func testAFailedTransactionIsWrittenWithItsExitAndLastErrorAndChecked(t *testing.T) { k := newKeeperBed(t) if _, err := k.ask(t, "node-a", ActUpgrade); err != nil { t.Fatal(err) @@ -259,7 +259,7 @@ func TestAFailedTransactionIsWrittenWithItsExitAndLastErrorAndChecked(t *testing } } -func TestATransactionThatCannotStartFailsAndIsChecked(t *testing.T) { +func testATransactionThatCannotStartFailsAndIsChecked(t *testing.T) { k := newKeeperBed(t) k.root.fail = errors.New("the root helper's socket is not there") if _, err := k.ask(t, "node-a", ActUpgrade); err != nil { @@ -271,7 +271,7 @@ func TestATransactionThatCannotStartFailsAndIsChecked(t *testing.T) { } } -func TestAnAskIsTakenByItsNodesKeeperAlone(t *testing.T) { +func testAnAskIsTakenByItsNodesKeeperAlone(t *testing.T) { k := newKeeperBed(t) key, err := k.ask(t, "node-b", ActUpgrade) if err != nil { @@ -288,7 +288,7 @@ func TestAnAskIsTakenByItsNodesKeeperAlone(t *testing.T) { } } -func TestAnAskARunAlreadyNamesIsDeletedAndNotRunAgain(t *testing.T) { +func testAnAskARunAlreadyNamesIsDeletedAndNotRunAgain(t *testing.T) { k := newKeeperBed(t) done := Run{Node: "node-a", Run: "20261004T115900Z", Kind: RunUpgrade, From: FromAsk, Ask: RunID(k.now) + "-abc123", Steps: []Step{{Step: StepRefused, At: k.now, Why: "x"}}} @@ -307,7 +307,7 @@ func TestAnAskARunAlreadyNamesIsDeletedAndNotRunAgain(t *testing.T) { } } -func TestARunDoesNotStartWhileTheDatabaseIsLockedOrAnotherRunGoesOn(t *testing.T) { +func testARunDoesNotStartWhileTheDatabaseIsLockedOrAnotherRunGoesOn(t *testing.T) { k := newKeeperBed(t) k.locked = true if _, err := k.ask(t, "node-a", ActUpgrade); err != nil { @@ -336,7 +336,7 @@ func TestARunDoesNotStartWhileTheDatabaseIsLockedOrAnotherRunGoesOn(t *testing.T } } -func TestAKeeperStartedAgainFollowsItsRunToItsEnd(t *testing.T) { +func testAKeeperStartedAgainFollowsItsRunToItsEnd(t *testing.T) { k := newKeeperBed(t) if _, err := k.ask(t, "node-a", ActUpgrade); err != nil { t.Fatal(err) @@ -357,7 +357,7 @@ func TestAKeeperStartedAgainFollowsItsRunToItsEnd(t *testing.T) { } } -func TestAKeeperStartedAgainTakesUpARunStoppedBetweenItsSteps(t *testing.T) { +func testAKeeperStartedAgainTakesUpARunStoppedBetweenItsSteps(t *testing.T) { // Stopped after the gate, before the transaction started: nothing started, so the run is refused. k := newKeeperBed(t) stopped := Run{Node: "node-a", Run: "20261004T115900Z", Kind: RunUpgrade, From: FromAsk, @@ -401,7 +401,7 @@ func TestAKeeperStartedAgainTakesUpARunStoppedBetweenItsSteps(t *testing.T) { } } -func TestPacmanUpgradeWritesAnAskForTheNodeAndStartsNothing(t *testing.T) { +func testPacmanUpgradeWritesAnAskForTheNodeAndStartsNothing(t *testing.T) { asks := newBucket() savedState, savedNode, savedFetch := upgradeState, thisNode, fetch upgradeState = func(name string) Bucket { @@ -445,3 +445,25 @@ func TestPacmanUpgradeWritesAnAskForTheNodeAndStartsNothing(t *testing.T) { } t.Fatal("no pacman_upgrade") } + +// TestAnAskedUpgradeIsARunWrittenAndEmittedAtEachStep runs this file's cases in order as one test, each a row: the check counts every top-level test against the +// module's budget of 50 (design 47 H2 §819). A failing case is named by the line logged before it. +func TestAnAskedUpgradeIsARunWrittenAndEmittedAtEachStep(t *testing.T) { + for _, c := range []struct { + name string + run func(*testing.T) + }{ + {"TestAnAskedUpgradeIsARunWrittenAndEmittedAtEachStep", testAnAskedUpgradeIsARunWrittenAndEmittedAtEachStep}, + {"TestAFailedTransactionIsWrittenWithItsExitAndLastErrorAndChecked", testAFailedTransactionIsWrittenWithItsExitAndLastErrorAndChecked}, + {"TestATransactionThatCannotStartFailsAndIsChecked", testATransactionThatCannotStartFailsAndIsChecked}, + {"TestAnAskIsTakenByItsNodesKeeperAlone", testAnAskIsTakenByItsNodesKeeperAlone}, + {"TestAnAskARunAlreadyNamesIsDeletedAndNotRunAgain", testAnAskARunAlreadyNamesIsDeletedAndNotRunAgain}, + {"TestARunDoesNotStartWhileTheDatabaseIsLockedOrAnotherRunGoesOn", testARunDoesNotStartWhileTheDatabaseIsLockedOrAnotherRunGoesOn}, + {"TestAKeeperStartedAgainFollowsItsRunToItsEnd", testAKeeperStartedAgainFollowsItsRunToItsEnd}, + {"TestAKeeperStartedAgainTakesUpARunStoppedBetweenItsSteps", testAKeeperStartedAgainTakesUpARunStoppedBetweenItsSteps}, + {"TestPacmanUpgradeWritesAnAskForTheNodeAndStartsNothing", testPacmanUpgradeWritesAnAskForTheNodeAndStartsNothing}, + } { + t.Logf("case %s", c.name) + c.run(t) + } +} diff --git a/modules/pacman/cmd/pacman-tools/upgrade_health_test.go b/modules/pacman/cmd/pacman-tools/upgrade_health_test.go index 866902368..884e60e93 100644 --- a/modules/pacman/cmd/pacman-tools/upgrade_health_test.go +++ b/modules/pacman/cmd/pacman-tools/upgrade_health_test.go @@ -32,14 +32,14 @@ func passedRun(node string, at time.Time) Run { {Step: StepFinished, At: at.Add(5 * time.Minute)}, {Step: StepCheckedAfter, At: at.Add(6 * time.Minute), Check: &Check{Passed: true}}}} } -func TestWithNoOrderEveryNodesHealthSaysWhy(t *testing.T) { +func testWithNoOrderEveryNodesHealthSaysWhy(t *testing.T) { m, _ := healthBed(t, nil) if h := m.UpgradeHealth(); h.Healthy || !strings.Contains(h.Why, "upgrade-order") { t.Fatalf("%+v", h) } } -func TestANodeWithNoRunYetIsHealthy(t *testing.T) { +func testANodeWithNoRunYetIsHealthy(t *testing.T) { m, _ := healthBed(t, []string{"node-a", "node-b"}) if h := m.UpgradeHealth(); !h.Healthy { t.Fatalf("%+v", h) @@ -48,7 +48,7 @@ func TestANodeWithNoRunYetIsHealthy(t *testing.T) { // issue 1579: a run that failed is unhealthy until a later run passes, whenever the node woke; a run read from // the log is history and judges nothing. -func TestTheLatestRunJudgesTheNodeNotAUnitsAge(t *testing.T) { +func testTheLatestRunJudgesTheNodeNotAUnitsAge(t *testing.T) { m, states := healthBed(t, []string{"node-a", "node-b"}) exit := 1 failedAt := healthNow.Add(-7 * 24 * time.Hour) @@ -68,7 +68,7 @@ func TestTheLatestRunJudgesTheNodeNotAUnitsAge(t *testing.T) { } } -func TestAFailedCheckIsUnhealthy(t *testing.T) { +func testAFailedCheckIsUnhealthy(t *testing.T) { m, states := healthBed(t, []string{"node-b"}) r := passedRun("node-b", healthNow.Add(-time.Hour)) r.Steps[len(r.Steps)-1].Check = &Check{Passed: false, Unbuilt: []DKMSBuild{{Module: "made-up-dkms", Version: "1.0", Kernel: "7.3.1-arch1-1"}}} @@ -78,7 +78,7 @@ func TestAFailedCheckIsUnhealthy(t *testing.T) { } } -func TestARunPastTheLimitIsHungAndOneWithinItIsHealthy(t *testing.T) { +func testARunPastTheLimitIsHungAndOneWithinItIsHealthy(t *testing.T) { m, states := healthBed(t, []string{"node-b"}) started := healthNow.Add(-3 * time.Hour) putRun(states, Run{Node: "node-b", Run: RunID(started), Kind: RunUpgrade, From: FromAsk, Steps: []Step{{Step: StepRecorded, At: started}, {Step: StepStarted, At: started, Unit: TransactionUnit(RunUpgrade, RunID(started))}}}) @@ -94,7 +94,7 @@ func TestARunPastTheLimitIsHungAndOneWithinItIsHealthy(t *testing.T) { } // 1841/AC4: a held kernel keeps the health unhealthy, naming it, until a later run upgrades it. -func TestAHeldKernelIsSaidUntilARunUpgradesIt(t *testing.T) { +func testAHeldKernelIsSaidUntilARunUpgradesIt(t *testing.T) { m, states := healthBed(t, []string{"node-b"}) r := passedRun("node-b", healthNow.Add(-2*time.Hour)) r.Steps[1].Held = []HeldKernel{{Package: "linux", Version: "7.3.1.arch1-1", Module: "zfs", Bound: "Linux-Maximum 7.2"}} @@ -111,7 +111,7 @@ func TestAHeldKernelIsSaidUntilARunUpgradesIt(t *testing.T) { // 1841/AC7: a turn not started within round-start-minutes makes the waiting node's health name the node it // waits for and since when; the stalled node's own health says it too. -func TestAStalledRoundIsSaidByTheNodesWaitingOnIt(t *testing.T) { +func testAStalledRoundIsSaidByTheNodesWaitingOnIt(t *testing.T) { order := []string{"node-a", "node-b", "node-c"} m, states := healthBed(t, order) round := RoundID(healthNow) @@ -131,7 +131,7 @@ func TestAStalledRoundIsSaidByTheNodesWaitingOnIt(t *testing.T) { } } -func TestARoundStoppedByAFailureIsSaidByTheNodesWaitingOnIt(t *testing.T) { +func testARoundStoppedByAFailureIsSaidByTheNodesWaitingOnIt(t *testing.T) { order := []string{"node-a", "node-b"} m, states := healthBed(t, order) round := RoundID(healthNow) @@ -145,3 +145,24 @@ func TestARoundStoppedByAFailureIsSaidByTheNodesWaitingOnIt(t *testing.T) { t.Fatalf("%+v", h) } } + +// TestWithNoOrderEveryNodesHealthSaysWhy runs this file's cases in order as one test, each a row: the check counts every top-level test against the +// module's budget of 50 (design 47 H2 §819). A failing case is named by the line logged before it. +func TestWithNoOrderEveryNodesHealthSaysWhy(t *testing.T) { + for _, c := range []struct { + name string + run func(*testing.T) + }{ + {"TestWithNoOrderEveryNodesHealthSaysWhy", testWithNoOrderEveryNodesHealthSaysWhy}, + {"TestANodeWithNoRunYetIsHealthy", testANodeWithNoRunYetIsHealthy}, + {"TestTheLatestRunJudgesTheNodeNotAUnitsAge", testTheLatestRunJudgesTheNodeNotAUnitsAge}, + {"TestAFailedCheckIsUnhealthy", testAFailedCheckIsUnhealthy}, + {"TestARunPastTheLimitIsHungAndOneWithinItIsHealthy", testARunPastTheLimitIsHungAndOneWithinItIsHealthy}, + {"TestAHeldKernelIsSaidUntilARunUpgradesIt", testAHeldKernelIsSaidUntilARunUpgradesIt}, + {"TestAStalledRoundIsSaidByTheNodesWaitingOnIt", testAStalledRoundIsSaidByTheNodesWaitingOnIt}, + {"TestARoundStoppedByAFailureIsSaidByTheNodesWaitingOnIt", testARoundStoppedByAFailureIsSaidByTheNodesWaitingOnIt}, + } { + t.Logf("case %s", c.name) + c.run(t) + } +} diff --git a/modules/pacman/cmd/pacman-tools/view_test.go b/modules/pacman/cmd/pacman-tools/view_test.go index 82c1f3a55..7b6dc1853 100644 --- a/modules/pacman/cmd/pacman-tools/view_test.go +++ b/modules/pacman/cmd/pacman-tools/view_test.go @@ -42,7 +42,7 @@ func viewStates(t *testing.T) (*fakeStates, *Machine) { return states, m } -func TestOnePackageOnEveryNodeWithWhoDeclaresIt(t *testing.T) { +func testOnePackageOnEveryNodeWithWhoDeclaresIt(t *testing.T) { _, m := viewStates(t) r, err := m.Packages("zsh", 500) if err != nil { @@ -99,7 +99,7 @@ func TestOnePackageOnEveryNodeWithWhoDeclaresIt(t *testing.T) { } } -func TestTheViewSaysWhatDiffersWhatIsOnSomeNodesWhatNoModuleDeclaresAndTheForeignApart(t *testing.T) { +func testTheViewSaysWhatDiffersWhatIsOnSomeNodesWhatNoModuleDeclaresAndTheForeignApart(t *testing.T) { _, m := viewStates(t) r, err := m.Packages("", 500) if err != nil { @@ -135,7 +135,7 @@ func TestTheViewSaysWhatDiffersWhatIsOnSomeNodesWhatNoModuleDeclaresAndTheForeig } } -func TestInstalledAndForeignAnswerFromTheBus(t *testing.T) { +func testInstalledAndForeignAnswerFromTheBus(t *testing.T) { _, m := viewStates(t) r, err := m.InstalledFrom("", "", "", false, 10) if err != nil || r["node"] != "node-a" || r["count"] != 4 || r["current"] != true { @@ -161,7 +161,7 @@ func TestInstalledAndForeignAnswerFromTheBus(t *testing.T) { } } -func TestUpdatesAnswerEveryNodeAndWhenItsListLastChanged(t *testing.T) { +func testUpdatesAnswerEveryNodeAndWhenItsListLastChanged(t *testing.T) { states, m := viewStates(t) up := states.store(StateUpdates) up.now = func() time.Time { return t0.Add(-2 * time.Hour) } @@ -183,7 +183,7 @@ func TestUpdatesAnswerEveryNodeAndWhenItsListLastChanged(t *testing.T) { } } -func TestANodeArgumentIsOneKeyToken(t *testing.T) { +func testANodeArgumentIsOneKeyToken(t *testing.T) { for _, bad := range []string{"node-a.not-current", "node-a.*", "a b"} { if _, err := nodeOf(map[string]any{"node": bad}); err == nil { t.Errorf("%q was taken as a node", bad) @@ -200,3 +200,21 @@ type failingReader struct{ err error } func (f failingReader) Get(string) (*stdio.StateEntry, error) { return nil, f.err } func (f failingReader) Keys() ([]string, error) { return nil, f.err } func (f failingReader) History(string) ([]stdio.StateRevision, error) { return nil, f.err } + +// TestOnePackageOnEveryNodeWithWhoDeclaresIt runs this file's cases in order as one test, each a row: the check counts every top-level test against the +// module's budget of 50 (design 47 H2 §819). A failing case is named by the line logged before it. +func TestOnePackageOnEveryNodeWithWhoDeclaresIt(t *testing.T) { + for _, c := range []struct { + name string + run func(*testing.T) + }{ + {"TestOnePackageOnEveryNodeWithWhoDeclaresIt", testOnePackageOnEveryNodeWithWhoDeclaresIt}, + {"TestTheViewSaysWhatDiffersWhatIsOnSomeNodesWhatNoModuleDeclaresAndTheForeignApart", testTheViewSaysWhatDiffersWhatIsOnSomeNodesWhatNoModuleDeclaresAndTheForeignApart}, + {"TestInstalledAndForeignAnswerFromTheBus", testInstalledAndForeignAnswerFromTheBus}, + {"TestUpdatesAnswerEveryNodeAndWhenItsListLastChanged", testUpdatesAnswerEveryNodeAndWhenItsListLastChanged}, + {"TestANodeArgumentIsOneKeyToken", testANodeArgumentIsOneKeyToken}, + } { + t.Logf("case %s", c.name) + c.run(t) + } +} -- 2.54.0 From 1d8839640efa86e796d4eb482f33e403b8002485 Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 22 Nov 2026 01:19:00 +0100 Subject: [PATCH 33/54] BM4b: the build modules keep an oversized end in steps-objects, read by the verb answer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The delivery SDK replaced the output chunks with the verb answer over the steps state's objects and output at form 2: each build module keeps an end over the steps state's value with KeepAnswerIn, serves AnswerTool and OutputTool over its steps state, and its steps name their outputs objects. Builds: design 47 H2 §791, H-A-412 addendum, H-A-412 second addendum (b), H-A-399 (1) --- modules/mesh-build-go/cmd/mesh-build-go/asks.go | 16 ++++++++++++++-- .../mesh-build-go/cmd/mesh-build-go/asks_test.go | 2 +- modules/mesh-build-go/cmd/mesh-build-go/main.go | 2 +- .../mesh-build-go/cmd/mesh-build-go/module.go | 6 +++++- modules/mesh-build-go/module.json | 3 ++- .../cmd/mesh-build-image/asks.go | 16 ++++++++++++++-- .../cmd/mesh-build-image/main.go | 2 +- .../cmd/mesh-build-image/manifest_test.go | 2 +- .../cmd/mesh-build-image/module.go | 6 +++++- modules/mesh-build-image/module.json | 2 +- .../mesh-build-node/cmd/mesh-build-node/asks.go | 16 ++++++++++++++-- .../mesh-build-node/cmd/mesh-build-node/main.go | 2 +- .../cmd/mesh-build-node/manifest_test.go | 2 +- .../cmd/mesh-build-node/module.go | 6 +++++- modules/mesh-build-node/module.json | 2 +- 15 files changed, 67 insertions(+), 18 deletions(-) diff --git a/modules/mesh-build-go/cmd/mesh-build-go/asks.go b/modules/mesh-build-go/cmd/mesh-build-go/asks.go index 7d8315498..7b07ee9e9 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/asks.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/asks.go @@ -10,6 +10,7 @@ import ( sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/steprun" stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/busstate" ) // **The module pulls its seat's asks** (novox/hq ADR 0525 §2-§6, ADR 0536 §4; issue 1887 A3go). It holds the seat @@ -50,9 +51,19 @@ func (m *Module) askSteps(steps []StepTool) steprun.AskSteps { Lint: m.lintStep, Node: m.Node, SayStep: m.sayStep(), + Outputs: m.outputs(), } } +// outputs are the outputs objects a test step keeps its run's output in and the verb output reads (design 47 H2 §791, +// §820): Outputs when a test sets it, else the module's own objects store on the bus. +func (m *Module) outputs() steprun.OutputObjects { + if m.Outputs != nil { + return m.Outputs + } + return busstate.ObjectsOf(steprun.OutputsStore) +} + // Puller is the module's pull of its seat's asks, on source (the runtime's seat pull when nil). func (m *Module) Puller(steps []StepTool, source steprun.AskSource) *steprun.Puller { if source == nil { @@ -88,7 +99,7 @@ func (m *Module) Puller(steps []StepTool, source steprun.AskSource) *steprun.Pul if m.SayEvent != nil { say = steprun.SayOn(sdkstep.AskGo, m.Node, m.SayEvent) } - return &steprun.Puller{KeepOutput: steprun.KeepOutputIn(m.Outputs), Source: source, Language: sdkstep.AskGo, Say: say, Prior: steprun.PriorRuns(m.ReadSteps.Get), Node: m.Node, Put: m.Steps, PausePut: m.Pauses, Load: load, ForgetPause: m.ForgetPause, WatchPauses: m.WatchPauses, PauseKeys: m.ReadPauses.Keys, + return &steprun.Puller{KeepAnswer: steprun.KeepAnswerIn(m.Answers), Source: source, Language: sdkstep.AskGo, Say: say, Prior: steprun.PriorRuns(m.ReadSteps.Get), Node: m.Node, Put: m.Steps, PausePut: m.Pauses, Load: load, ForgetPause: m.ForgetPause, WatchPauses: m.WatchPauses, PauseKeys: m.ReadPauses.Keys, Capacity: capacity, SettingsFile: settings, WatchConfig: watch, CoresPerStep: cores, BuildHours: hours, Logf: m.Say, Has: m.Has, Run: func(ctx context.Context, a sdkstep.StepAsk) (any, error) { if err := m.Check(ctx); err != nil { @@ -109,7 +120,8 @@ func (m *Module) Puller(steps []StepTool, source steprun.AskSource) *steprun.Pul // SeatTools are the seat's verbs on this node (ADR 0525 §10): what it runs, and its pause and resume. func (m *Module) SeatTools(p *steprun.Puller) []stdio.Tool { - return append(steprun.SeatTools(Seat, "Go", p), m.stepsTool(), m.keptArtifactVerb(), steprun.OutputTool(Seat, steprun.ReadOutputFrom(m.Outputs))) + return append(steprun.SeatTools(Seat, "Go", p), m.stepsTool(), m.keptArtifactVerb(), steprun.OutputTool(Seat, m.Node, m.outputs(), steprun.EndedHere(m.Node, m.ReadSteps.Keys, m.ReadSteps.Get)), + steprun.AnswerTool(Seat, steprun.ReadAnswerFrom(m.Answers))) } // StepEndTool ends one step left running (issue 1890); the module's steps state is the one it reads and writes. diff --git a/modules/mesh-build-go/cmd/mesh-build-go/asks_test.go b/modules/mesh-build-go/cmd/mesh-build-go/asks_test.go index 88542ab6b..955b8925a 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/asks_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/asks_test.go @@ -142,7 +142,7 @@ func mustJSON(v any) []byte { b, _ := json.Marshal(v); return b } // the node's build capacity as a setting, with the builder's default; the seat's verbs are served as its tools. func TestManifestClaimsTheGoBuildSeatAndReadsItsCapacity(t *testing.T) { m := readManifest(t) - if len(m.Claims) != 1 || m.Claims[0].Name != Seat || !slices.Equal(m.Claims[0].Serves, []string{"current", "drop", "pause", "resume", "steps", "kept-artifact", "output"}) { + if len(m.Claims) != 1 || m.Claims[0].Name != Seat || !slices.Equal(m.Claims[0].Serves, []string{"current", "drop", "pause", "resume", "steps", "kept-artifact", "output", "answer"}) { t.Fatalf("claims = %+v", m.Claims) } if len(m.Reads) != 0 { diff --git a/modules/mesh-build-go/cmd/mesh-build-go/main.go b/modules/mesh-build-go/cmd/mesh-build-go/main.go index 79a538789..9566774a1 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/main.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/main.go @@ -70,7 +70,7 @@ func main() { m.ReadEntry = askEntry m.AskInput = stdio.Ask m.KeptBuilds = busstate.ObjectsOf(KeptBuildsStore) - m.Outputs = busstate.ObjectsOf(steprun.OutputsStore) + m.Answers = busstate.ObjectsOf(steprun.AnswersStore) m.Self = strings.TrimSpace(os.Getenv("MESH_MODULE")) m.Probe = ProbeGo(m.Home, m.Cache) m.Bubblewrap = ProbeBubblewrap diff --git a/modules/mesh-build-go/cmd/mesh-build-go/module.go b/modules/mesh-build-go/cmd/mesh-build-go/module.go index 7e67a8b1b..150ab04e5 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/module.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/module.go @@ -124,7 +124,11 @@ type Module struct { // KeptBuilds is the object store of the builds kept for the artifact store's holder (keptbuilds.go); nil keeps none. // Self is this module's name as the runtime launched it (MESH_MODULE), the first part of the store's name on the bus. KeptBuilds KeptObjects - // Outputs is the object store of a run's answers over what step-ended carries inline (steprun.OutputsStore); nil keeps none. + // Answers is the steps state's objects (steprun.AnswersStore), where a step's end over what its key or step-ended + // carries is kept and the seat's verb answer reads it (design 47 H2 H-A-412 addendum); nil keeps none. + Answers steprun.OutputObjects + // Outputs are the outputs objects (steprun.OutputsStore) a test step keeps a run's output in, which the verb output + // reads (design 47 H2 §791); nil is the module's own on the bus (outputs), a test sets its own. Outputs steprun.OutputObjects Self string diff --git a/modules/mesh-build-go/module.json b/modules/mesh-build-go/module.json index 3198c8f59..4cebbdd0f 100644 --- a/modules/mesh-build-go/module.json +++ b/modules/mesh-build-go/module.json @@ -28,7 +28,8 @@ "resume", "steps", "kept-artifact", - "output" + "output", + "answer" ] } ], diff --git a/modules/mesh-build-image/cmd/mesh-build-image/asks.go b/modules/mesh-build-image/cmd/mesh-build-image/asks.go index 30ade441c..3c0c74a48 100644 --- a/modules/mesh-build-image/cmd/mesh-build-image/asks.go +++ b/modules/mesh-build-image/cmd/mesh-build-image/asks.go @@ -9,6 +9,7 @@ import ( sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/steprun" stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/busstate" ) // **The module pulls its seat's asks** (novox/hq ADR 0525 §2-§6; issue 1887 A3image), as mesh-build-go does: the seat @@ -33,9 +34,19 @@ func (m *Module) askSteps(steps []StepTool) steprun.AskSteps { Logf: m.Say, Node: m.Node, SayStep: m.sayStep(), + Outputs: m.outputs(), } } +// outputs are the outputs objects a test step keeps its run's output in and the verb output reads (design 47 H2 §791, +// §820): Outputs when a test sets it, else the module's own objects store on the bus. +func (m *Module) outputs() steprun.OutputObjects { + if m.Outputs != nil { + return m.Outputs + } + return busstate.ObjectsOf(steprun.OutputsStore) +} + // sayStep says a step ask's end on this node's subject of the seat (design 47 G §24): an image has no lint, unit or integration // step (§328), so none is asked of this seat, but the steps of every build module can say their ends, which the kit holds. func (m *Module) sayStep() func(sdkstep.StepEnded) error { @@ -77,7 +88,7 @@ func (m *Module) Puller(steps []StepTool, source steprun.AskSource) *steprun.Pul if m.SayEvent != nil { say = steprun.SayOn(sdkstep.AskImage, m.Node, m.SayEvent) } - return &steprun.Puller{KeepOutput: steprun.KeepOutputIn(m.Outputs), Source: source, Language: sdkstep.AskImage, Say: say, Prior: steprun.PriorRuns(m.ReadSteps.Get), Node: m.Node, Put: m.Steps, PausePut: m.Pauses, Load: load, ForgetPause: m.ForgetPause, WatchPauses: m.WatchPauses, PauseKeys: m.ReadPauses.Keys, + return &steprun.Puller{KeepAnswer: steprun.KeepAnswerIn(m.Answers), Source: source, Language: sdkstep.AskImage, Say: say, Prior: steprun.PriorRuns(m.ReadSteps.Get), Node: m.Node, Put: m.Steps, PausePut: m.Pauses, Load: load, ForgetPause: m.ForgetPause, WatchPauses: m.WatchPauses, PauseKeys: m.ReadPauses.Keys, Capacity: capacity, SettingsFile: settings, WatchConfig: watch, CoresPerStep: cores, BuildHours: hours, Eligible: m.Eligible, Logf: m.Say, Run: func(ctx context.Context, a sdkstep.StepAsk) (any, error) { if err := m.Check(ctx); err != nil { @@ -92,7 +103,8 @@ func (m *Module) Puller(steps []StepTool, source steprun.AskSource) *steprun.Pul // SeatTools are the seat's verbs on this node (ADR 0525 §10). func (m *Module) SeatTools(p *steprun.Puller) []stdio.Tool { - return append(steprun.SeatTools(Seat, "image", p), m.stepsTool(), m.keptArtifactVerb(), steprun.OutputTool(Seat, steprun.ReadOutputFrom(m.Outputs))) + return append(steprun.SeatTools(Seat, "image", p), m.stepsTool(), m.keptArtifactVerb(), steprun.OutputTool(Seat, m.Node, m.outputs(), steprun.EndedHere(m.Node, m.ReadSteps.Keys, m.ReadSteps.Get)), + steprun.AnswerTool(Seat, steprun.ReadAnswerFrom(m.Answers))) } // StepEndTool ends one step left running (issue 1890); the module's steps state is the one it reads and writes. diff --git a/modules/mesh-build-image/cmd/mesh-build-image/main.go b/modules/mesh-build-image/cmd/mesh-build-image/main.go index 61b05a9c8..f9bbd7227 100644 --- a/modules/mesh-build-image/cmd/mesh-build-image/main.go +++ b/modules/mesh-build-image/cmd/mesh-build-image/main.go @@ -67,7 +67,7 @@ func main() { logf("announcing the states it keeps: %v", err) os.Exit(1) } - m.Outputs = busstate.ObjectsOf(steprun.OutputsStore) + m.Answers = busstate.ObjectsOf(steprun.AnswersStore) steps := m.StepTools() booted := time.Now() puller := m.Puller(steps, nil) diff --git a/modules/mesh-build-image/cmd/mesh-build-image/manifest_test.go b/modules/mesh-build-image/cmd/mesh-build-image/manifest_test.go index a2f8be6c7..6c3966f78 100644 --- a/modules/mesh-build-image/cmd/mesh-build-image/manifest_test.go +++ b/modules/mesh-build-image/cmd/mesh-build-image/manifest_test.go @@ -136,7 +136,7 @@ func TestManifestKeepsItsStatesClaimsItsBuildSeatAndDeclaresItsHealthTool(t *tes if want := []string{"keys", MirrorWaitsState, "pauses"}; !slices.Equal(states, want) { t.Errorf("state = %v, want %v", states, want) } - if len(m.Claims) != 1 || m.Claims[0].Name != Seat || !slices.Equal(m.Claims[0].Serves, []string{"current", "drop", "pause", "resume", "steps", "kept-artifact", "output"}) { + if len(m.Claims) != 1 || m.Claims[0].Name != Seat || !slices.Equal(m.Claims[0].Serves, []string{"current", "drop", "pause", "resume", "steps", "kept-artifact", "output", "answer"}) { t.Errorf("claims = %v, want the seat %s alone", m.Claims, Seat) } raw, _ := os.ReadFile(filepath.Join(moduleDir, "module.json")) diff --git a/modules/mesh-build-image/cmd/mesh-build-image/module.go b/modules/mesh-build-image/cmd/mesh-build-image/module.go index a5679adca..2e24457e7 100644 --- a/modules/mesh-build-image/cmd/mesh-build-image/module.go +++ b/modules/mesh-build-image/cmd/mesh-build-image/module.go @@ -20,7 +20,11 @@ const CheckRound = 10 * time.Minute // Module is the image build module's one running state. type Module struct { - // Outputs is the object store of a run's answers over what step-ended carries inline (steprun.OutputsStore); nil keeps none. + // Answers is the steps state's objects (steprun.AnswersStore), where a step's end over what its key or step-ended + // carries is kept and the seat's verb answer reads it (design 47 H2 H-A-412 addendum); nil keeps none. + Answers steprun.OutputObjects + // Outputs are the outputs objects (steprun.OutputsStore) a test step keeps a run's output in, which the verb output + // reads (design 47 H2 §791); nil is the module's own on the bus (outputs), a test sets its own. Outputs steprun.OutputObjects // SayEvent publishes an event of the image build seat on a seat subject (stdio.SeatPublish; nil says none, a test's): the // puller's ask-taken, step-progress and step-ended (design 47 G §496, §602). diff --git a/modules/mesh-build-image/module.json b/modules/mesh-build-image/module.json index 6f99328dc..219d31c34 100644 --- a/modules/mesh-build-image/module.json +++ b/modules/mesh-build-image/module.json @@ -21,7 +21,7 @@ { "name": "node-build-image", "scope": "node", - "serves": ["current", "drop", "pause", "resume", "steps", "kept-artifact", "output"] + "serves": ["current", "drop", "pause", "resume", "steps", "kept-artifact", "output", "answer"] } ], "config": { diff --git a/modules/mesh-build-node/cmd/mesh-build-node/asks.go b/modules/mesh-build-node/cmd/mesh-build-node/asks.go index dc864e5e6..ddcf77176 100644 --- a/modules/mesh-build-node/cmd/mesh-build-node/asks.go +++ b/modules/mesh-build-node/cmd/mesh-build-node/asks.go @@ -10,6 +10,7 @@ import ( sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/steprun" stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/busstate" ) // **The module pulls its seat's asks** (novox/hq ADR 0525 §2-§6, ADR 0536 §4; issue 1887 A3node), as mesh-build-go @@ -45,9 +46,19 @@ func (m *Module) askSteps(steps []StepTool) steprun.AskSteps { Lint: m.lintStep, Node: m.Node, SayStep: m.sayStep(), + Outputs: m.outputs(), } } +// outputs are the outputs objects a test step keeps its run's output in and the verb output reads (design 47 H2 §791, +// §820): Outputs when a test sets it, else the module's own objects store on the bus. +func (m *Module) outputs() steprun.OutputObjects { + if m.Outputs != nil { + return m.Outputs + } + return busstate.ObjectsOf(steprun.OutputsStore) +} + // sayStep says a pipeline step's end on this node's subject of the seat (design 47 G §24, *Asked on a work queue, ended by an // event*): SayStep when a test sets it, else the runtime's seat publish. func (m *Module) sayStep() func(sdkstep.StepEnded) error { @@ -89,7 +100,7 @@ func (m *Module) Puller(steps []StepTool, source steprun.AskSource) *steprun.Pul if m.SayEvent != nil { say = steprun.SayOn(sdkstep.AskNode, m.Node, m.SayEvent) } - return &steprun.Puller{KeepOutput: steprun.KeepOutputIn(m.Outputs), Source: source, Language: sdkstep.AskNode, Say: say, Prior: steprun.PriorRuns(m.ReadSteps.Get), Node: m.Node, Put: m.Steps, PausePut: m.Pauses, Load: load, ForgetPause: m.ForgetPause, WatchPauses: m.WatchPauses, PauseKeys: m.ReadPauses.Keys, + return &steprun.Puller{KeepAnswer: steprun.KeepAnswerIn(m.Answers), Source: source, Language: sdkstep.AskNode, Say: say, Prior: steprun.PriorRuns(m.ReadSteps.Get), Node: m.Node, Put: m.Steps, PausePut: m.Pauses, Load: load, ForgetPause: m.ForgetPause, WatchPauses: m.WatchPauses, PauseKeys: m.ReadPauses.Keys, Capacity: capacity, SettingsFile: settings, WatchConfig: watch, CoresPerStep: cores, BuildHours: hours, Logf: m.Say, Run: func(ctx context.Context, a sdkstep.StepAsk) (any, error) { if err := m.Prepare(ctx); err != nil { @@ -104,7 +115,8 @@ func (m *Module) Puller(steps []StepTool, source steprun.AskSource) *steprun.Pul // SeatTools are the seat's verbs on this node (ADR 0525 §10). func (m *Module) SeatTools(p *steprun.Puller) []stdio.Tool { - return append(steprun.SeatTools(Seat, "Node", p), m.stepsTool(), m.keptArtifactVerb(), steprun.OutputTool(Seat, steprun.ReadOutputFrom(m.Outputs))) + return append(steprun.SeatTools(Seat, "Node", p), m.stepsTool(), m.keptArtifactVerb(), steprun.OutputTool(Seat, m.Node, m.outputs(), steprun.EndedHere(m.Node, m.ReadSteps.Keys, m.ReadSteps.Get)), + steprun.AnswerTool(Seat, steprun.ReadAnswerFrom(m.Answers))) } // StepEndTool ends one step left running (issue 1890); the module's steps state is the one it reads and writes. diff --git a/modules/mesh-build-node/cmd/mesh-build-node/main.go b/modules/mesh-build-node/cmd/mesh-build-node/main.go index 92a9ca911..b41d596b0 100644 --- a/modules/mesh-build-node/cmd/mesh-build-node/main.go +++ b/modules/mesh-build-node/cmd/mesh-build-node/main.go @@ -79,7 +79,7 @@ func main() { logf("announcing the states it keeps: %v", err) os.Exit(1) } - m.Outputs = busstate.ObjectsOf(steprun.OutputsStore) + m.Answers = busstate.ObjectsOf(steprun.AnswersStore) steps := m.StepTools() booted := time.Now() puller := m.Puller(steps, nil) diff --git a/modules/mesh-build-node/cmd/mesh-build-node/manifest_test.go b/modules/mesh-build-node/cmd/mesh-build-node/manifest_test.go index 6640f55a9..7372c3eaf 100644 --- a/modules/mesh-build-node/cmd/mesh-build-node/manifest_test.go +++ b/modules/mesh-build-node/cmd/mesh-build-node/manifest_test.go @@ -118,7 +118,7 @@ func TestManifestKeepsItsStatesClaimsItsBuildSeatAndDeclaresItsHealthTool(t *tes if want := []string{"keys", "pauses", "toolchains"}; !slices.Equal(states, want) { t.Errorf("state = %v, want %v", states, want) } - if len(m.Claims) != 1 || m.Claims[0].Name != Seat || !slices.Equal(m.Claims[0].Serves, []string{"current", "drop", "pause", "resume", "steps", "kept-artifact", "output"}) { + if len(m.Claims) != 1 || m.Claims[0].Name != Seat || !slices.Equal(m.Claims[0].Serves, []string{"current", "drop", "pause", "resume", "steps", "kept-artifact", "output", "answer"}) { t.Errorf("claims = %v, want the seat %s alone", m.Claims, Seat) } if !slices.Contains(m.Tools, HealthTool) { diff --git a/modules/mesh-build-node/cmd/mesh-build-node/module.go b/modules/mesh-build-node/cmd/mesh-build-node/module.go index ac30b5810..b121d41fd 100644 --- a/modules/mesh-build-node/cmd/mesh-build-node/module.go +++ b/modules/mesh-build-node/cmd/mesh-build-node/module.go @@ -25,7 +25,11 @@ const CheckRound = 10 * time.Minute // Module is the Node build module's one running state: the node it runs on, its recipe, the toolchain guard, the // toolchain directory and the states it writes (toolchains, steps). type Module struct { - // Outputs is the object store of a run's answers over what step-ended carries inline (steprun.OutputsStore); nil keeps none. + // Answers is the steps state's objects (steprun.AnswersStore), where a step's end over what its key or step-ended + // carries is kept and the seat's verb answer reads it (design 47 H2 H-A-412 addendum); nil keeps none. + Answers steprun.OutputObjects + // Outputs are the outputs objects (steprun.OutputsStore) a test step keeps a run's output in, which the verb output + // reads (design 47 H2 §791); nil is the module's own on the bus (outputs), a test sets its own. Outputs steprun.OutputObjects // SayEvent publishes an event of the node build seat on a seat subject (stdio.SeatPublish; nil says none, a test's): the // puller's ask-taken, step-progress and step-ended (design 47 G §496, §602). diff --git a/modules/mesh-build-node/module.json b/modules/mesh-build-node/module.json index 994095fa2..fdef48552 100644 --- a/modules/mesh-build-node/module.json +++ b/modules/mesh-build-node/module.json @@ -21,7 +21,7 @@ { "name": "node-build-node", "scope": "node", - "serves": ["current", "drop", "pause", "resume", "steps", "kept-artifact", "output"] + "serves": ["current", "drop", "pause", "resume", "steps", "kept-artifact", "output", "answer"] } ], "consumes": [ -- 2.54.0 From e335e859d57cdc35a4dc29fd91b8ceb1f2de873b Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 22 Nov 2026 01:24:44 +0100 Subject: [PATCH 34/54] BM3: the Node build module installs a module's npm tree and publishes it as :node-modules MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An install ask's body runs npm ci from the lock with no install script and no link, packs the tree alone under modules//node_modules, publishes it to the artifact store as the archive :node-modules under the ask's version and answers its reference, which the step's end carries. The package.json npm ci needs is written from the lock's root entry, a lock without one refused. The integration level's setup bound in code goes: half of the level's bound, which each level's script gets as MESH_TEST_LEVEL_BOUND. Builds: design 47 H2 §800; H-A-399 (2), H-A-403, H-A-427 --- .../cmd/mesh-build-node/asks.go | 1 + .../cmd/mesh-build-node/check.go | 32 +++-- .../cmd/mesh-build-node/check_test.go | 8 +- .../cmd/mesh-build-node/install.go | 121 ++++++++++++++++++ .../cmd/mesh-build-node/push_test.go | 74 ++++++++++- 5 files changed, 225 insertions(+), 11 deletions(-) create mode 100644 modules/mesh-build-node/cmd/mesh-build-node/install.go diff --git a/modules/mesh-build-node/cmd/mesh-build-node/asks.go b/modules/mesh-build-node/cmd/mesh-build-node/asks.go index ddcf77176..402ffcce3 100644 --- a/modules/mesh-build-node/cmd/mesh-build-node/asks.go +++ b/modules/mesh-build-node/cmd/mesh-build-node/asks.go @@ -43,6 +43,7 @@ func (m *Module) askSteps(steps []StepTool) steprun.AskSteps { Release: func(ws string) error { return m.spaces().Release(ws) }, Logf: m.Say, Has: m.Has, + Install: m.installTreeStep, Lint: m.lintStep, Node: m.Node, SayStep: m.sayStep(), diff --git a/modules/mesh-build-node/cmd/mesh-build-node/check.go b/modules/mesh-build-node/cmd/mesh-build-node/check.go index 0ecdd56bb..4f0b5b244 100644 --- a/modules/mesh-build-node/cmd/mesh-build-node/check.go +++ b/modules/mesh-build-node/cmd/mesh-build-node/check.go @@ -26,13 +26,28 @@ import ( "git.novox.be/novox/mesh-sdk/go/forgekey" ) -// The bounds of a level (ADR 0445 §4). +// The bounds of a level (ADR 0445 §4). The integration level's setup bound is half of its bound, never a value of its +// own (design 47 H2 §800: the setup bound in code goes), as mesh-sdk's test helpers take theirs from LevelBoundEnv. const ( - UnitBound = 5 * time.Second - IntegrationSetupBound = 30 * time.Second - IntegrationRunBound = 60 * time.Second + UnitBound = 5 * time.Second + IntegrationRunBound = 60 * time.Second ) +// LevelBoundEnv is the word each level's tests get its bound in, as a duration ("1m0s"; design 47 H2 §800): mesh-sdk's +// test helpers take a setup bound of half of it. +const LevelBoundEnv = "MESH_TEST_LEVEL_BOUND" + +// levelBound is a level's bound. +func levelBound(level string) time.Duration { + if level == "unit" { + return UnitBound + } + return IntegrationRunBound +} + +// integrationSetupBound is the integration level's setup bound: half of its bound (§800). +func integrationSetupBound() time.Duration { return IntegrationRunBound / 2 } + // TestResult is what a test step passes with, with the field names of mesh-build-go's (issue 1290). The step ends // passed whenever the level ran, whatever its verdict, and Passed is the verdict; it ends failed only when the tests // could not be run (no toolchain, a command that cannot start, a workspace that was never fetched). @@ -191,7 +206,7 @@ func (m *Module) testLevel(ctx context.Context, args map[string]any) (*TestResul report := filepath.Join(root, ".mesh-node-test-"+level+".tap") _ = os.Remove(report) env := []string{"PATH=" + binDir(root) + ":" + pathOrDefault(), "npm_config_cache=" + filepath.Join(m.Toolchain.Paths.Cache, "npm"), - "NODE_OPTIONS=" + steprun.NodeTestOptions(report)} + "NODE_OPTIONS=" + steprun.NodeTestOptions(report), LevelBoundEnv + "=" + levelBound(level).String()} r := m.runIn(ctx, root, false, tg.dir, env, "npm", scriptArgs(script)...) run += m.now().Sub(start) tap, _ := os.ReadFile(report) @@ -223,11 +238,12 @@ func (m *Module) testLevel(ctx context.Context, args map[string]any) (*TestResul } } else { res.Setup, res.Elapsed = check.Seconds(), run.Seconds() - res.SetupBound, res.Bound = IntegrationSetupBound.Seconds(), IntegrationRunBound.Seconds() - res.OverBound = check > IntegrationSetupBound || run > IntegrationRunBound + setup := integrationSetupBound() + res.SetupBound, res.Bound = setup.Seconds(), IntegrationRunBound.Seconds() + res.OverBound = check > setup || run > IntegrationRunBound if res.OverBound { why = append(why, fmt.Sprintf("the integration level took %.2f s of setup and %.2f s of running, over its bounds of %s and %s", - res.Setup, res.Elapsed, IntegrationSetupBound, IntegrationRunBound)) + res.Setup, res.Elapsed, setup, IntegrationRunBound)) } } res.Passed = len(failed) == 0 diff --git a/modules/mesh-build-node/cmd/mesh-build-node/check_test.go b/modules/mesh-build-node/cmd/mesh-build-node/check_test.go index fcc3a7ba4..ca66028cf 100644 --- a/modules/mesh-build-node/cmd/mesh-build-node/check_test.go +++ b/modules/mesh-build-node/cmd/mesh-build-node/check_test.go @@ -123,6 +123,10 @@ func TestTheIntegrationLevelRunsItsScriptAndReportsSetupAndRunningApart(t *testi if strings.Join(last.args, " ") != "run test:integration" { t.Errorf("ran %s, want npm run test:integration", joined(last)) } + // Design 47 H2 §800: the level's tests get its bound in the environment, the setup bound being half of it. + if !slices.Contains(last.env, "MESH_TEST_LEVEL_BOUND=1m0s") { + t.Errorf("the integration script ran with %v, not the level's bound MESH_TEST_LEVEL_BOUND=1m0s", last.env) + } if tsc := f.calls[0]; strings.Join(tsc.args[1:], " ") != "--noEmit -p ." { t.Errorf("a package with a tsconfig was checked with %s", joined(tsc)) } @@ -133,7 +137,7 @@ func TestTheIntegrationLevelRunsItsScriptAndReportsSetupAndRunningApart(t *testi } } -// The commands of a step get an environment built from scratch: no word of this process's. +// The commands of a step get an environment built from scratch: no word of this process's, the level's bound alone (§800). func TestTheStepsCommandsGetAnEnvironmentOfTheirOwn(t *testing.T) { t.Setenv("MESH_SECRET_TOKEN", "hunter2") m, f, work := newMachine(t) @@ -143,7 +147,7 @@ func TestTheStepsCommandsGetAnEnvironmentOfTheirOwn(t *testing.T) { } for _, c := range f.calls { for _, e := range c.env { - if strings.Contains(e, "hunter2") || strings.HasPrefix(e, "MESH_") { + if strings.Contains(e, "hunter2") || (strings.HasPrefix(e, "MESH_") && !strings.HasPrefix(e, LevelBoundEnv+"=")) { t.Errorf("%s was given %s", joined(c), e) } } diff --git a/modules/mesh-build-node/cmd/mesh-build-node/install.go b/modules/mesh-build-node/cmd/mesh-build-node/install.go new file mode 100644 index 000000000..25921b4ec --- /dev/null +++ b/modules/mesh-build-node/cmd/mesh-build-node/install.go @@ -0,0 +1,121 @@ +package main + +// **An install of a module's npm tree** (design 47 H2 §800; H-A-399 (2), H-A-403): the runner asks the seat node-build-node +// a StepAsk of the kind install, on the seat's install queue, for a module whose Go artifact's check-needs names this +// seat. The install takes the module's package lock at the commit, installs it from the npm registry seat with an +// integrity-checked install (npm ci) that runs no install script and makes no link (--ignore-scripts, --no-bin-links: the +// SDK's Pack and the unpack refuse a symlink, and the mesh creates none), packs the tree, publishes the archive to the +// artifact store as the module's artifact :node-modules under the ask's version, and answers its reference, which +// the step's end carries and the integration ask names for the Go build module to unpack. + +import ( + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + + artifactref "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/containerimageregistryclient/reference" + sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/steprun" +) + +// installKind is the artifact kind the install archive is published as (H-A-403: kind archive). +const installKind = "archive" + +// installTree is where the installed tree lies in the archive, from the workspace's tree: the module's directory's +// node_modules, which the Go build module unpacks at the tree's root (layInstall). +func installTree(module string) string { return filepath.Join("modules", module, "node_modules") } + +// installTreeStep is AskSteps.Install: the install body of an install ask (steprun's installStep gives its arguments). +func (m *Module) installTreeStep(ctx context.Context, args map[string]any) (any, error) { + module, _ := args["module"].(string) + lock, _ := args["lock"].(string) + artifact, _ := args["artifact"].(string) + version, _ := args["version"].(string) + name, ok := strings.CutPrefix(artifact, module+":") + switch { + case module == "" || !ok || artifact != sdkstep.InstallArtifact(module): + return nil, fmt.Errorf("the install of %q names the artifact %q, not %s", module, artifact, sdkstep.InstallArtifact(module)) + case version == "": + return nil, fmt.Errorf("the install of %s names no version to publish its archive under", module) + } + pkg, err := lockRoot(lock) + if err != nil { + return nil, fmt.Errorf("the install of %s: %w", module, err) + } + if err := os.MkdirAll(m.Work, 0o755); err != nil { + return nil, err + } + root, err := os.MkdirTemp(m.Work, "install-") + if err != nil { + return nil, err + } + defer os.RemoveAll(root) + dir := filepath.Join(root, "npm") + if err := os.MkdirAll(dir, 0o700); err != nil { + return nil, err + } + if err := os.WriteFile(filepath.Join(dir, "package.json"), pkg, 0o600); err != nil { + return nil, err + } + if err := os.WriteFile(filepath.Join(dir, "package-lock.json"), []byte(lock), 0o600); err != nil { + return nil, err + } + // The registry is given through a config file of its own, outside the workspace, removed after (npmrcIn). + npmrc, err := m.npmrcIn(root) + if err != nil { + return nil, fmt.Errorf("the install of %s: %w", module, err) + } + defer os.Remove(npmrc) + env := append(m.baseEnv(), "npm_config_cache="+m.npmCache()) + r := m.Run(ctx, env, "npm", "ci", "--ignore-scripts", "--no-bin-links", "--no-audit", "--no-fund", "--prefix", dir, "--userconfig", npmrc) + if why := ranFailed(r); why != "" { + return nil, steprun.Transport(fmt.Errorf("installing the npm tree of %s: %s", module, why)) + } + // Only the tree is archived, under the module's directory: never the package.json written here from the lock, which + // would lie over the module's own when unpacked. + packed := filepath.Join(root, "packed") + into := filepath.Join(packed, installTree(module)) + if err := os.MkdirAll(filepath.Dir(into), 0o755); err != nil { + return nil, err + } + if err := os.Rename(filepath.Join(dir, "node_modules"), into); err != nil { + return nil, fmt.Errorf("the install of %s made no node_modules: %w", module, err) + } + archive, digest, err := steprun.Pack(packed) + if err != nil { + return nil, fmt.Errorf("packing the npm tree of %s: %w", module, err) + } + if err := m.publishOn(ctx, steprun.ArtifactSeat, module+"/"+name, installKind, version, digest, archive); err != nil { + return nil, fmt.Errorf("publishing the npm tree of %s: %w", module, err) + } + reference := artifactref.SeatReference(steprun.ArtifactSeat, module, name, digest) + if m.Say != nil { + m.Say("installed the npm tree of %s at %s (%d bytes) as %s", module, version, len(archive), reference) + } + return map[string]any{"archive": reference}, nil +} + +// lockRoot is the package.json npm ci needs beside a lock, written from the lock's root package (packages[""]): its +// name, version and dependency maps, never a script. A lock with no root package (lockfileVersion 1) is refused. +func lockRoot(lock string) ([]byte, error) { + var l struct { + Packages map[string]map[string]any `json:"packages"` + } + if err := json.Unmarshal([]byte(lock), &l); err != nil { + return nil, fmt.Errorf("the package lock is not JSON: %w", err) + } + root, ok := l.Packages[""] + if !ok { + return nil, fmt.Errorf("the package lock names no root package (packages[\"\"]): a lock of lockfileVersion 2 or later is needed") + } + pkg := map[string]any{} + for _, k := range []string{"name", "version", "dependencies", "devDependencies", "optionalDependencies", "peerDependencies"} { + if v, ok := root[k]; ok { + pkg[k] = v + } + } + return json.Marshal(pkg) +} diff --git a/modules/mesh-build-node/cmd/mesh-build-node/push_test.go b/modules/mesh-build-node/cmd/mesh-build-node/push_test.go index fa5b44c01..6a2b2d95a 100644 --- a/modules/mesh-build-node/cmd/mesh-build-node/push_test.go +++ b/modules/mesh-build-node/cmd/mesh-build-node/push_test.go @@ -1,6 +1,9 @@ package main import ( + "archive/tar" + "bytes" + "compress/gzip" "context" "crypto/sha256" "encoding/base64" @@ -9,6 +12,8 @@ import ( "errors" "os" "path/filepath" + "reflect" + "slices" "strings" "sync" "testing" @@ -18,6 +23,7 @@ import ( builds "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/steprun" + "git.novox.be/novox/mesh-sdk/go/forgekey" ) // pushed are the blobs the fake holder took, by digest, for the tests that read an archive back. @@ -51,7 +57,7 @@ func withStore(t *testing.T, m *Module) *fakeStore { case "publish": var in registry.PublishInput _ = json.Unmarshal(raw, &in) - if (seat == steprun.ArtifactSeat) != (in.Kind == "bundle") || in.Transfer == "" || strings.ContainsAny(in.Transfer, "/. *>") { + if (seat == steprun.ArtifactSeat) != (in.Kind == "bundle" || in.Kind == "archive") || in.Transfer == "" || strings.ContainsAny(in.Transfer, "/. *>") { t.Errorf("publish on %s of kind %q, transfer %q", seat, in.Kind, in.Transfer) } s.open[in.Transfer] = in @@ -115,6 +121,72 @@ func TestADoneBundleAnswersTheReferenceOfTheArchiveItPushed(t *testing.T) { if len(store.published) != 1 || store.published[0].Version != "1.4.3" { t.Errorf("the bundle was published as %+v, want the ask's version 1.4.3", store.published) } + // Design 47 H2 §800, H-A-403, H-A-427: an install runs no script and makes no link; its tree alone, under the module's + // directory, is published as the archive :node-modules under the ask's version, and the end names its reference. + t.Run("an install runs no script", func(t *testing.T) { + m := &Module{Work: t.TempDir(), Toolchain: &Toolchain{Paths: Paths{Home: t.TempDir(), Cache: t.TempDir()}, + Registry: func() (Registry, error) { return Registry{}, nil }}} + store := withStore(t, m) + var ran []string + var written map[string]any + m.Run = func(_ context.Context, _ []string, name string, args ...string) forgekey.Ran { + ran = append([]string{name}, args...) + dir := args[slices.Index(args, "--prefix")+1] + raw, _ := os.ReadFile(filepath.Join(dir, "package.json")) + _ = json.Unmarshal(raw, &written) + os.MkdirAll(filepath.Join(dir, "node_modules", "left-pad"), 0o755) + os.WriteFile(filepath.Join(dir, "node_modules", "left-pad", "index.js"), []byte("module.exports = 1\n"), 0o644) + return forgekey.Ran{} + } + lock := `{"lockfileVersion": 3, "packages": {"": {"name": "@novox/module-gitea", "scripts": {"install": "touch ran"}, + "dependencies": {"left-pad": "^1.3.0"}}, "node_modules/left-pad": {"version": "1.3.0", "hasInstallScript": true}}}` + install := func(lock string) (any, error) { + return m.installTreeStep(context.Background(), map[string]any{"module": "gitea", "commit": strings.Repeat("a", 40), "fingerprint": "fp", + "pipeline": "p1", "lock": lock, "artifact": sdkstep.InstallArtifact("gitea"), "version": "0.4.2"}) + } + out, err := install(lock) + if err != nil { + t.Fatal(err) + } + if !slices.Contains(ran, "--ignore-scripts") || !slices.Contains(ran, "--no-bin-links") || ran[0] != "npm" || ran[1] != "ci" { + t.Errorf("the install ran %v, want npm ci --ignore-scripts --no-bin-links", ran) + } + if _, scripts := written["scripts"]; scripts || written["name"] != "@novox/module-gitea" { + t.Errorf("the package.json beside the lock is %v, want the lock's root with no script", written) + } + if len(store.published) != 1 || store.published[0].Name != "gitea/node-modules" || store.published[0].Kind != "archive" || store.published[0].Version != "0.4.2" { + t.Fatalf("published %+v, want gitea/node-modules of kind archive at 0.4.2", store.published) + } + digest := store.published[0].Digest + if want := map[string]any{"archive": "oci-artifact-registry/gitea/node-modules@" + digest}; !reflect.DeepEqual(out, want) { + t.Errorf("the install answered %v, want %v", out, want) + } + if names := archiveNames(t, store.blobs["gitea/node-modules@"+digest]); !slices.Equal(names, []string{"modules/gitea/node_modules/left-pad/index.js"}) { + t.Errorf("the archive holds %v, want the tree alone under the module's directory", names) + } + if _, err := install(`{"lockfileVersion": 1, "dependencies": {}}`); err == nil { + t.Error("a lock with no root package was installed") + } + }) +} + +// archiveNames are the file names a packed archive holds (steprun.Pack: tar in gzip). +func archiveNames(t *testing.T, archive []byte) []string { + t.Helper() + z, err := gzip.NewReader(bytes.NewReader(archive)) + if err != nil { + t.Fatal(err) + } + var names []string + r := tar.NewReader(z) + for { + h, err := r.Next() + if err != nil { + break + } + names = append(names, h.Name) + } + return names } // 1587-S7 and S8 (ADR 0498 §2, the lead's decision L9): a publish the holder refuses fails the step `not stored: -- 2.54.0 From 46990669deb1f9de7ac878ac20101d50bd00365b Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 22 Nov 2026 01:32:03 +0100 Subject: [PATCH 35/54] BM4: the bus module's keeper keeps an objects store's objects for its max-age-days MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The objects template takes max-age-days (absent keeps an object for good): the keeper plans it as the object store's max age, which the bus applies as the store's TTL, so a build module's outputs objects go at their seven days. Builds: design 47 H2 §820 (outputs: 7 days, the store's max-age), §791; H-A-434 --- modules/nats/internal/keeper/keeper_test.go | 3 +++ modules/nats/internal/keeper/plan.go | 7 ++++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/modules/nats/internal/keeper/keeper_test.go b/modules/nats/internal/keeper/keeper_test.go index 176994029..33593fc00 100644 --- a/modules/nats/internal/keeper/keeper_test.go +++ b/modules/nats/internal/keeper/keeper_test.go @@ -363,6 +363,9 @@ func TestEachTemplateIsPlannedUnderTheModulesNames(t *testing.T) { {name: "objects", entry: entry("files", catalogue.StorageObjectStore, catalogue.TemplateObjects, map[string]string{"max-mib": "32"}), objects: "object-store mesh-knowledge_files {32 MiB, age 0, history 0, new, size 0}", binding: "objects mesh-knowledge_files"}, + {name: "objects kept 7 days", entry: entry("outputs", catalogue.StorageObjectStore, catalogue.TemplateObjects, map[string]string{"max-mib": "1024", "max-age-days": "7"}), + objects: "object-store mesh-knowledge_outputs {1024 MiB, age 604800, history 0, new, size 0}", + binding: "objects mesh-knowledge_outputs"}, {name: "ledger", entry: ledger, objects: "stream mesh-knowledge_records [mesh.ledger.mesh-knowledge.records.history.>] {16 MiB, age 0, history 0, new, size 262144}; " + "key-value mesh-knowledge_records_state {64 MiB, age 0, history 1, new, size 262144}; " + diff --git a/modules/nats/internal/keeper/plan.go b/modules/nats/internal/keeper/plan.go index 12883bdfc..43320154a 100644 --- a/modules/nats/internal/keeper/plan.go +++ b/modules/nats/internal/keeper/plan.go @@ -211,8 +211,13 @@ func PlanOf(module string, e catalogue.BusStateEntry) (Plan, error) { if capMiB <= 0 { return Plan{}, fmt.Errorf("objects %q gives no %s, which the template objects requires", e.Name, catalogue.OptionCapMiB) } + // An object is kept for good unless the objects name max-age-days (design 47 H2 H-A-434): the store's max age. + age, err := opts.int(catalogue.OptionAgeDays, 0) + if err != nil { + return Plan{}, err + } p.Binding.ObjectStore = name - p.Objects = []Object{{Kind: KindObjects, Name: name, Settings: Settings{MaxBytes: capMiB * MiB, Discard: DiscardNew}}} + p.Objects = []Object{{Kind: KindObjects, Name: name, Settings: Settings{MaxBytes: capMiB * MiB, MaxAgeSeconds: days(age), Discard: DiscardNew}}} case e.Storage == catalogue.StorageLedger && e.Template == catalogue.TemplateLedger: history, err := logSettings(opts) if err != nil { -- 2.54.0 From 1697b7041c0bca7c1c613f15eeec4bff70995d59 Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 22 Nov 2026 01:32:09 +0100 Subject: [PATCH 36/54] BM4: each build module keeps a run's test output in its outputs objects MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Go build module's level answers its text in output (test2json's Output lines), the Node build module's what its commands wrote; the delivery SDK's step run keeps it as . in the module's outputs objects, which each module asks for in requires (1 GiB, kept 7 days) and reads through the verb output at form 2. Each module's kit test runs the SDK's SeatVerbs. Builds: design 47 H2 §791, §820; H-A-412 (1), H-A-412 second addendum (a), H-A-421, H-A-434 --- .../cmd/mesh-build-go/forge_test.go | 9 ++++-- .../cmd/mesh-build-go/kit_test.go | 27 ++++++++++++++++-- .../mesh-build-go/cmd/mesh-build-go/levels.go | 20 +++++++++++++ .../mesh-build-go/cmd/mesh-build-go/test.go | 3 ++ .../cmd/mesh-build-go/test_test.go | 4 +++ modules/mesh-build-go/module.json | 9 ++++++ .../cmd/mesh-build-image/forge_test.go | 8 ++++-- .../cmd/mesh-build-image/kit_test.go | 28 +++++++++++++++++-- .../cmd/mesh-build-image/manifest_test.go | 16 ++++++----- modules/mesh-build-image/module.json | 1 + .../cmd/mesh-build-node/check.go | 9 +++++- .../cmd/mesh-build-node/check_test.go | 5 ++++ .../cmd/mesh-build-node/kit_test.go | 25 ++++++++++++++++- .../cmd/mesh-build-node/manifest_test.go | 16 ++++++----- modules/mesh-build-node/module.json | 1 + 15 files changed, 154 insertions(+), 27 deletions(-) diff --git a/modules/mesh-build-go/cmd/mesh-build-go/forge_test.go b/modules/mesh-build-go/cmd/mesh-build-go/forge_test.go index a54618ce4..2e17a639b 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/forge_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/forge_test.go @@ -151,14 +151,17 @@ func TestManifestRequiresForgeAccountKeepsKeysAndTakesFilesFromTemplates(t *test // detected-steps: the state of the verb steps (design 47 G §326, §414), asked by the template state. // kept-builds: the builds kept for the artifact store's holder (keptbuilds.go), by the template objects (W12.5). kept := map[string]any{"name": "kept-builds", "storage": "object-store", "template": "objects", "options": map[string]any{"max-mib": float64(1024)}} - // outputs: a run's answer over what step-ended carries inline, in the steps state's objects store (design 47 G A-254, A-313). - outputs := map[string]any{"name": "steps", "storage": "key-value", "template": "state", "options": map[string]any{"ttl-seconds": float64(172800), "objects-max-mib": float64(256)}} + // steps: a step's end over what step-ended carries inline, in the steps state's objects store (design 47 G A-254, A-313; + // H2 H-A-412 addendum), read by the verb answer. + steps := map[string]any{"name": "steps", "storage": "key-value", "template": "state", "options": map[string]any{"ttl-seconds": float64(172800), "objects-max-mib": float64(256)}} + // outputs: a run's test output, read by the verb output (design 47 H2 §791, §820: 1 GiB, 7 days). + outputs := map[string]any{"name": "outputs", "storage": "object-store", "template": "objects", "options": map[string]any{"max-mib": float64(1024), "max-age-days": float64(7)}} detected := map[string]any{"name": "detected-steps", "storage": "key-value", "template": "state", "options": map[string]any{"history": float64(1)}} // The findings are asked in requires as a part that is no foundation part asks them (design 47 G §529, A-30). findings := map[string]any{"name": "findings", "storage": "key-value", "template": "state"} findingEvents := map[string]any{"name": "finding-events", "storage": "stream", "template": "events", "options": map[string]any{ "events": []any{"finding-written.*", "finding-cleared.*", "findings-stated.*"}, "max-age-days": float64(7), "max-mib": float64(16)}} - if want := []any{outputs, detected, kept, "artifact-store", "forge-account", "go-toolchain", "go-package-registry", "node-tools", findings, findingEvents}; !reflect.DeepEqual(m.Requires, want) { + if want := []any{steps, detected, outputs, kept, "artifact-store", "forge-account", "go-toolchain", "go-package-registry", "node-tools", findings, findingEvents}; !reflect.DeepEqual(m.Requires, want) { t.Errorf("requires = %v, want %v", m.Requires, want) } keeps := false diff --git a/modules/mesh-build-go/cmd/mesh-build-go/kit_test.go b/modules/mesh-build-go/cmd/mesh-build-go/kit_test.go index 3f1962c7d..fc8ed61e9 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/kit_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/kit_test.go @@ -3,14 +3,17 @@ package main import ( "context" "encoding/json" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/steprun" - "git.novox.be/novox/mesh-sdk/go/forgekey" + "errors" "os" "path/filepath" "strings" "testing" "time" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/steprun" + "git.novox.be/novox/mesh-sdk/go/buscalls" + "git.novox.be/novox/mesh-sdk/go/forgekey" + buildrun "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildrun" sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/steprun/conformance" @@ -87,6 +90,26 @@ func TestTheStepsSayAFailedNetworkAsATransportFault(t *testing.T) { // every step ask the runner sends it is refused ("this build module says no step's end"), as the first live pipeline met. func TestTheStepsCanSayAStepsEnd(t *testing.T) { conformance.StepEnds(t, (&Module{Node: "kitnode"}).askSteps(nil)) + // Design 47 H2 §791, §801, H-A-412, H-A-421: the seat's verbs current, output and answer keep the build-step part's form 2, + // and a pipeline step's run output is kept in the module's outputs objects and read back through output. + m := &Module{Node: "kitnode", Capacity: func() int { return 1 }, Outputs: kitObjects{}} + conformance.SeatVerbs(t, Seat, m.SeatTools(m.Puller(nil, steprun.NewFakeAsks())), m.askSteps(nil)) +} + +// kitObjects is an outputs store in memory, for the kit's SeatVerbs. +type kitObjects map[string][]byte + +func (k kitObjects) Put(name string, data []byte, _ string) (buscalls.ObjectInfo, error) { + k[name] = data + return buscalls.ObjectInfo{Name: name}, nil +} + +func (k kitObjects) Get(name string) (buscalls.ObjectData, error) { + d, ok := k[name] + if !ok { + return buscalls.ObjectData{}, errors.New("object not found") + } + return buscalls.ObjectData{Data: d}, nil } // A-489c (A-489a): the module's puller counts the runs its steps state says started, not the bus's deliveries (an ask given back before diff --git a/modules/mesh-build-go/cmd/mesh-build-go/levels.go b/modules/mesh-build-go/cmd/mesh-build-go/levels.go index 2d702ec0d..d002de777 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/levels.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/levels.go @@ -6,6 +6,7 @@ package main // step that could not run. import ( + "encoding/json" "fmt" "sort" "strings" @@ -63,6 +64,10 @@ type TestResult struct { // §800, H-A-294): counted apart from passed and failed, they leave the level passing, listed here, where the delivery // SDK's test step reads them into the step's end. NotRun []sdkstep.NotRun `json:"not-run,omitempty"` + // Output is the level's text (design 47 H2 §791, H-A-412 second addendum (a)): the Output lines of test2json, of + // every go.mod directory in the order they were tested. The delivery SDK's step run takes it out of the answer and + // keeps it as the run's output in the outputs objects; it is never part of a check's answer. + Output string `json:"output,omitempty"` // Store and Bus are what the integration level ran against: the postgres major version and the nats-server // version (ADR 0470 §4). // OverBound says a bound was kept no longer, and OverWhich which: setup, run or both. The level's verdict, Passed, is @@ -82,6 +87,21 @@ type TestResult struct { By string `json:"by,omitempty"` } +// goTestText is the text of `go test -json`'s output out: the Output of each of its events, in order (H-A-412 (a)); a line +// that is no event is none of it. +func goTestText(out string) string { + var text strings.Builder + for line := range strings.Lines(out) { + var ev struct { + Output string `json:"Output"` + } + if json.Unmarshal([]byte(line), &ev) == nil { + text.WriteString(ev.Output) + } + } + return text.String() +} + // readGoTest reads one go.mod directory's `go test -json` into res through the SDK's reader (issue 1901: one failure // contract for every build module, sdkstep.Failure), and answers the lines that were no event and how many packages // it named. diff --git a/modules/mesh-build-go/cmd/mesh-build-go/test.go b/modules/mesh-build-go/cmd/mesh-build-go/test.go index 0b7a0e0fc..543dd5222 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/test.go @@ -284,6 +284,7 @@ func (m *Module) runLevel(ctx context.Context, tree string, o levelOpts) (*TestR r runAnswer noise []string packages int + text string } var todo []string for _, d := range dirs { @@ -321,6 +322,7 @@ func (m *Module) runLevel(ctx context.Context, tree string, o levelOpts) (*TestR dr.res = &TestResult{} dr.r = run(ctx, append(append(m.goEnvFor(filepath.Join(tree, d)), o.env...), levelBoundEnv(o.bounds)...), "go", args...) dr.noise, dr.packages = readGoTest(dr.res, d, dr.r.Stdout) + dr.text = goTestText(dr.r.Stdout) }() } wg.Wait() @@ -339,6 +341,7 @@ func (m *Module) runLevel(ctx context.Context, tree string, o levelOpts) (*TestR res.NotRun = append(res.NotRun, dr.res.NotRun...) res.Packages = append(res.Packages, dr.res.Packages...) res.Slowest = append(res.Slowest, dr.res.Slowest...) + res.Output += dr.text if dr.res.Elapsed > res.Elapsed { res.Elapsed = dr.res.Elapsed } diff --git a/modules/mesh-build-go/cmd/mesh-build-go/test_test.go b/modules/mesh-build-go/cmd/mesh-build-go/test_test.go index afbded703..b3fe63370 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/test_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/test_test.go @@ -110,6 +110,10 @@ func TestUnitLevelNamesTheFailingTestAndFailsTheLevel(t *testing.T) { if st.State != sdkstep.StepPassed || !res.Passed || !slices.Equal(res.NotRun, want) { t.Errorf("step %s, passed %v, not run %+v; want the level passing with not run %+v", st.State, res.Passed, res.NotRun, want) } + // Design 47 H2 §791, H-A-412 (a): the level's answer carries its text in output, test2json's Output lines in order. + if !strings.HasPrefix(res.Output, "=== RUN TestTwo\n--- PASS: TestTwo (0.00s)\n") || !strings.Contains(res.Output, "NOT RUN: npm is not on this node") || strings.Contains(res.Output, `"Action"`) { + t.Errorf("the level's output is %q, want test2json's Output lines", res.Output) + } } // Issue 1921: a fixture with a failing test in each of two packages: one level's run names both, each with its own diff --git a/modules/mesh-build-go/module.json b/modules/mesh-build-go/module.json index 4cebbdd0f..9714631ce 100644 --- a/modules/mesh-build-go/module.json +++ b/modules/mesh-build-go/module.json @@ -111,6 +111,15 @@ "history": 1 } }, + { + "name": "outputs", + "storage": "object-store", + "template": "objects", + "options": { + "max-mib": 1024, + "max-age-days": 7 + } + }, { "name": "kept-builds", "storage": "object-store", diff --git a/modules/mesh-build-image/cmd/mesh-build-image/forge_test.go b/modules/mesh-build-image/cmd/mesh-build-image/forge_test.go index 7e37eedc5..0e188a830 100644 --- a/modules/mesh-build-image/cmd/mesh-build-image/forge_test.go +++ b/modules/mesh-build-image/cmd/mesh-build-image/forge_test.go @@ -145,9 +145,11 @@ func TestForgeFilesRenderFromTheBoundValuesInTheModulesOwnHome(t *testing.T) { // template in the files/ archive (novox/hq ADR 0349 §2). func TestManifestRequiresForgeAccountKeepsKeysAndTakesFilesFromTemplates(t *testing.T) { m := readForgeManifest(t) - // outputs: a run's answer over what step-ended carries inline, in the steps state's objects store (design 47 G A-254, A-313). - outputs := map[string]any{"name": "steps", "storage": "key-value", "template": "state", "options": map[string]any{"ttl-seconds": float64(172800), "objects-max-mib": float64(256)}} - if want := []any{outputs, "forge-account", "rootless-image-build", "node-tools"}; !reflect.DeepEqual(m.Requires, want) { + // steps: a step's end over what step-ended carries inline, in the steps state's objects store (design 47 G A-254, A-313). + steps := map[string]any{"name": "steps", "storage": "key-value", "template": "state", "options": map[string]any{"ttl-seconds": float64(172800), "objects-max-mib": float64(256)}} + // outputs: a run's test output, read by the verb output (design 47 H2 §791, §820: 1 GiB, 7 days). + outputs := map[string]any{"name": "outputs", "storage": "object-store", "template": "objects", "options": map[string]any{"max-mib": float64(1024), "max-age-days": float64(7)}} + if want := []any{steps, outputs, "forge-account", "rootless-image-build", "node-tools"}; !reflect.DeepEqual(m.Requires, want) { t.Errorf("requires = %v, want %v", m.Requires, want) } keeps := false diff --git a/modules/mesh-build-image/cmd/mesh-build-image/kit_test.go b/modules/mesh-build-image/cmd/mesh-build-image/kit_test.go index d151dcf28..4fde49f59 100644 --- a/modules/mesh-build-image/cmd/mesh-build-image/kit_test.go +++ b/modules/mesh-build-image/cmd/mesh-build-image/kit_test.go @@ -4,15 +4,17 @@ import ( "context" "encoding/json" "errors" - sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/steprun" - "git.novox.be/novox/mesh-sdk/go/forgekey" "os" "path/filepath" "strings" "testing" "time" + sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/steprun" + "git.novox.be/novox/mesh-sdk/go/buscalls" + "git.novox.be/novox/mesh-sdk/go/forgekey" + buildrun "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildrun" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/steprun/conformance" ) @@ -76,6 +78,26 @@ func TestTheStepsSayAFailedNetworkAsATransportFault(t *testing.T) { // every step ask the runner sends it is refused ("this build module says no step's end"), as the first live pipeline met. func TestTheStepsCanSayAStepsEnd(t *testing.T) { conformance.StepEnds(t, (&Module{Node: "kitnode"}).askSteps(nil)) + // Design 47 H2 §791, §801, H-A-412, H-A-421: the seat's verbs current, output and answer keep the build-step part's form 2, + // and a pipeline step's run output is kept in the module's outputs objects and read back through output. + m := &Module{Node: "kitnode", Capacity: func() int { return 1 }, Outputs: kitObjects{}} + conformance.SeatVerbs(t, Seat, m.SeatTools(m.Puller(nil, steprun.NewFakeAsks())), m.askSteps(nil)) +} + +// kitObjects is an outputs store in memory, for the kit's SeatVerbs. +type kitObjects map[string][]byte + +func (k kitObjects) Put(name string, data []byte, _ string) (buscalls.ObjectInfo, error) { + k[name] = data + return buscalls.ObjectInfo{Name: name}, nil +} + +func (k kitObjects) Get(name string) (buscalls.ObjectData, error) { + d, ok := k[name] + if !ok { + return buscalls.ObjectData{}, errors.New("object not found") + } + return buscalls.ObjectData{Data: d}, nil } // A-489c (A-489a): the module's puller counts the runs its steps state says started, not the bus's deliveries (an ask given back before diff --git a/modules/mesh-build-image/cmd/mesh-build-image/manifest_test.go b/modules/mesh-build-image/cmd/mesh-build-image/manifest_test.go index 6c3966f78..d345b6396 100644 --- a/modules/mesh-build-image/cmd/mesh-build-image/manifest_test.go +++ b/modules/mesh-build-image/cmd/mesh-build-image/manifest_test.go @@ -223,8 +223,9 @@ func TestTheModuleRequiresNodeToolsItPullsThrough(t *testing.T) { } } -// Design 47 G A-254, A-313, A-314: the module keeps a run's answer over what step-ended carries inline in the steps state's own -// objects store, sized in `requires` by the state option objects-max-mib, and serves it through the verb output. +// Design 47 G A-254, A-313, A-314: the module keeps a step's end over what step-ended carries inline in the steps state's own +// objects store, sized in `requires` by the state option objects-max-mib, and serves it through the verb answer (H2 H-A-412 +// addendum); and a run's test output in its outputs objects, 1 GiB kept 7 days, read by the verb output (H2 §791, §820). func TestManifestRequiresTheStepsStateWithItsObjectsStore(t *testing.T) { raw, err := os.ReadFile(moduleDir + "/module.json") if err != nil { @@ -236,11 +237,12 @@ func TestManifestRequiresTheStepsStateWithItsObjectsStore(t *testing.T) { if err := json.Unmarshal(raw, &doc); err != nil { t.Fatal(err) } - want := map[string]any{"name": "steps", "storage": "key-value", "template": "state", "options": map[string]any{"ttl-seconds": float64(172800), "objects-max-mib": float64(256)}} - for _, r := range doc.Requires { - if reflect.DeepEqual(r, want) { - return + for _, want := range []map[string]any{ + {"name": "steps", "storage": "key-value", "template": "state", "options": map[string]any{"ttl-seconds": float64(172800), "objects-max-mib": float64(256)}}, + {"name": "outputs", "storage": "object-store", "template": "objects", "options": map[string]any{"max-mib": float64(1024), "max-age-days": float64(7)}}, + } { + if !slices.ContainsFunc(doc.Requires, func(r any) bool { return reflect.DeepEqual(r, any(want)) }) { + t.Errorf("requires %v lack %v", doc.Requires, want) } } - t.Errorf("requires %v lack %v", doc.Requires, want) } diff --git a/modules/mesh-build-image/module.json b/modules/mesh-build-image/module.json index 219d31c34..ae2a83de5 100644 --- a/modules/mesh-build-image/module.json +++ b/modules/mesh-build-image/module.json @@ -65,6 +65,7 @@ ], "requires": [ {"name": "steps", "storage": "key-value", "template": "state", "options": {"ttl-seconds": 172800, "objects-max-mib": 256}}, + {"name": "outputs", "storage": "object-store", "template": "objects", "options": {"max-mib": 1024, "max-age-days": 7}}, "forge-account", "rootless-image-build", "node-tools" diff --git a/modules/mesh-build-node/cmd/mesh-build-node/check.go b/modules/mesh-build-node/cmd/mesh-build-node/check.go index 4f0b5b244..8f4a6ec57 100644 --- a/modules/mesh-build-node/cmd/mesh-build-node/check.go +++ b/modules/mesh-build-node/cmd/mesh-build-node/check.go @@ -75,6 +75,10 @@ type TestResult struct { // By is the build module and the SDK it was built with (sdkstep.By): the checker names it when a failure hides // its cause. By string `json:"by,omitempty"` + // Output is the level's text (design 47 H2 §791, H-A-412 second addendum (a)): what each command the level ran wrote, + // its stdout then its stderr, in the order they ran. The delivery SDK's step run takes it out of the answer and keeps + // it as the run's output in the outputs objects; it is never part of a check's answer. + Output string `json:"output,omitempty"` // OutOfStep are the generated TypeScript files of a contracts module out of step with the seat generator (CAT-BN2, // steprun.OutOfStep): the unit level fails on any, Why naming each. Absent when none is. OutOfStep []string `json:"out-of-step,omitempty"` @@ -156,6 +160,7 @@ func (m *Module) testLevel(ctx context.Context, args map[string]any) (*TestResul } var check, run time.Duration var failed []Failure + var text strings.Builder tested := false for _, tg := range targets { scripts, err := packageScripts(tg.dir) @@ -176,6 +181,8 @@ func (m *Module) testLevel(ctx context.Context, args map[string]any) (*TestResul } // a command that could not start is not a verdict: the step fails. One that ran and said no is. ran := func(name string, r forgekey.Ran, report string) error { + text.WriteString(r.Stdout) + text.WriteString(r.Stderr) if r.Err != "" { return fmt.Errorf("%s could not be run: status %d %s: %s", name, r.Status, r.Err, firstLine(r.Stderr)) } @@ -225,7 +232,7 @@ func (m *Module) testLevel(ctx context.Context, args map[string]any) (*TestResul } return res, nil } - res.Failed = failed + res.Failed, res.Output = failed, text.String() var why []string for _, f := range failed { why = append(why, failureWhy(f)) diff --git a/modules/mesh-build-node/cmd/mesh-build-node/check_test.go b/modules/mesh-build-node/cmd/mesh-build-node/check_test.go index ca66028cf..144e38a0f 100644 --- a/modules/mesh-build-node/cmd/mesh-build-node/check_test.go +++ b/modules/mesh-build-node/cmd/mesh-build-node/check_test.go @@ -111,6 +111,7 @@ func TestTheIntegrationLevelRunsItsScriptAndReportsSetupAndRunningApart(t *testi workspaceWith(t, work, map[string]string{"index.ts": `export const a = 1;`, "tsconfig.json": `{}`, "package.json": `{"scripts":{"test":"x","test:integration":"node --test int"}}`}) f.cost["tsc"], f.cost["npm"] = 2*time.Second, 40*time.Second + f.answer["npm"] = forgekey.Ran{Stdout: "ok 1 - joins\n", Stderr: "(node) warning\n"} out, err := m.test(context.Background(), testArgs("integration")) if err != nil { t.Fatal(err) @@ -127,6 +128,10 @@ func TestTheIntegrationLevelRunsItsScriptAndReportsSetupAndRunningApart(t *testi if !slices.Contains(last.env, "MESH_TEST_LEVEL_BOUND=1m0s") { t.Errorf("the integration script ran with %v, not the level's bound MESH_TEST_LEVEL_BOUND=1m0s", last.env) } + // Design 47 H2 §791, H-A-412 (a): the level's answer carries its text in output, what its commands wrote. + if !strings.HasSuffix(r.Output, "ok 1 - joins\n(node) warning\n") { + t.Errorf("the level's output is %q, want the script's stdout then its stderr", r.Output) + } if tsc := f.calls[0]; strings.Join(tsc.args[1:], " ") != "--noEmit -p ." { t.Errorf("a package with a tsconfig was checked with %s", joined(tsc)) } diff --git a/modules/mesh-build-node/cmd/mesh-build-node/kit_test.go b/modules/mesh-build-node/cmd/mesh-build-node/kit_test.go index 68d5e34a3..ddadfe488 100644 --- a/modules/mesh-build-node/cmd/mesh-build-node/kit_test.go +++ b/modules/mesh-build-node/cmd/mesh-build-node/kit_test.go @@ -3,13 +3,16 @@ package main import ( "context" "encoding/json" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/steprun" + "errors" "os" "path/filepath" "strings" "testing" "time" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/steprun" + "git.novox.be/novox/mesh-sdk/go/buscalls" + buildrun "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildrun" sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/steprun/conformance" @@ -81,6 +84,26 @@ func TestTheStepsSayAFailedNetworkAsATransportFault(t *testing.T) { // every step ask the runner sends it is refused ("this build module says no step's end"), as the first live pipeline met. func TestTheStepsCanSayAStepsEnd(t *testing.T) { conformance.StepEnds(t, (&Module{Node: "kitnode"}).askSteps(nil)) + // Design 47 H2 §791, §801, H-A-412, H-A-421: the seat's verbs current, output and answer keep the build-step part's form 2, + // and a pipeline step's run output is kept in the module's outputs objects and read back through output. + m := &Module{Node: "kitnode", Capacity: func() int { return 1 }, Outputs: kitObjects{}} + conformance.SeatVerbs(t, Seat, m.SeatTools(m.Puller(nil, steprun.NewFakeAsks())), m.askSteps(nil)) +} + +// kitObjects is an outputs store in memory, for the kit's SeatVerbs. +type kitObjects map[string][]byte + +func (k kitObjects) Put(name string, data []byte, _ string) (buscalls.ObjectInfo, error) { + k[name] = data + return buscalls.ObjectInfo{Name: name}, nil +} + +func (k kitObjects) Get(name string) (buscalls.ObjectData, error) { + d, ok := k[name] + if !ok { + return buscalls.ObjectData{}, errors.New("object not found") + } + return buscalls.ObjectData{Data: d}, nil } // A-489c (A-489a): the module's puller counts the runs its steps state says started, not the bus's deliveries (an ask given back before diff --git a/modules/mesh-build-node/cmd/mesh-build-node/manifest_test.go b/modules/mesh-build-node/cmd/mesh-build-node/manifest_test.go index 7372c3eaf..91fb3ac2b 100644 --- a/modules/mesh-build-node/cmd/mesh-build-node/manifest_test.go +++ b/modules/mesh-build-node/cmd/mesh-build-node/manifest_test.go @@ -290,8 +290,9 @@ func TestTheModuleRequiresNodeToolsItPullsThrough(t *testing.T) { } } -// Design 47 G A-254, A-313, A-314: the module keeps a run's answer over what step-ended carries inline in the steps state's own -// objects store, sized in `requires` by the state option objects-max-mib, and serves it through the verb output. +// Design 47 G A-254, A-313, A-314: the module keeps a step's end over what step-ended carries inline in the steps state's own +// objects store, sized in `requires` by the state option objects-max-mib, and serves it through the verb answer (H2 H-A-412 +// addendum); and a run's test output in its outputs objects, 1 GiB kept 7 days, read by the verb output (H2 §791, §820). func TestManifestRequiresTheStepsStateWithItsObjectsStore(t *testing.T) { raw, err := os.ReadFile(moduleDir + "/module.json") if err != nil { @@ -303,11 +304,12 @@ func TestManifestRequiresTheStepsStateWithItsObjectsStore(t *testing.T) { if err := json.Unmarshal(raw, &doc); err != nil { t.Fatal(err) } - want := map[string]any{"name": "steps", "storage": "key-value", "template": "state", "options": map[string]any{"ttl-seconds": float64(172800), "objects-max-mib": float64(256)}} - for _, r := range doc.Requires { - if reflect.DeepEqual(r, want) { - return + for _, want := range []map[string]any{ + {"name": "steps", "storage": "key-value", "template": "state", "options": map[string]any{"ttl-seconds": float64(172800), "objects-max-mib": float64(256)}}, + {"name": "outputs", "storage": "object-store", "template": "objects", "options": map[string]any{"max-mib": float64(1024), "max-age-days": float64(7)}}, + } { + if !slices.ContainsFunc(doc.Requires, func(r any) bool { return reflect.DeepEqual(r, any(want)) }) { + t.Errorf("requires %v lack %v", doc.Requires, want) } } - t.Errorf("requires %v lack %v", doc.Requires, want) } diff --git a/modules/mesh-build-node/module.json b/modules/mesh-build-node/module.json index fdef48552..e998f5ed2 100644 --- a/modules/mesh-build-node/module.json +++ b/modules/mesh-build-node/module.json @@ -70,6 +70,7 @@ "requires": [ {"name": "steps", "storage": "key-value", "template": "state", "options": {"ttl-seconds": 172800, "objects-max-mib": 256}}, {"name": "detected-steps", "storage": "key-value", "template": "state", "options": {"history": 1}}, + {"name": "outputs", "storage": "object-store", "template": "objects", "options": {"max-mib": 1024, "max-age-days": 7}}, "artifact-store", "forge-account", "npm-package-registry", -- 2.54.0 From d3d9ff5183e591145bb1876ae3cefedde8221864 Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 22 Nov 2026 01:39:49 +0100 Subject: [PATCH 37/54] M1: the build modules and the registry say their class and name every store's cap; detected-steps is pruned MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Each of the four is core. Every state that named no cap names one, from its live fill read 2026-11-22 01:4x CET (mesh-broker show, nats_streams on novox): mesh-build-go steps 16.0 MiB (16,783,695 bytes) -> 128 MiB; mesh-build-node steps 2.35 MiB (2,463,833 bytes) -> 16 MiB; every other state under 2 MiB -> 8 MiB (largest: distribution holds 1,844,287 bytes, mesh-build-go detected-steps 868,554, distribution transfers 846,359, findings 668,161, idle 401,736, mesh-build-go pauses 254,366). The Go build module's finding-events is 8 MiB for 7 days. The objects stores already name their caps; none keeps a log. The Go and Node build modules prune their state detected-steps (each key an artifact's steps as detected, detected again when gone) at their own writes, past prune-mark 70 % down to prune-target 50 %, and serve _state_prune_now and _state_keep; no other state of the four is prunable (each is truth: running steps, pauses, keys, toolchains, findings, holds, transfers, windows, mirrors, taken asks). Builds: design 47 H2 §812, §820; H-A-411 (1), (2), (3), H-A-437 --- .../internal/holder/holder_test.go | 14 +++++ modules/distribution/module.json | 29 ++++++++--- .../cmd/mesh-build-go/asks_test.go | 15 ++++++ .../cmd/mesh-build-go/detected.go | 35 +++++++++++++ .../cmd/mesh-build-go/forge_test.go | 11 ++-- .../mesh-build-go/cmd/mesh-build-go/main.go | 2 +- .../cmd/mesh-build-go/module_test.go | 2 +- .../mesh-build-go/cmd/mesh-build-go/tools.go | 1 + modules/mesh-build-go/module.json | 51 ++++++++++++++++--- .../cmd/mesh-build-image/forge_test.go | 5 +- .../cmd/mesh-build-image/manifest_test.go | 6 ++- modules/mesh-build-image/module.json | 9 ++-- .../cmd/mesh-build-node/detected.go | 35 +++++++++++++ .../cmd/mesh-build-node/forge_test.go | 3 +- .../cmd/mesh-build-node/main.go | 2 +- .../cmd/mesh-build-node/manifest_test.go | 25 ++++++++- .../cmd/mesh-build-node/module_test.go | 2 +- .../cmd/mesh-build-node/tools.go | 1 + modules/mesh-build-node/module.json | 33 +++++++++--- 19 files changed, 242 insertions(+), 39 deletions(-) create mode 100644 modules/mesh-build-go/cmd/mesh-build-go/detected.go create mode 100644 modules/mesh-build-node/cmd/mesh-build-node/detected.go diff --git a/modules/distribution/internal/holder/holder_test.go b/modules/distribution/internal/holder/holder_test.go index 069f08745..aa76a8bcb 100644 --- a/modules/distribution/internal/holder/holder_test.go +++ b/modules/distribution/internal/holder/holder_test.go @@ -929,10 +929,24 @@ func TestTheHolderOfTheTwoImageSeats(t *testing.T) { Name string `json:"name"` Port int `json:"port"` } `json:"listens"` + Class string `json:"class"` + State []struct { + Name string `json:"name"` + MaxMiB int `json:"max-mib"` + } `json:"state"` } if err := json.Unmarshal(raw, &m); err != nil { t.Fatal(err) } + // Design 47 H2 H-A-411 (2): a core module, and every state names its cap, 8 MiB. + if m.Class != "core" { + t.Errorf("class %q, want core", m.Class) + } + for _, st := range m.State { + if st.MaxMiB != 8 { + t.Errorf("the state %s names max-mib %d, want 8", st.Name, st.MaxMiB) + } + } // The reservation is asked through the seat mesh-builds, so the manifest takes no package of the builder (design 47 §381). if len(m.Packages) != 0 { t.Errorf("packages %v: want none", m.Packages) diff --git a/modules/distribution/module.json b/modules/distribution/module.json index 501826bf4..6941af2d1 100644 --- a/modules/distribution/module.json +++ b/modules/distribution/module.json @@ -1,6 +1,7 @@ { "module": "distribution", "version": "0.1", + "class": "core", "provides": [ { "name": "artifact-store", @@ -249,15 +250,31 @@ } ], "state": [ - "transfers", + { + "name": "transfers", + "max-mib": 8 + }, { "name": "idle", - "key-ttl": true + "key-ttl": true, + "max-mib": 8 + }, + { + "name": "holds", + "max-mib": 8 + }, + { + "name": "windows", + "max-mib": 8 + }, + { + "name": "mirrors", + "max-mib": 8 + }, + { + "name": "findings", + "max-mib": 8 }, - "holds", - "windows", - "mirrors", - "findings", { "name": "taken", "key-ttl": true, diff --git a/modules/mesh-build-go/cmd/mesh-build-go/asks_test.go b/modules/mesh-build-go/cmd/mesh-build-go/asks_test.go index 955b8925a..5dcf5bfe7 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/asks_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/asks_test.go @@ -13,6 +13,8 @@ import ( sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/steprun" + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/catalogue" ) const askCommit = "0123456789abcdef0123456789abcdef01234567" @@ -156,6 +158,19 @@ func TestManifestClaimsTheGoBuildSeatAndReadsItsCapacity(t *testing.T) { } `json:"config"` } _ = json.Unmarshal(raw, &doc) + // Design 47 H2 H-A-411 (2), (3): a core module; its state detected-steps is pruned at its own writes, with the two prune + // tools and the settings prune-mark (70 %) and prune-target (50 %). + if m.Class != catalogue.ClassCore { + t.Errorf("class = %q, want %s", m.Class, catalogue.ClassCore) + } + for _, tool := range []string{ModuleName + "_state_prune_now", ModuleName + "_state_keep"} { + if !slices.Contains(m.Tools, tool) { + t.Errorf("tools = %v lack %s", m.Tools, tool) + } + } + if mark, target := string(doc.Config[stdio.PruneMarkSetting].Default), string(doc.Config[stdio.PruneTargetSetting].Default); mark != "70" || target != "50" { + t.Errorf("prune-mark %s and prune-target %s, want 70 and 50", mark, target) + } // Issue 1806's follow-up: the cores a step needs is a setting of its own, in the settings file, with the SDK's default. if cores := string(doc.Config[steprun.CoresPerStepSetting].Default); cores != fmt.Sprint(steprun.DefaultCoresPerStep[sdkstep.AskGo]) { t.Errorf("the cores-per-step default = %s, want %g", cores, steprun.DefaultCoresPerStep[sdkstep.AskGo]) diff --git a/modules/mesh-build-go/cmd/mesh-build-go/detected.go b/modules/mesh-build-go/cmd/mesh-build-go/detected.go new file mode 100644 index 000000000..4bf0beabe --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/detected.go @@ -0,0 +1,35 @@ +package main + +import ( + "fmt" + "os" + "sync" + + seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo" + + "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient" + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/stateobjects" +) + +// ModuleName is this module's name, which its prune tools are named for (stateobjects.PruneTools). +const ModuleName = "mesh-build-go" + +// prunedDetected is the state detected-steps, pruned toward its cap at this module's own writes (design 47 H2 §812; +// H-A-411 (2), (3)): past prune-mark of its 8 MiB the oldest keys go down to prune-target, in one journal line. Each key +// is an artifact's steps as detected, which the verb steps detects again when its key is gone. A setting that is no +// percentage is said, and the defaults hold. +var prunedDetected = sync.OnceValue(func() *stdio.PrunedState { + p, err := stdio.PruneSettings() + if err != nil { + fmt.Fprintf(os.Stderr, "the prune of %s keeps its defaults (%d %% to %d %%): %v\n", seat.StateDetectedSteps, stdio.PruneMarkDefault, stdio.PruneTargetDefault, err) + p = stdio.Prune{Mark: stdio.PruneMarkDefault, Target: stdio.PruneTargetDefault} + } + return stdio.State(seat.StateDetectedSteps).Pruned(p) +}) + +// pruneTools are the module's two prune tools over detected-steps, each written in the hand-act log (H-A-380 (3)). +func pruneTools() []stdio.Tool { + return stateobjects.PruneTools(ModuleName, []stateobjects.Pruning{{Name: seat.StateDetectedSteps, State: prunedDetected()}}, + meshhandactsclient.Recorder{Ask: stdio.Ask}) +} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/forge_test.go b/modules/mesh-build-go/cmd/mesh-build-go/forge_test.go index 2e17a639b..8b6fbd31d 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/forge_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/forge_test.go @@ -153,20 +153,21 @@ func TestManifestRequiresForgeAccountKeepsKeysAndTakesFilesFromTemplates(t *test kept := map[string]any{"name": "kept-builds", "storage": "object-store", "template": "objects", "options": map[string]any{"max-mib": float64(1024)}} // steps: a step's end over what step-ended carries inline, in the steps state's objects store (design 47 G A-254, A-313; // H2 H-A-412 addendum), read by the verb answer. - steps := map[string]any{"name": "steps", "storage": "key-value", "template": "state", "options": map[string]any{"ttl-seconds": float64(172800), "objects-max-mib": float64(256)}} + steps := map[string]any{"name": "steps", "storage": "key-value", "template": "state", "options": map[string]any{"ttl-seconds": float64(172800), "objects-max-mib": float64(256), "max-mib": float64(128)}} // outputs: a run's test output, read by the verb output (design 47 H2 §791, §820: 1 GiB, 7 days). outputs := map[string]any{"name": "outputs", "storage": "object-store", "template": "objects", "options": map[string]any{"max-mib": float64(1024), "max-age-days": float64(7)}} - detected := map[string]any{"name": "detected-steps", "storage": "key-value", "template": "state", "options": map[string]any{"history": float64(1)}} + detected := map[string]any{"name": "detected-steps", "storage": "key-value", "template": "state", "options": map[string]any{"history": float64(1), "max-mib": float64(8)}} // The findings are asked in requires as a part that is no foundation part asks them (design 47 G §529, A-30). - findings := map[string]any{"name": "findings", "storage": "key-value", "template": "state"} + findings := map[string]any{"name": "findings", "storage": "key-value", "template": "state", "options": map[string]any{"max-mib": float64(8)}} findingEvents := map[string]any{"name": "finding-events", "storage": "stream", "template": "events", "options": map[string]any{ - "events": []any{"finding-written.*", "finding-cleared.*", "findings-stated.*"}, "max-age-days": float64(7), "max-mib": float64(16)}} + "events": []any{"finding-written.*", "finding-cleared.*", "findings-stated.*"}, "max-age-days": float64(7), "max-mib": float64(8)}} if want := []any{steps, detected, outputs, kept, "artifact-store", "forge-account", "go-toolchain", "go-package-registry", "node-tools", findings, findingEvents}; !reflect.DeepEqual(m.Requires, want) { t.Errorf("requires = %v, want %v", m.Requires, want) } keeps := false for _, s := range m.State { - if s == "keys" { + // Design 47 H2 H-A-411 (2): every state names its cap, 8 MiB. + if e, _ := s.(map[string]any); e["name"] == "keys" && e["max-mib"] == float64(8) { keeps = true } } diff --git a/modules/mesh-build-go/cmd/mesh-build-go/main.go b/modules/mesh-build-go/cmd/mesh-build-go/main.go index 9566774a1..6fe0401ab 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/main.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/main.go @@ -44,7 +44,7 @@ func main() { Toolchains: stdio.State(seat.StateToolchains).Put, SayToolchain: stdio.SeatPublish, Steps: stdio.State(seat.StateSteps).Put, - Detected: stdio.State(seat.StateDetectedSteps).Put, + Detected: prunedDetected().Put, ReadDetected: readOf(stdio.State(seat.StateDetectedSteps)).Get, ForgetDetected: stdio.State(seat.StateDetectedSteps).Delete, RecordAct: handActRecorder, diff --git a/modules/mesh-build-go/cmd/mesh-build-go/module_test.go b/modules/mesh-build-go/cmd/mesh-build-go/module_test.go index fa17102e2..94e7d4549 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/module_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/module_test.go @@ -116,7 +116,7 @@ func TestAGoThatCannotBeRunRefusesEveryStepAndHealthIsUnhealthy(t *testing.T) { {Name: "short_probe", Run: run}, }, m.StepTools()...) for _, tool := range m.Tools(steps) { - if tool.Name == StatusTool || tool.Name == HealthTool || tool.Name == StepReadTool || tool.Name == StepsReadTool || tool.Name == ToolchainsReadTool || tool.Name == ReleaseTool || tool.Name == KeptTool || tool.Name == KeptDropTool || tool.Name == StepEndTool { + if strings.HasPrefix(tool.Name, ModuleName+"_state_") || tool.Name == StatusTool || tool.Name == HealthTool || tool.Name == StepReadTool || tool.Name == StepsReadTool || tool.Name == ToolchainsReadTool || tool.Name == ReleaseTool || tool.Name == KeptTool || tool.Name == KeptDropTool || tool.Name == StepEndTool { continue } var reason string diff --git a/modules/mesh-build-go/cmd/mesh-build-go/tools.go b/modules/mesh-build-go/cmd/mesh-build-go/tools.go index 73047dc8d..9a5ef62c8 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/tools.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/tools.go @@ -45,6 +45,7 @@ func (m *Module) Tools(steps []StepTool) []stdio.Tool { tools = append(tools, m.ReadTools()...) tools = append(tools, steprun.StepEndTool(StepEndTool, m.Node, m.stepScan(), m.Held)) tools = append(tools, m.releaseTool()) + tools = append(tools, pruneTools()...) tools = append(tools, m.keptTools()...) return append(tools, stdio.Tool{Name: StatusTool, diff --git a/modules/mesh-build-go/module.json b/modules/mesh-build-go/module.json index 9714631ce..2de4a290b 100644 --- a/modules/mesh-build-go/module.json +++ b/modules/mesh-build-go/module.json @@ -1,6 +1,7 @@ { "module": "mesh-build-go", "version": "0.1", + "class": "core", "packages": { "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds": "0.1", "git.novox.be/novox/mesh-catalog/modules/ssh-client/sshkeys": "0.1" @@ -13,9 +14,18 @@ "user-namespaces" ], "state": [ - "toolchains", - "keys", - "pauses" + { + "name": "toolchains", + "max-mib": 8 + }, + { + "name": "keys", + "max-mib": 8 + }, + { + "name": "pauses", + "max-mib": 8 + } ], "claims": [ { @@ -75,6 +85,24 @@ "trusted": false, "runtime": true, "default": "15s,60s" + }, + "prune-mark": { + "type": "number", + "description": "The fill of mesh-build-go's state detected-steps, in percent of its cap, past which the module prunes it at its own write, oldest first (design 47 H2 §812): below the bus module's raise mark of 75 %.", + "trusted": false, + "runtime": true, + "minimum": 1, + "maximum": 100, + "default": 70 + }, + "prune-target": { + "type": "number", + "description": "The fill, in percent of the cap, a prune of mesh-build-go's state detected-steps takes it down to (design 47 H2 §812); below prune-mark.", + "trusted": false, + "runtime": true, + "minimum": 1, + "maximum": 100, + "default": 50 } }, "tools": [ @@ -91,7 +119,9 @@ "build_go_step_end", "build_go_toolchains", "build_go_status", - "build_go_health" + "build_go_health", + "mesh-build-go_state_keep", + "mesh-build-go_state_prune_now" ], "requires": [ { @@ -100,7 +130,8 @@ "template": "state", "options": { "ttl-seconds": 172800, - "objects-max-mib": 256 + "objects-max-mib": 256, + "max-mib": 128 } }, { @@ -108,7 +139,8 @@ "storage": "key-value", "template": "state", "options": { - "history": 1 + "history": 1, + "max-mib": 8 } }, { @@ -136,7 +168,10 @@ { "name": "findings", "storage": "key-value", - "template": "state" + "template": "state", + "options": { + "max-mib": 8 + } }, { "name": "finding-events", @@ -149,7 +184,7 @@ "findings-stated.*" ], "max-age-days": 7, - "max-mib": 16 + "max-mib": 8 } } ], diff --git a/modules/mesh-build-image/cmd/mesh-build-image/forge_test.go b/modules/mesh-build-image/cmd/mesh-build-image/forge_test.go index 0e188a830..84d6c2982 100644 --- a/modules/mesh-build-image/cmd/mesh-build-image/forge_test.go +++ b/modules/mesh-build-image/cmd/mesh-build-image/forge_test.go @@ -146,7 +146,7 @@ func TestForgeFilesRenderFromTheBoundValuesInTheModulesOwnHome(t *testing.T) { func TestManifestRequiresForgeAccountKeepsKeysAndTakesFilesFromTemplates(t *testing.T) { m := readForgeManifest(t) // steps: a step's end over what step-ended carries inline, in the steps state's objects store (design 47 G A-254, A-313). - steps := map[string]any{"name": "steps", "storage": "key-value", "template": "state", "options": map[string]any{"ttl-seconds": float64(172800), "objects-max-mib": float64(256)}} + steps := map[string]any{"name": "steps", "storage": "key-value", "template": "state", "options": map[string]any{"ttl-seconds": float64(172800), "objects-max-mib": float64(256), "max-mib": float64(8)}} // outputs: a run's test output, read by the verb output (design 47 H2 §791, §820: 1 GiB, 7 days). outputs := map[string]any{"name": "outputs", "storage": "object-store", "template": "objects", "options": map[string]any{"max-mib": float64(1024), "max-age-days": float64(7)}} if want := []any{steps, outputs, "forge-account", "rootless-image-build", "node-tools"}; !reflect.DeepEqual(m.Requires, want) { @@ -154,7 +154,8 @@ func TestManifestRequiresForgeAccountKeepsKeysAndTakesFilesFromTemplates(t *test } keeps := false for _, s := range m.State { - if s == "keys" { + // Design 47 H2 H-A-411 (2): every state names its cap, 8 MiB. + if e, _ := s.(map[string]any); e["name"] == "keys" && e["max-mib"] == float64(8) { keeps = true } } diff --git a/modules/mesh-build-image/cmd/mesh-build-image/manifest_test.go b/modules/mesh-build-image/cmd/mesh-build-image/manifest_test.go index d345b6396..82008d5a4 100644 --- a/modules/mesh-build-image/cmd/mesh-build-image/manifest_test.go +++ b/modules/mesh-build-image/cmd/mesh-build-image/manifest_test.go @@ -113,6 +113,10 @@ func TestWorkAndStoreDirectoriesLieOutsideTheHome(t *testing.T) { // and the registry's address is bound into a file of the home that the program reads. func TestManifestKeepsItsStatesClaimsItsBuildSeatAndDeclaresItsHealthTool(t *testing.T) { m := readManifest(t) + // Design 47 H2 H-A-411 (2): a core module. + if m.Class != catalogue.ClassCore { + t.Errorf("class = %q, want %s", m.Class, catalogue.ClassCore) + } // Design 47 G A-488a, A-488b: its runtime follows the seat list, so the module consumes the seat's two events (which makes its // durable consumer) and invokes the verb that reads the list. for _, ev := range []string{"mesh-catalog.seat-definition-changed", "mesh-catalog.seat-definition-removed"} { @@ -238,7 +242,7 @@ func TestManifestRequiresTheStepsStateWithItsObjectsStore(t *testing.T) { t.Fatal(err) } for _, want := range []map[string]any{ - {"name": "steps", "storage": "key-value", "template": "state", "options": map[string]any{"ttl-seconds": float64(172800), "objects-max-mib": float64(256)}}, + {"name": "steps", "storage": "key-value", "template": "state", "options": map[string]any{"ttl-seconds": float64(172800), "objects-max-mib": float64(256), "max-mib": float64(8)}}, {"name": "outputs", "storage": "object-store", "template": "objects", "options": map[string]any{"max-mib": float64(1024), "max-age-days": float64(7)}}, } { if !slices.ContainsFunc(doc.Requires, func(r any) bool { return reflect.DeepEqual(r, any(want)) }) { diff --git a/modules/mesh-build-image/module.json b/modules/mesh-build-image/module.json index ae2a83de5..4ef6ba7b6 100644 --- a/modules/mesh-build-image/module.json +++ b/modules/mesh-build-image/module.json @@ -1,6 +1,7 @@ { "module": "mesh-build-image", "version": "0.1", + "class": "core", "packages": { "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds": "0.1", "git.novox.be/novox/mesh-catalog/modules/ssh-client/sshkeys": "0.1" @@ -13,9 +14,9 @@ "user-namespaces" ], "state": [ - "keys", - "pauses", - {"name": "mirror-waits", "ttl-seconds": 172800} + {"name": "keys", "max-mib": 8}, + {"name": "pauses", "max-mib": 8}, + {"name": "mirror-waits", "ttl-seconds": 172800, "max-mib": 8} ], "claims": [ { @@ -64,7 +65,7 @@ "build_image_health" ], "requires": [ - {"name": "steps", "storage": "key-value", "template": "state", "options": {"ttl-seconds": 172800, "objects-max-mib": 256}}, + {"name": "steps", "storage": "key-value", "template": "state", "options": {"ttl-seconds": 172800, "objects-max-mib": 256, "max-mib": 8}}, {"name": "outputs", "storage": "object-store", "template": "objects", "options": {"max-mib": 1024, "max-age-days": 7}}, "forge-account", "rootless-image-build", diff --git a/modules/mesh-build-node/cmd/mesh-build-node/detected.go b/modules/mesh-build-node/cmd/mesh-build-node/detected.go new file mode 100644 index 000000000..c295130e3 --- /dev/null +++ b/modules/mesh-build-node/cmd/mesh-build-node/detected.go @@ -0,0 +1,35 @@ +package main + +import ( + "fmt" + "os" + "sync" + + seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildnode" + + "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient" + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/stateobjects" +) + +// ModuleName is this module's name, which its prune tools are named for (stateobjects.PruneTools). +const ModuleName = "mesh-build-node" + +// prunedDetected is the state detected-steps, pruned toward its cap at this module's own writes (design 47 H2 §812; +// H-A-411 (2), (3)): past prune-mark of its 8 MiB the oldest keys go down to prune-target, in one journal line. Each key +// is an artifact's steps as detected, which the verb steps detects again when its key is gone. A setting that is no +// percentage is said, and the defaults hold. +var prunedDetected = sync.OnceValue(func() *stdio.PrunedState { + p, err := stdio.PruneSettings() + if err != nil { + fmt.Fprintf(os.Stderr, "the prune of %s keeps its defaults (%d %% to %d %%): %v\n", seat.StateDetectedSteps, stdio.PruneMarkDefault, stdio.PruneTargetDefault, err) + p = stdio.Prune{Mark: stdio.PruneMarkDefault, Target: stdio.PruneTargetDefault} + } + return stdio.State(seat.StateDetectedSteps).Pruned(p) +}) + +// pruneTools are the module's two prune tools over detected-steps, each written in the hand-act log (H-A-380 (3)). +func pruneTools() []stdio.Tool { + return stateobjects.PruneTools(ModuleName, []stateobjects.Pruning{{Name: seat.StateDetectedSteps, State: prunedDetected()}}, + meshhandactsclient.Recorder{Ask: stdio.Ask}) +} diff --git a/modules/mesh-build-node/cmd/mesh-build-node/forge_test.go b/modules/mesh-build-node/cmd/mesh-build-node/forge_test.go index 26f8dfabb..f9ade8758 100644 --- a/modules/mesh-build-node/cmd/mesh-build-node/forge_test.go +++ b/modules/mesh-build-node/cmd/mesh-build-node/forge_test.go @@ -150,7 +150,8 @@ func TestManifestRequiresForgeAccountKeepsKeysAndTakesFilesFromTemplates(t *test } keeps := false for _, s := range m.State { - if s == "keys" { + // Design 47 H2 H-A-411 (2): every state names its cap, 8 MiB. + if e, _ := s.(map[string]any); e["name"] == "keys" && e["max-mib"] == float64(8) { keeps = true } } diff --git a/modules/mesh-build-node/cmd/mesh-build-node/main.go b/modules/mesh-build-node/cmd/mesh-build-node/main.go index b41d596b0..920a97dc8 100644 --- a/modules/mesh-build-node/cmd/mesh-build-node/main.go +++ b/modules/mesh-build-node/cmd/mesh-build-node/main.go @@ -51,7 +51,7 @@ func main() { SayToolchain: stdio.SeatPublish, Steps: stdio.State(seat.StateSteps).Put, ReadSteps: readOf(stdio.State(seat.StateSteps)), - Detected: stdio.State(seat.StateDetectedSteps).Put, + Detected: prunedDetected().Put, ReadDetected: readOf(stdio.State(seat.StateDetectedSteps)).Get, ForgetDetected: stdio.State(seat.StateDetectedSteps).Delete, RecordAct: handActRecorder, diff --git a/modules/mesh-build-node/cmd/mesh-build-node/manifest_test.go b/modules/mesh-build-node/cmd/mesh-build-node/manifest_test.go index 91fb3ac2b..8310d9478 100644 --- a/modules/mesh-build-node/cmd/mesh-build-node/manifest_test.go +++ b/modules/mesh-build-node/cmd/mesh-build-node/manifest_test.go @@ -10,6 +10,7 @@ import ( "strings" "testing" + stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-sdk/go/catalogue" ) @@ -121,10 +122,30 @@ func TestManifestKeepsItsStatesClaimsItsBuildSeatAndDeclaresItsHealthTool(t *tes if len(m.Claims) != 1 || m.Claims[0].Name != Seat || !slices.Equal(m.Claims[0].Serves, []string{"current", "drop", "pause", "resume", "steps", "kept-artifact", "output", "answer"}) { t.Errorf("claims = %v, want the seat %s alone", m.Claims, Seat) } + // Design 47 H2 H-A-411 (2), (3): a core module; its state detected-steps is pruned at its own writes, with the two prune + // tools and the settings prune-mark (70 %) and prune-target (50 %). + if m.Class != catalogue.ClassCore { + t.Errorf("class = %q, want %s", m.Class, catalogue.ClassCore) + } + for _, tool := range []string{ModuleName + "_state_prune_now", ModuleName + "_state_keep"} { + if !slices.Contains(m.Tools, tool) { + t.Errorf("tools = %v lack %s", m.Tools, tool) + } + } + raw, _ := os.ReadFile(filepath.Join(moduleDir, "module.json")) + var doc struct { + Config map[string]struct { + Default json.RawMessage `json:"default"` + } `json:"config"` + } + _ = json.Unmarshal(raw, &doc) + if mark, target := string(doc.Config[stdio.PruneMarkSetting].Default), string(doc.Config[stdio.PruneTargetSetting].Default); mark != "70" || target != "50" { + t.Errorf("prune-mark %s and prune-target %s, want 70 and 50", mark, target) + } if !slices.Contains(m.Tools, HealthTool) { t.Errorf("tools = %v lack %s", m.Tools, HealthTool) } - raw, _ := os.ReadFile(filepath.Join(moduleDir, "module.json")) + raw, _ = os.ReadFile(filepath.Join(moduleDir, "module.json")) if !strings.Contains(string(raw), `"tool": "`+HealthTool+`"`) { t.Errorf("runtime-health does not name %s", HealthTool) } @@ -305,7 +326,7 @@ func TestManifestRequiresTheStepsStateWithItsObjectsStore(t *testing.T) { t.Fatal(err) } for _, want := range []map[string]any{ - {"name": "steps", "storage": "key-value", "template": "state", "options": map[string]any{"ttl-seconds": float64(172800), "objects-max-mib": float64(256)}}, + {"name": "steps", "storage": "key-value", "template": "state", "options": map[string]any{"ttl-seconds": float64(172800), "objects-max-mib": float64(256), "max-mib": float64(16)}}, {"name": "outputs", "storage": "object-store", "template": "objects", "options": map[string]any{"max-mib": float64(1024), "max-age-days": float64(7)}}, } { if !slices.ContainsFunc(doc.Requires, func(r any) bool { return reflect.DeepEqual(r, any(want)) }) { diff --git a/modules/mesh-build-node/cmd/mesh-build-node/module_test.go b/modules/mesh-build-node/cmd/mesh-build-node/module_test.go index a04ed8e4f..99b460230 100644 --- a/modules/mesh-build-node/cmd/mesh-build-node/module_test.go +++ b/modules/mesh-build-node/cmd/mesh-build-node/module_test.go @@ -117,7 +117,7 @@ func TestANodeThatCannotBeRunRefusesEveryStepAndHealthIsUnhealthy(t *testing.T) {Name: "short_probe", Run: run}, }, (&Module{}).StepTools()...) for _, tool := range m.Tools(steps) { - if tool.Name == StatusTool || tool.Name == HealthTool || tool.Name == StepReadTool || tool.Name == StepsReadTool || tool.Name == ToolchainsReadTool || tool.Name == ReleaseTool || tool.Name == StepEndTool { + if strings.HasPrefix(tool.Name, ModuleName+"_state_") || tool.Name == StatusTool || tool.Name == HealthTool || tool.Name == StepReadTool || tool.Name == StepsReadTool || tool.Name == ToolchainsReadTool || tool.Name == ReleaseTool || tool.Name == StepEndTool { continue } var reason string diff --git a/modules/mesh-build-node/cmd/mesh-build-node/tools.go b/modules/mesh-build-node/cmd/mesh-build-node/tools.go index 604a034e3..ee0e9beaf 100644 --- a/modules/mesh-build-node/cmd/mesh-build-node/tools.go +++ b/modules/mesh-build-node/cmd/mesh-build-node/tools.go @@ -43,6 +43,7 @@ func (m *Module) Tools(steps []StepTool) []stdio.Tool { tools = append(tools, m.ReadTools()...) tools = append(tools, steprun.StepEndTool(StepEndTool, m.Node, m.stepScan(), m.Held)) tools = append(tools, m.releaseTool()) + tools = append(tools, pruneTools()...) return append(tools, stdio.Tool{Name: StatusTool, Description: "What the Node build module is on this machine: the Node version its recipe builds with and the one installed, " + diff --git a/modules/mesh-build-node/module.json b/modules/mesh-build-node/module.json index e998f5ed2..d3b769eb2 100644 --- a/modules/mesh-build-node/module.json +++ b/modules/mesh-build-node/module.json @@ -1,6 +1,7 @@ { "module": "mesh-build-node", "version": "0.1", + "class": "core", "packages": { "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds": "0.1", "git.novox.be/novox/mesh-catalog/modules/ssh-client/sshkeys": "0.1" @@ -13,9 +14,9 @@ "user-namespaces" ], "state": [ - "toolchains", - "keys", - "pauses" + {"name": "toolchains", "max-mib": 8}, + {"name": "keys", "max-mib": 8}, + {"name": "pauses", "max-mib": 8} ], "claims": [ { @@ -53,6 +54,24 @@ "trusted": false, "runtime": true, "default": "15s,60s" + }, + "prune-mark": { + "type": "number", + "description": "The fill of mesh-build-node's state detected-steps, in percent of its cap, past which the module prunes it at its own write, oldest first (design 47 H2 §812): below the bus module's raise mark of 75 %.", + "trusted": false, + "runtime": true, + "minimum": 1, + "maximum": 100, + "default": 70 + }, + "prune-target": { + "type": "number", + "description": "The fill, in percent of the cap, a prune of mesh-build-node's state detected-steps takes it down to (design 47 H2 §812); below prune-mark.", + "trusted": false, + "runtime": true, + "minimum": 1, + "maximum": 100, + "default": 50 } }, "tools": [ @@ -65,11 +84,13 @@ "build_node_step_end", "build_node_toolchains", "build_node_status", - "build_node_health" + "build_node_health", + "mesh-build-node_state_keep", + "mesh-build-node_state_prune_now" ], "requires": [ - {"name": "steps", "storage": "key-value", "template": "state", "options": {"ttl-seconds": 172800, "objects-max-mib": 256}}, - {"name": "detected-steps", "storage": "key-value", "template": "state", "options": {"history": 1}}, + {"name": "steps", "storage": "key-value", "template": "state", "options": {"ttl-seconds": 172800, "objects-max-mib": 256, "max-mib": 16}}, + {"name": "detected-steps", "storage": "key-value", "template": "state", "options": {"history": 1, "max-mib": 8}}, {"name": "outputs", "storage": "object-store", "template": "objects", "options": {"max-mib": 1024, "max-age-days": 7}}, "artifact-store", "forge-account", -- 2.54.0 From 39ed20c5bc502cea6a9d1f38719535a38337eb64 Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 22 Nov 2026 01:41:30 +0100 Subject: [PATCH 38/54] C4: the forge module's tokens live in the vault, one process mints at a time MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two processes that minted at once left the forge two tokens by one name, and a token kept in a file in the runtime state is a plain state. Both accounts' tokens (mesh-tools, mesh-packages) are now the module's own secrets in the vault, written through the seat mesh-vault's own-put and read through its get, sealed to the reading process's key. A refused or expired (token-ttl) token is minted again only by the holder of the lease mint in the state token-lease (mint-lease); each mint is named -, and the token it replaces is deleted by id only after own-put landed. The Go bundles read their tokens through get at start and at secret-changed; the MESH_GITEA_*_TOKEN_FILE words go. forge_token_lease reads the lease and each token's age, never the token. Builds: design 47 H2 §808, §822; H-A-407, H-A-422, H-A-426, H-A-430 (1) --- modules/gitea/README.md | 10 +- modules/gitea/cmd/forge/account_test.go | 60 +- modules/gitea/cmd/forge/forge.go | 5 +- modules/gitea/cmd/forge/forge_test.go | 10 +- modules/gitea/cmd/forge/main.go | 12 + modules/gitea/cmd/package-registry/forge.go | 12 +- modules/gitea/cmd/package-registry/main.go | 14 + .../cmd/package-registry/retention_test.go | 5 +- .../gitea/cmd/package-registry/seats_test.go | 56 +- .../whole_integration_test.go | 26 +- modules/gitea/cmd/repositories/main.go | 4 - modules/gitea/index.ts | 6 +- modules/gitea/internal/forgehttp/forgehttp.go | 100 ++- modules/gitea/internal/forgehttp/vault.go | 166 +++++ modules/gitea/kept.ts | 1 + modules/gitea/module.json | 29 +- modules/gitea/packages-account.ts | 34 +- modules/gitea/test/fakebus.ts | 96 +++ modules/gitea/test/kept.test.ts | 4 +- modules/gitea/test/token.test.ts | 275 ++++---- modules/gitea/test/tools-account.test.ts | 62 +- modules/gitea/token.ts | 614 ++++++++++-------- modules/gitea/tools/tokenlease.ts | 16 + 23 files changed, 1078 insertions(+), 539 deletions(-) create mode 100644 modules/gitea/internal/forgehttp/vault.go create mode 100644 modules/gitea/test/fakebus.ts create mode 100644 modules/gitea/tools/tokenlease.ts diff --git a/modules/gitea/README.md b/modules/gitea/README.md index c946a02b0..321c54e1f 100644 --- a/modules/gitea/README.md +++ b/modules/gitea/README.md @@ -100,7 +100,8 @@ search that reaches the call's deadline stops there and answers what it found, w It reaches the forge through the same HTTP client as the package-registry bundle (`internal/forgehttp`), as the tools' own account (novox/hq ADR 0365 §2, issue 469): the token the TypeScript bundle mints for `mesh-tools` and -keeps (`MESH_GITEA_TOKEN_FILE`), sent only when its scopes line names that account, never the admin login. One tool +keeps in the vault as the module's own (`module/gitea/mesh-tools`, design 47 H2 §808), read through the vault seat's +`get` at start and at each `secret-changed` naming it, never the admin login. One tool call takes at most 50 seconds. `go test ./cmd/forge` runs it against a fake forge: the files of one commit created, updated and deleted on a new branch from its base; an existing branch committed on; the default branch, a protected branch and @@ -247,9 +248,10 @@ version already gone counts as deleted. Over the forge's HTTP interface on the machine (`MESH_GITEA_URL`), never as the admin account (design 51: each holder keeps its write side as an own secret beside a read-only tool token). It **reads** as the tools' account, -with the token the TypeScript bundle keeps for it (`MESH_GITEA_TOKEN_FILE`, scopes including `read:package`), and -**writes** (a publish, a remove, retention's deletion) as the packages account `mesh-packages` -(`MESH_GITEA_PACKAGES_USER`), with its own token (`MESH_GITEA_PACKAGES_TOKEN_FILE`, the one scope `write:package`). +with the token the TypeScript bundle keeps for it in the vault (`module/gitea/mesh-tools`, scopes including +`read:package`), and **writes** (a publish, a remove, retention's deletion) as the packages account `mesh-packages` +(`MESH_GITEA_PACKAGES_USER`), with its own token (`module/gitea/mesh-packages`, the one scope `write:package`), both +read through the vault seat's `get` at start and at each `secret-changed` (design 47 H2 §808). That account is plain, a member of the team `mesh-packages` that writes the organisation's packages and nothing else; the watcher makes it with the administrator's login and mints and keeps its token at its start (`packages-account.ts`). No timer looks at it after that: when the forge refuses a write (401) this bundle raises the finding diff --git a/modules/gitea/cmd/forge/account_test.go b/modules/gitea/cmd/forge/account_test.go index d37c6594d..54e7213a3 100644 --- a/modules/gitea/cmd/forge/account_test.go +++ b/modules/gitea/cmd/forge/account_test.go @@ -1,35 +1,52 @@ package main // The commit and the reads act as the tools' own account (novox/hq ADR 0365 §2, issue 469): the token the module -// keeps for `mesh-tools`, never the admin login. That account is no repository's administrator, so the forge +// keeps for `mesh-tools` in the vault (design 47 H2 §808), never the admin login. That account is no repository's administrator, so the forge // refuses it the protection rules (403, as gitea's reqAdmin does); the commit then judges an existing branch by the // protection the forge says it applies, and leaves a branch not made yet to the forge's own judgement. import ( "context" + "encoding/json" + "fmt" "net/http" "net/http/httptest" - "os" - "path/filepath" "strings" "testing" + "git.novox.be/novox/mesh-sdk/go/seal" + "gitea/internal/forgehttp" ) const toolsToken = "tools-token-never-shown" -// asTools serves the fake forge to a bundle holding the tools' token: only that token is taken, the rules are -// refused as to a non-administrator, and the branches named protected say so as the forge does. -func asTools(t *testing.T, f *fakeForge, protected map[string]string, scopes string) (*Forge, *[]string) { - dir := t.TempDir() - file := filepath.Join(dir, "token") - if err := os.WriteFile(file, []byte(toolsToken+"\n"), 0o600); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(file+".scopes", []byte(scopes+"\n"), 0o600); err != nil { +// vaultHolding is the vault seat's get answering account's token sealed to the key it is asked with; "" holds none. +func vaultHolding(t *testing.T, account, token string) *forgehttp.VaultToken { + t.Helper() + v, err := forgehttp.NewVaultToken(account, func(address string, args any) (json.RawMessage, error) { + a := args.(map[string]any) + if address != forgehttp.VaultGet || a["name"] != forgehttp.VaultName(account) { + return nil, fmt.Errorf("asked %s for %v", address, a["name"]) + } + if token == "" { + return nil, fmt.Errorf("the vault holds no secret named %s", a["name"]) + } + box, err := seal.Seal(token, a["public_key"].(string)) + if err != nil { + return nil, err + } + return json.Marshal(map[string]any{"name": a["name"], "generation": 1, "sealed": box}) + }) + if err != nil { t.Fatal(err) } + return v +} + +// asTools serves the fake forge to a bundle holding the token the vault hands: only the tools' token is taken, the +// rules are refused as to a non-administrator, and the branches named protected say so as the forge does. +func asTools(t *testing.T, f *fakeForge, protected map[string]string, held string) (*Forge, *[]string) { var seen []string const repo = "/api/v1/repos/acme/app" srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -52,12 +69,12 @@ func asTools(t *testing.T, f *fakeForge, protected map[string]string, scopes str f.handle(w, r) })) t.Cleanup(srv.Close) - return &Forge{Client: forgehttp.Client{URL: srv.URL, TokenFile: file, Account: "mesh-tools"}}, &seen + return &Forge{Client: forgehttp.Client{URL: srv.URL, Token: vaultHolding(t, "mesh-tools", held), Account: "mesh-tools"}}, &seen } func TestTheToolsAccountCommitsOnANewBranchWithItsTokenThoughTheRulesAreNotItsToRead(t *testing.T) { f := newFake(t) - forge, _ := asTools(t, f, nil, "read:user,write:issue,write:repository for mesh-tools") + forge, _ := asTools(t, f, nil, toolsToken) got, err := commit(context.Background(), forge, ask(t, map[string]any{"branch": "fix/365", "files": filesArg(map[string]any{"path": "x", "content": "x"})})) if err != nil { @@ -75,7 +92,7 @@ func TestTheToolsAccountIsRefusedAProtectedBranchTheForgeSaysItGuardsAndTheDefau if rule != "" { protected[branch] = rule } - forge, _ := asTools(t, f, protected, "read:user,write:issue,write:repository for mesh-tools") + forge, _ := asTools(t, f, protected, toolsToken) _, err := commit(context.Background(), forge, ask(t, map[string]any{"branch": branch, "files": filesArg(map[string]any{"path": "x", "content": "x"})})) if err == nil || !strings.Contains(err.Error(), "refused") || !strings.Contains(err.Error(), "ADR 0266") { @@ -90,13 +107,13 @@ func TestTheToolsAccountIsRefusedAProtectedBranchTheForgeSaysItGuardsAndTheDefau } } -func TestATokenKeptForTheAdministratorIsNeverSent(t *testing.T) { +func TestNoTokenIsSentWhileTheVaultHoldsNoneForTheAccount(t *testing.T) { f := newFake(t) - // What the module kept before ADR 0365: the admin's token, its scopes line naming no account. - forge, seen := asTools(t, f, nil, "read:user,write:issue,write:repository") + // Design 47 H2 §808: the token is read through the vault seat's get; while the module has minted none, nothing goes. + forge, seen := asTools(t, f, nil, "") _, err := commit(context.Background(), forge, ask(t, map[string]any{"branch": "fix/365", "files": filesArg(map[string]any{"path": "x", "content": "x"})})) - if err == nil || !strings.Contains(err.Error(), "not mesh-tools's") { + if err == nil || !strings.Contains(err.Error(), "not kept yet") { t.Fatalf("%v", err) } if len(*seen) != 0 { @@ -104,9 +121,8 @@ func TestATokenKeptForTheAdministratorIsNeverSent(t *testing.T) { } } -func TestTheBundleHandedATokenFileNeedsNoAdminLogin(t *testing.T) { +func TestTheBundleReadsItsTokenFromTheVaultAndNeedsNoAdminLogin(t *testing.T) { t.Setenv("MESH_GITEA_URL", "http://127.0.0.1:3000") - t.Setenv("MESH_GITEA_TOKEN_FILE", "/state/token") t.Setenv("MESH_GITEA_TOOLS_USER", "") t.Setenv("MESH_GITEA_ADMIN_USER", "") t.Setenv("MESH_GITEA_ADMIN_PASSWORD_FILE", "") @@ -114,7 +130,7 @@ func TestTheBundleHandedATokenFileNeedsNoAdminLogin(t *testing.T) { if err != nil { t.Fatal(err) } - if f.TokenFile != "/state/token" || f.Account != "mesh-tools" || f.User != "" || f.PasswordFile != "" { + if f.Token == nil || f.Token.Account != "mesh-tools" || f.Account != "mesh-tools" || f.User != "" || f.PasswordFile != "" { t.Fatalf("%+v", f.Client) } } diff --git a/modules/gitea/cmd/forge/forge.go b/modules/gitea/cmd/forge/forge.go index a9d279306..1be60c00b 100644 --- a/modules/gitea/cmd/forge/forge.go +++ b/modules/gitea/cmd/forge/forge.go @@ -21,7 +21,10 @@ type Forge struct { // ForgeFromEnv is the forge as the manifest hands it to this bundle. func ForgeFromEnv() (*Forge, error) { - c, missing := forgehttp.FromEnv() + c, missing, err := forgehttp.ToolsFromEnv() + if err != nil { + return nil, err + } if len(missing) > 0 { return nil, fmt.Errorf("the forge cannot be reached: %s not set by the manifest", strings.Join(missing, ", ")) } diff --git a/modules/gitea/cmd/forge/forge_test.go b/modules/gitea/cmd/forge/forge_test.go index 427330873..73c0db149 100644 --- a/modules/gitea/cmd/forge/forge_test.go +++ b/modules/gitea/cmd/forge/forge_test.go @@ -691,8 +691,8 @@ func TestToolsAreTheOnesTheReadmeNamesAndTheManifestBuilds(t *testing.T) { // The tools' own account, never the admin login (novox/hq ADR 0365 §2); the package registry's bundle also writes // as the packages account (design 51: the write side an own secret beside a read-only tool token). wants := map[string][]string{ - "cmd/forge": {"MESH_GITEA_URL", "MESH_GITEA_TOOLS_USER", "MESH_GITEA_TOKEN_FILE"}, - "cmd/package-registry": {"MESH_GITEA_URL", "MESH_GITEA_TOOLS_USER", "MESH_GITEA_TOKEN_FILE", "MESH_GITEA_PACKAGES_USER", "MESH_GITEA_PACKAGES_TOKEN_FILE"}, + "cmd/forge": {"MESH_GITEA_URL", "MESH_GITEA_TOOLS_USER"}, + "cmd/package-registry": {"MESH_GITEA_URL", "MESH_GITEA_TOOLS_USER", "MESH_GITEA_PACKAGES_USER"}, } for _, a := range m.Build.Artifacts { want, ok := wants[a.From] @@ -713,6 +713,12 @@ func TestToolsAreTheOnesTheReadmeNamesAndTheManifestBuilds(t *testing.T) { t.Errorf("%s is given %s: it acts as an account of its own, not the administrator", a.From, k) } } + // Design 47 H2 §808, H-A-422 (1): the token is read through the vault seat's get, never from a file. + for _, k := range []string{"MESH_GITEA_TOKEN_FILE", "MESH_GITEA_PACKAGES_TOKEN_FILE"} { + if a.Env[k] != "" { + t.Errorf("%s is given %s: its token is read from the vault", a.From, k) + } + } } for from := range wants { t.Errorf("the manifest builds no %s bundle", from) diff --git a/modules/gitea/cmd/forge/main.go b/modules/gitea/cmd/forge/main.go index fadedecdd..ec2132339 100644 --- a/modules/gitea/cmd/forge/main.go +++ b/modules/gitea/cmd/forge/main.go @@ -14,6 +14,8 @@ import ( "strings" stdio "git.novox.be/novox/mesh-sdk/go" + + "gitea/internal/forgehttp" ) // ToolNames are the tools this bundle serves. The module's manifest lists no `tools` (see the package-registry @@ -21,6 +23,16 @@ import ( var ToolNames = []string{"gitea_commit", "gitea_commits", "gitea_compare", "gitea_pull_reviews", "gitea_list_dir", "gitea_search_code"} func main() { + // The tools' account's token, read from the vault at start and at each secret-changed naming it (design 47 H2 §808). + go func() { + account := os.Getenv("MESH_GITEA_TOOLS_USER") + if account == "" { + account = forgehttp.ToolsAccount + } + if err := forgehttp.Follow(func(f string, a ...any) { fmt.Fprintf(os.Stderr, f+"\n", a...) }, account); err != nil { + fmt.Fprintln(os.Stderr, "the vault's changes of the forge token are not heard:", err) + } + }() if err := stdio.Serve("", Tools(ForgeFromEnv)); err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) diff --git a/modules/gitea/cmd/package-registry/forge.go b/modules/gitea/cmd/package-registry/forge.go index 82fb0e69a..3823e2e5a 100644 --- a/modules/gitea/cmd/package-registry/forge.go +++ b/modules/gitea/cmd/package-registry/forge.go @@ -78,8 +78,14 @@ func (f *Forge) As(typ string) *Forge { // ForgeFromEnv is the forge as the manifest hands it to this bundle. func ForgeFromEnv() (*Forge, error) { - c, missing := forgehttp.FromEnv() - w, writeMissing := forgehttp.WriterFromEnv() + c, missing, err := forgehttp.ToolsFromEnv() + if err != nil { + return nil, err + } + w, writeMissing, err := forgehttp.WriterFromEnv() + if err != nil { + return nil, err + } for _, m := range writeMissing { if m != "MESH_GITEA_URL" { missing = append(missing, m) @@ -168,7 +174,7 @@ func (f *Forge) DistTags(ctx context.Context, name string) (map[string]string, e // write is the registry's write side, refused when the manifest handed none. func (f *Forge) write() (*forgehttp.Client, error) { if f.Writer == nil { - return nil, errors.New("the package registry's write side is not handed to this bundle (MESH_GITEA_PACKAGES_TOKEN_FILE); nothing was written") + return nil, errors.New("the package registry's write side is not handed to this bundle (the packages account's token); nothing was written") } return f.Writer, nil } diff --git a/modules/gitea/cmd/package-registry/main.go b/modules/gitea/cmd/package-registry/main.go index 02de1384c..96adda433 100644 --- a/modules/gitea/cmd/package-registry/main.go +++ b/modules/gitea/cmd/package-registry/main.go @@ -17,6 +17,7 @@ import ( stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-sdk/go/views" + "gitea/internal/forgehttp" "gitea/internal/packageseats" ) @@ -49,6 +50,19 @@ func main() { os.Exit(1) } }() + // Both accounts' tokens, read from the vault at start and at each secret-changed naming one (design 47 H2 §808). + go func() { + tools, packages := os.Getenv("MESH_GITEA_TOOLS_USER"), os.Getenv("MESH_GITEA_PACKAGES_USER") + if tools == "" { + tools = forgehttp.ToolsAccount + } + if packages == "" { + packages = forgehttp.PackagesAccount + } + if err := forgehttp.Follow(func(f string, a ...any) { fmt.Fprintf(os.Stderr, f+"\n", a...) }, tools, packages); err != nil { + fmt.Fprintln(os.Stderr, "the vault's changes of the forge tokens are not heard:", err) + } + }() forge := func() (*Forge, error) { return ForgeFromEnv() } holder := func() (*packageseats.Holder, error) { return busHolder(forge) } go followIdle(holder) diff --git a/modules/gitea/cmd/package-registry/retention_test.go b/modules/gitea/cmd/package-registry/retention_test.go index 11baa150d..9dd5ee5cd 100644 --- a/modules/gitea/cmd/package-registry/retention_test.go +++ b/modules/gitea/cmd/package-registry/retention_test.go @@ -496,7 +496,8 @@ func TestToolsAreTheOnesTheReadmeNamesAndSayWhatTheyDo(t *testing.T) { t.Errorf("transfers has options (a lifetime): %s", req) } // The retention asks the builder's versions; the forge's other bundles ask the catalog half and record a re-read. - if strings.Join(m.Invokes, ",") != "seat:mesh-catalog.revision,seat:mesh-catalog.registrations,seat:mesh-hand-acts.hand-act,seat:mesh-builds.reserved" { + // The forge module's tokens through the vault seat: own-put by the minter, get by every process (design 47 H2 §808). + if strings.Join(m.Invokes, ",") != "seat:mesh-catalog.revision,seat:mesh-catalog.registrations,seat:mesh-hand-acts.hand-act,seat:mesh-builds.reserved,seat:mesh-vault.own-put,seat:mesh-vault.get" { t.Errorf("the manifest invokes %v", m.Invokes) } if len(m.Tools) != 0 { @@ -506,7 +507,7 @@ func TestToolsAreTheOnesTheReadmeNamesAndSayWhatTheyDo(t *testing.T) { for _, a := range m.Build.Artifacts { if a.From == "cmd/package-registry" { found = true - for _, k := range []string{"MESH_GITEA_URL", "MESH_GITEA_TOKEN_FILE", "MESH_GITEA_PACKAGES_TOKEN_FILE", "MESH_NPM_OWNER"} { + for _, k := range []string{"MESH_GITEA_URL", "MESH_NPM_OWNER"} { if a.Env[k] == "" { t.Errorf("the bundle is not given %s", k) } diff --git a/modules/gitea/cmd/package-registry/seats_test.go b/modules/gitea/cmd/package-registry/seats_test.go index 13d4fbd0f..33cab8930 100644 --- a/modules/gitea/cmd/package-registry/seats_test.go +++ b/modules/gitea/cmd/package-registry/seats_test.go @@ -9,17 +9,17 @@ import ( "encoding/base64" "encoding/hex" "encoding/json" + "fmt" "io" "net/http" "net/http/httptest" - "os" - "path/filepath" "strings" "sync" "testing" "time" keyparts "git.novox.be/novox/mesh-sdk/go/keys" + "git.novox.be/novox/mesh-sdk/go/seal" "git.novox.be/novox/mesh-sdk/go/views" "gitea/internal/forgehttp" @@ -116,15 +116,10 @@ func TestThePackageSeatsStoreIntoTheForge(t *testing.T) { g := ®istry{uploaded: map[string][]byte{}, held: map[string][]byte{}} srv := httptest.NewServer(http.HandlerFunc(g.handle)) defer srv.Close() - dir := t.TempDir() - kept := func(name, token, line string) string { - file := filepath.Join(dir, name) - _ = os.WriteFile(file, []byte(token+"\n"), 0o600) - _ = os.WriteFile(file+".scopes", []byte(line+"\n"), 0o600) - return file - } - reader := forgehttp.Client{URL: srv.URL, TokenFile: kept("token", "reader", "read:package,read:user for mesh-tools"), Account: "mesh-tools"} - writer := forgehttp.Client{URL: srv.URL, TokenFile: kept("packages-token", "writer", "write:package for mesh-packages"), Account: "mesh-packages"} + vault := map[string]string{"mesh-tools": "reader", "mesh-packages": "writer"} + writerToken := vaultHolding(t, "mesh-packages", vault) + reader := forgehttp.Client{URL: srv.URL, Token: vaultHolding(t, "mesh-tools", vault), Account: "mesh-tools"} + writer := forgehttp.Client{URL: srv.URL, Token: writerToken, Account: "mesh-packages"} forge := &Forge{Client: reader, Writer: &writer, Owner: "acme"} if _, err := (seatStore{forge: func() (*Forge, error) { return &Forge{Client: reader, Owner: "acme"}, nil }}).Put(packageseats.GoSeat, packageseats.Publish{}, nil); err == nil || !strings.Contains(err.Error(), "write side") { @@ -138,12 +133,22 @@ func TestThePackageSeatsStoreIntoTheForge(t *testing.T) { if k := views.FindingKey("seat", keyparts.EscapeName(packageseats.Self)+".write", PackagesTokenRefused); k != PackagesTokenRefusedKey { t.Errorf("the refusal's key is %s, the watcher watches %s", k, PackagesTokenRefusedKey) } - _ = os.WriteFile(writer.TokenFile, []byte("stale\n"), 0o600) + vault["mesh-packages"] = "stale" + if read, err := writerToken.Changed([]byte(`{"name":"module/gitea/mesh-packages","generation":2}`)); !read || err != nil { + t.Fatalf("the secret-changed of the packages token was not read again: %v %v", read, err) + } if _, err := s.Put(packageseats.GoSeat, packageseats.Publish{Version: "0.1.15"}, moduleZip(t, "git.example/acme/sdk@v0.1.15")); err == nil || len(refused) != 1 || !refused[0] { t.Errorf("a write the forge refused with 401 raised no finding: %v %v", err, refused) } - _ = os.WriteFile(writer.TokenFile, []byte("writer\n"), 0o600) + vault["mesh-packages"] = "writer" + // Design 47 H2 §808: a secret-changed naming another secret reads nothing; the one naming this token reads it again. + if read, _ := writerToken.Changed([]byte(`{"name":"module/gitea/mesh-tools","generation":3}`)); read { + t.Error("a secret-changed of the tools' token read the packages token again") + } + if read, err := writerToken.Changed([]byte(`{"name":"module/gitea/mesh-packages","generation":3}`)); !read || err != nil { + t.Fatalf("the secret-changed of the packages token was not read again: %v %v", read, err) + } tgz := tarball(t, `{"name":"@acme/contracts","version":"0.1.15-feat-x.2","main":"index.js"}`) p := packageseats.Publish{Name: "@acme/contracts", Version: "0.1.15-feat-x.2", Tag: "feat-x", Kind: packageseats.KindPackage, Digest: "sha256:" + sha(tgz)} @@ -197,3 +202,28 @@ func TestThePackageSeatsStoreIntoTheForge(t *testing.T) { t.Errorf("a write the forge took after the refusal did not clear it: %v", refused) } } + +// vaultHolding is the vault seat's get answering account's token from held, sealed to the key it is asked with; an +// account held has no entry is a secret the vault holds none of. +func vaultHolding(t *testing.T, account string, held map[string]string) *forgehttp.VaultToken { + t.Helper() + v, err := forgehttp.NewVaultToken(account, func(address string, args any) (json.RawMessage, error) { + a := args.(map[string]any) + if address != forgehttp.VaultGet || a["name"] != forgehttp.VaultName(account) { + return nil, fmt.Errorf("asked %s for %v", address, a["name"]) + } + token, ok := held[account] + if !ok { + return nil, fmt.Errorf("the vault holds no secret named %s", a["name"]) + } + box, err := seal.Seal(token, a["public_key"].(string)) + if err != nil { + return nil, err + } + return json.Marshal(map[string]any{"name": a["name"], "generation": 1, "sealed": box}) + }) + if err != nil { + t.Fatal(err) + } + return v +} diff --git a/modules/gitea/cmd/package-registry/whole_integration_test.go b/modules/gitea/cmd/package-registry/whole_integration_test.go index b7b76057f..5f6848f07 100644 --- a/modules/gitea/cmd/package-registry/whole_integration_test.go +++ b/modules/gitea/cmd/package-registry/whole_integration_test.go @@ -40,6 +40,7 @@ import ( "github.com/nats-io/nats.go/jetstream" "git.novox.be/novox/mesh-sdk/go/buscalls" + "git.novox.be/novox/mesh-sdk/go/seal" "git.novox.be/novox/mesh-sdk/go/testbus" "gitea/internal/packageseats" @@ -65,6 +66,7 @@ type player struct { answers map[string]chan wire puts map[string][]byte // the put sessions open, by call gets map[string][]byte // the get sessions open, by call + vault map[string]string // the vault's secrets the seat's get hands, by name (design 47 H2 §808) } func (p *player) send(m map[string]any) { @@ -130,6 +132,19 @@ func (p *player) handle(m wire) { break } p.answer(m.ID, map[string]any{"module": "m", "version": q.Version, "artifacts": []map[string]any{{"artifact": "m:a", "digest": d}}}, nil) + case m.Method == "mesh/ask" && a.Key == "seat:mesh-vault.get": + var q struct { + Name string `json:"name"` + PublicKey string `json:"public_key"` + } + _ = json.Unmarshal(a.Body, &q) + token, ok := p.vault[q.Name] + if !ok { + p.answer(m.ID, nil, errors.New("the vault holds no secret named "+q.Name)) + break + } + box, err := seal.Seal(token, q.PublicKey) + p.answer(m.ID, map[string]any{"name": q.Name, "generation": 1, "sealed": box}, err) case m.Method == "mesh/seat.publish": p.answer(m.ID, map[string]any{}, p.nc.Publish(a.Subject, a.Body)) case m.Method == "mesh/capabilities": @@ -378,17 +393,9 @@ func TestThePackageSeatsWhole(t *testing.T) { } })) t.Cleanup(srv.Close) - dir := t.TempDir() - kept := func(name, token, line string) string { - file := filepath.Join(dir, name) - _ = os.WriteFile(file, []byte(token+"\n"), 0o600) - _ = os.WriteFile(file+".scopes", []byte(line+"\n"), 0o600) - return file - } cmd := exec.CommandContext(ctx, bin) cmd.Env = append(os.Environ(), "MESH_NODE=novox", "MESH_GITEA_URL="+srv.URL, "MESH_NPM_OWNER=acme", "MESH_GITEA_TOOLS_USER=mesh-tools", - "MESH_GITEA_TOKEN_FILE="+kept("token", "reader", "read:package,read:user,write:issue,write:repository for mesh-tools"), - "MESH_GITEA_PACKAGES_TOKEN_FILE="+kept("packages-token", "writer", "write:package for mesh-packages")) + "MESH_GITEA_PACKAGES_USER=mesh-packages") cmd.Stderr = os.Stderr in, _ := cmd.StdinPipe() out, _ := cmd.StdoutPipe() @@ -400,6 +407,7 @@ func TestThePackageSeatsWhole(t *testing.T) { zipped := bigZip(t, "git.example/acme/sdk@v0.1.15") digest := "sha256:" + sha(zipped) p := &player{ctx: ctx, t: t, nc: nc, js: js, in: in, answers: map[string]chan wire{}, + vault: map[string]string{"module/gitea/mesh-tools": "reader", "module/gitea/mesh-packages": "writer"}, numbered: map[string]string{"git.example/acme/sdk@0.1.15": digest}} go p.serve(out) p.send(map[string]any{"id": "init", "method": "initialize", "params": map[string]any{}}) diff --git a/modules/gitea/cmd/repositories/main.go b/modules/gitea/cmd/repositories/main.go index fea644bc2..a6b791ec0 100644 --- a/modules/gitea/cmd/repositories/main.go +++ b/modules/gitea/cmd/repositories/main.go @@ -36,10 +36,6 @@ func runSync() int { missing = append(missing, k) } } - if forge.TokenFile != "" { - // The tools' account cannot make or protect a repository, and must not (ADR 0428 §8). - missing = append(missing, "the setup login (MESH_GITEA_TOKEN_FILE is set instead)") - } if len(missing) > 0 { fmt.Fprintf(os.Stderr, "repositories: %v not set by the manifest; nothing was changed\n", missing) return 1 diff --git a/modules/gitea/index.ts b/modules/gitea/index.ts index 61792f30f..4e31b6d4b 100644 --- a/modules/gitea/index.ts +++ b/modules/gitea/index.ts @@ -48,6 +48,8 @@ import { Calls, callRetentionDays, sayUnsaid } from "./calls.js"; import { Once, askTakenBoundMs, takenState } from "./taken.js"; import { Partials } from "./partials.js"; import { callTools, mergedTool } from "./tools/calls.js"; +import { tokenLeaseTool } from "./tools/tokenlease.js"; +import { processVaultAndLease } from "./token.js"; import { seatPublish } from "@novox/mesh-sdk/runtime"; import { seat as GIT, events as GIT_EVENTS, type BranchMovedEvent, type PullClosedEvent, type PullMergedEvent, type PullUpdatedEvent } from "@novox/mesh-delivery-contracts-git"; import { seat as RUNNER, events as RUNNER_EVENTS, type StepEndedEvent } from "@novox/mesh-delivery-contracts-mesh-pipeline-runner"; @@ -60,7 +62,7 @@ import { announcementOf } from "./updated.js"; import { ReadAll, readAllTools } from "./readall.js"; import { DeliveredWrites, writeTools } from "./writes.js"; import { TOOLS_ACCOUNT, TOOLS_ORG, meshAccounts } from "./tools-account.js"; -import { PACKAGES_TOKEN_REFUSED, keepPackagesToken, packagesToken, renewOnRefusal } from "./packages-account.js"; +import { PACKAGES_ACCOUNT, PACKAGES_TOKEN_REFUSED, keepPackagesToken, packagesToken, renewOnRefusal } from "./packages-account.js"; // Without a way to a token — configured, or mintable with the admin account (token.ts) — there is // nothing to watch; log and stay quiet rather than crash the runtime. With one, the first poll mints @@ -305,6 +307,8 @@ registerModuleTools("gitea", () => [eventsTool(pace), ...readAllTools(readAll), watched: () => registrations.watched, calls, say: seatPublish, partials, once }) : []), ...callTools(calls, partials, seatPublish, async (act) => { await broker().request("seat:mesh-hand-acts.hand-act", act); }), ...(gitea ? [mergedTool(gitea, calls)] : []), + tokenLeaseTool(processVaultAndLease().vault, processVaultAndLease().lease, + [process.env.MESH_GITEA_TOOLS_USER || TOOLS_ACCOUNT, process.env.MESH_GITEA_PACKAGES_USER || PACKAGES_ACCOUNT]), ...failedStepsTools(failedSteps, async (act) => { await broker().request("seat:mesh-hand-acts.hand-act", act); })]); /** What the forge's event asks of the announcer at once (issue 729, 1422): see watch.ts. */ diff --git a/modules/gitea/internal/forgehttp/forgehttp.go b/modules/gitea/internal/forgehttp/forgehttp.go index 3951ce342..b9d5a77e0 100644 --- a/modules/gitea/internal/forgehttp/forgehttp.go +++ b/modules/gitea/internal/forgehttp/forgehttp.go @@ -3,13 +3,12 @@ // // Two ways in, by what the manifest hands the bundle: // -// - **as the tools' account** (novox/hq ADR 0365 §2), when MESH_GITEA_TOKEN_FILE names the token the module -// keeps for it (token.ts mints it, for `mesh-tools`, and keeps it at 0600 with its scopes line beside it). A -// token whose scopes line does not say it is the tools' account's — one kept from before, the administrator's — -// is never sent: the call fails, saying so, until the module has minted the account's own. This bundle mints -// nothing; a token the forge rejects is renewed by the module's next call, and said here. +// - **as the tools' account or the packages account** (novox/hq ADR 0365 §2; design 47 H2 §808), with the account's +// token the module keeps in the vault as its own (token.ts mints it; vault.go reads it through the vault seat's +// `get` at start and at `secret-changed`). This bundle mints nothing; a token the forge rejects is renewed by the +// module's next call, and said here. // - **as the admin account**, whose password the vault delivered into a file this module owns (novox/hq ADR -// 0086), by basic authentication, when no token file is named. +// 0086), by basic authentication, when no token is given. // // A credential is read when a call is made and never logged, answered or put in an error. package forgehttp @@ -35,12 +34,12 @@ type Client struct { URL string // e.g. http://127.0.0.1:3000 User string PasswordFile string - // TokenFile, when set, is the tools' account's token (MESH_GITEA_TOKEN_FILE), sent instead of the admin login; - // Account the account its scopes line must name (MESH_GITEA_TOOLS_USER, default mesh-tools). - TokenFile string - Account string - HTTP *http.Client // when nil, a client with Timeout - Timeout time.Duration // one request's limit when HTTP is nil; 20 s when zero + // Token, when set, is the account's token as the vault holds it (vault.go), sent instead of the admin login; + // Account the account it is (MESH_GITEA_TOOLS_USER, default mesh-tools; MESH_GITEA_PACKAGES_USER for the writer). + Token *VaultToken + Account string + HTTP *http.Client // when nil, a client with Timeout + Timeout time.Duration // one request's limit when HTTP is nil; 20 s when zero } // ToolsAccount is the account the forge's tools act as when the manifest names none (token.ts, tools-account.ts). @@ -52,37 +51,35 @@ const ToolsAccount = "mesh-tools" const PackagesAccount = "mesh-packages" // WriterFromEnv is the package registry's write side as the manifest hands it to a bundle: the packages account's -// token (MESH_GITEA_PACKAGES_TOKEN_FILE, kept by the module beside the tools' token) and the names of the variables -// it left unset. Never the admin login: a bundle handed no writer's token writes nothing. -func WriterFromEnv() (Client, []string) { - c := Client{URL: strings.TrimRight(os.Getenv("MESH_GITEA_URL"), "/"), TokenFile: os.Getenv("MESH_GITEA_PACKAGES_TOKEN_FILE"), - Account: os.Getenv("MESH_GITEA_PACKAGES_USER")} - if c.Account == "" { - c.Account = PackagesAccount - } - var missing []string - for _, kv := range [][2]string{{"MESH_GITEA_URL", c.URL}, {"MESH_GITEA_PACKAGES_TOKEN_FILE", c.TokenFile}} { - if kv[1] == "" { - missing = append(missing, kv[0]) - } - } - return c, missing +// token, read from the vault (vault.go), and the names of the variables it left unset. Never the admin login. +func WriterFromEnv() (Client, []string, error) { + return accountFromEnv(os.Getenv("MESH_GITEA_PACKAGES_USER"), PackagesAccount) } -// FromEnv is the forge as the manifest hands it to a bundle, and the names of the variables it left unset. A bundle -// handed MESH_GITEA_TOKEN_FILE acts as the tools' account and needs no admin login. -func FromEnv() (Client, []string) { - if file := os.Getenv("MESH_GITEA_TOKEN_FILE"); file != "" { - c := Client{URL: strings.TrimRight(os.Getenv("MESH_GITEA_URL"), "/"), TokenFile: file, - Account: os.Getenv("MESH_GITEA_TOOLS_USER")} - if c.Account == "" { - c.Account = ToolsAccount - } - if c.URL == "" { - return c, []string{"MESH_GITEA_URL"} - } - return c, nil +// ToolsFromEnv is the forge as the tools' account, its token read from the vault (vault.go), and the names of the +// variables the manifest left unset; it needs no admin login. +func ToolsFromEnv() (Client, []string, error) { + return accountFromEnv(os.Getenv("MESH_GITEA_TOOLS_USER"), ToolsAccount) +} + +func accountFromEnv(account, fallback string) (Client, []string, error) { + if account == "" { + account = fallback } + token, err := TokenOf(account) + if err != nil { + return Client{}, nil, err + } + c := Client{URL: strings.TrimRight(os.Getenv("MESH_GITEA_URL"), "/"), Token: token, Account: account} + if c.URL == "" { + return c, []string{"MESH_GITEA_URL"}, nil + } + return c, nil, nil +} + +// FromEnv is the forge as the admin account, as the manifest hands it to a bundle, and the names of the variables it +// left unset. +func FromEnv() (Client, []string) { c := Client{ URL: strings.TrimRight(os.Getenv("MESH_GITEA_URL"), "/"), User: os.Getenv("MESH_GITEA_ADMIN_USER"), @@ -132,27 +129,10 @@ func (c *Client) password() (string, error) { return strings.TrimSpace(string(raw)), nil } -// token is the tools' account's token, refused unless its scopes line names that account. -func (c *Client) token() (string, error) { - raw, err := os.ReadFile(c.TokenFile) - if err != nil { - return "", fmt.Errorf("the token for %s is not kept yet (the module mints it): %w", c.Account, err) - } - line, err := os.ReadFile(c.TokenFile + ".scopes") - if err != nil || !strings.HasSuffix(strings.TrimSpace(string(line)), " for "+c.Account) { - return "", fmt.Errorf("the token kept at %s is not %s's (novox/hq ADR 0365): the module mints that account's own at its next call; nothing was sent", c.TokenFile, c.Account) - } - token := strings.TrimSpace(string(raw)) - if token == "" { - return "", fmt.Errorf("the token kept at %s is empty; nothing was sent", c.TokenFile) - } - return token, nil -} - // authorize puts the bundle's credential on a request: the tools' token, or the admin login. func (c *Client) authorize(req *http.Request) error { - if c.TokenFile != "" { - token, err := c.token() + if c.Token != nil { + token, err := c.Token.Token() if err != nil { return err } @@ -212,7 +192,7 @@ func (c *Client) Send(ctx context.Context, method, path string, raw []byte, cont return nil, fmt.Errorf("the forge's answer to %s %s is larger than %d MiB and was not read whole", method, path, limit>>20) } switch { - case res.StatusCode == http.StatusUnauthorized && c.TokenFile != "": + case res.StatusCode == http.StatusUnauthorized && c.Token != nil: return nil, fmt.Errorf("the forge rejected %s's token for %s (401): the module mints it afresh; nothing was done: %w", c.Account, method+" "+path, &StatusError{method, path, res.StatusCode, "unauthorized"}) case res.StatusCode == http.StatusNotFound: diff --git a/modules/gitea/internal/forgehttp/vault.go b/modules/gitea/internal/forgehttp/vault.go new file mode 100644 index 000000000..b6ad605c3 --- /dev/null +++ b/modules/gitea/internal/forgehttp/vault.go @@ -0,0 +1,166 @@ +package forgehttp + +// The forge module's tokens as its Go bundles read them (design 47 H2 §808; H-A-422 (1), H-A-426): kept in the vault as +// the module's own, `module/gitea/`, written by the TypeScript bundle's minter through own-put, and read here +// through the seat `mesh-vault`'s `get`, sealed to this process's own key (the Go SDK's seal opens it), at start and at +// each `secret-changed` that names it. Never a file: the bundle holds the token in memory only, and says nothing of it. + +import ( + "encoding/json" + "errors" + "fmt" + "strings" + "sync" + + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/seal" +) + +// VaultGet is the vault seat's verb a token is read by, and SecretChanged the vault's event that a secret changed. +const ( + VaultGet = "seat:mesh-vault.get" + SecretChanged = "mesh-vault.secret-changed" +) + +// VaultName is the vault's name of an account's token (H-A-426). +func VaultName(account string) string { return "module/gitea/" + account } + +// Ask asks one seat verb and answers its reply (stdio.Ask). +type Ask func(address string, args any) (json.RawMessage, error) + +// VaultToken is one account's token as this process holds it, read from the vault. +type VaultToken struct { + Account string + ask Ask + keys seal.KeyPair + + mu sync.Mutex + token string + generation int +} + +// NewVaultToken is an account's token read through ask, sealed to a key made for this process. +func NewVaultToken(account string, ask Ask) (*VaultToken, error) { + keys, err := seal.GenerateKeyPair() + if err != nil { + return nil, fmt.Errorf("no key to read %s's token with: %w", account, err) + } + return &VaultToken{Account: account, ask: ask, keys: keys}, nil +} + +// Load reads the account's token from the vault again. +func (t *VaultToken) Load() error { + raw, err := t.ask(VaultGet, map[string]any{"name": VaultName(t.Account), "public_key": t.keys.PublicKey}) + if err != nil { + return fmt.Errorf("the vault did not hand %s: %w", VaultName(t.Account), err) + } + var got struct { + Generation int `json:"generation"` + Sealed *seal.Box `json:"sealed"` + } + if err := json.Unmarshal(raw, &got); err != nil || got.Sealed == nil { + return fmt.Errorf("the vault answered no sealed %s", VaultName(t.Account)) + } + token, err := seal.Open(*got.Sealed, t.keys.PrivateKey) + if err != nil { + return fmt.Errorf("%s does not open with this process's key: %w", VaultName(t.Account), err) + } + token = strings.TrimSpace(token) + if token == "" { + return fmt.Errorf("the vault's %s is empty", VaultName(t.Account)) + } + t.mu.Lock() + t.token, t.generation = token, got.Generation + t.mu.Unlock() + return nil +} + +// Token is the token held; read from the vault when none is yet (the module mints it first). +func (t *VaultToken) Token() (string, error) { + t.mu.Lock() + token := t.token + t.mu.Unlock() + if token != "" { + return token, nil + } + if err := t.Load(); err != nil { + return "", fmt.Errorf("the token for %s is not kept yet (the module mints it): %w", t.Account, err) + } + t.mu.Lock() + defer t.mu.Unlock() + return t.token, nil +} + +// Generation is the vault's generation of the token held, 0 when none is held. +func (t *VaultToken) Generation() int { + t.mu.Lock() + defer t.mu.Unlock() + return t.generation +} + +// Changed takes one `secret-changed`: the token is read again when it names this account's; answers whether it did. +func (t *VaultToken) Changed(body []byte) (bool, error) { + var e struct { + Name string `json:"name"` + } + if err := json.Unmarshal(body, &e); err != nil { + return false, fmt.Errorf("%s is not a secret's change: %w", SecretChanged, err) + } + if e.Name != VaultName(t.Account) { + return false, nil + } + return true, t.Load() +} + +var ( + processTokens = map[string]*VaultToken{} + processTokensMu sync.Mutex +) + +// TokenOf is this process's one token of account, read through the runtime's seat calls. +func TokenOf(account string) (*VaultToken, error) { + processTokensMu.Lock() + defer processTokensMu.Unlock() + if t, ok := processTokens[account]; ok { + return t, nil + } + t, err := NewVaultToken(account, stdio.Ask) + if err != nil { + return nil, err + } + processTokens[account] = t + return t, nil +} + +// Follow reads each account's token once the runtime serves, and again at each `secret-changed` naming it, saying +// through logf what it read (never the token). It returns when the subscription ends. +func Follow(logf func(format string, args ...any), accounts ...string) error { + var held []*VaultToken + for _, a := range accounts { + t, err := TokenOf(a) + if err != nil { + return err + } + held = append(held, t) + } + <-stdio.Served() + for _, t := range held { + if err := t.Load(); err != nil { + logf("%s is not read at the start (the module mints it): %v", VaultName(t.Account), err) + } else { + logf("read %s at generation %d", VaultName(t.Account), t.Generation()) + } + } + return stdio.Subscribe(SecretChanged, func(e stdio.Envelope) error { + var errs []error + for _, t := range held { + if read, err := t.Changed(e.Body); err != nil { + errs = append(errs, err) + logf("%s changed and could not be read again: %v", VaultName(t.Account), err) + } else if read { + logf("read %s again at generation %d", VaultName(t.Account), t.Generation()) + } + } + return errors.Join(errs...) + }) +} diff --git a/modules/gitea/kept.ts b/modules/gitea/kept.ts index 368cdc388..deba4f89d 100644 --- a/modules/gitea/kept.ts +++ b/modules/gitea/kept.ts @@ -13,6 +13,7 @@ export const KEPT: readonly { readonly name: string; readonly readable?: boolean { name: "first-views" }, { name: "idle" }, { name: "taken" }, + { name: "token-lease" }, { name: "transfers" }, ]; diff --git a/modules/gitea/module.json b/modules/gitea/module.json index 6d490aedc..04d479f64 100644 --- a/modules/gitea/module.json +++ b/modules/gitea/module.json @@ -245,6 +245,20 @@ "trusted": false, "runtime": true, "default": "10m" + }, + "token-ttl": { + "type": "string", + "description": "How old the forge module's token of an account may grow, as a duration such as 24h (design 47 H2 \u00a7808): a token the vault has kept longer than this, by its changed-at, is minted again, one process at a time, and the one it replaces is deleted at the forge.", + "trusted": false, + "runtime": true, + "default": "24h" + }, + "mint-lease": { + "type": "string", + "description": "How long one process of the forge module may hold the lease to mint a token, as a duration such as 60s (design 47 H2 \u00a7808): the lease key mint in the state token-lease lives this long, so a process that died holding it is replaced after it, and a process that finds it taken waits at most this long.", + "trusted": false, + "runtime": true, + "default": "60s" } }, "facts": { @@ -527,9 +541,7 @@ "env": { "MESH_GITEA_URL": "http://127.0.0.1:${port:3000}", "MESH_GITEA_TOOLS_USER": "mesh-tools", - "MESH_GITEA_TOKEN_FILE": "${dir:runtime-state}/token", "MESH_GITEA_PACKAGES_USER": "mesh-packages", - "MESH_GITEA_PACKAGES_TOKEN_FILE": "${dir:runtime-state}/packages-token", "MESH_NPM_OWNER": "novox" } }, @@ -546,7 +558,6 @@ "env": { "MESH_GITEA_URL": "http://127.0.0.1:${port:3000}", "MESH_GITEA_TOOLS_USER": "mesh-tools", - "MESH_GITEA_TOKEN_FILE": "${dir:runtime-state}/token", "MESH_GITEA_PATCH_DIR": "${dir:forge-patches}" } }, @@ -664,6 +675,11 @@ "name": "taken", "key-ttl": true, "max-mib": 8 + }, + { + "name": "token-lease", + "key-ttl": true, + "max-mib": 1 } ], "reads": [ @@ -678,7 +694,9 @@ "seat:mesh-catalog.revision", "seat:mesh-catalog.registrations", "seat:mesh-hand-acts.hand-act", - "seat:mesh-builds.reserved" + "seat:mesh-builds.reserved", + "seat:mesh-vault.own-put", + "seat:mesh-vault.get" ], "consumes": [ "mesh-catalog.catalog-revision", @@ -688,7 +706,8 @@ "git.pull-updated", "git.pull-closed", "git.pull-merged", - "mesh-conditions.findings-asked" + "mesh-conditions.findings-asked", + "mesh-vault.secret-changed" ], "streams": [ { diff --git a/modules/gitea/packages-account.ts b/modules/gitea/packages-account.ts index ac0ec296b..bc264d44d 100644 --- a/modules/gitea/packages-account.ts +++ b/modules/gitea/packages-account.ts @@ -9,28 +9,25 @@ // makes `mesh-tools`; its password random, sent once and kept nowhere. // - **the team**: `mesh-packages` in the organisation that owns the registry, writing packages and nothing else: // no code, no issues, no pull requests, no administration. -// - **the token**: minted for the account by the administrator's login with the one scope `write:package`, kept at -// 0600 in the module's runtime state as `packages-token`, its scopes line beside it saying whose it is -// (MintedToken: one mint at a time across the bundle's processes, re-minted when the kept one is gone or other). +// - **the token**: minted for the account by the administrator's login with the one scope `write:package`, kept in +// the vault as the module's own `module/gitea/mesh-packages` (design 47 H2 §808, H-A-422, H-A-426; MintedToken: one +// mint at a time across the bundle's processes, a name per mint, re-minted when refused or older than token-ttl). // // The watcher mints it at its start (keepPackagesToken). No timer looks at it after that: the Go bundle mints nothing, // and when the forge refuses its write it raises the finding PACKAGES_TOKEN_REFUSED in the module's findings state; the // watcher, watching that key, looks again (a refused token is minted afresh) and clears the finding (renewOnRefusal). import { randomBytes } from "node:crypto"; -import { join } from "node:path"; -import { MintedToken } from "./token.js"; +import { adminFromEnv, MintedToken } from "./token.js"; import { TOOLS_ORG, type ToolsForge } from "./tools-account.js"; /** The account the package seats write as (internal/forgehttp PackagesAccount). */ export const PACKAGES_ACCOUNT = "mesh-packages"; -/** Its team in the registry's organisation, and the token's name in the account's list. */ +/** Its team in the registry's organisation, and the token's fixed name in the account's list before a name per mint. */ export const PACKAGES_TEAM = "mesh-packages"; export const PACKAGES_TOKEN_NAME = "mesh-packages"; /** The one scope the write side needs; reads go with the tools' token. */ export const PACKAGES_TOKEN_SCOPES: readonly string[] = ["write:package"]; -/** Where the token is kept in the module's runtime state (the manifest's MESH_GITEA_PACKAGES_TOKEN_FILE). */ -export const PACKAGES_TOKEN_FILE = "packages-token"; /** What the team grants: write on the organisation's packages, and no more. */ export const PACKAGES_TEAM_UNITS = { @@ -47,26 +44,19 @@ export async function setUpPackagesAccount(forge: ToolsForge, account = PACKAGES } /** The packages account's token, minted by the administrator's login from what the manifest hands the watcher. */ -export function packagesToken(url: string, env: NodeJS.ProcessEnv = process.env, log?: (line: string) => void): MintedToken { - const admin = env.MESH_GITEA_ADMIN_USER; - const passwordFile = env.MESH_GITEA_ADMIN_PASSWORD_FILE; - const stateDir = env.MESH_GITEA_STATE_DIR; - const missing = [ - admin ? null : "MESH_GITEA_ADMIN_USER", - passwordFile ? null : "MESH_GITEA_ADMIN_PASSWORD_FILE", - stateDir ? null : "MESH_GITEA_STATE_DIR", - ].filter((v): v is string => v !== null); - if (missing.length) throw new Error(`the package seats' write token cannot be minted: ${missing.join(", ")} not set by the manifest`); +export function packagesToken(url: string, env: NodeJS.ProcessEnv = process.env, log?: (line: string) => void, + over: Partial[0], "vault" | "lease" | "fetch" | "now">> = {}): MintedToken { + const { admin, passwordFile } = adminFromEnv(env, "the package seats' write token cannot be minted:"); const account = env.MESH_GITEA_PACKAGES_USER || PACKAGES_ACCOUNT; if (account === admin) throw new Error(`MESH_GITEA_PACKAGES_USER names the admin account "${admin}"; the package seats write as an account of their own`); + const base = over.vault && over.lease ? {} : MintedToken.settingsFromEnv(env); return new MintedToken({ - url, admin: admin!, passwordFile: passwordFile!, file: join(stateDir!, PACKAGES_TOKEN_FILE), account, - name: PACKAGES_TOKEN_NAME, scopes: PACKAGES_TOKEN_SCOPES, log, + ...base, ...over, url, admin, passwordFile, account, oldName: PACKAGES_TOKEN_NAME, scopes: PACKAGES_TOKEN_SCOPES, log, setUp: async (password) => { const { GiteaAdmin } = await import("./client.js"); - await setUpPackagesAccount(new GiteaAdmin(url, admin!, password), account, () => randomBytes(24).toString("base64url")); + await setUpPackagesAccount(new GiteaAdmin(url, admin, password), account, () => randomBytes(24).toString("base64url")); }, - }); + } as ConstructorParameters[0]); } /** diff --git a/modules/gitea/test/fakebus.ts b/modules/gitea/test/fakebus.ts new file mode 100644 index 000000000..3c218e648 --- /dev/null +++ b/modules/gitea/test/fakebus.ts @@ -0,0 +1,96 @@ +// The bus as the forge module's token needs it, in memory (design 47 H2 §808, §822): the vault seat's `own-put` in its +// two sealed steps and its `get` sealed to the asker's key, and the runtime's state with a create-only key that lives +// its TTL, as the runtime that launched the bundle answers them. `order` records each own-put and each forge delete a +// test also writes into it, so the delete's place after the own-put can be read. + +import nacl from "tweetnacl"; +import blakejs from "blakejs"; +import { seal } from "@novox/mesh-sdk/seal"; +import { useBroker, type Broker } from "@novox/mesh-sdk/messaging"; +import { useRuntime } from "@novox/mesh-sdk/runtime"; +import { useStateChannel, type StateChange } from "@novox/mesh-sdk/state"; + +export interface Held { value: string; generation: number; changedAt: string } + +export function fakeBus(now: () => Date = () => new Date(), order: string[] = []) { + const vault = new Map(); + const calls = new Map(); + const kv = new Map(); + const watchers: { state: string; key: string; deliver: (c: StateChange) => Promise }[] = []; + let revision = 0; + const keyOf = (state: string, key: string) => `${state}/${key}`; + const live = (k: string) => { + const e = kv.get(k); + if (e && e.until !== null && e.until <= now().getTime()) kv.delete(k); + return kv.get(k); + }; + const tell = async (state: string, key: string, op: "put" | "delete", value?: unknown) => { + revision++; + for (const w of watchers) if (w.state === state && (w.key === "" || w.key === key)) await w.deliver({ state, key, op, value, revision, current: false }); + }; + const broker: Broker = { + async request(key: string, body: Req): Promise { + const b = body as Record; + if (key === "seat:mesh-vault.own-put" && b.call === undefined) { + const keys = nacl.box.keyPair(); + const call = `call-${calls.size + 1}`; + calls.set(call, { name: `module/gitea/${b.name}`, secretKey: keys.secretKey }); + return { call, seal_to: Buffer.from(keys.publicKey).toString("base64"), secret: `module/gitea/${b.name}` } as Res; + } + if (key === "seat:mesh-vault.own-put") { + const c = calls.get(b.call); + if (!c) throw new Error(`no own-put call ${b.call}`); + calls.delete(b.call); + const boxed = new Uint8Array(Buffer.from(b.sealed, "base64")); + const eph = boxed.subarray(0, 32); + const pub = nacl.box.keyPair.fromSecretKey(c.secretKey).publicKey; + const nonce = blakejs.blake2b(Buffer.concat([eph, pub]), undefined, 24); + const opened = nacl.box.open(boxed.subarray(32), nonce, eph, c.secretKey); + if (!opened) throw new Error("the sealed value does not open"); + const generation = (vault.get(c.name)?.generation ?? 0) + 1; + vault.set(c.name, { value: Buffer.from(opened).toString("utf8"), generation, changedAt: now().toISOString() }); + order.push(`own-put ${c.name} ${generation}`); + return { name: c.name, generation } as Res; + } + if (key === "seat:mesh-vault.get") { + const held = vault.get(b.name); + if (!held) throw new Error(`the vault holds no secret named ${b.name}`); + return { name: b.name, generation: held.generation, "changed-at": held.changedAt, sealed: seal(held.value, b.public_key) } as Res; + } + throw new Error(`not played here: ${key}`); + }, + handle: async () => () => {}, + publish: async () => {}, + subscribe: async () => () => {}, + close: async () => {}, + }; + const ask = async (method: string, params: unknown): Promise => { + const p = params as { state: string; key: string; value?: unknown; ttl_seconds?: number }; + const k = keyOf(p.state, p.key); + switch (method) { + case "mesh/state.create": + if (live(k)) throw Object.assign(new Error("the key has a value"), { code: -32010 }); + kv.set(k, { value: p.value, until: p.ttl_seconds ? now().getTime() + p.ttl_seconds * 1000 : null }); + await tell(p.state, p.key, "put", p.value); + return { revision }; + case "mesh/state.get": { + const e = live(k); + return e ? { key: p.key, value: e.value, revision } : null; + } + case "mesh/state.put": + kv.set(k, { value: p.value, until: null }); + await tell(p.state, p.key, "put", p.value); + return { revision }; + case "mesh/state.delete": + kv.delete(k); + await tell(p.state, p.key, "delete"); + return null; + default: + throw new Error(`not played here: ${method}`); + } + }; + useBroker(() => broker); + useRuntime(ask); + useStateChannel({ ask, watch: async (params, deliver) => { const w = params as { state: string; key: string }; watchers.push({ ...w, deliver }); } }); + return { vault, kv, order, ask }; +} diff --git a/modules/gitea/test/kept.test.ts b/modules/gitea/test/kept.test.ts index f7d7639de..5595321c0 100644 --- a/modules/gitea/test/kept.test.ts +++ b/modules/gitea/test/kept.test.ts @@ -16,8 +16,8 @@ test("the states the module announces are the states its manifest declares", asy test("the announcement has the layout mesh-sdk's go/announce reads: version 1, the module, the node, the states sorted by name", async () => { const { statesAnnouncement } = await import("../kept.ts"); - assert.deepEqual(statesAnnouncement("gitea", "ace"), { v: 1, module: "gitea", node: "ace", states: [{ name: "announced" }, { name: "bound", readable: true }, { name: "calls" }, { name: "failed-steps" }, { name: "findings", readable: true }, { name: "first-views" }, { name: "idle" }, { name: "taken" }, { name: "transfers" }] }); - assert.deepEqual(statesAnnouncement("gitea", undefined), { v: 1, module: "gitea", states: [{ name: "announced" }, { name: "bound", readable: true }, { name: "calls" }, { name: "failed-steps" }, { name: "findings", readable: true }, { name: "first-views" }, { name: "idle" }, { name: "taken" }, { name: "transfers" }] }); + assert.deepEqual(statesAnnouncement("gitea", "ace"), { v: 1, module: "gitea", node: "ace", states: [{ name: "announced" }, { name: "bound", readable: true }, { name: "calls" }, { name: "failed-steps" }, { name: "findings", readable: true }, { name: "first-views" }, { name: "idle" }, { name: "taken" }, { name: "token-lease" }, { name: "transfers" }] }); + assert.deepEqual(statesAnnouncement("gitea", undefined), { v: 1, module: "gitea", states: [{ name: "announced" }, { name: "bound", readable: true }, { name: "calls" }, { name: "failed-steps" }, { name: "findings", readable: true }, { name: "first-views" }, { name: "idle" }, { name: "taken" }, { name: "token-lease" }, { name: "transfers" }] }); }); test("index.ts says the announcement as the event `states` before the tools are served", () => { diff --git a/modules/gitea/test/token.test.ts b/modules/gitea/test/token.test.ts index 289020c57..7231822c3 100644 --- a/modules/gitea/test/token.test.ts +++ b/modules/gitea/test/token.test.ts @@ -10,15 +10,19 @@ import { test, after } from "node:test"; import assert from "node:assert/strict"; import { createServer, type IncomingMessage, type ServerResponse } from "node:http"; -import { mkdir, mkdtemp, readFile, rm, stat, writeFile } from "node:fs/promises"; +import { mkdtemp, stat, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { collectTools } from "@novox/mesh-sdk/tools"; -import { AdminRefused, MintedToken, TOKEN_SCOPES } from "../dist/token.js"; +import { AdminRefused, MintedToken, TOKEN_NAME, TOKEN_SCOPES, leaseOverState, vaultOverBus, type MintedTokenOptions } from "../dist/token.js"; +import { tokenLeaseTool } from "../dist/tools/tokenlease.js"; +import { fakeBus } from "./fakebus.ts"; import { GiteaClient } from "../dist/client.js"; import { getGiteaTools } from "../dist/tools/index.js"; +/** What the vault and the forge were asked, in order: each own-put and each token deleted (design 47 H2 §808). */ +const ORDER: string[] = []; const ADMIN = "mesh-admin"; const PASSWORD = "the-vault-minted-this"; @@ -126,6 +130,7 @@ function fakeForge(): Promise { if (matches.length > 1) return json(res, 422, { message: `multiple matches for token name '${key}'` }); const t = matches[0]; forge.deletedBy.push(byId ? "id" : "name"); + ORDER.push(`delete ${t.name}`); const twin = forge.twins.findIndex((w) => w.id === t.id); if (twin >= 0) forge.twins.splice(twin, 1); else forge.tokens.delete(t.name); @@ -253,58 +258,59 @@ function fakeForge(): Promise { // ---- What the runtime's environment gives the module. ---- -async function delivered(forge: Forge): Promise<{ env: NodeJS.ProcessEnv; file: string; logs: string[] }> { +async function delivered(forge: Forge, now: () => Date = () => new Date()) { const dir = await mkdtemp(join(tmpdir(), "gitea-")); const passwordFile = join(dir, "admin.secret"); await writeFile(passwordFile, PASSWORD + "\n", { mode: 0o600 }); - const state = join(dir, "state"); + ORDER.length = 0; return { env: { MESH_GITEA_URL: forge.url, MESH_GITEA_ADMIN_USER: ADMIN, MESH_GITEA_ADMIN_PASSWORD_FILE: passwordFile, - MESH_GITEA_STATE_DIR: state, - }, - file: join(state, "token"), - logs: [], + MESH_GITEA_STATE_DIR: join(dir, "state"), + } as NodeJS.ProcessEnv, + bus: fakeBus(now, ORDER), + logs: [] as string[], }; } -/** A client as a fresh process would build it: a new source over the kept file, its log captured. */ -function minted(env: NodeJS.ProcessEnv, logs: string[]): GiteaClient { - const source = new MintedToken({ - url: env.MESH_GITEA_URL!, - admin: env.MESH_GITEA_ADMIN_USER!, - passwordFile: env.MESH_GITEA_ADMIN_PASSWORD_FILE!, - file: join(env.MESH_GITEA_STATE_DIR!, "token"), - log: (l) => logs.push(l), - }); - return new GiteaClient(env.MESH_GITEA_URL!, source); +/** A client as a fresh process would build it: a new source over the vault and the lease on the bus, its log captured. */ +function minted(env: NodeJS.ProcessEnv, logs: string[], extra: Partial = {}): GiteaClient { + return new GiteaClient(env.MESH_GITEA_URL!, source(env, logs, extra)); } +function source(env: NodeJS.ProcessEnv, logs: string[], extra: Partial = {}): MintedToken { + return new MintedToken({ + url: env.MESH_GITEA_URL!, admin: env.MESH_GITEA_ADMIN_USER!, passwordFile: env.MESH_GITEA_ADMIN_PASSWORD_FILE!, + account: "mesh-tools", oldName: TOKEN_NAME, vault: vaultOverBus(), lease: leaseOverState(), log: (l) => logs.push(l), ...extra, + }); +} + +const VAULT = "module/gitea/mesh-tools"; const forge = await fakeForge(); after(() => forge.close()); -test("first start: mints with the admin account, keeps the token at 0600, asks for the tools' scopes only", async () => { - const { env, file, logs } = await delivered(forge); +test("first start: mints mesh-tools-1 with the admin account, writes it through own-put, asks for the tools' scopes only", async () => { + const { env, bus, logs } = await delivered(forge); + forge.tokens.clear(); const repos = await minted(env, logs).listRepos(); assert.equal(repos[0]?.full_name, "novox/hq"); - assert.equal(forge.mints, 1); // Never write:admin (novox/hq ADR 0266): the forge's administration is the operator's, not a tool's. assert.deepEqual(forge.lastScopes, ["write:repository", "write:issue", "read:user", "read:package"]); assert.deepEqual(forge.lastScopes, [...TOKEN_SCOPES]); - const token = forge.tokens.get("mesh-tools")!; - assert.equal(await readFile(file, "utf8"), token + "\n"); - assert.equal(await readFile(file + ".scopes", "utf8"), "read:package,read:user,write:issue,write:repository\n"); - assert.equal((await stat(file)).mode & 0o777, 0o600); - // Said that it minted, and where it keeps it — never what it is. - assert.ok(logs.some((l) => l.startsWith("minted a token")), logs.join("\n")); + // Design 47 H2 §808, H-A-422 (2), H-A-426: named per mint, kept as the module's own secret, never a file. + const token = forge.tokens.get("mesh-tools-1")!; + assert.ok(token); + assert.deepEqual([bus.vault.get(VAULT)?.value, bus.vault.get(VAULT)?.generation], [token, 1]); + await assert.rejects(stat(join(env.MESH_GITEA_STATE_DIR!, "token")), /ENOENT/); + assert.ok(logs.some((l) => l.startsWith("minted mesh-tools-1")), logs.join("\n")); assert.ok(logs.every((l) => !l.includes(token) && !l.includes(PASSWORD)), logs.join("\n")); }); -test("second start: reuses the kept token, mints nothing", async () => { +test("second start: reuses the vault's token, mints nothing", async () => { const { env, logs } = await delivered(forge); await minted(env, logs).listRepos(); const before = forge.mints; @@ -313,54 +319,83 @@ test("second start: reuses the kept token, mints nothing", async () => { await minted(env, again).listRepos(); assert.equal(forge.mints, before); - assert.ok(again.some((l) => l.startsWith("reusing the token kept at")), again.join("\n")); - assert.ok(again.every((l) => !l.includes(forge.tokens.get("mesh-tools")!)), again.join("\n")); + assert.ok(again.some((l) => l.startsWith(`reusing the token the vault keeps as ${VAULT}`)), again.join("\n")); }); -test("the forge rejects the kept token (its data was restored): minted afresh, once, and the call goes through", async () => { - const { env, file, logs } = await delivered(forge); +test("the forge rejects the vault's token: mesh-tools-2 is minted once, written through own-put, and only then is mesh-tools-1 deleted by its id", async () => { + const { env, bus, logs } = await delivered(forge); + forge.tokens.clear(); const client = minted(env, logs); await client.listRepos(); const before = forge.mints; + ORDER.length = 0; + forge.deletedBy.length = 0; - forge.tokens.clear(); // the forge no longer knows any token — a restore from the predecessor + forge.tokens.set("mesh-tools-1", "revoked-meanwhile"); // the forge no longer takes the vault's token const repos = await client.listRepos(); assert.equal(repos.length, 1); assert.equal(forge.mints, before + 1); - assert.equal(await readFile(file, "utf8"), forge.tokens.get("mesh-tools") + "\n"); - assert.ok(logs.some((l) => l.startsWith("the forge rejected the kept token")), logs.join("\n")); + assert.equal(bus.vault.get(VAULT)?.value, forge.tokens.get("mesh-tools-2")); + // H-A-422 (2): the own-put first, the replaced token's delete after, by id; the forge holds one tools' token after. + assert.deepEqual(ORDER, [`own-put ${VAULT} 2`, "delete mesh-tools-1"]); + assert.deepEqual(forge.deletedBy, ["id"]); + assert.deepEqual([...forge.tokens.keys()].filter((n) => n.startsWith("mesh-tools")), ["mesh-tools-2"]); + assert.ok(logs.some((l) => l.startsWith("the forge rejected the vault's token")), logs.join("\n")); }); -test("a token kept with write:admin by an earlier build is minted again without it at the next start, and the forge drops the old one", async () => { - const { env, file, logs } = await delivered(forge); - await minted(env, logs).listRepos(); - const old = forge.tokens.get("mesh-tools")!; - forge.scopesOf.set(old, ["write:repository", "write:issue", "read:user", "write:admin"]); - await rm(file + ".scopes"); // an earlier build kept no scopes beside it +test("an own-put that fails leaves the old token standing, and the next mint replaces the unused one it left", async () => { + const { env, bus, logs } = await delivered(forge); + forge.tokens.clear(); + const client = minted(env, logs); + await client.listRepos(); + const old = forge.tokens.get("mesh-tools-1")!; + forge.tokens.set("mesh-tools-1", "revoked-meanwhile"); + ORDER.length = 0; + let failPut = true; + const failing = minted(env, logs, { vault: { get: vaultOverBus().get, put: async (a, t) => { + if (failPut) { failPut = false; throw new Error("the vault is away"); } + return vaultOverBus().put(a, t); + } } }); + + await assert.rejects(failing.listRepos(), /the vault is away/); + assert.deepEqual(ORDER, [], "nothing deleted while the vault holds the old token"); + assert.equal(bus.vault.get(VAULT)?.generation, 1); + assert.ok(forge.tokens.has("mesh-tools-2"), "the unused token of the failed mint stands"); + + assert.equal((await failing.listRepos()).length, 1); + assert.deepEqual(ORDER, ["delete mesh-tools-2", `own-put ${VAULT} 2`, "delete mesh-tools-1"]); + assert.notEqual(bus.vault.get(VAULT)?.value, old); +}); + +test("a token older than token-ttl is minted again, and the one it replaces deleted", async () => { + let clock = Date.parse("2026-11-22T00:00:00Z"); + const now = () => new Date(clock); + const { env, bus, logs } = await delivered(forge, now); + forge.tokens.clear(); + const client = minted(env, logs, { now, ttlMs: 24 * 3600_000 }); + await client.listRepos(); const before = forge.mints; - const again: string[] = []; - await minted(env, again).listRepos(); + clock += 23 * 3600_000; + await client.listRepos(); + assert.equal(forge.mints, before, "a token within token-ttl is used"); + clock += 2 * 3600_000; + await client.listRepos(); assert.equal(forge.mints, before + 1); - assert.deepEqual(forge.lastScopes, [...TOKEN_SCOPES]); - assert.ok(!forge.lastScopes!.includes("write:admin")); - assert.notEqual(forge.tokens.get("mesh-tools"), old); - assert.ok(![...forge.tokens.values()].includes(old), "the old token still stands on the forge"); - assert.ok(again.some((l) => l.includes("minted with other scopes")), again.join("\n")); - // An admin route through the escape hatch is refused before it is sent. - const tool = getGiteaTools(minted(env, again)).find((t) => t.name === "gitea_api")!; - await assert.rejects(tool.run({ path: "/admin/users/mesh_novox_builder", method: "PATCH", body: { admin: true } }), - /not a write gitea_api makes/); + assert.equal(bus.vault.get(VAULT)?.generation, 2); + assert.deepEqual([...forge.tokens.keys()].filter((n) => n.startsWith("mesh-tools")), ["mesh-tools-2"]); + assert.ok(logs.some((l) => l.includes("older than token-ttl")), logs.join("\n")); }); -test("a kept token lacking a scope the tools need: the forge refuses the call for the scope, the token is re-minted with the whole list, and the call goes through", async () => { - const { env, file, logs } = await delivered(forge); +test("a token lacking a scope the tools need: the forge refuses the call for the scope, the token is re-minted with the whole list, and the call goes through", async () => { + const { env, bus, logs } = await delivered(forge); + forge.tokens.clear(); const client = minted(env, logs); await client.listRepos(); const before = forge.mints; - const old = forge.tokens.get("mesh-tools")!; + const old = forge.tokens.get("mesh-tools-1")!; forge.scopesOf.set(old, ["write:repository", "write:issue"]); // minted before read:user was added const repos = await client.api("/user/repos"); @@ -368,37 +403,11 @@ test("a kept token lacking a scope the tools need: the forge refuses the call fo assert.equal(repos.length, 1); assert.equal(forge.mints, before + 1); assert.deepEqual(forge.lastScopes, [...TOKEN_SCOPES]); - assert.notEqual(forge.tokens.get("mesh-tools"), old); - assert.equal(await readFile(file, "utf8"), forge.tokens.get("mesh-tools") + "\n"); - assert.ok(logs.some((l) => l.startsWith("the forge rejected the kept token")), logs.join("\n")); + assert.equal(bus.vault.get(VAULT)?.value, forge.tokens.get("mesh-tools-2")); }); -test("the kept file is gone but the forge still holds a token by that name: replaced, not refused", async () => { - const { env, file, logs } = await delivered(forge); - await minted(env, logs).listRepos(); - const before = forge.mints; - await rm(file); - - const repos = await minted(env, logs).listRepos(); - - assert.equal(repos.length, 1); - assert.equal(forge.mints, before + 1); - assert.equal([...forge.tokens.keys()].filter((n) => n === "mesh-tools").length, 1); - assert.ok(logs.some((l) => l.includes('already holds a token named "mesh-tools"')), logs.join("\n")); -}); - -test("concurrent first calls share one mint", async () => { +test("the first mint deletes by id every token under the old fixed name, two raced ones too, and keeps the account's others", async () => { const { env, logs } = await delivered(forge); - const client = minted(env, logs); - const before = forge.mints; - - await Promise.all([client.listRepos(), client.listRepos(), client.listRepos()]); - - assert.equal(forge.mints, before + 1); -}); - -test("two tokens hold the name (two mints raced): both are deleted by their id, and one fresh token is minted", async () => { - const { env, file, logs } = await delivered(forge); forge.tokens.clear(); forge.twins.length = 0; for (const other of ["laptop", "backup", "ci"]) forge.plant(other); // the account's other tokens: kept @@ -411,44 +420,68 @@ test("two tokens hold the name (two mints raced): both are deleted by their id, assert.equal(repos.length, 1); assert.equal(forge.mints, before + 1); + // H-A-430 (1): the tokens the module kept before under the old fixed name are the ones the first mint replaces. assert.deepEqual(forge.deletedBy, ["id", "id"]); + assert.deepEqual(ORDER, [`own-put ${VAULT} 1`, "delete mesh-tools", "delete mesh-tools"]); assert.equal(forge.twins.length, 0); - assert.deepEqual([...forge.tokens.keys()].sort(), ["backup", "ci", "laptop", "mesh-tools"]); - assert.equal(await readFile(file, "utf8"), forge.tokens.get("mesh-tools") + "\n"); - assert.ok(logs.some((l) => l.includes('holds 2 tokens named "mesh-tools"')), logs.join("\n")); + assert.deepEqual([...forge.tokens.keys()].sort(), ["backup", "ci", "laptop", "mesh-tools-1"]); }); -test("two processes on one kept file, both without a token, mint once: the second waits for the first and reuses it", async () => { +test("concurrent first calls share one mint", async () => { const { env, logs } = await delivered(forge); forge.tokens.clear(); - forge.twins.length = 0; + const client = minted(env, logs); const before = forge.mints; - // Two sources over one state directory, as the tools bundle and the watcher are two processes. - const first = minted(env, logs); - const second = minted(env, logs); - await Promise.all([first.listRepos(), second.listRepos()]); + await Promise.all([client.listRepos(), client.listRepos(), client.listRepos()]); assert.equal(forge.mints, before + 1); - assert.equal(forge.twins.length, 0); - assert.ok(logs.some((l) => l.includes("another process minted it")), logs.join("\n")); }); -test("a lock left by a process that died mid-mint is taken over, not waited on for ever", async () => { - const { env, file, logs } = await delivered(forge); +test("two processes, one holding the old token, both refused, mint one token in all: the second waits for the lease and uses the vault's newer token", async () => { + const { env, bus, logs } = await delivered(forge); forge.tokens.clear(); forge.twins.length = 0; - await mkdir(join(env.MESH_GITEA_STATE_DIR!), { recursive: true }); - await writeFile(`${file}.lock`, `2147483646 ${Date.now()} gone\n`); // a pid no process holds + const first = minted(env, logs); + await first.listRepos(); + const second = minted(env, logs); + await second.listRepos(); // both hold mesh-tools-1 const before = forge.mints; - assert.equal((await minted(env, logs).listRepos()).length, 1); + forge.tokens.set("mesh-tools-1", "revoked-meanwhile"); + await Promise.all([first.listRepos(), second.listRepos()]); + + // Design 47 H2 §808: one mint in all, and the forge holds one token for the account after. assert.equal(forge.mints, before + 1); - await assert.rejects(stat(`${file}.lock`), /ENOENT/); + assert.deepEqual([...forge.tokens.keys()].filter((n) => n.startsWith("mesh-tools")), ["mesh-tools-2"]); + assert.equal(bus.vault.get(VAULT)?.generation, 2); + assert.ok(logs.some((l) => l.includes("another process holds the mint lease")), logs.join("\n")); + assert.equal(bus.kv.size, 0, "the lease is deleted after the mint"); +}); + +test("a minter killed while holding the lease is replaced after mint-lease: the waiter creates the lease itself and mints", async () => { + let clock = Date.parse("2026-11-22T00:00:00Z"); + const now = () => new Date(clock); + const { env, bus, logs } = await delivered(forge, now); + forge.tokens.clear(); + // A process that took the lease and died: its key lives mint-lease (60 s) and nobody deletes it. + await bus.ask("mesh/state.create", { state: "token-lease", key: "mint", value: { holder: "ace/4242", since: now().toISOString() }, ttl_seconds: 60 }); + const waited: number[] = []; + const lease = leaseOverState(); + // Waiting for the key to change: nothing changes, so the wait lasts its bound, the clock moving on by it. + const before = forge.mints; + + const repos = await minted(env, logs, { lease: { ...lease, changed: async (ms) => { waited.push(ms); clock += ms; } } }).listRepos(); + + assert.equal(repos.length, 1); + assert.deepEqual(waited, [60_000], "the waiter waited mint-lease once"); + assert.equal(forge.mints, before + 1); + assert.ok(logs.some((l) => l.includes("another process holds the mint lease (ace/4242 since")), logs.join("\n")); }); test("the admin account is refused: said plainly, nothing kept, and the next call fails the same way rather than crashing", async () => { - const { env, file, logs } = await delivered(forge); + const { env, bus, logs } = await delivered(forge); + forge.tokens.clear(); forge.admins.delete(ADMIN); // the forge's data came from a predecessor; mesh-admin was never created there try { const client = minted(env, logs); @@ -464,7 +497,7 @@ test("the admin account is refused: said plainly, nothing kept, and the next cal }); await assert.rejects(client.listRepos(), AdminRefused); assert.equal(forge.mints, before); - await assert.rejects(stat(file), /ENOENT/); + assert.equal(bus.vault.size, 0); // The account appears (the operator created it): the very next call mints and works. forge.admins.set(ADMIN, PASSWORD); @@ -475,25 +508,47 @@ test("the admin account is refused: said plainly, nothing kept, and the next cal } }); -test("one process shares one source per kept file — the watcher and the tools never renew against each other", async () => { +test("one process shares one source per account — the watcher and the tools never renew against each other", async () => { const { env } = await delivered(forge); assert.equal(MintedToken.fromEnv(env.MESH_GITEA_URL!, env), MintedToken.fromEnv(env.MESH_GITEA_URL!, env)); }); -test("a second process finds the token the first renewed, and reuses it instead of minting over it", async () => { +test("a second process finds the token the first renewed in the vault, and reuses it instead of minting over it", async () => { const { env, logs } = await delivered(forge); + forge.tokens.clear(); const first = minted(env, logs); const second = minted(env, logs); await first.listRepos(); - await second.listRepos(); // both hold the same kept token + await second.listRepos(); // both hold the same token const before = forge.mints; - forge.tokens.clear(); + forge.tokens.set("mesh-tools-1", "revoked-meanwhile"); await first.listRepos(); // renews: one mint - await second.listRepos(); // rejected too — but the kept file already carries the renewed one + await second.listRepos(); // rejected too — but the vault already holds the renewed one assert.equal(forge.mints, before + 1); - assert.ok(logs.some((l) => l.includes("is newer — reusing it")), logs.join("\n")); + assert.ok(logs.some((l) => l.includes("is newer (generation 2) — reusing it")), logs.join("\n")); +}); + +test("forge_token_lease says the lease's holder and since when, and each token's age from the vault, never the token", async () => { + let clock = Date.parse("2026-11-22T00:00:00Z"); + const now = () => new Date(clock); + const { env, bus, logs } = await delivered(forge, now); + forge.tokens.clear(); + await minted(env, logs, { now }).listRepos(); + await bus.ask("mesh/state.create", { state: "token-lease", key: "mint", value: { holder: "ace/4242", since: "2026-11-22T00:00:30Z" }, ttl_seconds: 60 }); + clock += 90_000; + const tool = tokenLeaseTool(vaultOverBus(), leaseOverState(), ["mesh-tools", "mesh-packages"], now); + + const said = await tool.run({}, undefined as never) as any; + + assert.equal(tool.name, "forge_token_lease"); + assert.deepEqual(said.tokens.map((t: any) => [t.account, t.held, t.generation, t["age-seconds"]]), + [["mesh-tools", true, 1, 90], ["mesh-packages", false, undefined, undefined]]); + assert.ok(!JSON.stringify(said).includes(bus.vault.get(VAULT)!.value), "the token is never said"); + // The lease lives mint-lease: past it, nobody holds it. + clock += 30_000; + assert.deepEqual((await tool.run({}, undefined as never) as any).lease, { held: false }); }); test("a configured token wins, and is reported rather than minted over when the forge rejects it", async () => { @@ -510,8 +565,8 @@ test("a configured token wins, and is reported rather than minted over when the test("nothing to mint with and no token: the client says what is missing", async () => { assert.throws( - () => GiteaClient.fromEnv({ MESH_GITEA_URL: forge.url, MESH_GITEA_ADMIN_USER: ADMIN }), - /set GITEA_TOKEN, or MESH_GITEA_ADMIN_PASSWORD_FILE, MESH_GITEA_STATE_DIR/, + () => GiteaClient.fromEnv({ MESH_GITEA_URL: forge.url, MESH_GITEA_ADMIN_USER: ADMIN, MESH_GITEA_TOOLS_USER: "mesh-tools-nothing" }), + /set GITEA_TOKEN, or MESH_GITEA_ADMIN_PASSWORD_FILE/, ); }); diff --git a/modules/gitea/test/tools-account.test.ts b/modules/gitea/test/tools-account.test.ts index 637d9ed1a..f486ae51b 100644 --- a/modules/gitea/test/tools-account.test.ts +++ b/modules/gitea/test/tools-account.test.ts @@ -10,11 +10,12 @@ import { test, after } from "node:test"; import assert from "node:assert/strict"; import { createServer, type IncomingMessage, type ServerResponse } from "node:http"; -import { mkdtemp, readFile, writeFile, mkdir } from "node:fs/promises"; +import { mkdtemp, readFile, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { MintedToken, TOKEN_SCOPES } from "../dist/token.js"; +import { MintedToken, TOKEN_NAME, TOKEN_SCOPES, leaseOverState, vaultOverBus } from "../dist/token.js"; +import { fakeBus } from "./fakebus.ts"; import { GiteaClient } from "../dist/client.js"; import { TOOLS_ACCOUNT, TOOLS_TEAM_UNITS, BUILDER_ACCOUNT, AGENT_ACCOUNT, meshAccounts } from "../dist/tools-account.js"; import { PACKAGES_ACCOUNT, PACKAGES_TEAM_UNITS, PACKAGES_TOKEN_REFUSED, keepPackagesToken, packagesToken, renewOnRefusal } from "../dist/packages-account.js"; @@ -266,7 +267,12 @@ function fakeForge() { const forge = await fakeForge(); after(() => forge.close()); +/** The bus of the environment made last: the vault's secrets and the lease (design 47 H2 §808). */ +let bus = fakeBus(); +const vaulted = (account: string) => bus.vault.get(`module/gitea/${account}`)?.value; + async function environment(keepMainRules = false): Promise { + bus = fakeBus(); const dir = await mkdtemp(join(tmpdir(), "gitea-tools-account-")); const passwordFile = join(dir, "admin.secret"); await writeFile(passwordFile, PASSWORD + "\n", { mode: 0o600 }); @@ -281,7 +287,7 @@ async function environment(keepMainRules = false): Promise t.name === "mesh-tools"); - assert.deepEqual(held.map((t) => t.user), [TOOLS_ACCOUNT], "the one token is the tools' account's, none is the administrator's"); - assert.equal(await readFile(join(env.MESH_GITEA_STATE_DIR, "token"), "utf8"), held[0].sha1 + "\n"); + const held = forge.tokens.filter((t) => t.name === "mesh-tools-1" || t.name === "mesh-tools"); + assert.deepEqual(held.map((t) => [t.user, t.name]), [[TOOLS_ACCOUNT, "mesh-tools-1"]], "the one token is the tools' account's, none is the administrator's"); + assert.equal(vaulted(TOOLS_ACCOUNT), held[0].sha1, "kept in the vault as the module's own (design 47 H2 §808)"); const team = forge.teams.find((t) => t.name === "mesh-tools"); assert.ok(team, "the team is made in the mesh's organisation"); @@ -334,17 +340,13 @@ test("the administrator's token from before goes: a kept admin token is minted a // What the module kept before ADR 0365: a token of the administrator's own, by the same name, its scopes beside it. const old = { id: 900, name: "mesh-tools", sha1: "sha-admin-old", user: ADMIN, scopes: ["write:repository", "write:issue", "read:user"] }; forge.tokens.push(old); - await mkdir(env.MESH_GITEA_STATE_DIR, { recursive: true }); - await writeFile(join(env.MESH_GITEA_STATE_DIR, "token"), old.sha1 + "\n", { mode: 0o600 }); - await writeFile(join(env.MESH_GITEA_STATE_DIR, "token.scopes"), "read:user,write:issue,write:repository\n", { mode: 0o600 }); await fresh(env).listRepos(); assert.ok(!forge.tokens.some((t) => t.user === ADMIN), "the administrator holds no tools' token any more"); - const kept = (await readFile(join(env.MESH_GITEA_STATE_DIR, "token"), "utf8")).trim(); + const kept = vaulted(TOOLS_ACCOUNT)!; assert.notEqual(kept, old.sha1); assert.equal(forge.tokens.find((t) => t.sha1 === kept)?.user, TOOLS_ACCOUNT); - assert.match(await readFile(join(env.MESH_GITEA_STATE_DIR, "token.scopes"), "utf8"), / for mesh-tools\n$/); }); test("a tools' account someone made an administrator is made a plain account again at the next mint", async () => { @@ -363,17 +365,24 @@ test("the manifest hands the tools their account, and the Go forge bundle holds const manifest = JSON.parse(await readFile(new URL("../module.json", import.meta.url), "utf8")); const code = manifest.build.artifacts.find((a: any) => a.name === "code"); assert.equal(code.env.MESH_GITEA_TOOLS_USER, TOOLS_ACCOUNT); - // The Go forge bundle reads the token the tools' account holds, never the administrator's password. + // The Go forge bundle reads the token the tools' account holds, never the administrator's password; through the + // vault seat's get, never a file (design 47 H2 §808, H-A-422 (1)). const forgeBundle = manifest.build.artifacts.find((a: any) => a.name === "forge"); - assert.ok(forgeBundle.env.MESH_GITEA_TOKEN_FILE); + assert.equal(forgeBundle.env.MESH_GITEA_TOOLS_USER, TOOLS_ACCOUNT); assert.equal(forgeBundle.env.MESH_GITEA_ADMIN_PASSWORD_FILE, undefined); // The package registry's bundle reads with the tools' token and writes with the packages account's (design 51). const packages = manifest.build.artifacts.find((a: any) => a.name === "package-registry"); - assert.equal(packages.env.MESH_GITEA_TOKEN_FILE, forgeBundle.env.MESH_GITEA_TOKEN_FILE); assert.equal(packages.env.MESH_GITEA_PACKAGES_USER, PACKAGES_ACCOUNT); - assert.match(packages.env.MESH_GITEA_PACKAGES_TOKEN_FILE, /^\$\{dir:runtime-state\}\/packages-token$/); assert.equal(packages.env.MESH_GITEA_ADMIN_PASSWORD_FILE, undefined); - assert.equal(code.env.MESH_GITEA_STATE_DIR, "${dir:runtime-state}", "the watcher mints the packages token where the bundle reads it"); + for (const a of manifest.build.artifacts) { + assert.equal(a.env?.MESH_GITEA_TOKEN_FILE ?? a.env?.MESH_GITEA_PACKAGES_TOKEN_FILE, undefined, `${a.name} is handed a token file`); + } + assert.ok(manifest.invokes.includes("seat:mesh-vault.own-put") && manifest.invokes.includes("seat:mesh-vault.get"), manifest.invokes.join(",")); + assert.ok(manifest.consumes.includes("mesh-vault.secret-changed"), "the Go bundles hear the vault's changes"); + // The lease and the two settings (§808, §820). + assert.deepEqual(manifest.state.find((s: any) => s.name === "token-lease"), { name: "token-lease", "key-ttl": true, "max-mib": 1 }); + assert.deepEqual([manifest.config["token-ttl"]?.default, manifest.config["token-ttl"]?.runtime], ["24h", true]); + assert.deepEqual([manifest.config["mint-lease"]?.default, manifest.config["mint-lease"]?.runtime], ["60s", true]); }); test("the package seats write as an account of their own: plain, in a team writing only packages, its token write:package alone, renewed when the forge rejects it", async () => { @@ -383,10 +392,10 @@ test("the package seats write as an account of their own: plain, in a team writi const account = forge.users.get(PACKAGES_ACCOUNT); assert.ok(account, "the module made the packages account"); assert.equal(account!.admin, false, "the packages account is no administrator"); - const held = forge.tokens.filter((t) => t.name === "mesh-packages"); - assert.deepEqual(held.map((t) => [t.user, t.scopes]), [[PACKAGES_ACCOUNT, ["write:package"]]]); - assert.equal(await readFile(join(env.MESH_GITEA_STATE_DIR, "packages-token"), "utf8"), held[0].sha1 + "\n"); - assert.equal(await readFile(join(env.MESH_GITEA_STATE_DIR, "packages-token.scopes"), "utf8"), "write:package for mesh-packages\n"); + const named = (t: { name: string }) => t.name === "mesh-packages" || t.name.startsWith("mesh-packages-"); + const held = forge.tokens.filter(named); + assert.deepEqual(held.map((t) => [t.user, t.name, t.scopes]), [[PACKAGES_ACCOUNT, "mesh-packages-1", ["write:package"]]]); + assert.equal(vaulted(PACKAGES_ACCOUNT), held[0].sha1, "kept in the vault as module/gitea/mesh-packages (H-A-426)"); const team = forge.teams.find((t) => t.name === "mesh-packages"); assert.ok(team?.members.has(PACKAGES_ACCOUNT)); assert.deepEqual(team!.units.units_map, PACKAGES_TEAM_UNITS.units_map, "the team writes packages and nothing else"); @@ -398,11 +407,12 @@ test("the package seats write as an account of their own: plain, in a team writi const renew = renewOnRefusal(source, forge.url, { delete: async (k) => { cleared.push(k); } }, (l) => said.push(l)); await renew({ key: "seat.gitea.other", op: "put" }); await renew({ key: PACKAGES_TOKEN_REFUSED, op: "delete" }); - assert.equal(forge.tokens.filter((t) => t.name === "mesh-packages").length, 0, "nothing renews but the refusal"); + assert.equal(forge.tokens.filter(named).length, 0, "nothing renews but the refusal"); await renew({ key: PACKAGES_TOKEN_REFUSED, op: "put" }); assert.deepEqual(said, ["the forge rejected the package seats' write token; minted afresh"]); assert.deepEqual(cleared, [PACKAGES_TOKEN_REFUSED]); - assert.equal(forge.tokens.filter((t) => t.name === "mesh-packages").length, 1); + assert.deepEqual(forge.tokens.filter(named).map((t) => t.name), ["mesh-packages-2"]); + assert.equal(vaulted(PACKAGES_ACCOUNT), forge.tokens.find(named)!.sha1); // The tools' token reads packages too: read:package (design 51's read-only tool token). assert.ok(TOKEN_SCOPES.includes("read:package")); }); @@ -433,12 +443,12 @@ test("an old token that cannot be deleted after the mint is said, the new token } finally { forge.faults.deleteToken = false; } - const kept = (await readFile(join(env.MESH_GITEA_STATE_DIR, "token"), "utf8")).trim(); + const kept = vaulted(TOOLS_ACCOUNT)!; assert.equal(forge.tokens.find((t) => t.sha1 === kept)?.user, TOOLS_ACCOUNT, "the account's token is kept and works"); assert.ok(forge.tokens.some((t) => t.sha1 === old.sha1)); - // The next process reuses the kept token and retires the old one it could not before. + // The next process reuses the vault's token and retires the old one it could not before. const next = new MintedToken({ url: forge.url, admin: ADMIN, passwordFile: env.MESH_GITEA_ADMIN_PASSWORD_FILE!, - file: join(env.MESH_GITEA_STATE_DIR, "token"), account: TOOLS_ACCOUNT, log: () => {} }); + account: TOOLS_ACCOUNT, oldName: TOKEN_NAME, vault: vaultOverBus(), lease: leaseOverState(), stateDir: env.MESH_GITEA_STATE_DIR, log: () => {} }); assert.equal(await next.current(), kept); assert.ok(!forge.tokens.some((t) => t.sha1 === old.sha1), "the admin's old token was deleted at the next start"); }); @@ -548,7 +558,7 @@ test("the start's pass gives each of the mesh's accounts read, as a collaborator assert.deepEqual((await fresh(env).listAllRepos()).map((r) => r.full_name).sort(), ["jschoubben/txt-game", "novox/hq"]); // The account's own token cannot grant itself: the forge asks the repository's administration (the reason for the admin login). - const own = (await readFile(join(env.MESH_GITEA_STATE_DIR, "token"), "utf8")).trim(); + const own = vaulted(TOOLS_ACCOUNT)!; const self = await fetch(`${forge.url}/api/v1/repos/jschoubben/car-hunter/collaborators/${TOOLS_ACCOUNT}`, { method: "PUT", headers: { Authorization: `token ${own}`, "Content-Type": "application/json" }, body: JSON.stringify({ permission: "read" }) }); assert.equal(self.status, 404, "a repository the account cannot read is not even there for it"); diff --git a/modules/gitea/token.ts b/modules/gitea/token.ts index 5e079e8bf..9bfec2ca3 100644 --- a/modules/gitea/token.ts +++ b/modules/gitea/token.ts @@ -4,45 +4,85 @@ // paste in, and pasting one into settings would put a secret in the inventory in plaintext. What // the mesh does deliver is the admin account: a login the manifest names and a password the vault // minted and the host unsealed into a file (novox/hq ADR 0086). That account is enough to mint a -// token, so the module mints its own (hq issue 100, the forge's tools): +// token, so the module mints its own (hq issue 100, the forge's tools), for each of its two accounts, the tools' +// (`mesh-tools`) and the package seats' (`mesh-packages`): // -// - at first use, when none is kept: POST /users/{admin}/tokens over basic auth, with the two -// scopes the tools and the watcher need, and no more; -// - kept in the module's own state, a 0600 file, and read back on the next start — the forge -// hands a token's value out exactly once, so a token not kept is a token lost; -// - re-minted when the forge rejects it (401) or the kept file is gone. The one case that is not -// a fault: the forge's data was restored from a predecessor and the token the file names never -// existed there; -// - minted by one process at a time. The runtime starts each of the bundle's entrypoints as a process -// of its own (novox/hq ADR 0193): the watcher (index.js) and the tools (tools/index.js) are two -// processes over one kept file. Two that minted at once left the forge two tokens by one name, which -// gitea then refuses to delete by name — for ever, and every forge tool with it (the 2026-10-14 -// outage). So a mint holds a lock file beside the kept token, and reads the kept file again once it -// holds it: what another process minted meanwhile is reused, not minted over; -// - replaced by id: an old token by this name is found in the account's list and deleted by its id, -// every one of them, so a name held twice is cleared rather than refused. -// -// - minted for the tools' account, not the administrator (novox/hq ADR 0365 §2, issue 469): the administrator's -// login mints it for `mesh-tools` (tools-account.ts), after making that account and its team, and deletes any -// token by this name the administrator itself still holds — the token the tools used before. A token kept -// for another account than the one asked (the scopes line says whose) is minted again at the next start. +// - **kept as a secret, the module's own in the vault** (design 47 H2 §808, §822; H-A-422, H-A-426): written through +// the seat `mesh-vault`'s two-step sealed write `own-put` under the account's name, so the vault keeps it as +// `module/gitea/`, its newest generation only, and read through the seat's `get`, sealed to this process's +// key. Never a file in the runtime state and never plain in a state: the forge hands a token's value out exactly +// once, and the vault is where the module keeps what it must not lose; +// - **one name per mint**, `-`, the generation the one this own-put makes (get's generation plus +// one; 1 when the vault holds none; H-A-430 (1)). The token minted is written through own-put first, and only then +// is the token it replaces deleted at the forge, by its id: no moment is without a valid token, and a mint that +// fails leaves the old one standing. The first mint replaces the tokens the module kept before under the account's +// old fixed name; +// - **minted by one process at a time** (§808). The runtime starts each of the bundle's entrypoints as a process of its +// own (novox/hq ADR 0193), and two that minted at once left the forge two tokens by one name (the 2026-10-14 outage). +// A process the forge refuses first reads the vault's token; only when the refused token is the vault's current one +// does it take the lease — the key `mint` in the module's state `token-lease`, created only, living `mint-lease` (a +// setting, default 60 s) — and mint. A process that finds the lease taken waits for that key to change, at most the +// lease, then reads the vault again; past it, the lease has expired and the waiter creates it itself; +// - **minted again when older than `token-ttl`** (a setting, default 24 hours), its age the vault's `changed-at`; +// - minted for the account, not the administrator (novox/hq ADR 0365 §2, issue 469): the administrator's login mints +// it for the account, after making that account and its team, and deletes any token named `mesh-tools` the +// administrator itself still holds — the token the tools used before. // // An explicitly configured token still wins, and is never minted over: if it is rejected, that is // reported, not repaired — somebody chose it. // -// The token is never logged. Lines say that one was minted, reused or renewed, and where it is -// kept; never what it is. +// The token is never logged. Lines say that one was minted, reused or renewed, and under which name; +// never what it is. -import { chmodSync, closeSync, existsSync, mkdirSync, openSync, readFileSync, renameSync, rmSync, statSync, unlinkSync, writeFileSync, writeSync } from "node:fs"; +import { existsSync, mkdirSync, rmSync, writeFileSync, readFileSync } from "node:fs"; import { randomBytes } from "node:crypto"; -import { dirname, join } from "node:path"; +import { join } from "node:path"; +import { generateKeyPair, open, sealAnonymous, type Box, type KeyPair } from "@novox/mesh-sdk/seal"; +import { broker } from "@novox/mesh-sdk/messaging"; +import { askRuntime } from "@novox/mesh-sdk/runtime"; +import { state, type State } from "@novox/mesh-sdk/state"; import { setUpToolsAccount, TOOLS_ACCOUNT, type AdminCall } from "./tools-account.js"; - -/** The name the token carries in the forge's own list — one per mesh runtime, found by name. */ import { keepMainRules } from "./mainrules.js"; +import { durationMs } from "./taken.js"; +/** The tools' token's old fixed name in the forge's list, the one the admin's own token carried too (ADR 0365 §2). */ export const TOKEN_NAME = "mesh-tools"; +/** The module whose own secrets the vault keeps its tokens as (§822: the module named by the caller's account). */ +export const VAULT_MODULE = "gitea"; + +/** The vault's name of an account's token: `module/gitea/` (H-A-426). */ +export function vaultName(account: string): string { + return `module/${VAULT_MODULE}/${account}`; +} + +/** The forge's name of an account's token of one generation: `-` (H-A-422 (2)). */ +export function mintName(account: string, generation: number): string { + return `${account}-${generation}`; +} + +/** The state the mint's lease is kept in, and its one key (§808, §820). */ +export const LEASE_STATE = "token-lease"; +export const LEASE_KEY = "mint"; + +/** token-ttl's and mint-lease's defaults (§808). */ +export const TOKEN_TTL_DEFAULT = "24h"; +export const MINT_LEASE_DEFAULT = "60s"; + +/** A setting the node-engine composes as `MESH_CONFIG_`, a duration, in milliseconds; the default when unset, and + * when it is no duration (said). */ +export function settingMs(env: NodeJS.ProcessEnv, setting: string, fallback: string, log: (line: string) => void = () => {}): number { + const raw = env[`MESH_CONFIG_${setting.toUpperCase().replace(/-/g, "_")}`]; + const dflt = durationMs(fallback)!; + if (raw === undefined || raw.trim() === "") return dflt; + const ms = durationMs(raw); + if (ms === null) { + log(`the setting ${setting} is ${JSON.stringify(raw)}, no duration: the default ${fallback} holds`); + return dflt; + } + return ms; +} + /** * The least the fifteen tools and the watcher need (gitea's route groups, 1.20+ scoped tokens): * write:repository — create/delete repositories, pull requests (list/get/open/merge); @@ -58,8 +98,6 @@ export const TOKEN_NAME = "mesh-tools"; * **Not write:admin** (novox/hq ADR 0266). It was here so a person could settle the forge's users through * `gitea_api` (novox/hq 229); whoever may call a tool includes agents, and the forge's administration is the * operator's. The provisioner settles users with the admin account's own login (GiteaAdmin), never this token. - * A token kept with a different list is minted again with this one at the next start (the scopes are kept - * beside it), and the old one is deleted from the forge by its id — so the scope a kept token had does not outlive this. * * A token kept from before a scope was added lacks it: the forge answers such a call with * `403 token does not have at least one of required scope(s)`, and the client treats that like a @@ -100,6 +138,103 @@ export class AdminRefused extends Error { } } +/** A token as the vault holds it: the value, the generation and when it changed (`get`'s answer, opened). */ +export interface VaultToken { + readonly token: string; + readonly generation: number; + readonly changedAt: string; +} + +/** The vault as the minter reaches it: the seat `mesh-vault`'s `get` and `own-put` (§808, §822). */ +export interface TokenVault { + /** The account's token the vault holds, or null when it holds none. */ + get(account: string): Promise; + /** Write the account's token as the module's own; answers the generation the vault kept. */ + put(account: string, token: string): Promise; +} + +/** The lease `mint` in the state `token-lease` (§808). */ +export interface MintLease { + /** Create the lease for `holder`, living `ms`; false when another holds it. */ + take(holder: string, ms: number): Promise; + /** The lease's holder and since when, or null when nobody holds it. */ + read(): Promise<{ holder: string; since: string } | null>; + /** Delete the lease, when `holder` still holds it. */ + release(holder: string): Promise; + /** Resolves when the lease's key changes (deleted, or written again), or after `ms`, whichever is first. */ + changed(ms: number): Promise; +} + +/** The vault's words when it holds no secret by the name asked (mesh-vault's Get). */ +const HOLDS_NONE = /the vault holds no secret named/; + +/** The vault over the bus: `get` sealed to this process's own key, made once a process; `own-put` in its two steps. */ +export function vaultOverBus(request: (verb: string, body: unknown) => Promise = (v, b) => broker().request(v, b)): TokenVault { + let keys: KeyPair | undefined; + return { + async get(account) { + keys ??= generateKeyPair(); + let answer: { generation?: number; "changed-at"?: string; sealed?: Box }; + try { + answer = (await request("seat:mesh-vault.get", { name: vaultName(account), public_key: keys.publicKey })) as typeof answer; + } catch (err) { + if (HOLDS_NONE.test(err instanceof Error ? err.message : String(err))) return null; + throw err; + } + if (!answer?.sealed) throw new Error(`the vault answered no sealed ${vaultName(account)}`); + return { token: open(answer.sealed, keys.privateKey), generation: answer.generation ?? 0, changedAt: answer["changed-at"] ?? "" }; + }, + async put(account, token) { + const opened = (await request("seat:mesh-vault.own-put", { name: account })) as { call?: string; seal_to?: string }; + if (!opened?.call || !opened.seal_to) throw new Error(`the vault opened no own-put call for ${vaultName(account)}`); + const kept = (await request("seat:mesh-vault.own-put", { call: opened.call, sealed: sealAnonymous(token, opened.seal_to) })) as { generation?: number }; + return kept?.generation ?? 0; + }, + }; +} + +/** The runtime's refusal of a create whose key has a value (mesh-sdk's CODE_CHANGED). */ +const CODE_CHANGED = -32010; + +/** The lease in the module's state `token-lease` over the runtime: a create with the lease as its time to live, and one + * watch of the key a process, handing each change to the waiters. */ +export function leaseOverState(st: State<{ holder: string; since: string }> = state(LEASE_STATE), + ask: (method: string, params: unknown) => Promise = askRuntime): MintLease { + const waiters = new Set<() => void>(); + let watching: Promise | undefined; + return { + async take(holder, ms) { + try { + await ask("mesh/state.create", { state: LEASE_STATE, key: LEASE_KEY, value: { holder, since: new Date().toISOString() }, + ttl_seconds: Math.max(1, Math.ceil(ms / 1000)) }); + return true; + } catch (err) { + if ((err as { code?: number }).code === CODE_CHANGED) return false; + throw err; + } + }, + async read() { + return (await st.get(LEASE_KEY))?.value ?? null; + }, + async release(holder) { + if ((await st.get(LEASE_KEY))?.value?.holder === holder) await st.delete(LEASE_KEY); + }, + async changed(ms) { + watching ??= st.watch((change) => { + if (!change.current) for (const wake of [...waiters]) wake(); + }, { key: LEASE_KEY }); + await watching; + await new Promise((resolve) => { + const wake = () => { clearTimeout(timer); waiters.delete(wake); resolve(); }; + const timer = setTimeout(wake, ms); + waiters.add(wake); + // Released between the refused create and this wait: nothing to wait for. + void st.get(LEASE_KEY).then((e) => { if (!e) wake(); }, () => {}); + }); + }, + }; +} + export interface MintedTokenOptions { /** The forge, e.g. http://127.0.0.1:3000. */ readonly url: string; @@ -107,115 +242,118 @@ export interface MintedTokenOptions { readonly admin: string; /** The file the host unsealed the admin password into (ADR 0086). Read at mint time, so a rotation takes. */ readonly passwordFile: string; - /** Where the token is kept: a 0600 file in the module's own state. */ - readonly file: string; - readonly name?: string; + /** The account the token is minted for, and the vault name it is kept under (H-A-426). */ + readonly account: string; + /** The token's fixed name before a name per mint: the first mint deletes the account's tokens by it (H-A-430 (1)). */ + readonly oldName: string; readonly scopes?: readonly string[]; - /** - * The account the token is minted for: the tools' own (novox/hq ADR 0365 §2). The admin login mints it, after - * `setUp` made the account, and the admin's own tokens by this name are deleted. Left out, the token is the - * admin's own, as it was before. - */ - readonly account?: string; - /** Run with the admin's password before each mint for `account`: makes the account and its team (tools-account.ts). */ + /** Run with the admin's password before each mint: makes the account and its team (tools-account.ts). */ readonly setUp?: (password: string) => Promise; + /** Where the module keeps its token, and the mint's lease (§808). */ + readonly vault: TokenVault; + readonly lease: MintLease; + /** token-ttl and mint-lease, in milliseconds (§808). */ + readonly ttlMs?: number; + readonly leaseMs?: number; + /** Who holds the lease when this process does: the node and the process. */ + readonly holder?: string; + /** The module's runtime state directory, where a retirement of the admin's old token that failed is remembered. */ + readonly stateDir?: string; + readonly now?: () => Date; readonly log?: (line: string) => void; readonly fetch?: typeof fetch; } -/** The token the module mints for itself, kept in its state and renewed when the forge rejects it. */ +/** The token the module mints for itself, kept in the vault and renewed when the forge rejects it. */ export class MintedToken implements TokenSource { - private held: string | null = null; - private readFile = false; + private held: VaultToken | null = null; private inflight: Promise | null = null; - private readonly name: string; private readonly scopes: readonly string[]; - private readonly account: string; + private readonly ttlMs: number; + private readonly leaseMs: number; + private readonly holder: string; + private readonly now: () => Date; private readonly log: (line: string) => void; private readonly fetchImpl: typeof fetch; constructor(private readonly opts: MintedTokenOptions) { - this.name = opts.name ?? TOKEN_NAME; this.scopes = opts.scopes ?? TOKEN_SCOPES; - this.account = opts.account ?? opts.admin; + this.ttlMs = opts.ttlMs ?? durationMs(TOKEN_TTL_DEFAULT)!; + this.leaseMs = opts.leaseMs ?? durationMs(MINT_LEASE_DEFAULT)!; + this.holder = opts.holder ?? `${process.env.MESH_NODE || "unknown"}/${process.pid}`; + this.now = opts.now ?? (() => new Date()); this.log = opts.log ?? ((line) => console.log(`[gitea] ${line}`)); this.fetchImpl = opts.fetch ?? fetch; } + /** The account this source mints for. */ + get account(): string { + return this.opts.account; + } + /** - * Build from the runtime's environment: the forge's URL, the admin login and password file the - * manifest hands the runtime, and the module's state directory (MESH_GITEA_STATE_DIR, a directory - * the runtime mounts writable). Throws, naming what is missing, rather than hand back a source - * that cannot mint. + * The tools' token from the runtime's environment: the forge's URL, the admin login and password file the manifest + * hands the runtime, the vault and the lease over the bus. Throws, naming what is missing, rather than hand back a + * source that cannot mint. One source per account and runtime state in a process: two would each renew on a 401. */ static fromEnv(url: string, env: NodeJS.ProcessEnv = process.env): MintedToken { - const opts = MintedToken.optionsFromEnv(url, env); - // One source per kept file in a process: two sources in one process would each renew on a 401. - // This covers one process only. The watcher and the tools are two processes (the runtime starts - // each entrypoint as its own), and those are kept apart by the mint's lock file (mint, below). - const shared = MintedToken.shared.get(opts.file); + const account = env.MESH_GITEA_TOOLS_USER || TOOLS_ACCOUNT; + const { admin, passwordFile } = adminFromEnv(env, "no Gitea token, and nothing to mint one with — set GITEA_TOKEN, or"); + const key = `${account} ${env.MESH_GITEA_STATE_DIR ?? ""}`; + const shared = MintedToken.shared.get(key); if (shared) return shared; - const source = new MintedToken(opts); - MintedToken.shared.set(opts.file, source); + if (account === admin) throw new Error(`MESH_GITEA_TOOLS_USER names the admin account "${admin}"; the tools act as an account of their own (novox/hq ADR 0365)`); + const source = new MintedToken({ + ...MintedToken.settingsFromEnv(env), url, admin, passwordFile, account, oldName: TOKEN_NAME, + // GiteaAdmin is loaded when first used: client.ts imports this file. + setUp: async (password) => { + const { GiteaAdmin } = await import("./client.js"); + await setUpToolsAccount(new GiteaAdmin(url, admin, password), account, () => randomBytes(24).toString("base64url"), keepMainRules(env)); + }, + }); + MintedToken.shared.set(key, source); return source; } private static readonly shared = new Map(); - private static optionsFromEnv(url: string, env: NodeJS.ProcessEnv): MintedTokenOptions { - const admin = env.MESH_GITEA_ADMIN_USER; - const passwordFile = env.MESH_GITEA_ADMIN_PASSWORD_FILE; - const stateDir = env.MESH_GITEA_STATE_DIR; - const missing = [ - admin ? null : "MESH_GITEA_ADMIN_USER", - passwordFile ? null : "MESH_GITEA_ADMIN_PASSWORD_FILE", - stateDir ? null : "MESH_GITEA_STATE_DIR", - ].filter((v): v is string => v !== null); - if (missing.length) { - throw new Error(`no Gitea token, and nothing to mint one with — set GITEA_TOKEN, or ${missing.join(", ")}`); - } - // The tools' own account, never the administrator (novox/hq ADR 0365 §2): MESH_GITEA_TOOLS_USER names it. - const account = env.MESH_GITEA_TOOLS_USER || TOOLS_ACCOUNT; - if (account === admin) throw new Error(`MESH_GITEA_TOOLS_USER names the admin account "${admin}"; the tools act as an account of their own (novox/hq ADR 0365)`); + /** What every account's source takes from the environment: the vault and lease over the bus, the two settings. */ + static settingsFromEnv(env: NodeJS.ProcessEnv): Pick { + const log = (line: string) => console.log(`[gitea] ${line}`); return { - url, admin: admin!, passwordFile: passwordFile!, file: join(stateDir!, "token"), account, - // GiteaAdmin is loaded when first used: client.ts imports this file. - setUp: async (password) => { - const { GiteaAdmin } = await import("./client.js"); - await setUpToolsAccount(new GiteaAdmin(url, admin!, password), account, () => randomBytes(24).toString("base64url"), keepMainRules(env)); - }, + vault: sharedVault(), lease: sharedLease(), + ttlMs: settingMs(env, "token-ttl", TOKEN_TTL_DEFAULT, log), leaseMs: settingMs(env, "mint-lease", MINT_LEASE_DEFAULT, log), + stateDir: env.MESH_GITEA_STATE_DIR || undefined, }; } + /** The token held, read from the vault when none is, minted when the vault holds none or one older than token-ttl. */ async current(): Promise { - if (this.held !== null) return this.held; - if (!this.readFile) { - this.readFile = true; - const kept = this.read(); - if (kept !== null && this.keptScopes() === this.scopesLine()) { - this.held = kept; - this.log(`reusing the token kept at ${this.opts.file}`); - // A retirement of the admin's old token that failed after the mint is tried again, once a process. - if (existsSync(this.retireFile)) await this.locked(() => this.retireAdmin()).catch((err) => this.log(`${err}`)); - return kept; - } - if (kept !== null) return this.mint("the kept token was minted with other scopes than the tools take now — minting one with them", kept); + if (this.held !== null && !this.expired(this.held)) return this.held.token; + const kept = await this.opts.vault.get(this.opts.account); + if (kept === null) return this.mint("the vault holds no token for this account — minting one", null); + if (this.expired(kept)) return this.mint(`the vault's token is older than token-ttl (${this.ttlMs / 1000}s) — minting one`, kept.token); + if (this.held === null) { + this.log(`reusing the token the vault keeps as ${vaultName(this.opts.account)} (generation ${kept.generation})`); + // A retirement of the admin's old token that failed after the mint is tried again, once a process. + if (this.retireFile && existsSync(this.retireFile)) await this.retireAdmin().catch((err) => this.log(`${err}`)); } - return this.mint("no token kept — minting one", null); + this.held = kept; + return kept.token; } async renew(rejected: string): Promise { // Another caller already renewed while this one was in flight with the old token. - if (this.held !== null && this.held !== rejected) return this.held; - // Or another process did, and kept it: use what is kept before minting over it. - const kept = this.read(); - if (kept !== null && kept !== rejected) { + if (this.held !== null && this.held.token !== rejected) return this.held.token; + // A refusal of a stale token mints nothing: the vault's newer one is used (§808). + const kept = await this.opts.vault.get(this.opts.account); + if (kept !== null && kept.token !== rejected) { this.held = kept; - this.log(`the forge rejected the token held; the kept one at ${this.opts.file} is newer — reusing it`); - return kept; + this.log(`the forge rejected the token held; the vault's ${vaultName(this.opts.account)} is newer (generation ${kept.generation}) — reusing it`); + return kept.token; } this.held = null; - return this.mint("the forge rejected the kept token — minting a fresh one", rejected); + return this.mint("the forge rejected the vault's token — minting a fresh one", rejected); } /** What the forge's scoped tokens say when a kept token predates a scope the tools now need. */ @@ -223,132 +361,55 @@ export class MintedToken implements TokenSource { return status === 403 && /required scope/i.test(body); } + private expired(t: VaultToken): boolean { + const at = Date.parse(t.changedAt); + return Number.isFinite(at) && this.now().getTime() - at > this.ttlMs; + } + /** - * One mint at a time: concurrent first calls in this process share it, and a mint in another process - * on this machine is waited for (the lock file). Once the lock is held, the kept file is read again: - * a token there that is not `stale` (the one rejected, or kept with other scopes) and carries the - * scopes asked for was minted by the process that held the lock before — it is reused. + * One mint at a time: concurrent calls in this process share it, and a mint in another process is waited for (the + * lease). Holding the lease, the vault is read again: a token there that is not `stale` (the one refused or expired) + * was minted by the process that held the lease before — it is used, not minted over. */ private mint(why: string, stale: string | null): Promise { if (this.inflight === null) { this.log(why); - this.inflight = this.locked(async () => { - const kept = this.read(); - if (kept !== null && kept !== stale && this.keptScopes() === this.scopesLine()) { - this.held = kept; - this.log(`another process minted it meanwhile — reusing the token kept at ${this.opts.file}`); - return kept; - } - return this.doMint(); - }).finally(() => { + this.inflight = this.leased(stale).finally(() => { this.inflight = null; }); } return this.inflight; } - /** How long a mint may hold the lock before another process takes it as left behind, and how long one waits. */ - static readonly LOCK_STALE_MS = 120_000; - static readonly LOCK_WAIT_MS = 150_000; - - /** - * Run `work` holding the lock file beside the kept token: created exclusively, holding this process's - * pid, the time and a nonce; removed after, only when it is still ours. A lock whose process is gone, - * or older than LOCK_STALE_MS, was left by a process that died mid-mint and is taken over — never - * waited on for ever, which would be the same lock-out by another road. - */ - private async locked(work: () => Promise): Promise { - const lock = `${this.opts.file}.lock`; - mkdirSync(dirname(this.opts.file), { recursive: true, mode: 0o700 }); - const mine = `${process.pid} ${Date.now()} ${randomBytes(8).toString("hex")}\n`; - const deadline = Date.now() + MintedToken.LOCK_WAIT_MS; + private async leased(stale: string | null): Promise { for (;;) { - try { - const fd = openSync(lock, "wx", 0o600); - try { writeSync(fd, mine); } finally { closeSync(fd); } - break; - } catch (err) { - if ((err as NodeJS.ErrnoException).code !== "EEXIST") { - throw new Error(`cannot take the mint lock at ${lock}: ${(err as Error).message}`); + if (await this.opts.lease.take(this.holder, this.leaseMs)) { + try { + const kept = await this.opts.vault.get(this.opts.account); + if (kept !== null && kept.token !== stale && !this.expired(kept)) { + this.held = kept; + this.log(`another process minted it meanwhile — reusing the vault's generation ${kept.generation}`); + return kept.token; + } + return await this.doMint(kept); + } finally { + await this.opts.lease.release(this.holder).catch((err) => + this.log(`the mint lease could not be deleted; it ends with mint-lease — ${err instanceof Error ? err.message : err}`)); } } - if (this.leftBehind(lock)) { - this.log(`the mint lock at ${lock} was left by a process that is gone — taking it over`); - try { unlinkSync(lock); } catch { /* another waiter took it over first */ } - continue; + const other = await this.opts.lease.read().catch(() => null); + this.log(`another process holds the mint lease${other ? ` (${other.holder} since ${other.since})` : ""} — waiting for it, at most mint-lease (${this.leaseMs / 1000}s)`); + await this.opts.lease.changed(this.leaseMs); + const kept = await this.opts.vault.get(this.opts.account); + if (kept !== null && kept.token !== stale && !this.expired(kept)) { + this.held = kept; + this.log(`the process that held the mint lease minted generation ${kept.generation} — reusing it`); + return kept.token; } - if (Date.now() > deadline) { - throw new Error(`another process has held the mint lock at ${lock} for over ${MintedToken.LOCK_WAIT_MS / 1000}s — not minting beside it`); - } - await new Promise((r) => setTimeout(r, 50 + Math.floor(Math.random() * 50))); - } - try { - return await work(); - } finally { - try { - if (readFileSync(lock, "utf8") === mine) unlinkSync(lock); - } catch { /* already gone */ } } } - /** Whether the lock's holder is gone: its pid answers no signal, or it is older than LOCK_STALE_MS. */ - private leftBehind(lock: string): boolean { - let text: string; - let age: number; - try { - text = readFileSync(lock, "utf8"); - age = Date.now() - statSync(lock).mtimeMs; - } catch { - return false; // released meanwhile: the next attempt takes it - } - if (age > MintedToken.LOCK_STALE_MS) return true; - const pid = Number(text.split(" ")[0]); - if (!Number.isInteger(pid) || pid <= 0) return age > 1_000; // half-written, or not ours: give it a moment - try { - process.kill(pid, 0); - return false; - } catch (err) { - return (err as NodeJS.ErrnoException).code === "ESRCH"; - } - } - - private read(): string | null { - try { - const token = readFileSync(this.opts.file, "utf8").replace(/\n$/, ""); - return token.length ? token : null; - } catch (err) { - if ((err as NodeJS.ErrnoException).code === "ENOENT") return null; - throw new Error(`cannot read the kept Gitea token at ${this.opts.file}: ${(err as Error).message}`); - } - } - - /** The scopes as kept beside the token: sorted, one line, and whose it is when it is not the admin's. */ - private scopesLine(): string { - const scopes = [...this.scopes].sort().join(","); - return this.account === this.opts.admin ? scopes : `${scopes} for ${this.account}`; - } - - /** The scopes the kept token was minted with, "" when not kept (a token kept before they were). */ - private keptScopes(): string { - try { - return readFileSync(`${this.opts.file}.scopes`, "utf8").trim(); - } catch { - return ""; - } - } - - /** Write the token at 0600, whole or not at all: a temp file beside it, then a rename. Its scopes beside it. */ - private keep(token: string): void { - mkdirSync(dirname(this.opts.file), { recursive: true, mode: 0o700 }); - const tmp = `${this.opts.file}.tmp`; - writeFileSync(tmp, token + "\n", { mode: 0o600 }); - chmodSync(tmp, 0o600); - renameSync(tmp, this.opts.file); - writeFileSync(`${this.opts.file}.scopes.tmp`, this.scopesLine() + "\n", { mode: 0o600 }); - renameSync(`${this.opts.file}.scopes.tmp`, `${this.opts.file}.scopes`); - } - - /** One call to the forge's API with the admin login, the password read now. */ + /** The admin password, read now. */ private password(): string { try { return readFileSync(this.opts.passwordFile, "utf8").replace(/\n$/, ""); @@ -357,6 +418,7 @@ export class MintedToken implements TokenSource { } } + /** One call to the forge's API with the admin login. */ private adminCall(password = this.password()): AdminCall { const authorization = "Basic " + Buffer.from(`${this.opts.admin}:${password}`).toString("base64"); const api = `${this.opts.url.replace(/\/+$/, "")}/api/v1`; @@ -377,71 +439,81 @@ export class MintedToken implements TokenSource { } /** Where a retirement of the admin's old token that failed is remembered, to be tried again (retireAdmin). */ - private get retireFile(): string { - return `${this.opts.file}.retire-admin`; + private get retireFile(): string | null { + return this.opts.stateDir ? join(this.opts.stateDir, `${this.opts.account}.retire-admin`) : null; } /** - * The admin's own token by this name goes: what the tools used before must not outlive the move (novox/hq ADR - * 0365 §2). Run after the account's token is minted and kept, so a failure here never leaves the tools without - * a token: it is said, remembered beside the kept token, and tried again when a process next reads that token. + * The admin's own token named `mesh-tools` goes: what the tools used before must not outlive the move (novox/hq ADR + * 0365 §2). Run after the account's token is minted and kept, so a failure here never leaves the tools without a + * token: it is said, remembered, and tried again when a process next reads the vault's token. */ private async retireAdmin(admin: AdminCall = this.adminCall()): Promise { - if (this.account === this.opts.admin) return; + if (this.opts.oldName !== TOKEN_NAME) return; try { - const retired = await this.dropNamed(admin, `/users/${encodeURIComponent(this.opts.admin)}/tokens`, this.opts.admin); - if (retired > 0) this.log(`deleted ${retired} token(s) named "${this.name}" the admin account "${this.opts.admin}" held: the tools act as "${this.account}" now`); - rmSync(this.retireFile, { force: true }); + const retired = await this.dropNamed(admin, `/users/${encodeURIComponent(this.opts.admin)}/tokens`, this.opts.admin, TOKEN_NAME); + if (retired > 0) this.log(`deleted ${retired} token(s) named "${TOKEN_NAME}" the admin account "${this.opts.admin}" held: the tools act as "${this.opts.account}" now`); + if (this.retireFile) rmSync(this.retireFile, { force: true }); } catch (err) { - writeFileSync(this.retireFile, `${new Date().toISOString()}\n`, { mode: 0o600 }); - this.log(`FAILED to delete the token(s) named "${this.name}" the admin account "${this.opts.admin}" holds — tried again at the next start: ${(err as Error).message}`); + if (this.retireFile) { + mkdirSync(this.opts.stateDir!, { recursive: true, mode: 0o700 }); + writeFileSync(this.retireFile, `${new Date().toISOString()}\n`, { mode: 0o600 }); + } + this.log(`FAILED to delete the token(s) named "${TOKEN_NAME}" the admin account "${this.opts.admin}" holds — tried again at the next start: ${(err as Error).message}`); } } - private async doMint(): Promise { + /** + * Mint `-` (the generation this own-put makes), write it through own-put, and only then delete + * at the forge, by id, the token it replaces: `replaced`'s by its name, or at the first mint the tokens under the + * account's old fixed name (H-A-422 (2), H-A-430 (1)). + */ + private async doMint(replaced: VaultToken | null): Promise { const password = this.password(); const admin = this.adminCall(password); - const tokensOf = (account: string) => `/users/${encodeURIComponent(account)}/tokens`; - const call = (method: string, path = "", body?: unknown) => admin(method, tokensOf(this.account) + path, body); + const account = this.opts.account; + const tokens = `/users/${encodeURIComponent(account)}/tokens`; + const generation = (replaced?.generation ?? 0) + 1; + const name = mintName(account, generation); - if (this.account !== this.opts.admin) { - // The tools' account and its team first (novox/hq ADR 0365 §2): a failure here fails the mint, tried again - // at the next call. - const me = await admin("GET", "/user"); - if (me.status === 401 || me.status === 403) throw new AdminRefused(this.opts.admin, me.status); - await this.opts.setUp?.(password); - } + // The account and its team first (novox/hq ADR 0365 §2): a failure here fails the mint, tried again at the next call. + const me = await admin("GET", "/user"); + if (me.status === 401 || me.status === 403) throw new AdminRefused(this.opts.admin, me.status); + await this.opts.setUp?.(password); - let res = await call("POST", "", { name: this.name, scopes: this.scopes }); + let res = await admin("POST", tokens, { name, scopes: this.scopes }); if (res.status === 401 || res.status === 403) throw new AdminRefused(this.opts.admin, res.status); if (res.status === 400 || res.status === 422) { - // The forge still holds a token by this name whose value we no longer have — the kept file - // went while the forge's data stayed, or two processes once minted at once and left two. It is - // ours to replace: find every one by this name in the account's list and delete each by its id - // (gitea refuses a delete by a name held twice: "multiple matches for token name"), then mint. - await this.dropNamed(admin, tokensOf(this.account), this.account, true); - res = await call("POST", "", { name: this.name, scopes: this.scopes }); + // A mint of this generation whose own-put never landed left its token at the forge, unused: deleted by its id + // (gitea refuses a delete by a name held twice), then minted again. + await this.dropNamed(admin, tokens, account, name, true); + res = await admin("POST", tokens, { name, scopes: this.scopes }); } if (res.status !== 201 && res.status !== 200) { - throw new Error(`Gitea POST /users/${this.account}/tokens: ${res.status} ${detail(res.body)}`); + throw new Error(`Gitea POST /users/${account}/tokens: ${res.status} ${detail(res.body)}`); } const token = typeof res.body?.sha1 === "string" ? res.body.sha1 : null; - if (!token) throw new Error(`Gitea POST /users/${this.account}/tokens: ${res.status} but no token in the reply`); + if (!token) throw new Error(`Gitea POST /users/${account}/tokens: ${res.status} but no token in the reply`); - this.keep(token); - this.held = token; - this.log(`minted a token for "${this.account}" (${this.scopes.join(", ")}), kept at ${this.opts.file}`); - // Only now, with the account's own token kept, does the admin's old one go. + const kept = await this.opts.vault.put(account, token); + if (kept !== generation) this.log(`the vault kept ${vaultName(account)} at generation ${kept}, the forge names it ${name}`); + this.held = { token, generation: kept, changedAt: this.now().toISOString() }; + this.log(`minted ${name} for "${account}" (${this.scopes.join(", ")}), kept in the vault as ${vaultName(account)}`); + + // Only now, with the new token kept, does the one it replaces go. + const gone = replaced ? mintName(account, replaced.generation) : this.opts.oldName; + try { + const dropped = await this.dropNamed(admin, tokens, account, gone); + if (dropped > 0) this.log(`deleted ${dropped} token(s) named "${gone}" "${account}" held: ${name} replaces them`); + } catch (err) { + this.log(`FAILED to delete the token "${gone}" ${name} replaces; the forge holds both until the next mint — ${(err as Error).message}`); + } await this.retireAdmin(admin); return token; } - /** - * Delete every token by this name an account holds, each by its id (gitea refuses a delete by a name held twice: - * "multiple matches for token name"); answers how many. `replacing` says the line for a mint that found its name - * taken. - */ - private async dropNamed(admin: AdminCall, tokens: string, account: string, replacing = false): Promise { + /** Delete every token named `name` an account holds, each by its id; answers how many. */ + private async dropNamed(admin: AdminCall, tokens: string, account: string, name: string, replacing = false): Promise { const held: number[] = []; for (let page = 1; page <= 100; page++) { const listed = await admin("GET", `${tokens}?page=${page}&limit=50`); @@ -449,23 +521,59 @@ export class MintedToken implements TokenSource { throw new Error(`Gitea GET /users/${account}/tokens: ${listed.status} ${detail(listed.body)}`); } if (listed.body.length === 0) break; - for (const t of listed.body) if (t?.name === this.name && typeof t.id === "number") held.push(t.id); - } - if (replacing) { - this.log(held.length === 1 - ? `the forge already holds a token named "${this.name}" — replacing it` - : `the forge holds ${held.length} tokens named "${this.name}" — deleting each by its id and minting one`); + for (const t of listed.body) if (t?.name === name && typeof t.id === "number") held.push(t.id); } + if (replacing) this.log(`the forge already holds ${held.length} token(s) named "${name}" — deleting each by its id and minting one`); for (const id of held) { const dropped = await admin("DELETE", `${tokens}/${id}`); if (dropped.status !== 204 && dropped.status !== 404) { - throw new Error(`Gitea DELETE /users/${account}/tokens/${id} (named ${this.name}): ${dropped.status} ${detail(dropped.body)}`); + throw new Error(`Gitea DELETE /users/${account}/tokens/${id} (named ${name}): ${dropped.status} ${detail(dropped.body)}`); } } return held.length; } } +/** The admin login and password file the manifest hands the runtime, or an error naming what is missing. */ +export function adminFromEnv(env: NodeJS.ProcessEnv, missingWords: string): { admin: string; passwordFile: string } { + const admin = env.MESH_GITEA_ADMIN_USER; + const passwordFile = env.MESH_GITEA_ADMIN_PASSWORD_FILE; + const missing = [admin ? null : "MESH_GITEA_ADMIN_USER", passwordFile ? null : "MESH_GITEA_ADMIN_PASSWORD_FILE"] + .filter((v): v is string => v !== null); + if (missing.length) throw new Error(`${missingWords} ${missing.join(", ")}`); + return { admin: admin!, passwordFile: passwordFile! }; +} + +// One vault and one lease a process, shared by both accounts' sources: one key and one watch. +let vaultOnce: TokenVault | undefined; +let leaseOnce: MintLease | undefined; +function sharedVault(): TokenVault { + return (vaultOnce ??= vaultOverBus()); +} +function sharedLease(): MintLease { + return (leaseOnce ??= leaseOverState()); +} + +/** The tool forge_token_lease's reading (§808): the lease's holder and since when, and each account's token age, read + * from the vault's `changed-at`; never the token. */ +export async function tokenLease(vault: TokenVault, lease: MintLease, accounts: readonly string[], now: Date = new Date()) { + const held = await lease.read(); + const out: { account: string; vault: string; generation?: number; "changed-at"?: string; "age-seconds"?: number; held: boolean }[] = []; + for (const account of accounts) { + const t = await vault.get(account); + out.push(t === null ? { account, vault: vaultName(account), held: false } : { + account, vault: vaultName(account), held: true, generation: t.generation, "changed-at": t.changedAt, + "age-seconds": Math.max(0, Math.floor((now.getTime() - Date.parse(t.changedAt)) / 1000)), + }); + } + return { lease: held === null ? { held: false } : { held: true, holder: held.holder, since: held.since }, tokens: out }; +} + +/** The vault and the lease this process mints through, for the tool forge_token_lease. */ +export function processVaultAndLease(): { vault: TokenVault; lease: MintLease } { + return { vault: sharedVault(), lease: sharedLease() }; +} + function detail(body: unknown): string { return typeof body === "string" ? body : JSON.stringify(body); } diff --git a/modules/gitea/tools/tokenlease.ts b/modules/gitea/tools/tokenlease.ts new file mode 100644 index 000000000..fc8a4f40f --- /dev/null +++ b/modules/gitea/tools/tokenlease.ts @@ -0,0 +1,16 @@ +// The tool forge_token_lease (design 47 H2 §808, the tools table): the mint lease's holder and since when, and each +// account's token age, read from the vault's `changed-at`; never the token. + +import type { ToolDefinition } from "@novox/mesh-sdk/tools"; +import { tokenLease, type MintLease, type TokenVault } from "../token.js"; + +export function tokenLeaseTool(vault: TokenVault, lease: MintLease, accounts: readonly string[], now: () => Date = () => new Date()): ToolDefinition { + return { + name: "forge_token_lease", + description: "The forge token's mint lease and each account's token age (design 47 H2 §808): who holds the lease `mint` in the " + + "state token-lease and since when, and for each account (" + accounts.join(", ") + ") the vault's generation of its token " + + "and when it changed, never the token itself.", + input: { type: "object", properties: {} }, + run: async () => tokenLease(vault, lease, accounts, now()), + }; +} -- 2.54.0 From 872ed19648eb481fad470d9baa46170889bc8a74 Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 22 Nov 2026 01:44:03 +0100 Subject: [PATCH 39/54] M1: every module's findings stream holds 8 MiB, not only the delivery modules' MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The eleven other modules that bind finding-events (claude-code, mesh-healers, mesh-knowledge, mesh-provisioner, mesh-recovery, mesh-self-check, mesh-watcher, minio, nfs-server, restic, zfs) set max-mib 8 for 7 days, as the delivery modules' already do, so the indexer's finding-events rule (I6) judges every manifest alike. Builds: design 47 H2 §783; H-A-411 (1), H-A-430 (3) --- modules/claude-code/module.json | 2 +- modules/mesh-healers/module.json | 2 +- modules/mesh-knowledge/module.json | 2 +- modules/mesh-provisioner/module.json | 2 +- modules/mesh-recovery/module.json | 2 +- modules/mesh-self-check/module.json | 2 +- modules/mesh-watcher/module.json | 2 +- modules/minio/module.json | 2 +- modules/nfs-server/module.json | 2 +- modules/restic/module.json | 2 +- modules/zfs/module.json | 2 +- 11 files changed, 11 insertions(+), 11 deletions(-) diff --git a/modules/claude-code/module.json b/modules/claude-code/module.json index cbc6cab7d..fd0a9fc56 100644 --- a/modules/claude-code/module.json +++ b/modules/claude-code/module.json @@ -24,7 +24,7 @@ "findings-stated.*" ], "max-age-days": 7, - "max-mib": 16 + "max-mib": 8 } }, { diff --git a/modules/mesh-healers/module.json b/modules/mesh-healers/module.json index 66bfbe727..930c16ec9 100644 --- a/modules/mesh-healers/module.json +++ b/modules/mesh-healers/module.json @@ -94,7 +94,7 @@ "findings-stated.*" ], "max-age-days": 7, - "max-mib": 16 + "max-mib": 8 } } ] diff --git a/modules/mesh-knowledge/module.json b/modules/mesh-knowledge/module.json index 339872e77..43fbf81f2 100644 --- a/modules/mesh-knowledge/module.json +++ b/modules/mesh-knowledge/module.json @@ -82,7 +82,7 @@ {"name": "reviews", "storage": "stream", "configuration": {"when-full": "refuse", "max-mib": 512, "max-age-days": "none", "grant": "readers", "events": ["part-reviewed"]}}, { "name": "finding-events", "storage": "stream", "template": "events", - "options": {"events": ["finding-written.*", "finding-cleared.*", "findings-stated.*"], "max-age-days": 7, "max-mib": 16} + "options": {"events": ["finding-written.*", "finding-cleared.*", "findings-stated.*"], "max-age-days": 7, "max-mib": 8} } ], "contributes": { diff --git a/modules/mesh-provisioner/module.json b/modules/mesh-provisioner/module.json index 2d4e79405..dd980a5c0 100644 --- a/modules/mesh-provisioner/module.json +++ b/modules/mesh-provisioner/module.json @@ -119,7 +119,7 @@ "findings-stated.*" ], "max-age-days": 7, - "max-mib": 16 + "max-mib": 8 } }, { diff --git a/modules/mesh-recovery/module.json b/modules/mesh-recovery/module.json index 3e138a482..2719831e7 100644 --- a/modules/mesh-recovery/module.json +++ b/modules/mesh-recovery/module.json @@ -94,7 +94,7 @@ "findings-stated.*" ], "max-age-days": 7, - "max-mib": 16 + "max-mib": 8 } } ], diff --git a/modules/mesh-self-check/module.json b/modules/mesh-self-check/module.json index 16ee72e74..ec9dfb550 100644 --- a/modules/mesh-self-check/module.json +++ b/modules/mesh-self-check/module.json @@ -116,7 +116,7 @@ "findings-stated.*" ], "max-age-days": 7, - "max-mib": 16 + "max-mib": 8 } } ] diff --git a/modules/mesh-watcher/module.json b/modules/mesh-watcher/module.json index 9ffeaa193..23877ef24 100644 --- a/modules/mesh-watcher/module.json +++ b/modules/mesh-watcher/module.json @@ -93,7 +93,7 @@ "findings-stated.*" ], "max-age-days": 7, - "max-mib": 16 + "max-mib": 8 } } ] diff --git a/modules/minio/module.json b/modules/minio/module.json index 9a7ed812f..956a69fb6 100644 --- a/modules/minio/module.json +++ b/modules/minio/module.json @@ -37,7 +37,7 @@ "findings-stated.*" ], "max-age-days": 7, - "max-mib": 16 + "max-mib": 8 } } ], diff --git a/modules/nfs-server/module.json b/modules/nfs-server/module.json index 650a0aa40..b3f027d18 100644 --- a/modules/nfs-server/module.json +++ b/modules/nfs-server/module.json @@ -217,7 +217,7 @@ "findings-stated.*" ], "max-age-days": 7, - "max-mib": 16 + "max-mib": 8 } } ] diff --git a/modules/restic/module.json b/modules/restic/module.json index 54ac3c376..1a006be0e 100644 --- a/modules/restic/module.json +++ b/modules/restic/module.json @@ -262,7 +262,7 @@ "findings-stated.*" ], "max-age-days": 7, - "max-mib": 16 + "max-mib": 8 } } ] diff --git a/modules/zfs/module.json b/modules/zfs/module.json index 0594e54c8..9e41b0586 100644 --- a/modules/zfs/module.json +++ b/modules/zfs/module.json @@ -119,7 +119,7 @@ "findings-stated.*" ], "max-age-days": 7, - "max-mib": 16 + "max-mib": 8 } } ], -- 2.54.0 From 3ec18c9cd2bfa4ed190ff51b614262004632470d Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 22 Nov 2026 01:45:28 +0100 Subject: [PATCH 40/54] M1: nftables' setting allowances-max is marked runtime The packet filter holder's tools read allowances-max themselves (config at tools/index.ts); no file places it, so the manifest says "runtime": true and the judge's placed-nowhere rule holds it as read by the module's code. Builds: design 47 H2 H-A-430 (4) --- modules/nftables/module.json | 1 + modules/nftables/test/allowances.test.ts | 1 + 2 files changed, 2 insertions(+) diff --git a/modules/nftables/module.json b/modules/nftables/module.json index 33c275f66..f4b31b727 100644 --- a/modules/nftables/module.json +++ b/modules/nftables/module.json @@ -95,6 +95,7 @@ "type": "number", "description": "The most allowances this machine keeps, which allow refuses past (design 33 control-verbs §3): it keeps the rendered chain readable and allowed in one answer.", "trusted": false, + "runtime": true, "default": 100 } }, diff --git a/modules/nftables/test/allowances.test.ts b/modules/nftables/test/allowances.test.ts index 44b0342a5..7bd8d219c 100644 --- a/modules/nftables/test/allowances.test.ts +++ b/modules/nftables/test/allowances.test.ts @@ -353,4 +353,5 @@ test("the manifest serves the six verbs, requires the state per machine, invokes assert.deepEqual(m.invokes, ["seat:node-engine.converged", "seat:mesh-hand-acts.hand-act"]); assert.equal(m.emits, undefined, "the claim says the seat's event (H-A-317)"); assert.equal(m.config["allowances-max"].default, MOST); + assert.equal(m.config["allowances-max"].runtime, true, "the holder's code reads it itself; no file places it (H-A-430 (4))"); }); -- 2.54.0 From 575fe562434748b688ef62e8c3708f1f86420587 Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 22 Nov 2026 01:46:45 +0100 Subject: [PATCH 41/54] M1: the gate's and artifact-store-tools' caps stand above their live fills MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Read on novox at 2026-11-22 (the seat mesh-broker's show; the gate's old-section state source through the bus module's nats_streams, since the bus module made no entry for it): mesh-delivery-gate heads 582 bytes, 5 keys cap 8 MiB mesh-delivery-gate findings 24905 bytes, 27 keys cap 8 MiB mesh-delivery-gate source 70166 bytes, 68 msgs cap 8 MiB artifact-store-tools findings 8199 bytes, 8 keys cap 8 MiB Each fill is below a quarter of 8 MiB (2 MiB), so each cap M1 wrote stays at 8 MiB; none is raised. Builds: design 47 H2 §812; H-A-411 (2), H-A-437 -- 2.54.0 From ed14a100064fbd955d62c9cabbc100936ca1c092 Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 22 Nov 2026 01:48:18 +0100 Subject: [PATCH 42/54] GT: each act the gate owes names its delivery record MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The gate's owed event carries record, the delivery record's source revision . (the record's key), said and kept in the record's acts alike, so the forge bridge keeps it with the act and bridge_owed_again finds a record's acts in its own state. Built against the seat mesh-delivery's Owed.record on mesh-delivery feat/47-H2-a. Builds: design 47 H2 §807; H-A-432 (1) --- modules/mesh-delivery-gate/cmd/mesh-delivery-gate/owed.go | 3 +++ .../cmd/mesh-delivery-gate/owedacts_test.go | 4 ++++ 2 files changed, 7 insertions(+) diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/owed.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/owed.go index 938dd0466..4e52798cd 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/owed.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/owed.go @@ -48,6 +48,9 @@ func (h *Holder) owe(r *Record, o seat.Owed) error { return errors.New("no seat to say owed on") } now := h.Now().UTC() + // The act names its delivery record by its source revision, ., so the forge bridge keeps it with the act + // and bridge_owed_again finds a record's acts in its own state (design 47 H2 §807, H-A-432 (1)). + o.Record = r.Milestone if had, ok := r.Acts[o.ID]; ok && had.Owed.Said != nil { o.Said = had.Owed.Said } else { diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/owedacts_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/owedacts_test.go index 29bea18c0..5635de6c9 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/owedacts_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/owedacts_test.go @@ -54,6 +54,10 @@ func TestARecordsChangesOweTheNoteTheViewAndWhenFailedOrHeldAComment(t *testing. if _, ok := r.Acts[o.ID]; !ok { t.Fatalf("act %s not kept in the record", o.ID) } + // H-A-432 (1): each act names its record by its source revision, said and kept alike. + if o.Record != "main.47" || r.Acts[o.ID].Owed.Record != o.Record { + t.Fatalf("act %s names the record %q, kept %q, not main.47", o.ID, o.Record, r.Acts[o.ID].Owed.Record) + } if o.Act == seat.OwedActNote && (o.Line != "2026-11-11T21:00:00Z record "+r.Instance+": mesh-x on ace build built (built it)" || o.Commit != head) { t.Fatalf("note %+v", o) } -- 2.54.0 From 22b92b827de605191ae66b01fe9d1ecd14f222c7 Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 22 Nov 2026 01:50:30 +0100 Subject: [PATCH 43/54] BM publish version: the build modules' bundle asks name the ask's version, as the SDK now refuses one without A bundle publishes under the version its ask reserved, and the delivery SDK's publish refuses an empty one loudly; the tests' bundle asks carry a version, a bundle asked with none is shown refused, and an archive still publishes by its digest alone. Builds: design 47 H2 H-A-373, H-A-373 addendum --- .../cmd/mesh-build-go/build_test.go | 32 +++++++++++++------ .../cmd/mesh-build-go/isolate_test.go | 2 +- .../mesh-build-go/pins_integration_test.go | 2 +- .../cmd/mesh-build-go/publish_test.go | 8 ++--- .../mesh-build-go/store_integration_test.go | 9 ++++-- .../test_real_integration_test.go | 2 +- .../mesh-build-node/build_integration_test.go | 2 +- .../cmd/mesh-build-node/build_test.go | 27 ++++++++++------ .../cmd/mesh-build-node/push_test.go | 1 - .../mesh-build-node/store_integration_test.go | 2 +- 10 files changed, 56 insertions(+), 31 deletions(-) diff --git a/modules/mesh-build-go/cmd/mesh-build-go/build_test.go b/modules/mesh-build-go/cmd/mesh-build-go/build_test.go index e26f61f48..e56bac62e 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/build_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/build_test.go @@ -61,6 +61,16 @@ func runBuild(t *testing.T, m *Module, states *keptStates, artifact string) (Bui return runBuildArgs(t, m, states, map[string]any{"workspace": "ws1", "module": "modules/demo", "artifact": artifact}) } +// askedVersion is the version a test's bundle ask reserved: every bundle publishes under the ask's version, and the +// delivery SDK refuses one that names none (design 47 H, H-A-373); an archive may still publish by its digest alone. +const askedVersion = "1.4.3" + +// runBundle runs the build tool for a bundle artifact under the ask's version, as the builder asks it. +func runBundle(t *testing.T, m *Module, states *keptStates, artifact string) (BuildResult, sdkstep.Step) { + t.Helper() + return runBuildArgs(t, m, states, map[string]any{"workspace": "ws1", "module": "modules/demo", "artifact": artifact, "version": askedVersion}) +} + // runBuildArgs runs the build tool with args, such as an ask's version, and answers what its step ended with. func runBuildArgs(t *testing.T, m *Module, states *keptStates, args map[string]any) (BuildResult, sdkstep.Step) { t.Helper() @@ -111,7 +121,7 @@ func goEntry(t *testing.T) (compile []string, linker []string, stamp string) { // GOFLAGS=-trimpath -buildvcs=false, go build, -ldflags -s -w -X main.builtFor=, -o /, the package. func TestTheBuildCommandCarriesTodaysFlagsAndEnvironment(t *testing.T) { m, states, calls := buildModule(t, manifestWith(goBundle), "binary") - res, st := runBuild(t, m, states, "tools") + res, st := runBundle(t, m, states, "tools") if st.State != sdkstep.StepPassed { t.Fatalf("the step ended %s: %s", st.State, st.Reason) } @@ -135,7 +145,7 @@ func TestTheBuildCommandCarriesTodaysFlagsAndEnvironment(t *testing.T) { } dir := filepath.Join(m.Work, "ws1", "tree", "modules", "demo") out := filepath.Join(dir, ".mesh-build", "tools") - want := "go -C " + dir + " build -ldflags " + strings.Join(linker, " ") + " -X " + stamp + "=arch -o " + filepath.Join(out, "demo-bin") + " ./cmd/demo-tools" + want := "go -C " + dir + " build -ldflags " + strings.Join(linker, " ") + " -X " + stamp + "=arch -X " + versionStamp + "=" + askedVersion + " -o " + filepath.Join(out, "demo-bin") + " ./cmd/demo-tools" if c.line != want { t.Errorf("command = %q\nwant %q", c.line, want) } @@ -148,8 +158,8 @@ func TestTheBuildCommandCarriesTodaysFlagsAndEnvironment(t *testing.T) { // are one digest. func TestTheArchiveHoldsOneExecutableAndTwoBuildsAreOneDigest(t *testing.T) { m, states, _ := buildModule(t, manifestWith(goBundle), "the program") - first, _ := runBuild(t, m, states, "tools") - second, _ := runBuild(t, m, states, "tools") + first, _ := runBundle(t, m, states, "tools") + second, _ := runBundle(t, m, states, "tools") if first.Digest != second.Digest || first.Reference != second.Reference { t.Errorf("two builds: %+v and %+v", first, second) } @@ -182,7 +192,7 @@ func TestAnArtifactOfAnotherLanguageFailsTheStepNamingIt(t *testing.T) { `{"name":"tools","kind":"image","from":"files"}`: "image", } { m, states, calls := buildModule(t, manifestWith(artifact), "x") - _, st := runBuild(t, m, states, "tools") + _, st := runBundle(t, m, states, "tools") if st.State != sdkstep.StepFailed || !strings.Contains(st.Reason, "tools") || !strings.Contains(st.Reason, want) || len(*calls) != 0 { t.Errorf("%s: the step ended %s: %q after %d commands", artifact, st.State, st.Reason, len(*calls)) } @@ -202,7 +212,7 @@ func TestTheBinaryIsNamedAsTheBuildAgentNamesIt(t *testing.T) { `{"name":"tools","kind":"bundle","language":"go","system":"arch","from":".","loads":["tools"]}`: "tools", } { m, states, _ := buildModule(t, manifestWith(artifact), "x") - if res, st := runBuild(t, m, states, "tools"); res.Binary != want { + if res, st := runBundle(t, m, states, "tools"); res.Binary != want { t.Errorf("%s: binary %q, want %q (%s %s)", artifact, res.Binary, want, st.State, st.Reason) } } @@ -235,7 +245,7 @@ func TestABuildOutputDirectoryLinkedOutsideTheModuleIsRefused(t *testing.T) { if err := os.Symlink(outside, filepath.Join(m.Work, "ws1", "tree", "modules", "demo", ".mesh-build")); err != nil { t.Fatal(err) } - _, st := runBuild(t, m, states, "tools") + _, st := runBundle(t, m, states, "tools") if st.State != sdkstep.StepFailed || !strings.Contains(st.Reason, "outside") || !strings.Contains(st.Reason, outside) || len(*calls) != 0 { t.Errorf("the step ended %s: %q after %d commands", st.State, st.Reason, len(*calls)) } @@ -253,7 +263,7 @@ func TestAManifestFallingBackToAPredefinedKeyBuilds(t *testing.T) { t.Fatal("the captured manifest no longer falls back to a predefined key") } m, states, _ := buildModule(t, string(raw), "binary") - res, st := runBuild(t, m, states, "tools") + res, st := runBundle(t, m, states, "tools") if st.State != sdkstep.StepPassed { t.Fatalf("the step ended %s: %s", st.State, st.Reason) } @@ -274,8 +284,10 @@ func TestTheBuildGetsTheVersionTheAskCarriesAsMeshVersion(t *testing.T) { } } } - if st := states.waitEnd(t); st.State != sdkstep.StepPassed { - t.Fatalf("the step ended %s: %s", st.State, st.Reason) + // A bundle asked with no version is built, stamped with none, and its publish refused loudly (H-A-373). + if st := states.waitEnd(t); (version != "") != (st.State == sdkstep.StepPassed) || + version == "" && !strings.Contains(st.Reason, "names no version") { + t.Fatalf("ask version %q: the step ended %s: %s", version, st.State, st.Reason) } got := "" for _, r := range *calls { diff --git a/modules/mesh-build-go/cmd/mesh-build-go/isolate_test.go b/modules/mesh-build-go/cmd/mesh-build-go/isolate_test.go index 4ef09ae75..c3e9920df 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/isolate_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/isolate_test.go @@ -167,7 +167,7 @@ func TestABuildRunsShutOffAndOnlyATrunkBuildWritesTheCaches(t *testing.T) { var seen []shutOff m.Run = throughBubblewrap(t, m.Run, &seen) run := stepTool(t, m, "build_go_build") - args := map[string]any{"workspace": "ws1", "module": "modules/demo", "artifact": "tools"} + args := map[string]any{"workspace": "ws1", "module": "modules/demo", "artifact": "tools", "version": askedVersion} if publish { args["publish"] = true } diff --git a/modules/mesh-build-go/cmd/mesh-build-go/pins_integration_test.go b/modules/mesh-build-go/cmd/mesh-build-go/pins_integration_test.go index 1e2e79b36..998c89a94 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/pins_integration_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/pins_integration_test.go @@ -59,7 +59,7 @@ func TestRealFetchAndBuildOfAMemberAskedAPullHeadItsGoModDoesNotName(t *testing. // The build runs shut off in bubblewrap (1309), which a build node has and build-agent's check container has not: // there the build half is not run, said loudly; the fetch half above, the pin's own part (issue 1148), always is. needTools(t, "the shut-off build of a member pinning a pull head", "bwrap") - built, err := m.build(context.Background(), map[string]any{"workspace": res.Workspace, "module": ".", "artifact": "tools"}) + built, err := m.build(context.Background(), map[string]any{"workspace": res.Workspace, "module": ".", "artifact": "tools", "version": askedVersion}) if err != nil { t.Fatal(fmt.Errorf("the build: %w", err)) } diff --git a/modules/mesh-build-go/cmd/mesh-build-go/publish_test.go b/modules/mesh-build-go/cmd/mesh-build-go/publish_test.go index 06ac02449..72850b9fd 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/publish_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/publish_test.go @@ -136,13 +136,13 @@ func TestARefusedPushFailsTheStepNotStored(t *testing.T) { m, states, _ := buildModule(t, manifestWith(goBundle), "the program") store := withStore(t, m) store.fail = errors.New("refused: the forge did not take it: disk full") - if _, st := runBuild(t, m, states, "tools"); st.State != sdkstep.StepFailed || !strings.HasPrefix(st.Reason, builds.NotStored+"tools: ") || !strings.Contains(st.Reason, "disk full") { + if _, st := runBundle(t, m, states, "tools"); st.State != sdkstep.StepFailed || !strings.HasPrefix(st.Reason, builds.NotStored+"tools: ") || !strings.Contains(st.Reason, "disk full") { t.Errorf("the step ended %s: %q", st.State, st.Reason) } m.Call = func(string, any) (json.RawMessage, error) { return nil, errors.New("nats: no responders available for request") } - if _, st := runBuild(t, m, states, "tools"); st.State != sdkstep.StepFailed || !strings.HasPrefix(st.Reason, builds.NotStored+"tools: ") || + if _, st := runBundle(t, m, states, "tools"); st.State != sdkstep.StepFailed || !strings.HasPrefix(st.Reason, builds.NotStored+"tools: ") || !strings.Contains(st.Reason, "oci-artifact-registry") { t.Errorf("no holder: the step ended %s: %q", st.State, st.Reason) } @@ -151,7 +151,7 @@ func TestARefusedPushFailsTheStepNotStored(t *testing.T) { m.Call = func(key string, _ any) (json.RawMessage, error) { return nil, errors.New("nats: no responders available for request") } - if _, st := runBuild(t, m, states, "tools"); st.State != sdkstep.StepFailed || !st.Passing || !strings.Contains(st.Reason, "publish: ") { + if _, st := runBundle(t, m, states, "tools"); st.State != sdkstep.StepFailed || !st.Passing || !strings.Contains(st.Reason, "publish: ") { t.Errorf("the step ended %s: %q (passing %v)", st.State, st.Reason, st.Passing) } } @@ -162,7 +162,7 @@ func TestThePackedFileLeavesTheNodeOnceItsPushAnswered(t *testing.T) { for _, fail := range []error{nil, errors.New("refused")} { m, states, _ := buildModule(t, manifestWith(goBundle), "the program") withStore(t, m).fail = fail - runBuild(t, m, states, "tools") + runBundle(t, m, states, "tools") var archives []string filepath.WalkDir(m.Work, func(p string, d os.DirEntry, err error) error { if err == nil && strings.HasSuffix(p, ".tar.gz") { diff --git a/modules/mesh-build-go/cmd/mesh-build-go/store_integration_test.go b/modules/mesh-build-go/cmd/mesh-build-go/store_integration_test.go index f33613e2d..402f2e463 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/store_integration_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/store_integration_test.go @@ -139,8 +139,13 @@ func TestADoneBuildsArtifactsResolveInARealArtifactStore(t *testing.T) { m.Run = StepRunner(home) addr := realStore(t, m) done := builds.BuilderBuild{V: 1, State: builds.BuildDone, Manifest: strings.Repeat("0", 64)} - for _, a := range []struct{ name, kind string }{{"tools", "bundle"}, {"files", "archive"}} { - out, err := m.build(context.Background(), map[string]any{"workspace": "ws1", "module": ".", "artifact": a.name}) + // The bundle publishes under the ask's version, the archive by its digest alone (H-A-373 and its addendum). + for _, a := range []struct{ name, kind, version string }{{"tools", "bundle", askedVersion}, {"files", "archive", ""}} { + args := map[string]any{"workspace": "ws1", "module": ".", "artifact": a.name} + if a.version != "" { + args["version"] = a.version + } + out, err := m.build(context.Background(), args) if err != nil { t.Fatalf("%s: %v", a.name, err) } diff --git a/modules/mesh-build-go/cmd/mesh-build-go/test_real_integration_test.go b/modules/mesh-build-go/cmd/mesh-build-go/test_real_integration_test.go index 7645d90bb..6cc6ae493 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/test_real_integration_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/test_real_integration_test.go @@ -206,7 +206,7 @@ func TestBuildTwiceGivesOneDigestAndOneExecutable(t *testing.T) { m := &Module{Home: home, Work: work, Cache: t.TempDir()} m.Run = StepRunner(home) withStore(t, m) - args := map[string]any{"workspace": "ws1", "module": ".", "artifact": "tools"} + args := map[string]any{"workspace": "ws1", "module": ".", "artifact": "tools", "version": askedVersion} // a bundle publishes under the ask's version (H-A-373) first, err := m.build(context.Background(), args) if err != nil { t.Fatal(err) diff --git a/modules/mesh-build-node/cmd/mesh-build-node/build_integration_test.go b/modules/mesh-build-node/cmd/mesh-build-node/build_integration_test.go index b80ac17d0..44cb3a6d7 100644 --- a/modules/mesh-build-node/cmd/mesh-build-node/build_integration_test.go +++ b/modules/mesh-build-node/cmd/mesh-build-node/build_integration_test.go @@ -117,7 +117,7 @@ func TestTheFixturePassesTheUnitLevelAndBuildsTwiceToOneDigest(t *testing.T) { } var digests []string for _, id := range []string{"w1", "w2"} { - out, err := m.build(context.Background(), map[string]any{"workspace": id, "module": "modules/m", "artifact": "tools"}) + out, err := m.build(context.Background(), map[string]any{"workspace": id, "module": "modules/m", "artifact": "tools", "version": askedVersion}) if err != nil { t.Fatal(err) } diff --git a/modules/mesh-build-node/cmd/mesh-build-node/build_test.go b/modules/mesh-build-node/cmd/mesh-build-node/build_test.go index b3621b268..328d80ae4 100644 --- a/modules/mesh-build-node/cmd/mesh-build-node/build_test.go +++ b/modules/mesh-build-node/cmd/mesh-build-node/build_test.go @@ -55,8 +55,13 @@ func esbuildEffect(t *testing.T) func(c call) { } } +// askedVersion is the version a test's bundle ask reserved: every bundle publishes under the ask's version, and the +// delivery SDK refuses one that names none (design 47 H, H-A-373). +const askedVersion = "1.4.3" + +// buildArgs is a bundle's build ask, under the version the ask reserved. func buildArgs() map[string]any { - return map[string]any{"workspace": "w1", "module": "modules/m", "artifact": "tools"} + return map[string]any{"workspace": "w1", "module": "modules/m", "artifact": "tools", "version": askedVersion} } func builtMachine(t *testing.T, extra string) (*Module, *fakeMachine, string) { @@ -309,11 +314,13 @@ func TestTheBundleBuildGetsTheVersionTheAskCarriesAsMeshVersion(t *testing.T) { for _, version := range []string{"1.4.3", ""} { m, f, _ := builtMachine(t, "") args := buildArgs() - if version != "" { - args["version"] = version + args["version"] = version + if version == "" { + delete(args, "version") } - if _, err := m.build(context.Background(), args); err != nil { - t.Fatal(err) + // A bundle asked with no version is built with none, and its publish refused loudly (H-A-373). + if _, err := m.build(context.Background(), args); (err != nil) != (version == "") || err != nil && !strings.Contains(err.Error(), "names no version") { + t.Fatalf("ask version %q: %v", version, err) } for _, c := range f.calls { got := "" @@ -336,11 +343,13 @@ func TestTheBundleCarriesTheAsksVersionInItsPackageJSON(t *testing.T) { m, f, work := builtMachine(t, "") workspaceWith(t, work, map[string]string{"modules/m/package.json": `{"name":"m","version":"0.0.0"}`}) args := buildArgs() - if c.version != "" { - args["version"] = c.version + args["version"] = c.version + if c.version == "" { + delete(args, "version") } - if _, err := m.build(context.Background(), args); err != nil { - t.Fatal(err) + // A bundle asked with no version keeps the package.json's, and its publish is refused loudly (H-A-373). + if _, err := m.build(context.Background(), args); (err != nil) != (c.version == "") || err != nil && !strings.Contains(err.Error(), "names no version") { + t.Fatalf("ask version %q: %v", c.version, err) } _ = f raw, err := os.ReadFile(filepath.Join(work, "w1", "stage-tools", "modules", "module", "package.json")) diff --git a/modules/mesh-build-node/cmd/mesh-build-node/push_test.go b/modules/mesh-build-node/cmd/mesh-build-node/push_test.go index 6a2b2d95a..b3d0dd3a1 100644 --- a/modules/mesh-build-node/cmd/mesh-build-node/push_test.go +++ b/modules/mesh-build-node/cmd/mesh-build-node/push_test.go @@ -106,7 +106,6 @@ func TestADoneBundleAnswersTheReferenceOfTheArchiveItPushed(t *testing.T) { m, _, _ := builtMachine(t, ``) store := withStore(t, m) args := buildArgs() - args["version"] = "1.4.3" out, err := m.build(context.Background(), args) if err != nil { t.Fatal(err) diff --git a/modules/mesh-build-node/cmd/mesh-build-node/store_integration_test.go b/modules/mesh-build-node/cmd/mesh-build-node/store_integration_test.go index 65634fb0a..558c27337 100644 --- a/modules/mesh-build-node/cmd/mesh-build-node/store_integration_test.go +++ b/modules/mesh-build-node/cmd/mesh-build-node/store_integration_test.go @@ -96,7 +96,7 @@ func TestADoneBundleResolvesInARealArtifactStore(t *testing.T) { needBubblewrap(t) m, _ := realMachine(t) addr := realStore(t, m) - out, err := m.build(context.Background(), map[string]any{"workspace": "w1", "module": "modules/m", "artifact": "tools"}) + out, err := m.build(context.Background(), map[string]any{"workspace": "w1", "module": "modules/m", "artifact": "tools", "version": askedVersion}) if err != nil { t.Fatal(err) } -- 2.54.0 From 280adcce685df68c94a9ffe525f8533e1d7049cc Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 22 Nov 2026 01:52:24 +0100 Subject: [PATCH 44/54] C4: the vault keeps the forge token with the scopes it was minted with MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The file beside the token that kept its scopes went with C4, and with it the re-mint of a token carrying a scope beyond the tools' (write:admin). The value written through own-put now holds the token and its scopes; a token minted with other scopes, or a value naming none, is minted again unasked and the one it replaces deleted at the forge. The Go bundles read the token out of that value. The lost test is back as a row of the scope test in test/token.test.ts. Builds: design 47 H2 §808; H-A-439 --- modules/gitea/cmd/forge/account_test.go | 4 +- .../gitea/cmd/package-registry/seats_test.go | 4 +- .../whole_integration_test.go | 4 +- modules/gitea/internal/forgehttp/vault.go | 11 ++++- modules/gitea/test/token.test.ts | 47 ++++++++++++++---- modules/gitea/test/tools-account.test.ts | 4 +- modules/gitea/token.ts | 49 +++++++++++++++---- 7 files changed, 97 insertions(+), 26 deletions(-) diff --git a/modules/gitea/cmd/forge/account_test.go b/modules/gitea/cmd/forge/account_test.go index 54e7213a3..f0995aa60 100644 --- a/modules/gitea/cmd/forge/account_test.go +++ b/modules/gitea/cmd/forge/account_test.go @@ -32,7 +32,9 @@ func vaultHolding(t *testing.T, account, token string) *forgehttp.VaultToken { if token == "" { return nil, fmt.Errorf("the vault holds no secret named %s", a["name"]) } - box, err := seal.Seal(token, a["public_key"].(string)) + // The value the minter writes: the token and the scopes it was minted with (H-A-439). + value, _ := json.Marshal(map[string]any{"token": token, "scopes": []string{"write:repository", "write:issue", "read:user", "read:package"}}) + box, err := seal.Seal(string(value), a["public_key"].(string)) if err != nil { return nil, err } diff --git a/modules/gitea/cmd/package-registry/seats_test.go b/modules/gitea/cmd/package-registry/seats_test.go index 33cab8930..5d30aff98 100644 --- a/modules/gitea/cmd/package-registry/seats_test.go +++ b/modules/gitea/cmd/package-registry/seats_test.go @@ -216,7 +216,9 @@ func vaultHolding(t *testing.T, account string, held map[string]string) *forgeht if !ok { return nil, fmt.Errorf("the vault holds no secret named %s", a["name"]) } - box, err := seal.Seal(token, a["public_key"].(string)) + // The value the minter writes: the token and the scopes it was minted with (H-A-439). + value, _ := json.Marshal(map[string]any{"token": token, "scopes": []string{"write:repository", "write:issue", "read:user", "read:package"}}) + box, err := seal.Seal(string(value), a["public_key"].(string)) if err != nil { return nil, err } diff --git a/modules/gitea/cmd/package-registry/whole_integration_test.go b/modules/gitea/cmd/package-registry/whole_integration_test.go index 5f6848f07..1f4f511f4 100644 --- a/modules/gitea/cmd/package-registry/whole_integration_test.go +++ b/modules/gitea/cmd/package-registry/whole_integration_test.go @@ -143,7 +143,9 @@ func (p *player) handle(m wire) { p.answer(m.ID, nil, errors.New("the vault holds no secret named "+q.Name)) break } - box, err := seal.Seal(token, q.PublicKey) + // The value the minter writes: the token and the scopes it was minted with (H-A-439). + value, _ := json.Marshal(map[string]any{"token": token, "scopes": []string{"write:repository", "write:issue", "read:user", "read:package"}}) + box, err := seal.Seal(string(value), q.PublicKey) p.answer(m.ID, map[string]any{"name": q.Name, "generation": 1, "sealed": box}, err) case m.Method == "mesh/seat.publish": p.answer(m.ID, map[string]any{}, p.nc.Publish(a.Subject, a.Body)) diff --git a/modules/gitea/internal/forgehttp/vault.go b/modules/gitea/internal/forgehttp/vault.go index b6ad605c3..2fec629c6 100644 --- a/modules/gitea/internal/forgehttp/vault.go +++ b/modules/gitea/internal/forgehttp/vault.go @@ -61,11 +61,18 @@ func (t *VaultToken) Load() error { if err := json.Unmarshal(raw, &got); err != nil || got.Sealed == nil { return fmt.Errorf("the vault answered no sealed %s", VaultName(t.Account)) } - token, err := seal.Open(*got.Sealed, t.keys.PrivateKey) + opened, err := seal.Open(*got.Sealed, t.keys.PrivateKey) if err != nil { return fmt.Errorf("%s does not open with this process's key: %w", VaultName(t.Account), err) } - token = strings.TrimSpace(token) + // The value holds the token and the scopes it was minted with (H-A-439); the minter judges the scopes, this reads the token. + var value struct { + Token string `json:"token"` + } + if err := json.Unmarshal([]byte(opened), &value); err != nil { + return fmt.Errorf("the vault's %s is not a token with its scopes: %w", VaultName(t.Account), err) + } + token := strings.TrimSpace(value.Token) if token == "" { return fmt.Errorf("the vault's %s is empty", VaultName(t.Account)) } diff --git a/modules/gitea/test/token.test.ts b/modules/gitea/test/token.test.ts index 7231822c3..0bd3e0008 100644 --- a/modules/gitea/test/token.test.ts +++ b/modules/gitea/test/token.test.ts @@ -15,7 +15,7 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { collectTools } from "@novox/mesh-sdk/tools"; -import { AdminRefused, MintedToken, TOKEN_NAME, TOKEN_SCOPES, leaseOverState, vaultOverBus, type MintedTokenOptions } from "../dist/token.js"; +import { AdminRefused, MintedToken, TOKEN_NAME, TOKEN_SCOPES, leaseOverState, openValue, vaultOverBus, vaultValue, type MintedTokenOptions } from "../dist/token.js"; import { tokenLeaseTool } from "../dist/tools/tokenlease.js"; import { fakeBus } from "./fakebus.ts"; import { GiteaClient } from "../dist/client.js"; @@ -288,6 +288,8 @@ function source(env: NodeJS.ProcessEnv, logs: string[], extra: Partial) => openValue(bus.vault.get(VAULT)?.value ?? "").token; const forge = await fakeForge(); after(() => forge.close()); @@ -304,7 +306,8 @@ test("first start: mints mesh-tools-1 with the admin account, writes it through // Design 47 H2 §808, H-A-422 (2), H-A-426: named per mint, kept as the module's own secret, never a file. const token = forge.tokens.get("mesh-tools-1")!; assert.ok(token); - assert.deepEqual([bus.vault.get(VAULT)?.value, bus.vault.get(VAULT)?.generation], [token, 1]); + assert.deepEqual([vaulted(bus), bus.vault.get(VAULT)?.generation], [token, 1]); + assert.deepEqual(openValue(bus.vault.get(VAULT)!.value).scopes, [...TOKEN_SCOPES], "kept with the scopes it was minted with (H-A-439)"); await assert.rejects(stat(join(env.MESH_GITEA_STATE_DIR!, "token")), /ENOENT/); assert.ok(logs.some((l) => l.startsWith("minted mesh-tools-1")), logs.join("\n")); assert.ok(logs.every((l) => !l.includes(token) && !l.includes(PASSWORD)), logs.join("\n")); @@ -336,7 +339,7 @@ test("the forge rejects the vault's token: mesh-tools-2 is minted once, written assert.equal(repos.length, 1); assert.equal(forge.mints, before + 1); - assert.equal(bus.vault.get(VAULT)?.value, forge.tokens.get("mesh-tools-2")); + assert.equal(vaulted(bus), forge.tokens.get("mesh-tools-2")); // H-A-422 (2): the own-put first, the replaced token's delete after, by id; the forge holds one tools' token after. assert.deepEqual(ORDER, [`own-put ${VAULT} 2`, "delete mesh-tools-1"]); assert.deepEqual(forge.deletedBy, ["id"]); @@ -353,9 +356,9 @@ test("an own-put that fails leaves the old token standing, and the next mint rep forge.tokens.set("mesh-tools-1", "revoked-meanwhile"); ORDER.length = 0; let failPut = true; - const failing = minted(env, logs, { vault: { get: vaultOverBus().get, put: async (a, t) => { + const failing = minted(env, logs, { vault: { get: vaultOverBus().get, put: async (a, t, scopes) => { if (failPut) { failPut = false; throw new Error("the vault is away"); } - return vaultOverBus().put(a, t); + return vaultOverBus().put(a, t, scopes); } } }); await assert.rejects(failing.listRepos(), /the vault is away/); @@ -365,7 +368,7 @@ test("an own-put that fails leaves the old token standing, and the next mint rep assert.equal((await failing.listRepos()).length, 1); assert.deepEqual(ORDER, ["delete mesh-tools-2", `own-put ${VAULT} 2`, "delete mesh-tools-1"]); - assert.notEqual(bus.vault.get(VAULT)?.value, old); + assert.notEqual(vaulted(bus), old); }); test("a token older than token-ttl is minted again, and the one it replaces deleted", async () => { @@ -389,7 +392,7 @@ test("a token older than token-ttl is minted again, and the one it replaces dele assert.ok(logs.some((l) => l.includes("older than token-ttl")), logs.join("\n")); }); -test("a token lacking a scope the tools need: the forge refuses the call for the scope, the token is re-minted with the whole list, and the call goes through", async () => { +test("a token lacking a scope the tools need: the forge refuses the call for the scope, the token is re-minted with the whole list, and the call goes through", async (t) => { const { env, bus, logs } = await delivered(forge); forge.tokens.clear(); const client = minted(env, logs); @@ -403,7 +406,33 @@ test("a token lacking a scope the tools need: the forge refuses the call for the assert.equal(repos.length, 1); assert.equal(forge.mints, before + 1); assert.deepEqual(forge.lastScopes, [...TOKEN_SCOPES]); - assert.equal(bus.vault.get(VAULT)?.value, forge.tokens.get("mesh-tools-2")); + assert.equal(vaulted(bus), forge.tokens.get("mesh-tools-2")); + + // Design 47 H2 §808, H-A-439: the vault keeps the scopes beside the token, so one carrying a scope beyond the tools' + // is minted again unasked, as when they were kept in a file beside it. + await t.test("a token kept with write:admin is minted again without it at the next start, and the forge drops the old one", async () => { + const { env, bus, logs } = await delivered(forge); + forge.tokens.clear(); + await minted(env, logs).listRepos(); + const old = forge.tokens.get("mesh-tools-1")!; + // An earlier mint asked write:admin too, and the vault keeps what it asked. + bus.vault.get(VAULT)!.value = vaultValue(old, [...TOKEN_SCOPES, "write:admin"]); + forge.scopesOf.set(old, [...TOKEN_SCOPES, "write:admin"]); + const before = forge.mints; + + const again: string[] = []; + await minted(env, again).listRepos(); + + assert.equal(forge.mints, before + 1); + assert.deepEqual(forge.lastScopes, [...TOKEN_SCOPES]); + assert.ok(![...forge.tokens.values()].includes(old), "the old token still stands on the forge"); + assert.deepEqual(openValue(bus.vault.get(VAULT)!.value), { token: forge.tokens.get("mesh-tools-2"), scopes: [...TOKEN_SCOPES] }); + assert.ok(again.some((l) => l.includes("minted with other scopes")), again.join("\n")); + // An admin route through the escape hatch is refused before it is sent. + const tool = getGiteaTools(minted(env, again)).find((x) => x.name === "gitea_api")!; + await assert.rejects(tool.run({ path: "/admin/users/mesh_novox_builder", method: "PATCH", body: { admin: true } }, undefined as never), + /not a write gitea_api makes/); + }); }); test("the first mint deletes by id every token under the old fixed name, two raced ones too, and keeps the account's others", async () => { @@ -545,7 +574,7 @@ test("forge_token_lease says the lease's holder and since when, and each token's assert.equal(tool.name, "forge_token_lease"); assert.deepEqual(said.tokens.map((t: any) => [t.account, t.held, t.generation, t["age-seconds"]]), [["mesh-tools", true, 1, 90], ["mesh-packages", false, undefined, undefined]]); - assert.ok(!JSON.stringify(said).includes(bus.vault.get(VAULT)!.value), "the token is never said"); + assert.ok(!JSON.stringify(said).includes(vaulted(bus)), "the token is never said"); // The lease lives mint-lease: past it, nobody holds it. clock += 30_000; assert.deepEqual((await tool.run({}, undefined as never) as any).lease, { held: false }); diff --git a/modules/gitea/test/tools-account.test.ts b/modules/gitea/test/tools-account.test.ts index f486ae51b..2ef7431a9 100644 --- a/modules/gitea/test/tools-account.test.ts +++ b/modules/gitea/test/tools-account.test.ts @@ -14,7 +14,7 @@ import { mkdtemp, readFile, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { MintedToken, TOKEN_NAME, TOKEN_SCOPES, leaseOverState, vaultOverBus } from "../dist/token.js"; +import { MintedToken, TOKEN_NAME, TOKEN_SCOPES, leaseOverState, openValue, vaultOverBus } from "../dist/token.js"; import { fakeBus } from "./fakebus.ts"; import { GiteaClient } from "../dist/client.js"; import { TOOLS_ACCOUNT, TOOLS_TEAM_UNITS, BUILDER_ACCOUNT, AGENT_ACCOUNT, meshAccounts } from "../dist/tools-account.js"; @@ -269,7 +269,7 @@ after(() => forge.close()); /** The bus of the environment made last: the vault's secrets and the lease (design 47 H2 §808). */ let bus = fakeBus(); -const vaulted = (account: string) => bus.vault.get(`module/gitea/${account}`)?.value; +const vaulted = (account: string) => { const v = bus.vault.get(`module/gitea/${account}`)?.value; return v === undefined ? undefined : openValue(v).token; }; async function environment(keepMainRules = false): Promise { bus = fakeBus(); diff --git a/modules/gitea/token.ts b/modules/gitea/token.ts index 9bfec2ca3..c02c3fa01 100644 --- a/modules/gitea/token.ts +++ b/modules/gitea/token.ts @@ -24,6 +24,9 @@ // setting, default 60 s) — and mint. A process that finds the lease taken waits for that key to change, at most the // lease, then reads the vault again; past it, the lease has expired and the waiter creates it itself; // - **minted again when older than `token-ttl`** (a setting, default 24 hours), its age the vault's `changed-at`; +// - **minted again when minted with other scopes** (H-A-439): the value the vault keeps holds the token and the scopes +// it was minted with, so a token carrying a scope beyond the ones the tools take (such as write:admin) is minted again +// unasked, and the one it replaces deleted, as when the scopes were kept in a file beside the token; // - minted for the account, not the administrator (novox/hq ADR 0365 §2, issue 469): the administrator's login mints // it for the account, after making that account and its team, and deletes any token named `mesh-tools` the // administrator itself still holds — the token the tools used before. @@ -138,19 +141,42 @@ export class AdminRefused extends Error { } } -/** A token as the vault holds it: the value, the generation and when it changed (`get`'s answer, opened). */ +/** A token as the vault holds it: the token, the scopes it was minted with, the generation and when it changed (`get`'s + * answer, opened). */ export interface VaultToken { readonly token: string; + readonly scopes: readonly string[]; readonly generation: number; readonly changedAt: string; } +/** The value the vault keeps for an account: the token and the scopes it was minted with (§808, H-A-439). */ +export function vaultValue(token: string, scopes: readonly string[]): string { + return JSON.stringify({ token, scopes: [...scopes] }); +} + +/** A value the vault keeps, opened: its token and scopes. A value of another form names no scopes, so it is minted again. */ +export function openValue(value: string): { token: string; scopes: readonly string[] } { + try { + const v = JSON.parse(value); + if (v && typeof v.token === "string" && Array.isArray(v.scopes)) return { token: v.token, scopes: v.scopes.map(String) }; + } catch { + // not the token and its scopes + } + return { token: value.trim(), scopes: [] }; +} + +/** Whether two lists name the same scopes, in any order. */ +function sameScopes(a: readonly string[], b: readonly string[]): boolean { + return [...a].sort().join(",") === [...b].sort().join(","); +} + /** The vault as the minter reaches it: the seat `mesh-vault`'s `get` and `own-put` (§808, §822). */ export interface TokenVault { /** The account's token the vault holds, or null when it holds none. */ get(account: string): Promise; - /** Write the account's token as the module's own; answers the generation the vault kept. */ - put(account: string, token: string): Promise; + /** Write the account's token and the scopes it was minted with as the module's own; answers the generation the vault kept. */ + put(account: string, token: string, scopes: readonly string[]): Promise; } /** The lease `mint` in the state `token-lease` (§808). */ @@ -182,12 +208,12 @@ export function vaultOverBus(request: (verb: string, body: unknown) => Promise Date: Sun, 22 Nov 2026 01:54:05 +0100 Subject: [PATCH 45/54] GT: an act the forge bridge ends not-needed is ended in its record MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The forge bridge ends an act whose pull request closed unmerged with the outcome not-needed. The gate keeps it in the record's acts as ended, as it keeps performed and could not be performed, and no longer counts it as owed. The gate's code takes any outcome forgemsg reads, so the change is forgemsg's outcome list at mesh-delivery feat/47-H2-a. The row in the owed-acts test fails where forgemsg refuses not-needed. Builds: design 47 H2 §807; H-A-432 (2) --- .../cmd/mesh-delivery-gate/owedacts_test.go | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/owedacts_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/owedacts_test.go index 5635de6c9..41338b81c 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/owedacts_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/owedacts_test.go @@ -6,6 +6,7 @@ import ( "time" seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" ) type sayerFake struct{ said []seat.Owed } @@ -70,6 +71,34 @@ func TestARecordsChangesOweTheNoteTheViewAndWhenFailedOrHeldAComment(t *testing. t.Fatalf("said again: %v %d", err, len(f.said)) } }) + // Design 47 H2 §807, H-A-432 (2): an act whose pull request closed unmerged ends not-needed, kept in the record's acts as + // ended like performed and could not be performed, and no longer owed. + t.Run("an act the forge bridge ends not-needed is ended in the record and owed no more", func(t *testing.T) { + h, rs, r, f := setup(t) + h.Logf = func(string, ...any) {} + if _, err := rs.Outcome(r, RecordOutcome{Module: "mesh-x", Fingerprint: "src2:f1", Node: "ace", Step: "build", Outcome: OutcomeBuilt, Why: "built it"}); err != nil { + t.Fatal(err) + } + id := f.said[0].ID + body, err := forgemsg.EncodeOwedEnded(forgemsg.OwedEnded{V: 1, Of: seat.Seat, ID: id, Outcome: forgemsg.OwedNotNeeded, + Words: "novox/app#7 closed unmerged", At: at}) + if err != nil { + t.Fatal(err) + } + if err := h.OwedEnded(body); err != nil { + t.Fatal(err) + } + kept, err := rs.Kept() + if err != nil || len(kept) != 1 { + t.Fatalf("kept %d records: %v", len(kept), err) + } + if a := kept[0].Acts[id]; a.Outcome != forgemsg.OwedNotNeeded || a.Words != "novox/app#7 closed unmerged" { + t.Fatalf("the act is kept as %+v", a) + } + if owed := h.Status("")["owed"]; owed != 1 { + t.Fatalf("%v acts owed, not the one left", owed) + } + }) t.Run("a record ended failed says a comment naming why; ended healthy says none", func(t *testing.T) { _, rs, r, f := setup(t) if _, err := rs.Outcome(r, RecordOutcome{Module: "mesh-x", Fingerprint: "src2:f1", Node: "ace", Step: "build", Outcome: OutcomeFailed, Why: "tests red"}); err != nil { -- 2.54.0 From 30a3e665e0070ae5b0ebfede985da5759af8affd Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 22 Nov 2026 02:06:45 +0100 Subject: [PATCH 46/54] GT: an owed act ended not-needed is ended, never said as a refusal MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A pull request closed unmerged is a fact of the forge, not a fault of the act, so the gate no longer keeps a refusal for an act the forge bridge ends not-needed: the record's acts and gate_record show it ended with that outcome, and delivery_status says no refusal of it. Every other outcome other than performed is still said as a refusal. Builds: design 47 H2 §807; H-A-432 (2), H-A-444 --- .../cmd/mesh-delivery-gate/owedacts_test.go | 22 +++++++++++++++++++ .../cmd/mesh-delivery-gate/owedended.go | 8 ++++--- 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/owedacts_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/owedacts_test.go index 41338b81c..f7331bc9e 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/owedacts_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/owedacts_test.go @@ -98,6 +98,28 @@ func TestARecordsChangesOweTheNoteTheViewAndWhenFailedOrHeldAComment(t *testing. if owed := h.Status("")["owed"]; owed != 1 { t.Fatalf("%v acts owed, not the one left", owed) } + // H-A-444: ended not-needed is a fact of the forge, never a refusal: gate_record shows it ended with that outcome, + // and delivery_status says no refusal of it. + views, err := rs.RecordOf(r.Instance, "") + if err != nil || len(views) != 1 || views[0].Record.Acts[id].Outcome != forgemsg.OwedNotNeeded { + t.Fatalf("gate_record shows %+v: %v", views, err) + } + if refused, ok := h.Status("")["refused"]; ok { + t.Fatalf("delivery_status says a not-needed act as refused: %v", refused) + } + // an act that could not be performed is a fault, and stays said as refused + other := f.said[1].ID + body, err = forgemsg.EncodeOwedEnded(forgemsg.OwedEnded{V: 1, Of: seat.Seat, ID: other, Outcome: forgemsg.OwedCouldNotBePerformed, + Words: "the forge answered 500", At: at}) + if err != nil { + t.Fatal(err) + } + if err := h.OwedEnded(body); err != nil { + t.Fatal(err) + } + if refused, _ := h.Status("")["refused"].([]string); len(refused) != 1 || !strings.Contains(refused[0], "the forge answered 500") { + t.Fatalf("delivery_status's refusals: %v", h.Status("")["refused"]) + } }) t.Run("a record ended failed says a comment naming why; ended healthy says none", func(t *testing.T) { _, rs, r, f := setup(t) diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/owedended.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/owedended.go index 009902462..1d6d5551f 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/owedended.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/owedended.go @@ -14,8 +14,8 @@ var owedEndedEvent = declared(forgemsg.OwedEndedEvent) // OwedEnded takes the forge bridge's outcome of one act this owner said as owed, and writes it into the delivery record that // holds the act (A-35): an act is answered only once its outcome is recorded. An outcome heard twice, or of an id no record -// holds, changes nothing. An outcome other than performed is said. An error leaves the event to be heard again: the record -// could not take it. +// holds, changes nothing. An outcome other than performed or not-needed is said as a refusal. An error leaves the event to +// be heard again: the record could not take it. func (h *Holder) OwedEnded(body []byte) error { e, err := forgemsg.DecodeOwedEnded(body) if err != nil { @@ -41,7 +41,9 @@ func (h *Holder) OwedEnded(body []byte) error { if !ok { continue } - if e.Outcome != forgemsg.OwedPerformed { + // Not-needed (its pull request closed unmerged) is a fact of the forge, not a fault of the act: ended with that + // outcome in the record, never said as a refusal (design 47 H2 §807, H-A-444). + if e.Outcome != forgemsg.OwedPerformed && e.Outcome != forgemsg.OwedNotNeeded { h.Refuse(fmt.Errorf("%s: the %s owed on %s#%d at %s was %s: %s", r.Instance, held.Act, held.Repository, held.Number, shortOf(held.Head, 12), e.Outcome, e.Words)) } -- 2.54.0 From 98da0e0b68d62564817bd8bd784d296364a55fb2 Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 22 Nov 2026 02:27:36 +0100 Subject: [PATCH 47/54] M1 check-needs: a package outside every Go artifact's from takes the manifest's own check-needs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The forge module's internal/seatgit whole test needs the Node build seat, and no Go artifact's from holds it, so nothing named its need and the Go build seat ran it on a node without the Node toolchain. The manifest now says the need at its top level, read by the Go build module for such packages; gitea declares node-build-node (review round 1, mesh-catalog, finding on §800; H-A-453 (2)). Builds: design 47 H2 §800, H-A-453 (2) --- modules/gitea/module.json | 3 +++ .../mesh-build-go/cmd/mesh-build-go/digest.go | 23 +++++++++++++------ .../cmd/mesh-build-go/digest_test.go | 22 ++++++++++++++++++ .../cmd/mesh-build-go/manifest_test.go | 11 +++++++++ 4 files changed, 52 insertions(+), 7 deletions(-) diff --git a/modules/gitea/module.json b/modules/gitea/module.json index 04d479f64..9eec20044 100644 --- a/modules/gitea/module.json +++ b/modules/gitea/module.json @@ -3,6 +3,9 @@ "version": "0.1", "class": "core", "moved": true, + "check-needs": [ + "node-build-node" + ], "requires": [ "postgres-database", "route", diff --git a/modules/mesh-build-go/cmd/mesh-build-go/digest.go b/modules/mesh-build-go/cmd/mesh-build-go/digest.go index ed77b326d..5a29a290a 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/digest.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/digest.go @@ -61,8 +61,9 @@ const ReadsOutsideMarker = "reads-outside" // readNeeds is what a package's integration tests need of the node they run on (issue 890; mesh-delivery-sdk // steprun.AskNeeds): the seats its module's manifest names in `check-needs` on each Go artifact whose path holds the // package's directory (design 47 H2 §800; H-A-350 DMC-6, H-A-399 (3)), `node-build-node` for tests that read the Node -// build module's installed toolchain. The manifest is the nearest module.json in dir or above it within the tree; nil -// when there is none, or none of its Go artifacts holds dir. The digest reports it (DigestResult.Needs); a check ask +// build module's installed toolchain, and for a package none of its Go artifacts holds, the manifest's own top-level +// `check-needs` (H-A-453 (2)). The manifest is the nearest module.json in dir or above it within the tree; nil when +// there is none, or it names no needs for dir. The digest reports it (DigestResult.Needs); a check ask // leaves such a package untested on a node that lacks it, and the checker asks it again of a node that has it. The unit // level is never given anything of the node (ADR 0533 §1), so the field binds the integration level alone. func readNeeds(tree, dir string) ([]string, error) { @@ -82,17 +83,25 @@ func readNeeds(tree, dir string) ([]string, error) { return nil, fmt.Errorf("the manifest %s: %w", filepath.Join(at, "module.json"), err) } var needs []string + held := false if manifest.Build != nil { for _, a := range manifest.Build.Artifacts { - if a.Language == "go" && len(a.CheckNeeds) > 0 && within(filepath.Join(at, filepath.FromSlash(a.From)), dir) { - for _, n := range a.CheckNeeds { - if !slices.Contains(needs, n) { - needs = append(needs, n) - } + if a.Language != "go" || !within(filepath.Join(at, filepath.FromSlash(a.From)), dir) { + continue + } + held = true + for _, n := range a.CheckNeeds { + if !slices.Contains(needs, n) { + needs = append(needs, n) } } } } + if !held { + // A package no Go artifact's from holds (the forge module's internal/seatgit) takes the manifest's own + // check-needs (H-A-453 (2)). + needs = slices.Clone(manifest.CheckNeeds) + } return needs, nil } return nil, nil diff --git a/modules/mesh-build-go/cmd/mesh-build-go/digest_test.go b/modules/mesh-build-go/cmd/mesh-build-go/digest_test.go index ecca63a39..ef7414428 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/digest_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/digest_test.go @@ -365,6 +365,28 @@ func TestAListedReadsOutsideMarkerKeepsTheDigestOfOnlyWhatItLists(t *testing.T) if n := digests(t, m, LevelUnit).Needs; len(n) != 0 { t.Errorf("the unit level names needs: %+v", n) } + + // H-A-453 (2): the manifest's own check-needs name the seats of a package no Go artifact's from holds (the forge + // module's internal/seatgit), and never those of a package an artifact holds, which keeps the artifact's. + for _, row := range []struct { + manifest, cNeeds string + }{ + {`{"module":"digest","check-needs":["node-build-node"],"build":{"artifacts":[` + + `{"name":"tools","kind":"bundle","language":"go","from":"a","check-needs":["node-build-node"]},` + + `{"name":"other","kind":"bundle","language":"go","from":"c"}]}}`, ""}, + {`{"module":"digest","check-needs":["node-build-node"],"build":{"artifacts":[` + + `{"name":"tools","kind":"bundle","language":"go","from":"a","check-needs":["node-build-node"]},` + + `{"name":"code","kind":"bundle","language":"typescript"}]}}`, "node-build-node"}, + } { + edit(t, tree, "module.json", row.manifest) + got := map[string]string{} + for _, n := range digests(t, m, LevelIntegration).Needs { + got[n.Package] = strings.Join(n.Needs, ",") + } + if got[a] != "node-build-node" || got[c] != row.cNeeds { + t.Errorf("%s: the integration level's needs = %v, want %s and %s needing %q", row.manifest, got, a, c, row.cNeeds) + } + } } // Issue 1935: a catalogue module's packages read ../../module.json and the module's files: a change to them changes the diff --git a/modules/mesh-build-go/cmd/mesh-build-go/manifest_test.go b/modules/mesh-build-go/cmd/mesh-build-go/manifest_test.go index 396c469ec..a9a75d265 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/manifest_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/manifest_test.go @@ -7,6 +7,7 @@ import ( "strings" "testing" + sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep" "git.novox.be/novox/mesh-sdk/go/catalogue" ) @@ -157,6 +158,16 @@ func TestGiteaReadsTheBuildModulesKeys(t *testing.T) { t.Errorf("gitea reads %v, lacks %s", gitea.Reads, want) } } + // H-A-453 (2): the forge module's internal/seatgit, which no Go artifact's from holds, takes the manifest's own + // check-needs, so its whole test is asked of a node with the Node build seat and given the Node build module's install. + root, err := filepath.Abs(catalogueRoot) + if err != nil { + t.Fatal(err) + } + if needs, err := readNeeds(root, filepath.Join(root, "modules", "gitea", "internal", "seatgit")); err != nil || + !slices.Equal(needs, []string{sdkstep.NeedNodeBuildNode}) { + t.Errorf("the forge module's seatgit package needs %v (%v), want %s", needs, err, sdkstep.NeedNodeBuildNode) + } } // 1309-S1: every run step runs in bubblewrap, so the module declares its package (the lead's decision: each build module -- 2.54.0 From 5dc8cb343bd2b955c27890391c3723e261708073 Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 22 Nov 2026 02:34:05 +0100 Subject: [PATCH 48/54] GT2 whole: the gate's whole test ends a record and reads its delivery time and class MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The review found §810's whole criterion proven only at the unit level (H-A-453 (3)): the whole test now ends its record through the wants seat's wanted and pipeline-wanted and the node-engine seat's running, over the real bus, and reads the head's delivery-time and class core from a core mover's catalog entry. Builds: design 47 H2 §810, H-A-453 (3) --- .../cmd/mesh-delivery-gate/bus_test.go | 9 ++++-- .../mesh-delivery-gate/mergecall_bus_test.go | 30 ++++++++++++++++++- 2 files changed, 36 insertions(+), 3 deletions(-) diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/bus_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/bus_test.go index 84e5ac42c..a6af04418 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/bus_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/bus_test.go @@ -106,7 +106,8 @@ func newTestBus(t *testing.T) *testBus { // The streams the events are on (mesh.mod..event., mesh.seat..event.). if _, err := js.CreateStream(context.Background(), jetstream.StreamConfig{Name: "EVENTS", Subjects: []string{"mesh.seat.git.event.>", "mesh.seat." + oc.Seat + ".event.>", - "mesh.mod.mesh-delivery-forge-bridge.event.>", "mesh.mod.mesh-delivery-indexer.event.>"}}); err != nil { + "mesh.mod.mesh-delivery-forge-bridge.event.>", "mesh.mod.mesh-delivery-indexer.event.>", + "mesh.seat.mesh-pipeline-runner.event.>", "mesh.seat.mesh-wants.event.>", "mesh.seat.node-engine.event.>"}}); err != nil { t.Fatal(err) } return b @@ -377,7 +378,11 @@ func (r *runtime) answer(m rpcMessage) { case "seat:mesh-hand-acts.hand-act": return toolAnswer(map[string]any{"id": "hand-act-1"}), nil case "seat:" + meshcatalog.Seat + "." + meshcatalog.VerbEntry: - // A mover's catalog entry, read once for its class (design 47 H2 §810): this test's modules are leaves. + // A mover's catalog entry, read once for its class (design 47 H2 §810): a fingerprint of this test opening with + // "core" is a core module's, every other a leaf's. + if fp, _ := a.Body["fingerprint"].(string); strings.HasPrefix(fp, "core") { + return toolAnswer(map[string]any{"entry": map[string]any{"manifest": map[string]any{"class": "core"}}}), nil + } return toolAnswer(map[string]any{"entry": map[string]any{"manifest": map[string]any{}}}), nil case "gitea.gitea_note_append", "gitea.gitea_delivery_view", "gitea.gitea_commit_status", "gitea.gitea_merge_pull_request", "gitea.gitea_add_comment": diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/mergecall_bus_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/mergecall_bus_test.go index 899aa9b22..62f9e33bb 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/mergecall_bus_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/mergecall_bus_test.go @@ -71,7 +71,7 @@ func TestAGateWholeTakesAMergeCallAndWatchesTheHeadsOverARealBus(t *testing.T) { } b.publish(t, "mesh.mod.mesh-delivery-indexer.event.take-in.taken", map[string]any{"branch": branch, "source": 1, "revision": 1, - "call": call, "entries": []any{map[string]any{"module": "app", "commit": merge, "fingerprint": "f", "repository": "novox/app"}}}) + "call": call, "entries": []any{map[string]any{"module": "app", "commit": merge, "fingerprint": "core-f", "repository": "novox/app"}}}) rt.until("the take-in's mover in the record", eventWithin, func() bool { b.mu.Lock() defer b.mu.Unlock() @@ -82,6 +82,34 @@ func TestAGateWholeTakesAMergeCallAndWatchesTheHeadsOverARealBus(t *testing.T) { t.Fatalf("the merge call heard twice wrote source.%s %d times", escaped, e.Revision()) } + // GT2 (design 47 H2 §810; H-A-453 (3)): at its end the record keeps on its head the delivery time, from merge-called to + // its last mover's running, and its class, core for a record that moves a core module. + b.publish(t, "mesh.seat.mesh-wants.event.wanted.test.app", map[string]any{"v": 1, "module": "app", "node": "test", + "fp": "core-f", "pipeline": recordMilestone(branch, 1), "revision": 7}) + b.publish(t, "mesh.seat.node-engine.event.running.test.app", map[string]any{"v": 1, "module": "app", "node": "test", + "fingerprint": "core-f", "at": time.Now().UTC().Format(time.RFC3339)}) + rt.until("app running on test", eventWithin, func() bool { + b.mu.Lock() + defer b.mu.Unlock() + mover, _ := MoverInstance(recordMilestone(branch, 1), "app") + return strings.Contains(string(b.ledger.states[mover]), `"step":"`+StepRunning+`"`) + }) + b.publish(t, "mesh.seat.mesh-wants.event.pipeline-wanted."+escaped+".1", map[string]any{"v": 1, "branch": branch, + "source": 1, "pipeline": recordMilestone(branch, 1), "outcome": "running"}) + var head Record + rt.until("the record ended with its delivery time and class", eventWithin, func() bool { + b.mu.Lock() + defer b.mu.Unlock() + head = Record{} + return json.Unmarshal(b.ledger.states[instance], &head) == nil && head.Ended != nil + }) + took, err := time.ParseDuration(head.DeliveryTime) + if want := time.Since(time.Date(2026, 11, 20, 3, 0, 0, 0, time.UTC)); err != nil || took < want-time.Minute || took > want || + head.Class != "core" { + t.Errorf("the ended record keeps the delivery time %q and the class %q, want about %s and core", head.DeliveryTime, head.Class, + want.Round(time.Minute)) + } + // The seed (§768): every watched branch's first source revision at its head; its head-moved then holds nothing. seeded := strings.Repeat("5", 40) b.mu.Lock() -- 2.54.0 From 739fb6bf47d19fc7b6047e42e3ba834d8005370a Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 22 Nov 2026 02:35:31 +0100 Subject: [PATCH 49/54] M1: the forge module's caps stand above their live fills MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Read on novox at 2026-11-22 01:35 UTC through the bus module's nats_streams (the review's finding on H-A-437; H-A-453): gitea bound 16703 bytes, 28 msgs cap 8 MiB gitea announced 98 bytes, 1 msg cap 8 MiB gitea findings 565816 bytes, 2013 msgs cap 8 MiB gitea first-views 369 bytes, 1 msg cap 8 MiB gitea transfers 396894 bytes, 2015 msgs cap 8 MiB gitea idle 376879 bytes, 1041 msgs cap 8 MiB gitea failed-steps 255007 bytes, 754 msgs cap 8 MiB gitea calls 36362 bytes, 93 msgs cap 8 MiB gitea taken no bucket on the bus yet cap 8 MiB gitea-findings log 0 bytes, 0 msgs cap 8 MiB Each fill is below a quarter of 8 MiB (2 MiB), the largest 565,816 bytes, so each cap stays at 8 MiB; none is raised. Builds: design 47 H2 §812; H-A-411 (2), H-A-437, H-A-453 -- 2.54.0 From 2b06ab31d7f3cfa4057401db24f41993d2e372ba Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 22 Nov 2026 02:36:15 +0100 Subject: [PATCH 50/54] M1: the forge module declares tweetnacl, which its test bus imports MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit test/fakebus.ts imports tweetnacl to seal as the vault does; it came only through mesh-sdk's own dependency, so the forge module's tests named nothing they import (the review's finding; H-A-453). Pinned as mesh-sdk pins it, an external package as the rule allows. Builds: design 47 H2 §808, §822; H-A-407, H-A-453 --- modules/gitea/package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/gitea/package.json b/modules/gitea/package.json index 8ca41df32..85f0f46e9 100644 --- a/modules/gitea/package.json +++ b/modules/gitea/package.json @@ -16,6 +16,7 @@ }, "devDependencies": { "@types/node": "^22.0.0", + "tweetnacl": "1.0.3", "typescript": "^5.6.0" } } -- 2.54.0 From 4150402020700b0bf5cc85f6385944e7de83343c Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 22 Nov 2026 02:37:08 +0100 Subject: [PATCH 51/54] SC1 whole: the self-check's whole test raises and clears over-budget from the gate's times MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The review found §810's probe proven only at the unit level (H-A-453 (3)): the whole test now enables the budget probe, answers gate_times with a core delivery 11 minutes from its merge call, reads over-budget for core naming its record and longest phase, and a core delivery within its budget at the next run clears it. Builds: design 47 H2 §810, H-A-453 (3) --- .../mesh-self-check/whole_integration_test.go | 49 ++++++++++++++++++- 1 file changed, 47 insertions(+), 2 deletions(-) diff --git a/modules/mesh-self-check/cmd/mesh-self-check/whole_integration_test.go b/modules/mesh-self-check/cmd/mesh-self-check/whole_integration_test.go index fcb372384..07a99b160 100644 --- a/modules/mesh-self-check/cmd/mesh-self-check/whole_integration_test.go +++ b/modules/mesh-self-check/cmd/mesh-self-check/whole_integration_test.go @@ -49,6 +49,8 @@ type player struct { in io.Writer next int pending map[string]chan wire + // times is what the gate's gate_times answers the budget probe (design 47 H2 §810). + times map[string]any } func (p *player) send(m map[string]any) { @@ -147,7 +149,7 @@ func (p *player) handle(m wire) { } _ = json.Unmarshal(m.Params, &g) if g.Kind == "config" && g.Key == "probes" { - p.answer(m.ID, []string{"D2", "D7", "DW", "D15"}, nil) + p.answer(m.ID, []string{"D2", "D7", "DW", "D15", "D-budget"}, nil) return } p.answer(m.ID, nil, fmt.Errorf("no %s value %s here", g.Kind, g.Key)) @@ -179,6 +181,13 @@ func (p *player) handle(m wire) { Key string `json:"key"` } _ = json.Unmarshal(m.Params, &ask) + if ask.Key == TimesTool { + p.mu.Lock() + times := p.times + p.mu.Unlock() + p.answer(m.ID, times, nil) + return + } if ask.Key != StreamsTool { p.answer(m.ID, nil, errors.New("nats: no responders available for request")) return @@ -268,7 +277,14 @@ func TestTheSelfCheckWholeRunsOnTheBus(t *testing.T) { t.Fatal(err) } t.Cleanup(func() { _ = stdin.Close(); cancel(); _ = cmd.Wait() }) - p := &player{ctx: ctx, nc: nc, js: js, in: stdin, pending: map[string]chan wire{}} + // SC1 (design 47 H2 §810; H-A-453 (3)): the gate's newest core delivery took 11 minutes from its merge call, over its + // budget of ten. + coreTimes := func(took string, over bool) map[string]any { + return map[string]any{"core": map[string]any{"budget": "10m0s", "last": map[string]any{"record": "record/main_7", + "milestone": "main.7", "ended": "2026-11-22T01:00:00Z", "delivery-time": took, "longest-phase": "placing", + "longest-phase-time": "6m0s", "over": over}}, "leaf": map[string]any{"budget": "3m0s"}} + } + p := &player{ctx: ctx, nc: nc, js: js, in: stdin, pending: map[string]chan wire{}, times: coreTimes("11m0s", true)} go p.serve(stdout) raw, err := p.call(DoctorTool, map[string]any{"run": true}) @@ -323,4 +339,33 @@ func TestTheSelfCheckWholeRunsOnTheBus(t *testing.T) { if raw, err := p.call(VerdictsTool, map[string]any{}); err != nil || !strings.Contains(string(raw), answer.Run.Run) { t.Errorf("the week's verdicts answered %s, %v", raw, err) } + + // SC1: the over-budget core delivery raised over-budget for the class core, naming its record and longest phase; the next + // core delivery within its budget clears it. + overBudget := func() (string, bool) { + keys, err := findings.Keys(ctx) + if err != nil { + return "", false + } + for _, k := range keys { + if strings.Contains(k, ".core.") && strings.HasSuffix(k, "."+kindOverBudget) { + e, err := findings.Get(ctx, k) + return string(e.Value()), err == nil + } + } + return "", false + } + if raised, ok := overBudget(); verdicts["D-budget"] != VerdictFail || !ok || !strings.Contains(raised, "main.7") || + !strings.Contains(raised, "placing") { + t.Errorf("the over-budget core delivery: the probe's verdict %q, the finding %q (%v)", verdicts["D-budget"], raised, ok) + } + p.mu.Lock() + p.times = coreTimes("4m0s", false) + p.mu.Unlock() + if _, err := p.call(DoctorTool, map[string]any{"run": true}); err != nil { + t.Fatal(err) + } + if raised, ok := overBudget(); ok { + t.Errorf("a core delivery within its budget left over-budget raised: %s", raised) + } } -- 2.54.0 From 3dd92f1c4b242fc335b4d7ca2730e77f74f3faa6 Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 22 Nov 2026 02:49:12 +0100 Subject: [PATCH 52/54] C4 whole: the forge holds one token for the account after the bundle's own mint MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The seat git's whole test configures no token: the bundle mints the tools' account's token at the fixture forge with the admin's login, writes it through the vault seat's two-step own-put (sealed to the call's key) and reads it through get (sealed to the process's key), both answered by the test, with the mint's lease watched as the runtime delivers it. After the mint the forge holds exactly one token for the account, mesh-tools-1, the one the vault keeps at generation 1; the old fixed-name token is gone. Builds: design 47 H2 §808 (whole: the forge holds one token for the account after), H-A-422, H-A-430 (1), H-A-439, H-A-453 (3) --- .../seatgit/whole_integration_test.go | 274 +++++++++++++++++- 1 file changed, 270 insertions(+), 4 deletions(-) diff --git a/modules/gitea/internal/seatgit/whole_integration_test.go b/modules/gitea/internal/seatgit/whole_integration_test.go index eaa20face..f8bc98ae0 100644 --- a/modules/gitea/internal/seatgit/whole_integration_test.go +++ b/modules/gitea/internal/seatgit/whole_integration_test.go @@ -16,10 +16,16 @@ package seatgit // - With the forge's store stopped a merge call answers away, and called again once it is back it lands (H2 §776). // - A forge that refuses the second merge gives one merge-called with the first member and `refused`, and merge-partial // raised; a later call landing the refused member clears it (§767, test 115). +// - No token is configured: the bundle mints the tools' account's token at the fixture forge with the admin's login, +// writes it through the vault seat's two-step sealed own-put and reads it through its sealed get, both answered +// here, and the forge holds one token for the account after, the old fixed-name token deleted (H2 §808, H-A-422, +// H-A-430 (1), H-A-453 (3)). import ( "bufio" "context" + "crypto/rand" + "encoding/base64" "encoding/json" "errors" "fmt" @@ -35,9 +41,11 @@ import ( "testing" "time" + "git.novox.be/novox/mesh-sdk/go/seal" "git.novox.be/novox/mesh-sdk/go/testbus" "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" + "golang.org/x/crypto/nacl/box" ) const within = 30 * time.Second @@ -58,8 +66,107 @@ type forge struct { paths []string // away is the forge's store stopped (§776): the API answers 503 and the version read names the store unavailable. away bool + // admin is the admin's login, which mints (§808); tokens what each account holds; every other call is refused 401 + // unless it carries one of them. + admin string + tokens map[string][]forgeToken + nextID int } +// forgeToken is one access token the fixture forge minted. +type forgeToken struct { + ID int `json:"id"` + Name string `json:"name"` + sha1 string +} + +// authorized is the admin's basic login or a token the forge holds. +func (f *forge) authorized(r *http.Request) bool { + if u, p, ok := r.BasicAuth(); ok { + return u == f.admin && p == adminPassword + } + given := strings.TrimPrefix(r.Header.Get("Authorization"), "token ") + for _, held := range f.tokens { + for _, t := range held { + if given != "" && t.sha1 == given { + return true + } + } + } + return false +} + +// held answers the names of the tokens an account holds and their values. +func (f *forge) held(account string) (names, values []string) { + f.mu.Lock() + defer f.mu.Unlock() + for _, t := range f.tokens[account] { + names, values = append(names, t.Name), append(values, t.sha1) + } + return names, values +} + +// serveAdmin answers the admin API the mint uses: the admin's own read, the account and its team, and the account's +// tokens (mint, list, delete). False when the path is not one of them. +func (f *forge) serveAdmin(w http.ResponseWriter, r *http.Request) bool { + switch p := r.URL.Path; { + case p == "/api/v1/user": + _ = json.NewEncoder(w).Encode(map[string]any{"login": f.admin}) + case p == "/api/v1/admin/users" && r.Method == http.MethodPost: + w.WriteHeader(http.StatusCreated) + case p == "/api/v1/orgs/novox/teams" && r.Method == http.MethodGet: + _, _ = w.Write([]byte("[]")) + case p == "/api/v1/orgs/novox/teams" && r.Method == http.MethodPost: + w.WriteHeader(http.StatusCreated) + _, _ = w.Write([]byte(`{"id":1}`)) + case strings.HasPrefix(p, "/api/v1/teams/1/members/"): + w.WriteHeader(http.StatusNoContent) + default: + m := regexp.MustCompile(`^/api/v1/users/([^/]+)/tokens(?:/(\d+))?$`).FindStringSubmatch(p) + if m == nil { + return false + } + account := m[1] + switch { + case r.Method == http.MethodPost: + var asked struct { + Name string `json:"name"` + } + _ = json.NewDecoder(r.Body).Decode(&asked) + for _, t := range f.tokens[account] { + if t.Name == asked.Name { + http.Error(w, `{"message":"access token name has been used already"}`, http.StatusUnprocessableEntity) + return true + } + } + f.nextID++ + t := forgeToken{ID: f.nextID, Name: asked.Name, sha1: fmt.Sprintf("%040x", 0xa0000+f.nextID)} + f.tokens[account] = append(f.tokens[account], t) + w.WriteHeader(http.StatusCreated) + _ = json.NewEncoder(w).Encode(map[string]any{"id": t.ID, "name": t.Name, "sha1": t.sha1}) + case r.Method == http.MethodGet: + if r.URL.Query().Get("page") != "1" { + _, _ = w.Write([]byte("[]")) + return true + } + _ = json.NewEncoder(w).Encode(f.tokens[account]) + case r.Method == http.MethodDelete: + kept := f.tokens[account][:0] + for _, t := range f.tokens[account] { + if fmt.Sprint(t.ID) != m[2] { + kept = append(kept, t) + } + } + f.tokens[account] = kept + w.WriteHeader(http.StatusNoContent) + } + } + return true +} + +// adminPassword is the admin's password the fixture forge takes, in the file the host would unseal it into. +const adminPassword = "admin-password" + func (f *forge) serve(w http.ResponseWriter, r *http.Request) { f.mu.Lock() defer f.mu.Unlock() @@ -76,6 +183,13 @@ func (f *forge) serve(w http.ResponseWriter, r *http.Request) { http.Error(w, "Service Unavailable", http.StatusServiceUnavailable) return } + if !f.authorized(r) { + http.Error(w, `{"message":"token is required"}`, http.StatusUnauthorized) + return + } + if f.serveAdmin(w, r) { + return + } m := regexp.MustCompile(`^/api/v1/repos/([^/]+/[^/]+)/pulls/(\d+)(/merge)?$`).FindStringSubmatch(r.URL.Path) if m != nil { p := f.pulls[m[1]+"#"+m[2]] @@ -136,6 +250,76 @@ type runtime struct { // regs are what the seat mesh-catalog's registrations answers; subs the patterns the bundle subscribed. regs []map[string]any subs []string + // vault is the seat mesh-vault as this test answers it (§808, §822): the module's own secrets by name, and the + // own-put calls opened, each with the key its value is sealed to. + vault map[string]vaulted + puts map[string]openPut +} + +// openPut is an own-put call its first step opened: the name it puts and the key its value is sealed to. +type openPut struct { + name string + pub, priv *[32]byte +} + +// vaulted is one secret the test's vault keeps: its newest value and generation. +type vaulted struct { + value string + generation int +} + +// vaultAsk answers the seat mesh-vault's get, sealed to the asking process's key, and its two-step own-put, whose value +// comes sealed to the key the first step made (§822). +func (r *runtime) vaultAsk(key string, body json.RawMessage) (any, error) { + var b struct { + Name string `json:"name"` + PublicKey string `json:"public_key"` + Call string `json:"call"` + Sealed string `json:"sealed"` + } + _ = json.Unmarshal(body, &b) + r.mu.Lock() + defer r.mu.Unlock() + switch { + case key == "seat:mesh-vault.get": + v, ok := r.vault[b.Name] + if !ok { + return nil, fmt.Errorf("the vault holds no secret named %s", b.Name) + } + sealed, err := seal.Seal(v.value, b.PublicKey) + if err != nil { + return nil, err + } + return map[string]any{"generation": v.generation, "changed-at": time.Now().UTC().Format(time.RFC3339), "sealed": sealed}, nil + case b.Call == "": + pub, priv, err := box.GenerateKey(rand.Reader) + if err != nil { + return nil, err + } + r.next++ + call := fmt.Sprintf("put-%d", r.next) + r.puts[call] = openPut{name: b.Name, pub: pub, priv: priv} + return map[string]any{"call": call, "seal_to": base64.StdEncoding.EncodeToString(pub[:])}, nil + default: + put, ok := r.puts[b.Call] + if !ok { + return nil, fmt.Errorf("no own-put call %s is open", b.Call) + } + raw, err := base64.StdEncoding.DecodeString(b.Sealed) + if err != nil { + return nil, err + } + value, ok := box.OpenAnonymous(nil, raw, put.pub, put.priv) + if !ok { + return nil, fmt.Errorf("the own-put value of %s is not sealed to its call's key", b.Call) + } + // The module named by the caller's account, never by an argument: here always the forge module (§822). + name := "module/gitea/" + put.name + delete(r.puts, b.Call) + v := vaulted{value: string(value), generation: r.vault[name].generation + 1} + r.vault[name] = v + return map[string]any{"generation": v.generation}, nil + } } func (r *runtime) subscribed(pattern string) bool { @@ -208,7 +392,15 @@ func (r *runtime) answer(id json.RawMessage, method string, params json.RawMessa r.subs = append(r.subs, p.Pattern) r.mu.Unlock() return map[string]any{}, nil - case "mesh/publish", "mesh/state.watch", "mesh/seat.record": + case "mesh/state.watch": + // The mint's lease is watched for its release (§808): its changes are delivered as the runtime delivers them. + if p.State == "token-lease" { + var watch string + _ = json.Unmarshal(id, &watch) + return map[string]any{}, r.watchState(kv, p.Key, watch) + } + return map[string]any{}, nil + case "mesh/publish", "mesh/seat.record": return map[string]any{}, nil case "mesh/capabilities": return map[string]any{"words": []string{}}, nil @@ -226,6 +418,8 @@ func (r *runtime) answer(id json.RawMessage, method string, params json.RawMessa return map[string]any{"registrations": r.regs}, nil case "seat:mesh-hand-acts.hand-act": return map[string]any{}, nil + case "seat:mesh-vault.get", "seat:mesh-vault.own-put": + return r.vaultAsk(a.Key, p.Body) } return nil, fmt.Errorf("the runtime answers no ask %s in this test", a.Key) case "mesh/state.get": @@ -240,6 +434,15 @@ func (r *runtime) answer(id json.RawMessage, method string, params json.RawMessa return nil, err } return map[string]any{"key": p.Key, "value": json.RawMessage(e.Value()), "revision": e.Revision()}, nil + case "mesh/state.create": + if kv == nil { + return nil, fmt.Errorf("the state %q is not declared", p.State) + } + rev, err := kv.Create(ctx, p.Key, p.Value) + if errors.Is(err, jetstream.ErrKeyExists) { + return nil, changed{err} + } + return map[string]any{"revision": rev}, err case "mesh/state.put": if kv == nil { return nil, fmt.Errorf("the state %q is not declared", p.State) @@ -269,12 +472,46 @@ func (r *runtime) answer(id json.RawMessage, method string, params json.RawMessa return } if err != nil { - r.say(map[string]any{"id": id, "error": map[string]any{"code": -32000, "message": err.Error()}}) + code := -32000 + if errors.As(err, new(changed)) { + code = -32010 + } + r.say(map[string]any{"id": id, "error": map[string]any{"code": code, "message": err.Error()}}) return } r.say(map[string]any{"id": id, "result": result}) } +// watchState delivers each later change of one key of a state to the bundle's watch, as `mesh/state` requests naming +// the watch, a delete without its current value. +func (r *runtime) watchState(kv jetstream.KeyValue, key, watch string) error { + w, err := kv.Watch(context.Background(), key, jetstream.UpdatesOnly()) + if err != nil { + return err + } + r.t.Cleanup(func() { _ = w.Stop() }) + go func() { + for e := range w.Updates() { + if e == nil { + continue + } + change := map[string]any{"key": e.Key()} + if e.Operation() == jetstream.KeyValuePut { + change["current"] = map[string]any{"value": json.RawMessage(e.Value()), "revision": e.Revision()} + } + r.mu.Lock() + r.next++ + n := r.next + r.mu.Unlock() + r.say(map[string]any{"id": fmt.Sprintf("rt-watch-%d", n), "method": "mesh/state", "params": map[string]any{"watch": watch, "change": change}}) + } + }() + return nil +} + +// changed is the runtime's refusal of a create whose key has a value (mesh-sdk's CODE_CHANGED, -32010). +type changed struct{ error } + // ask sends the bundle one request and answers its result. func (r *runtime) ask(method string, params map[string]any) json.RawMessage { r.t.Helper() @@ -399,7 +636,7 @@ func TestTheSeatGitWholeOverARealBus(t *testing.T) { t.Fatal(err) } kvs := map[string]jetstream.KeyValue{} - for _, name := range []string{"bound", "announced", "findings", "first-views", "transfers", "idle", "failed-steps", "calls"} { + for _, name := range []string{"bound", "announced", "findings", "first-views", "transfers", "idle", "failed-steps", "calls", "token-lease"} { kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: "gitea_" + name}) if err != nil { t.Fatal(err) @@ -425,15 +662,22 @@ func TestTheSeatGitWholeOverARealBus(t *testing.T) { "novox/lib#10": {repo: "novox/lib", base: "main", head: B, number: 10}, "novox/hq#11": {repo: "novox/hq", base: "main", head: H, number: 11}, }} + // The forge holds the tools' account's token under its old fixed name, which the first mint replaces (H-A-430 (1)). + f.admin, f.nextID = "mesh-admin", 1 + f.tokens = map[string][]forgeToken{"mesh-tools": {{ID: 1, Name: "mesh-tools", sha1: strings.Repeat("0", 40)}}} srv := httptest.NewServer(http.HandlerFunc(f.serve)) t.Cleanup(srv.Close) state := t.TempDir() + passwordFile := filepath.Join(state, "admin.secret") + if err := os.WriteFile(passwordFile, []byte(adminPassword+"\n"), 0o600); err != nil { + t.Fatal(err) + } launch := `const sdk = await import("@novox/mesh-sdk/stdio"); await import("./dist/index.js"); await import("./dist/tools/index.js"); await sdk.serveRegisteredOverStdio("gitea");` cmd := exec.Command("node", "--input-type=module", "-e", launch) cmd.Dir = dir cmd.Env = append(os.Environ(), "MESH_SERVED_MODULE=gitea", "MESH_MODULE=gitea", "MESH_NODE=test", "MESH_GITEA_URL="+srv.URL, - "GITEA_TOKEN=t", "MESH_GITEA_STATE_DIR="+state, "MESH_GITEA_HOOK_PORT=0", "MESH_GITEA_CONFIG_FILE="+filepath.Join(state, "config.json")) + "MESH_GITEA_ADMIN_USER=mesh-admin", "MESH_GITEA_ADMIN_PASSWORD_FILE="+passwordFile, "MESH_GITEA_STATE_DIR="+state, "MESH_GITEA_HOOK_PORT=0", "MESH_GITEA_CONFIG_FILE="+filepath.Join(state, "config.json")) in, _ := cmd.StdinPipe() out, _ := cmd.StdoutPipe() cmd.Stderr = testLog{t} @@ -447,6 +691,7 @@ func TestTheSeatGitWholeOverARealBus(t *testing.T) { // A source-only registration (H2 §815): no module, one watched branch, its check a build seat's step. {"origin": "novox/hq", "branches": []string{"main"}, "policy": map[string]any{"steps": map[string]any{"unit": "required"}}, "check": map[string]any{"seat": "node-build-go", "step": "unit"}}}} + rt.vault, rt.puts = map[string]vaulted{}, map[string]openPut{} go rt.read(out) // At the start the stopped call says one merge-called naming the two that landed (§766). @@ -531,6 +776,27 @@ func TestTheSeatGitWholeOverARealBus(t *testing.T) { _, err := kvs["findings"].Get(ctx, key) return errors.Is(err, jetstream.ErrKeyNotFound) }) + + // No token was configured: the calls above ran on the token the bundle minted with the admin's login, wrote through + // the two-step sealed own-put and read through the sealed get; the forge holds one token for the account after, the + // one the vault keeps, and the old fixed-name token is gone (§808, H-A-430 (1)). + until(t, "the forge holds one token for the tools' account", func() bool { + names, _ := f.held("mesh-tools") + return len(names) == 1 + }) + names, values := f.held("mesh-tools") + rt.mu.Lock() + kept := rt.vault["module/gitea/mesh-tools"] + rt.mu.Unlock() + var opened struct { + Token string `json:"token"` + Scopes []string `json:"scopes"` + } + _ = json.Unmarshal([]byte(kept.value), &opened) + if fmt.Sprint(names) != "[mesh-tools-1]" || opened.Token != values[0] || kept.generation != 1 || len(opened.Scopes) == 0 { + t.Fatalf("the forge holds %v for the tools' account; the vault keeps generation %d of a token %t to it, scopes %v", + names, kept.generation, opened.Token == values[0], opened.Scopes) + } } type testLog struct{ t *testing.T } -- 2.54.0 From 758a8ad6532625f2d439944836ba98240ce0a37c Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 22 Nov 2026 02:58:39 +0100 Subject: [PATCH 53/54] GT1: each owed act is a ledger entry of its own; every ledger write is measured MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An act the gate owes the forge is now the entry act/ (type act, index acts on its record's key in the manifest), removed when the forge bridge performed it; the head entry holds the record's fields alone, and the record's acts are joined from their entries when read. The record type keeps no act operation: a head written before this change is read for its fields and the acts embedded in it are ignored (they are notes and views of the few records in flight at the delivery; losing them changes nothing the operator relies on). A view act's text is the record's summary, the head's fields and a count of movers per outcome, so it fits 64 KiB for any record; gate_record still shows the whole record. The 64 KiB refusal measures every write, the head's, a mover's and an act's, raising record-too-large, which the record's end clears for every entry of it. The gate's whole test gains the step: 126 movers write 127 entries plus their acts, none over 64 KiB, the head counting 126 and holding no act; a take-in said again with a why past 64 KiB raises record-too-large and the record's end clears it. Builds: design 47 H2 §809, H-A-385, H-A-391, H-A-453 (3), H-A-455, H-A-457, H-A-459 --- .../cmd/mesh-delivery-gate/bus_test.go | 10 +- .../mesh-delivery-gate/mergecall_bus_test.go | 70 ++++++++++++ .../cmd/mesh-delivery-gate/movers.go | 78 ++++++++++---- .../cmd/mesh-delivery-gate/owedacts.go | 38 ++++++- .../cmd/mesh-delivery-gate/owedacts_test.go | 40 ++++++- .../cmd/mesh-delivery-gate/owedended.go | 78 +++++++++++++- .../cmd/mesh-delivery-gate/record.go | 55 +++++----- .../cmd/mesh-delivery-gate/record_test.go | 11 +- .../cmd/mesh-delivery-gate/recordevents.go | 23 +++- .../cmd/mesh-delivery-gate/recovery.go | 4 +- .../cmd/mesh-delivery-gate/times.go | 6 +- modules/mesh-delivery-gate/module.json | 101 +++++++++++------- 12 files changed, 406 insertions(+), 108 deletions(-) diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/bus_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/bus_test.go index a6af04418..e525e2424 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/bus_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/bus_test.go @@ -395,7 +395,7 @@ func (r *runtime) answer(m rpcMessage) { return nil, fmt.Errorf("the runtime answers no ask %s in this test", a.Key) } switch m.Method { - case buscalls.LedgerApply, buscalls.LedgerGet, buscalls.LedgerQuery: + case buscalls.LedgerApply, buscalls.LedgerGet, buscalls.LedgerQuery, buscalls.LedgerRemove: // The gate's ledger `records` (W11.1), as the tool runner keeps it: the record test's ledger in memory. r.b.mu.Lock() defer r.b.mu.Unlock() @@ -414,6 +414,14 @@ func (r *runtime) answer(m rpcMessage) { } return a, err } + if m.Method == buscalls.LedgerRemove { + // An act performed is removed (design 47 H2 §809, H-A-455 (1)), as the tool runner's ledger removal does. + var in buscalls.LedgerRemoveRequest + if err := json.Unmarshal(m.Params, &in); err != nil { + return nil, err + } + return buscalls.LedgerRemoved{}, r.b.ledger.Remove(in.Instance) + } if m.Method == buscalls.LedgerGet { var in buscalls.InstanceRequest if err := json.Unmarshal(m.Params, &in); err != nil { diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/mergecall_bus_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/mergecall_bus_test.go index 62f9e33bb..c21674dd2 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/mergecall_bus_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/mergecall_bus_test.go @@ -5,6 +5,7 @@ package main import ( "context" "encoding/json" + "fmt" "strings" "testing" "time" @@ -110,6 +111,75 @@ func TestAGateWholeTakesAMergeCallAndWatchesTheHeadsOverARealBus(t *testing.T) { want.Round(time.Minute)) } + // GT1 (design 47 H2 §809; H-A-385, H-A-391, H-A-453 (3), H-A-455): a record of 126 movers writes 127 entries, its head + // and one per mover, and its acts owed the forge each an entry of its own, none over 64 KiB, the head counting 126 and + // holding no act; an entry made too large raises record-too-large, and the record's end clears it. + call2, merge2 := "00000000000000c2", strings.Repeat("e", 40) + instance2, _ := RecordInstance(recordMilestone(branch, 2)) + b.publish(t, "mesh.seat.git.event.merge-called."+call2, map[string]any{"call": call2, "by": "lead@shanks", "account": "acct-s", + "at": time.Now().UTC().Format(time.RFC3339), "members": []any{map[string]any{"repository": "novox/app", "number": 6, + "branch": branch, "head": strings.Repeat("f", 40), "commit": merge2}}}) + rt.until("the record of the second merge call", eventWithin, func() bool { + b.mu.Lock() + defer b.mu.Unlock() + return b.ledger.states[instance2] != nil + }) + var taken []any + for i := range 126 { + module := "mesh-module-" + strings.Repeat("x", 3) + fmt.Sprintf("%03d", i) + taken = append(taken, map[string]any{"module": module, "commit": merge2, "fingerprint": fmt.Sprintf("%064x", i), "repository": "novox/app"}) + } + b.publish(t, "mesh.mod.mesh-delivery-indexer.event.take-in.taken", map[string]any{"branch": branch, "source": 2, "revision": 2, + "call": call2, "entries": taken}) + rt.until("the record's 126 movers taken in", eventWithin, func() bool { + b.mu.Lock() + defer b.mu.Unlock() + last, _ := MoverInstance(recordMilestone(branch, 2), taken[125].(map[string]any)["module"].(string)) + return strings.Contains(string(b.ledger.states[instance2]), `"movers":126`) && + strings.Contains(string(b.ledger.states[last]), `"outcome":"`+OutcomeTakenIn+`"`) + }) + b.mu.Lock() + entries, acts, over := 0, 0, []string{} + for inst, raw := range b.ledger.states { + if string(raw) == "null" { + continue + } + var of struct { + Milestone string `json:"milestone"` + Record string `json:"record"` + } + _ = json.Unmarshal(raw, &of) + if of.Milestone != recordMilestone(branch, 2) && of.Record != recordMilestone(branch, 2) { + continue + } + if strings.HasPrefix(inst, TypeAct+"/") { + acts++ + } else { + entries++ + } + if len(raw) > EntryLimit { + over = append(over, fmt.Sprintf("%s holds %d bytes", inst, len(raw))) + } + } + headRaw := string(b.ledger.states[instance2]) + b.mu.Unlock() + if entries != 127 || acts == 0 || len(over) > 0 || strings.Contains(headRaw, `"acts"`) { + t.Fatalf("the record of 126 movers writes %d entries and %d acts, over 64 KiB: %v; its head: %.300s", entries, acts, over, headRaw) + } + if findingHeld(ctx, b, RecordTooLarge, "") { + t.Fatal("record-too-large is raised for a record whose entries are within 64 KiB") + } + // The first mover's take-in said again with a why past 64 KiB: its entry's write is refused and raises record-too-large; + // the record's end clears it. + first := taken[0].(map[string]any) + tooLarge, _ := MoverInstance(recordMilestone(branch, 2), first["module"].(string)) + b.publish(t, "mesh.mod.mesh-delivery-indexer.event.take-in.taken", map[string]any{"branch": branch, "source": 2, "revision": 2, + "call": call2, "why": strings.Repeat("w", EntryLimit), "entries": []any{first}}) + rt.until("record-too-large raised for the mover's entry", eventWithin, func() bool { return findingHeld(ctx, b, RecordTooLarge, tooLarge) }) + b.publish(t, "mesh.seat.mesh-wants.event.pipeline-wanted."+escaped+".2", map[string]any{"v": 1, "branch": branch, + "source": 2, "pipeline": recordMilestone(branch, 2), "outcome": "running"}) + rt.until("record-too-large cleared at the record's end", eventWithin, func() bool { return !findingHeld(ctx, b, RecordTooLarge, "") }) + // The seed (§768): every watched branch's first source revision at its head; its head-moved then holds nothing. seeded := strings.Repeat("5", 40) b.mu.Lock() diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/movers.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/movers.go index 4fc43b265..f70f710ed 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/movers.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/movers.go @@ -1,12 +1,13 @@ package main // A record's entries, one per module (design 47 H2 §809; H-A-385, H-A-391): the head entry `record/` keeps -// every record-wide field, its outcome, the count of its movers and §810's times, nothing per module; each mover, the -// catalog half's too, has its own entry `mover/` with its pipeline, version and the module's step -// outcomes per node (its machines and their running times among them). No entry names two modules. A record is joined -// from its head and the movers the ledger's index `movers` names under the record's key, one query. The gate refuses an -// entry over 64 KiB with the finding record-too-large, cleared when that entry is next written within 64 KiB or at the -// record's end. +// every record-wide field, its outcome, the count of its movers and §810's times, nothing per module and no act; each +// mover, the catalog half's too, has its own entry `mover/` with its pipeline, version and the +// module's step outcomes per node (its machines and their running times among them); each act owed the forge has its own +// entry `act/`, removed when performed (H-A-455 (1)). No entry names two modules. A record is joined +// from its head and the movers and acts the ledger's indexes `movers` and `acts` name under the record's key, one query +// each. The gate refuses a write over 64 KiB to any entry, the head's, a mover's or an act's, with the finding +// record-too-large, cleared when that entry is next written within 64 KiB or at the record's end (H-A-455 (3)). import ( "encoding/json" @@ -85,7 +86,7 @@ func MoverInstance(milestone, module string) (string, error) { return inst, nil } -// moversKey is the index key that names every mover of a record: its key's token, then the rest. +// moversKey is the index key that names every mover or act of a record: its key's token, then the rest. func moversKey(milestone string) string { return strings.TrimSuffix(buscalls.IndexKey([]string{milestone}, "x"), ".x") + ".>" } @@ -160,7 +161,7 @@ func (rs *Records) mover(r *Record, module, fingerprint string) (*Mover, error) class = firstOf(c, catalogue.ClassLeaf) } m := &Mover{Instance: inst, Record: r.Milestone, Module: module, Class: class} - seq, err := rs.Ledger.Apply(inst, OpMover, map[string]any{"record": r.Milestone, "module": module, "class": class}) + seq, err := rs.write(r, inst, OpMover, map[string]any{"record": r.Milestone, "module": module, "class": class}, m) if err != nil { return nil, err } @@ -169,7 +170,9 @@ func (rs *Records) mover(r *Record, module, fingerprint string) (*Mover, error) r.movers = map[string]*Mover{} } r.movers[module] = m - hseq, err := rs.Ledger.Apply(r.Instance, OpMovers, map[string]any{"movers": len(r.movers)}) + next := *r + next.MoverCount = len(r.movers) + hseq, err := rs.write(r, r.Instance, OpMovers, map[string]any{"movers": len(r.movers)}, next.head()) if err != nil { return m, fmt.Errorf("the mover %s is written and the record %s's count of its movers is not: %w", module, r.Instance, err) } @@ -188,21 +191,33 @@ func (rs *Records) SetMover(r *Record, module, fingerprint, pipeline, version st if next.Pipeline == m.Pipeline && next.Version == m.Version { return nil } - if err := rs.fits(m.Instance, next); err != nil { - return err - } - seq, err := rs.Ledger.Apply(m.Instance, OpMover, map[string]any{"record": r.Milestone, "module": module, "class": m.Class, - "pipeline": next.Pipeline, "version": next.Version}) + seq, err := rs.write(r, m.Instance, OpMover, map[string]any{"record": r.Milestone, "module": module, "class": m.Class, + "pipeline": next.Pipeline, "version": next.Version}, next) if err != nil { return err } m.Pipeline, m.Version, m.Sequence = next.Pipeline, next.Version, seq - rs.fitted(m.Instance) return nil } -// fits refuses an entry whose state would pass EntryLimit, raising record-too-large for it (§809). -func (rs *Records) fits(instance string, state any) error { +// write is one write to an entry of a record, measured first: the entry's state after it (the head's without its movers +// and acts) is refused over EntryLimit with record-too-large, and a write taken clears the entry's finding (§809, H-A-455 +// (3)). +func (rs *Records) write(r *Record, instance, operation string, body, state any) (uint64, error) { + if err := rs.fits(r, instance, state); err != nil { + return 0, err + } + seq, err := rs.Ledger.Apply(instance, operation, body) + if err != nil { + return 0, err + } + rs.fitted(instance) + return seq, nil +} + +// fits refuses an entry of r whose state would pass EntryLimit, raising record-too-large for it (§809); the entry is +// remembered with its record, so the record's end clears it whichever entry it was (H-A-455 (3)). +func (rs *Records) fits(r *Record, instance string, state any) error { raw, err := json.Marshal(state) if err != nil { return err @@ -210,6 +225,12 @@ func (rs *Records) fits(instance string, state any) error { if len(raw) <= EntryLimit { return nil } + rs.mu.Lock() + if rs.refused == nil { + rs.refused = map[string]string{} + } + rs.refused[instance] = r.Milestone + rs.mu.Unlock() if rs.TooLarge != nil { rs.TooLarge(instance, len(raw), true) } @@ -223,10 +244,27 @@ func (rs *Records) fitted(instance string) { } } -// clearTooLarge clears record-too-large for every entry of a record at its end. +// clearTooLarge clears record-too-large for every entry of a record at its end: its head, its movers, its acts and every +// entry of it a write was refused for. func (rs *Records) clearTooLarge(r *Record) { - rs.fitted(r.Instance) + insts := []string{r.Instance} for _, m := range r.movers { - rs.fitted(m.Instance) + insts = append(insts, m.Instance) + } + for id := range r.Acts { + if inst, err := ActInstance(r.Milestone, id); err == nil { + insts = append(insts, inst) + } + } + rs.mu.Lock() + for inst, of := range rs.refused { + if of == r.Milestone { + insts = append(insts, inst) + delete(rs.refused, inst) + } + } + rs.mu.Unlock() + for _, inst := range insts { + rs.fitted(inst) } } diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/owedacts.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/owedacts.go index e27918d7d..b297ff9dc 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/owedacts.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/owedacts.go @@ -12,9 +12,9 @@ import ( // What the gate owes the forge for a change of a delivery record (design 47 G A-39, A-35, §158, §161 to §167): every change of // a record, a step's outcome applied or the record ended, says a line on the commit's note of each member pull request's head -// (` record : ()`) and replaces each member's delivery view with what `show` answers for the -// record; a comment is said only when the record ends failed or held, naming why. Each act goes through owe (owed.go): kept in -// the record's acts, said as the seat's event owed, and ended on owed.ended. +// (` record : ()`) and replaces each member's delivery view with the record's summary, its head's +// fields and a count of its movers per outcome (H-A-459); a comment is said only when the record ends failed or held, naming why. Each act goes through owe (owed.go): kept as +// an entry of its own among the record's acts, said as the seat's event owed, and ended on owed.ended. // RecordChange is one change of a record: what changed and why; Ended when it is the record's end. type RecordChange struct { @@ -39,7 +39,7 @@ func (h *Holder) recordChanged(r *Record, c RecordChange) { } now := h.Now().UTC() line := fmt.Sprintf("%s record %s: %s (%s)", now.Format("2006-01-02T15:04:05Z"), r.Instance, c.What, c.Why) - view, err := json.MarshalIndent(map[string]any{"records": []recordView{viewOf(r)}}, "", " ") + view, err := json.MarshalIndent(map[string]any{"records": []recordSummary{summaryOf(r)}}, "", " ") if err != nil { h.Refuse(fmt.Errorf("the record %s cannot be shown on its pull requests: %w", r.Instance, err)) return @@ -70,6 +70,36 @@ func (h *Holder) recordChanged(r *Record, c RecordChange) { } } +// recordSummary is the record a view act shows (design 47 H2 §809, H-A-455 (2), H-A-459): the head's fields, without its +// acts, and a count of its movers per outcome, never the movers' entries, so a view act of any record fits 64 KiB; the +// whole record stays readable through gate_record. +type recordSummary struct { + recordView + MoversPerOutcome map[string]int `json:"movers-per-outcome,omitempty"` +} + +// summaryOf is the record a view act shows. +func summaryOf(r *Record) recordSummary { + v := viewOf(r) + head := r.head() + v.Record = &head + per := map[string]map[string]bool{} + for _, o := range r.Outcomes { + if per[o.Outcome] == nil { + per[o.Outcome] = map[string]bool{} + } + per[o.Outcome][o.Module] = true + } + out := recordSummary{recordView: v} + for outcome, movers := range per { + if out.MoversPerOutcome == nil { + out.MoversPerOutcome = map[string]int{} + } + out.MoversPerOutcome[outcome] = len(movers) + } + return out +} + // endedFailedOrHeld is the comment a record ended says when one of its outcomes is failed or held, naming why; "" otherwise. func endedFailedOrHeld(r *Record) string { var lines []string diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/owedacts_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/owedacts_test.go index f7331bc9e..30c7c83b0 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/owedacts_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/owedacts_test.go @@ -1,6 +1,7 @@ package main import ( + "encoding/json" "strings" "testing" "time" @@ -41,7 +42,7 @@ func TestARecordsChangesOweTheNoteTheViewAndWhenFailedOrHeldAComment(t *testing. return } t.Run("an outcome applied says the note and the view, no comment", func(t *testing.T) { - _, rs, r, f := setup(t) + h, rs, r, f := setup(t) if _, err := rs.Outcome(r, RecordOutcome{Module: "mesh-x", Fingerprint: "src2:f1", Node: "ace", Step: "build", Outcome: OutcomeBuilt, Why: "built it"}); err != nil { t.Fatal(err) } @@ -62,7 +63,8 @@ func TestARecordsChangesOweTheNoteTheViewAndWhenFailedOrHeldAComment(t *testing. if o.Act == seat.OwedActNote && (o.Line != "2026-11-11T21:00:00Z record "+r.Instance+": mesh-x on ace build built (built it)" || o.Commit != head) { t.Fatalf("note %+v", o) } - if o.Act == seat.OwedActView && !strings.Contains(o.Text, r.Instance) { + // H-A-459: the view is the record's head fields and a count of movers per outcome, never the movers' entries. + if o.Act == seat.OwedActView && (!strings.Contains(o.Text, r.Instance) || !strings.Contains(strings.Join(strings.Fields(o.Text), ""), `"movers-per-outcome":{"built":1}`) || strings.Contains(o.Text, `"outcomes"`) || strings.Contains(o.Text, `"entries"`)) { t.Fatalf("view %q", o.Text) } } @@ -70,6 +72,36 @@ func TestARecordsChangesOweTheNoteTheViewAndWhenFailedOrHeldAComment(t *testing. if _, err := rs.Outcome(r, RecordOutcome{Module: "mesh-x", Fingerprint: "src2:f1", Node: "ace", Step: "build", Outcome: OutcomeBuilt, Why: "built it"}); err != nil || len(f.said) != 2 { t.Fatalf("said again: %v %d", err, len(f.said)) } + // Design 47 H2 §809, H-A-455 (1): each act is an entry of its own, act/, naming its record; the + // head entry holds none, and an act performed is removed. + l := rs.Ledger.(*ledgerFake) + for _, o := range f.said { + inst, err := ActInstance(r.Milestone, o.ID) + if err != nil || !strings.Contains(string(l.states[inst]), `"record":"main.47"`) { + t.Fatalf("the act %s has no entry of its own at %s: %s (%v)", o.ID, inst, l.states[inst], err) + } + } + if strings.Contains(string(l.states[r.Instance]), `"acts"`) { + t.Fatalf("the head entry holds acts: %s", l.states[r.Instance]) + } + note := f.said[0] + body, err := forgemsg.EncodeOwedEnded(forgemsg.OwedEnded{V: 1, Of: seat.Seat, ID: note.ID, Outcome: forgemsg.OwedPerformed, At: at}) + if err != nil { + t.Fatal(err) + } + if err := h.OwedEnded(body); err != nil { + t.Fatal(err) + } + inst, _ := ActInstance(r.Milestone, note.ID) + if string(l.states[inst]) != "null" || h.Status("")["owed"] != 1 { + t.Fatalf("the act performed is kept as %s, %v owed", l.states[inst], h.Status("")["owed"]) + } + // H-A-457: a head written before, its acts embedded, is read for its fields; those acts are ignored. + l.states[r.Instance] = json.RawMessage(strings.Replace(string(l.states[r.Instance]), `{`, `{"acts":{"old":{"id":"old","act":"view"}},`, 1)) + read, err := rs.Read(r.Instance) + if _, old := read.Acts["old"]; err != nil || old || len(read.Acts) != 1 || read.Milestone != "main.47" { + t.Fatalf("an old head's embedded acts are read: %+v (%v)", read.Acts, err) + } }) // Design 47 H2 §807, H-A-432 (2): an act whose pull request closed unmerged ends not-needed, kept in the record's acts as // ended like performed and could not be performed, and no longer owed. @@ -133,6 +165,10 @@ func TestARecordsChangesOweTheNoteTheViewAndWhenFailedOrHeldAComment(t *testing. t.Fatalf("said %+v", f.said) } for _, o := range f.said { + // H-A-455 (2): the view is the record without its acts, the end's too, which follows the outcome's acts. + if o.Act == seat.OwedActView && strings.Contains(o.Text, `"acts"`) { + t.Fatalf("the view shows the record's acts: %s", o.Text) + } if o.Act == seat.OwedActComment && (!strings.Contains(o.Text, "tests red") || !strings.Contains(o.Text, "never runs")) { t.Fatalf("comment %q", o.Text) } diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/owedended.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/owedended.go index 1d6d5551f..8424195e9 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/owedended.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/owedended.go @@ -1,7 +1,9 @@ package main import ( + "encoding/json" "fmt" + "strings" "time" seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery" @@ -55,8 +57,11 @@ func (h *Holder) OwedEnded(body []byte) error { // RecordAct is one act the gate owes the forge for a pull request a record holds: said, with when (At, while Outcome is empty) // and as the seat's event says it (Owed, which saying it again at the gate's start repeats), then ended with the forge -// bridge's outcome and when (A-35, design 47 §165). +// bridge's outcome and when (A-35, design 47 §165). Each is an entry of its own, `act/`, its record's +// key the index's field, removed when performed (design 47 H2 §809, H-A-455 (1)). type RecordAct struct { + // Record is its record's key, `.`, the index acts's field. + Record string `json:"record,omitempty"` ID string `json:"id"` Act string `json:"act"` Repository string `json:"repository"` @@ -68,22 +73,87 @@ type RecordAct struct { Owed seat.Owed `json:"owed"` } -// Act writes one act into the record: said (no outcome), or ended; nothing when the record holds it so already. +// TypeAct is the ledger type of an act's entry, and ActsIndex the ledger's index of the acts' entries on their record's key +// (manifest `requires` records' types and indexes; H-A-455 (1)). +const ( + TypeAct = "act" + ActsIndex = "acts" +) + +// ActOperations are an act's operations, in the order the manifest's type act allows them. +var ActOperations = []string{OpAct} + +// ActInstance is the entry of a record's act: `act/`, escaped as a name. +func ActInstance(milestone, id string) (string, error) { + if strings.TrimSpace(id) == "" { + return "", fmt.Errorf("an act of the record %s names its id", milestone) + } + inst := TypeAct + "/" + milestoneToken(milestone+".act."+id) + if !segmentRe.MatchString(strings.TrimPrefix(inst, TypeAct+"/")) { + return "", fmt.Errorf("the act %s of %s cannot be named: %q", id, milestone, inst) + } + return inst, nil +} + +// Act writes one act's entry: said (no outcome), or ended; removed when the forge bridge performed it; nothing when the +// record holds it so already. Its write is measured as every entry's is (H-A-455 (3)). func (rs *Records) Act(r *Record, a RecordAct) error { if had, ok := r.Acts[a.ID]; ok && had.Outcome == a.Outcome { return nil } - seq, err := rs.Ledger.Apply(r.Instance, OpAct, a) + inst, err := ActInstance(r.Milestone, a.ID) if err != nil { return err } + if a.Outcome == forgemsg.OwedPerformed { + if err := rs.Ledger.Remove(inst); err != nil { + return err + } + delete(r.Acts, a.ID) + rs.fitted(inst) + return nil + } + a.Record = r.Milestone + if _, err := rs.write(r, inst, OpAct, a, a); err != nil { + return err + } if r.Acts == nil { r.Acts = map[string]RecordAct{} } - r.Acts[a.ID], r.Sequence = a, seq + r.Acts[a.ID] = a return nil } +// acts reads the acts' entries of a record through the index, one query, and each entry; nil when it has none. +func (rs *Records) acts(milestone string) (map[string]RecordAct, error) { + insts, err := rs.Ledger.Query(ActsIndex, moversKey(milestone)) + if err != nil { + return nil, fmt.Errorf("the acts of the record %s cannot be read: %w", milestone, err) + } + var out map[string]RecordAct + for _, inst := range insts { + raw, _, err := rs.Ledger.Get(inst) + if err != nil { + return nil, err + } + if raw == nil { + continue + } + var a RecordAct + if err := json.Unmarshal(raw, &a); err != nil { + return nil, fmt.Errorf("the act %s is not readable: %w", inst, err) + } + if a.Record != milestone || a.ID == "" { + continue + } + if out == nil { + out = map[string]RecordAct{} + } + out[a.ID] = a + } + return out, nil +} + // owedBound is how long an act said may wait for its owed.ended before it is stalled: many times a forge call's time, and // short beside the check's hour (design 47 *What the gate owes the forge*). const owedBound = 15 * time.Minute diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record.go index 26ed54871..e80ff1ea7 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record.go @@ -32,7 +32,7 @@ const RecordsLedger = "records" // RecordForm is the form of the records' layout (design 47 *A module's kept data on the bus*): raised whenever the gate // changes how it writes them. -const RecordForm = 2 +const RecordForm = 3 // The ledger's two types: a record, and the list of the records opened in one month (the ledger is listed through it, // since a bundle has no listing of a ledger's instances; a month of about 600 records keeps it far under a value's 256 @@ -53,13 +53,13 @@ const ( OpAway = "away" // set-entry into away, by node OpBack = "back" // remove from away, by node OpEnd = "end" // set-fields: ended - OpAct = "act" // set-entry into acts, by id: an act owed the forge with its outcome + OpAct = "act" // an act's entry (TypeAct): set-fields, an act owed the forge with its outcome OpHold = "hold" // a month's list: set-entry into records, by instance OpDrop = "drop" // a month's list: remove from records, by instance ) // RecordOperations are a record's operations, in the order the manifest's type allows them. -var RecordOperations = []string{OpOpen, OpGroup, OpTrace, OpMovers, OpPhases, OpAway, OpBack, OpAct, OpEnd} +var RecordOperations = []string{OpOpen, OpGroup, OpTrace, OpMovers, OpPhases, OpAway, OpBack, OpEnd} // HeldOperations are the month's list's operations. var HeldOperations = []string{OpHold, OpDrop} @@ -95,7 +95,8 @@ type Record struct { Outcomes map[string]RecordOutcome `json:"outcomes,omitempty"` // Away are the nodes away, by node, each with since when. Away map[string]NodeAway `json:"away,omitempty"` - // Acts are the acts owed the forge for the record's pull requests, by id, with their outcomes (owedended.go). + // Acts are the acts owed the forge for the record's pull requests, by id, with their outcomes (owedended.go): each an + // entry of its own, joined when read, never on the head entry (design 47 H2 §809, H-A-455 (1)). Acts map[string]RecordAct `json:"acts,omitempty"` Ended *RecordEnded `json:"ended,omitempty"` // Recovery marks the merge its author named a recovery by the seat's verb recovery (design 47 G §81, A-23). @@ -380,6 +381,8 @@ type Records struct { Class func(fingerprint string) (string, error) // TooLarge raises (raised) or clears record-too-large for an entry (§809); nil raises none. TooLarge func(instance string, bytes int, raised bool) + // refused are the entries a write was refused for over EntryLimit, each with its record's key, until the record ends. + refused map[string]string // held are the records kept, by instance, read once from the ledger (Current) and kept as the gate writes them: the // gate is their one writer. @@ -494,6 +497,11 @@ func (rs *Records) Read(instance string) (*Record, error) { return nil, err } r.join(ms) + // A head written before the acts had entries of their own is read for its fields; the acts embedded in it are ignored + // (design 47 H2 §809, H-A-457): the record's acts are its act entries alone. + if r.Acts, err = rs.acts(r.Milestone); err != nil { + return nil, err + } return &r, nil } @@ -549,7 +557,7 @@ func (rs *Records) Open(o RecordOpening) (r *Record, opened bool, err error) { } body := map[string]any{"milestone": milestone, "branch": o.Branch, "source": o.Source, "call": o.Call, "heads": o.Heads, "opened": now, "form": RecordForm, "merge-called": called} - if _, err := rs.Ledger.Apply(inst, OpOpen, body); err != nil { + if _, err := rs.write(&Record{Milestone: milestone}, inst, OpOpen, body, body); err != nil { return nil, false, err } if _, err := rs.Ledger.Apply(heldInstance(now), OpHold, heldEntry{Instance: inst, Opened: now}); err != nil { @@ -677,6 +685,15 @@ func (rs *Records) Sweep(now time.Time) error { return err } } + for id := range r.Acts { + inst, err := ActInstance(r.Milestone, id) + if err != nil { + return err + } + if err := rs.Ledger.Remove(inst); err != nil { + return err + } + } if err := rs.Ledger.Remove(e.Instance); err != nil { return err } @@ -697,15 +714,11 @@ func (rs *Records) SetGroup(r *Record, g RecordGroup) error { } next := *r next.Group = &g - if err := rs.fits(r.Instance, next.head()); err != nil { - return err - } - seq, err := rs.Ledger.Apply(r.Instance, OpGroup, map[string]any{"group": g}) + seq, err := rs.write(r, r.Instance, OpGroup, map[string]any{"group": g}, next.head()) if err != nil { return err } r.Group, r.Sequence = &g, seq - rs.fitted(r.Instance) return nil } @@ -716,15 +729,11 @@ func (rs *Records) SetTrace(r *Record, t RecordTrace) error { } next := *r next.Trace = &t - if err := rs.fits(r.Instance, next.head()); err != nil { - return err - } - seq, err := rs.Ledger.Apply(r.Instance, OpTrace, map[string]any{"trace": t}) + seq, err := rs.write(r, r.Instance, OpTrace, map[string]any{"trace": t}, next.head()) if err != nil { return err } r.Trace, r.Sequence = &t, seq - rs.fitted(r.Instance) return nil } @@ -744,15 +753,11 @@ func (rs *Records) Outcome(r *Record, o RecordOutcome) (applied bool, err error) } next := *m next.Outcomes = withEntry(m.Outcomes, o.Key, o) - if err := rs.fits(m.Instance, next); err != nil { - return false, err - } - seq, err := rs.Ledger.Apply(m.Instance, OpOutcome, o) + seq, err := rs.write(r, m.Instance, OpOutcome, o, next) if err != nil { return false, err } m.Outcomes, m.Sequence = next.Outcomes, seq - rs.fitted(m.Instance) if r.Outcomes == nil { r.Outcomes = map[string]RecordOutcome{} } @@ -775,15 +780,11 @@ func (rs *Records) AddEntry(r *Record, e RecordEntry) error { } next := *m next.Entries = withEntry(m.Entries, e.Key, e) - if err := rs.fits(m.Instance, next); err != nil { - return err - } - seq, err := rs.Ledger.Apply(m.Instance, OpEntry, e) + seq, err := rs.write(r, m.Instance, OpEntry, e, next) if err != nil { return err } m.Entries, m.Sequence = next.Entries, seq - rs.fitted(m.Instance) if r.Entries == nil { r.Entries = map[string]RecordEntry{} } @@ -801,9 +802,9 @@ func withEntry[V any](m map[string]V, k string, v V) map[string]V { return out } -// head is the record's head entry as the ledger holds it: every record-wide field, none of a mover's. +// head is the record's head entry as the ledger holds it: every record-wide field, none of a mover's and no act. func (r Record) head() Record { - r.Entries, r.Outcomes, r.movers = nil, nil, nil + r.Entries, r.Outcomes, r.movers, r.Acts = nil, nil, nil, nil return r } diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record_test.go index e0262dc92..e31a8b5d0 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record_test.go @@ -248,6 +248,11 @@ func TestTheDeliveryRecordIsALedgerInstanceKeyedByItsSourceRevision(t *testing.T Outcome: OutcomeFailed, Why: why, At: at}); err == nil || !strings.Contains(err.Error(), RecordTooLarge) || len(found) != 1 { t.Fatalf("an entry over 64 KiB was taken (%v), findings %v", err, found) } + // H-A-455 (3): an act's write is measured as a mover's is, and the record's end clears its finding too. + if err := rs.Act(read, RecordAct{ID: "view@1.a", Act: "view", Repository: "novox/app", Number: 3, Head: heads["novox/app@3"], + Owed: meshdelivery.Owed{Act: meshdelivery.OwedActView, Text: why}}); err == nil || !strings.Contains(err.Error(), RecordTooLarge) || len(found) != 2 { + t.Fatalf("an act over 64 KiB was taken (%v), findings %v", err, found) + } if err := rs.End(read, "every present node runs its entries", ""); err != nil || len(found) != 0 { t.Fatalf("the record's end left %v (%v)", found, err) } @@ -457,7 +462,8 @@ func TestTheDeliveryRecordIsALedgerInstanceKeyedByItsSourceRevision(t *testing.T // W11.1: the manifest's ledger types allow exactly the operations the gate applies, and the ledger is capped at 768 MiB. func TestTheManifestsLedgerTypesAreTheRecordsOperations(t *testing.T) { l := newLedgerFake(t) - for name, want := range map[string][]string{TypeRecord: RecordOperations, TypeHeld: HeldOperations, TypeMover: MoverOperations} { + for name, want := range map[string][]string{TypeRecord: RecordOperations, TypeHeld: HeldOperations, TypeMover: MoverOperations, + TypeAct: ActOperations} { var got []string for _, op := range l.types[name].Operations { got = append(got, op.Name) @@ -473,6 +479,9 @@ func TestTheManifestsLedgerTypesAreTheRecordsOperations(t *testing.T) { if ix := l.types[TypeMover].Indexes; len(ix) != 1 || ix[0].Name != MoversIndex || !slices.Equal(ix[0].On, []string{"record"}) { t.Errorf("the type mover's indexes are %+v, not one %s on its record's key", ix, MoversIndex) } + if ix := l.types[TypeAct].Indexes; len(ix) != 1 || ix[0].Name != ActsIndex || !slices.Equal(ix[0].On, []string{"record"}) { + t.Errorf("the type act's indexes are %+v, not one %s on its record's key (H-A-455 (1))", ix, ActsIndex) + } } // W11.2 (design 47 *The delivery record*): the record's outcomes arrive as events through the gate's consumer: each diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/recordevents.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/recordevents.go index 86e785113..5d5bc3640 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/recordevents.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/recordevents.go @@ -38,7 +38,9 @@ func (rs *Records) NodeAway(r *Record, node string, since time.Time) error { } if _, ok := r.Away[node]; !ok { a := NodeAway{Node: node, Since: since.UTC()} - seq, err := rs.Ledger.Apply(r.Instance, OpAway, a) + next := *r + next.Away = withEntry(r.Away, node, a) + seq, err := rs.write(r, r.Instance, OpAway, a, next.head()) if err != nil { return err } @@ -56,7 +58,14 @@ func (rs *Records) NodeBack(r *Record, node string) error { if _, ok := r.Away[node]; !ok { return nil } - seq, err := rs.Ledger.Apply(r.Instance, OpBack, map[string]any{"node": node}) + next := *r + next.Away = map[string]NodeAway{} + for n, a := range r.Away { + if n != node { + next.Away[n] = a + } + } + seq, err := rs.write(r, r.Instance, OpBack, map[string]any{"node": node}, next.head()) if err != nil { return err } @@ -141,7 +150,12 @@ func (rs *Records) end(r *Record, why, by string) error { body["delivery-time"], body["class"] = took.String(), class } } - seq, err := rs.Ledger.Apply(r.Instance, OpEnd, body) + next := *r + next.Ended = &e + if took > 0 { + next.DeliveryTime, next.Class = took.String(), class + } + seq, err := rs.write(r, r.Instance, OpEnd, body, next.head()) if err != nil { return err } @@ -149,11 +163,12 @@ func (rs *Records) end(r *Record, why, by string) error { if took > 0 { r.DeliveryTime, r.Class = took.String(), class } - rs.clearTooLarge(r) rs.keep(r) if rs.Changed != nil { rs.Changed(r, RecordChange{What: "ended", Why: why, Ended: true}) } + // The end clears record-too-large of every entry of the record, the acts its end owed among them (§809, H-A-455 (3)). + rs.clearTooLarge(r) return nil } diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/recovery.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/recovery.go index 3259ce5a1..a6a931398 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/recovery.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/recovery.go @@ -31,7 +31,9 @@ func (rs *Records) Recovery(repository, commit, why, by string) (meshdelivery.Re if r.Recovery != nil && *r.Recovery == want { return meshdelivery.RecoveryAnswer{By: by, Commit: head, Recovery: true, Repository: repository}, nil } - seq, err := rs.Ledger.Apply(r.Instance, OpTrace, map[string]any{"recovery": want}) + next := *r + next.Recovery = &want + seq, err := rs.write(r, r.Instance, OpTrace, map[string]any{"recovery": want}, next.head()) if err != nil { return meshdelivery.RecoveryAnswer{}, err } diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/times.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/times.go index da2bde271..da3762abf 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/times.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/times.go @@ -57,15 +57,11 @@ func (rs *Records) SetPhases(r *Record, p *runner.PipelineEndedPhases) error { } next := *r next.Phases = p - if err := rs.fits(r.Instance, next.head()); err != nil { - return err - } - seq, err := rs.Ledger.Apply(r.Instance, OpPhases, map[string]any{"phases": p}) + seq, err := rs.write(r, r.Instance, OpPhases, map[string]any{"phases": p}, next.head()) if err != nil { return err } r.Phases, r.Sequence = p, seq - rs.fitted(r.Instance) return nil } diff --git a/modules/mesh-delivery-gate/module.json b/modules/mesh-delivery-gate/module.json index 37f784c26..a6cb5797e 100644 --- a/modules/mesh-delivery-gate/module.json +++ b/modules/mesh-delivery-gate/module.json @@ -103,30 +103,9 @@ "ended": { "type": "object" }, - "id": { - "type": "string" - }, - "act": { - "type": "string" - }, - "repository": { - "type": "string" - }, - "number": { - "type": "integer" - }, - "head": { - "type": "string" - }, - "words": { - "type": "string" - }, "recovery": { "type": "object" }, - "owed": { - "type": "object" - }, "movers": { "type": "integer" }, @@ -208,24 +187,6 @@ "into": "away", "key": "node" }, - { - "name": "act", - "effect": "set-entry", - "fields": [ - "id", - "act", - "repository", - "number", - "head", - "outcome", - "words", - "at", - "owed" - ], - "into": "acts", - "key": "id", - "sequence": true - }, { "name": "end", "effect": "set-fields", @@ -335,6 +296,68 @@ } ] }, + { + "name": "act", + "schema": { + "record": { + "type": "string" + }, + "id": { + "type": "string" + }, + "act": { + "type": "string" + }, + "repository": { + "type": "string" + }, + "number": { + "type": "integer" + }, + "head": { + "type": "string" + }, + "outcome": { + "type": "string" + }, + "words": { + "type": "string" + }, + "at": { + "type": "string" + }, + "owed": { + "type": "object" + } + }, + "operations": [ + { + "name": "act", + "effect": "set-fields", + "fields": [ + "record", + "id", + "act", + "repository", + "number", + "head", + "outcome", + "words", + "at", + "owed" + ], + "sequence": true + } + ], + "indexes": [ + { + "name": "acts", + "on": [ + "record" + ] + } + ] + }, { "name": "held", "schema": { -- 2.54.0 From 1a54a432cfc0ae84051a1f716fb06b0e45c6eb08 Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 22 Nov 2026 03:10:24 +0100 Subject: [PATCH 54/54] GT1 review round 2: the record's end clears its record-too-large from the findings state MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The entries whose refusal a record's end must clear were kept in process memory, so after a restart a refusal of an entry never written (an act's first write, a mover's) was never cleared. A record-too-large finding now names its record as its subject (its id stays the refused entry), and the record's end clears every held one whose subject is the record, read from the gate's own findings state; the in-memory map goes. A finding raised before this change names its entry as its subject and clears when that entry is next written within 64 KiB. Review finding: record-too-large cleared at the record's end only from process memory (movers.go:228-233, Records.refused). Builds: design 47 H2 §809, H-A-455 (3), H-A-460 --- .../cmd/mesh-delivery-gate/findings.go | 30 ++++++---- .../cmd/mesh-delivery-gate/movers.go | 55 +++++++------------ .../cmd/mesh-delivery-gate/owedended.go | 2 +- .../cmd/mesh-delivery-gate/record.go | 7 +-- .../cmd/mesh-delivery-gate/record_test.go | 36 ++++++++---- 5 files changed, 70 insertions(+), 60 deletions(-) diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/findings.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/findings.go index 1ac846fc4..e9c5afe16 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/findings.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/findings.go @@ -32,32 +32,40 @@ func NewFindingWriter() (*views.HeldWriter[meshconditions.Finding, meshcondition Asked: meshconditions.SubjectFindingsAsked, Module: Module, State: stdio.State("findings"), Kinds: []string{HeadUntaken, RecordTooLarge}}) } -// recordTooLarge raises record-too-large for a delivery record's entry the gate refused over EntryLimit, or clears it when -// the writer holds it (design 47 H2 §809): the record's records hook. -func (h *Holder) recordTooLarge(instance string, bytes int, raised bool) { +// recordTooLarge raises record-too-large for a delivery record's entry the gate refused over EntryLimit, its subject the +// record (design 47 H2 §809; H-A-460), or clears it when the writer holds it; with no instance it clears every one the +// writer holds whose subject is the record (its end, H-A-455 (3)): the record's records hook. +func (h *Holder) recordTooLarge(record, instance string, bytes int, raised bool) { if h.Findings == nil { return } - held := false + var held []string for _, f := range h.Findings.Held() { - if f.Scope == FindingScope && f.ID == instance && f.Kind == RecordTooLarge { - held = true + if f.Scope != FindingScope || f.Kind != RecordTooLarge { + continue + } + if instance == "" && f.Subject.ID == record || instance != "" && f.ID == instance { + held = append(held, f.ID) } } var err error switch { - case raised && !held: + case raised && len(held) == 0: now := h.Now().UTC() err = h.Findings.Write(context.Background(), meshconditions.Finding{Scope: FindingScope, ID: instance, Kind: RecordTooLarge, Observer: Module, Severity: meshconditions.FindingSeverityWarning, Resolver: "operator", Since: now, At: now, Summary: fmt.Sprintf("%s would hold %d bytes, over the %d one entry holds: the write is refused", instance, bytes, EntryLimit), - Headline: "A delivery record's entry is too large", Subject: meshconditions.Subject{Scope: FindingScope, ID: instance}, + Headline: "A delivery record's entry is too large", Subject: meshconditions.Subject{Scope: FindingScope, ID: record}, Explanation: "The gate keeps one entry per record and one per mover, each within 64 KiB (design 47 H2 §809); " + "this entry would pass it, so the write is refused. It clears when the entry is next written within 64 KiB or the record ends."}) - case !raised && held: - err = h.Findings.Clear(context.Background(), FindingScope, instance, RecordTooLarge, "the entry is written within 64 KiB, or its record ended") + case !raised: + for _, id := range held { + if e := h.Findings.Clear(context.Background(), FindingScope, id, RecordTooLarge, "the entry is written within 64 KiB, or its record ended"); e != nil { + err = e + } + } } if err != nil && h.Logf != nil { - h.Logf("[mesh-delivery] record-too-large of %s is not said: %v", instance, err) + h.Logf("[mesh-delivery] record-too-large of %s (%s) is not said: %v", instance, record, err) } } diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/movers.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/movers.go index f70f710ed..8e3ca90ea 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/movers.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/movers.go @@ -211,12 +211,21 @@ func (rs *Records) write(r *Record, instance, operation string, body, state any) if err != nil { return 0, err } - rs.fitted(instance) + rs.fitted(r, instance) return seq, nil } -// fits refuses an entry of r whose state would pass EntryLimit, raising record-too-large for it (§809); the entry is -// remembered with its record, so the record's end clears it whichever entry it was (H-A-455 (3)). +// recordOf is the instance of r's head entry, the subject of its entries' findings (H-A-460). +func recordOf(r *Record) string { + if r.Instance != "" { + return r.Instance + } + inst, _ := RecordInstance(r.Milestone) + return inst +} + +// fits refuses an entry of r whose state would pass EntryLimit, raising record-too-large for it with its record as the +// finding's subject (§809; H-A-460), so the record's end clears it whichever entry it was (H-A-455 (3)). func (rs *Records) fits(r *Record, instance string, state any) error { raw, err := json.Marshal(state) if err != nil { @@ -225,46 +234,24 @@ func (rs *Records) fits(r *Record, instance string, state any) error { if len(raw) <= EntryLimit { return nil } - rs.mu.Lock() - if rs.refused == nil { - rs.refused = map[string]string{} - } - rs.refused[instance] = r.Milestone - rs.mu.Unlock() if rs.TooLarge != nil { - rs.TooLarge(instance, len(raw), true) + rs.TooLarge(recordOf(r), instance, len(raw), true) } return fmt.Errorf("the delivery record's entry %s would hold %d bytes, over the %d an entry holds: refused (%s)", instance, len(raw), EntryLimit, RecordTooLarge) } -// fitted clears record-too-large for an entry written within EntryLimit (TooLarge clears only one it holds). -func (rs *Records) fitted(instance string) { +// fitted clears record-too-large for an entry of r written within EntryLimit (TooLarge clears only one it holds). +func (rs *Records) fitted(r *Record, instance string) { if rs.TooLarge != nil { - rs.TooLarge(instance, 0, false) + rs.TooLarge(recordOf(r), instance, 0, false) } } -// clearTooLarge clears record-too-large for every entry of a record at its end: its head, its movers, its acts and every -// entry of it a write was refused for. +// clearTooLarge clears, at a record's end, every record-too-large the gate's findings hold whose subject is the record, +// whichever entry it was refused for and whenever it was raised: read from the findings state, nothing kept in process +// memory (§809, H-A-455 (3), H-A-460). func (rs *Records) clearTooLarge(r *Record) { - insts := []string{r.Instance} - for _, m := range r.movers { - insts = append(insts, m.Instance) - } - for id := range r.Acts { - if inst, err := ActInstance(r.Milestone, id); err == nil { - insts = append(insts, inst) - } - } - rs.mu.Lock() - for inst, of := range rs.refused { - if of == r.Milestone { - insts = append(insts, inst) - delete(rs.refused, inst) - } - } - rs.mu.Unlock() - for _, inst := range insts { - rs.fitted(inst) + if rs.TooLarge != nil { + rs.TooLarge(recordOf(r), "", 0, false) } } diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/owedended.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/owedended.go index 8424195e9..d8e6e6319 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/owedended.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/owedended.go @@ -110,7 +110,7 @@ func (rs *Records) Act(r *Record, a RecordAct) error { return err } delete(r.Acts, a.ID) - rs.fitted(inst) + rs.fitted(r, inst) return nil } a.Record = r.Milestone diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record.go index e80ff1ea7..b7f4804ee 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record.go @@ -379,10 +379,9 @@ type Records struct { // Class reads a mover's class from its catalog entry by its fingerprint (design 47 H2 §810; H-A-394 (2)), once, when its // entry opens; nil reads none, every mover a leaf. Class func(fingerprint string) (string, error) - // TooLarge raises (raised) or clears record-too-large for an entry (§809); nil raises none. - TooLarge func(instance string, bytes int, raised bool) - // refused are the entries a write was refused for over EntryLimit, each with its record's key, until the record ends. - refused map[string]string + // TooLarge raises (raised) or clears record-too-large for an entry of the record named by its head's instance, the + // finding's subject (§809; H-A-460); an empty instance clears every one held of the record (its end). nil raises none. + TooLarge func(record, instance string, bytes int, raised bool) // held are the records kept, by instance, read once from the ledger (Current) and kept as the gate writes them: the // gate is their one writer. diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record_test.go index e31a8b5d0..513150b59 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record_test.go @@ -13,6 +13,7 @@ import ( "time" "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient" + "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery" stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-sdk/go/buscalls" @@ -199,14 +200,11 @@ func TestTheDeliveryRecordIsALedgerInstanceKeyedByItsSourceRevision(t *testing.T // 64 KiB, the head counting 126; an entry made too large is refused with record-too-large, cleared at the record's end. t.Run("a record of 126 movers writes 127 entries, each within 64 KiB", func(t *testing.T) { l := newLedgerFake(t) - found := map[string]bool{} - rs := &Records{Ledger: l, Now: func() time.Time { return at }, TooLarge: func(inst string, _ int, raised bool) { - if raised { - found[inst] = true - } else { - delete(found, inst) - } - }} + // The findings are the gate's own findings state (H-A-460): a fresh Records, as after a restart, reads them there. + fw := &findingsFake{held: map[string]meshconditions.Finding{}} + h := &Holder{Findings: fw, Now: func() time.Time { return at }} + found := fw.held + rs := &Records{Ledger: l, Now: func() time.Time { return at }, TooLarge: h.recordTooLarge} r, _, err := rs.Open(opening("main", 90, heads)) if err != nil { t.Fatal(err) @@ -253,8 +251,26 @@ func TestTheDeliveryRecordIsALedgerInstanceKeyedByItsSourceRevision(t *testing.T Owed: meshdelivery.Owed{Act: meshdelivery.OwedActView, Text: why}}); err == nil || !strings.Contains(err.Error(), RecordTooLarge) || len(found) != 2 { t.Fatalf("an act over 64 KiB was taken (%v), findings %v", err, found) } - if err := rs.End(read, "every present node runs its entries", ""); err != nil || len(found) != 0 { - t.Fatalf("the record's end left %v (%v)", found, err) + for _, f := range found { + if f.Subject.ID != read.Instance { + t.Fatalf("a refusal of %s names %q as its subject, not its record %s", f.ID, f.Subject.ID, read.Instance) + } + } + // Another record's refusal is not this record's to clear. + other, _ := RecordInstance("main.other") + h.recordTooLarge(other, "mover/main_other_x", EntryLimit+1, true) + // A fresh Records, as after a restart, ends the record: every refusal of it clears, the act never written among + // them, read from the findings state alone (§809, H-A-455 (3), H-A-460). + restarted := &Records{Ledger: l, Now: func() time.Time { return at }, TooLarge: h.recordTooLarge} + again, err := restarted.Read(r.Instance) + if err != nil { + t.Fatal(err) + } + if err := restarted.End(again, "every present node runs its entries", ""); err != nil || len(found) != 1 { + t.Fatalf("the record's end after a restart left %v (%v)", found, err) + } + if _, ok := found[FindingScope+".mover/main_other_x."+RecordTooLarge]; !ok { + t.Fatalf("the record's end cleared another record's refusal: %v", found) } }) -- 2.54.0