Merge branch 'feat/47-G-build' into feat/47-G-build-assigner
This commit is contained in:
@@ -5,6 +5,8 @@ import (
|
||||
"encoding/json"
|
||||
"git.novox.be/novox/mesh-contracts/go/seats/meshbroker"
|
||||
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery"
|
||||
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshpipelinerunner"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
@@ -41,6 +43,8 @@ func consumed(w *watch.Watchman, subject string, body []byte, logf func(string,
|
||||
return
|
||||
}
|
||||
w.Accepted(a)
|
||||
case pipelineSubject(w, subject, body):
|
||||
case recordSubject(w, subject, body):
|
||||
case brokerSubject(subject) && strings.HasSuffix(subject, ".event."+eventCreated):
|
||||
if s, err := watch.ReadBrokerSaid(body); err != nil {
|
||||
logf("%v; left", err)
|
||||
@@ -140,3 +144,41 @@ func hearConsumed(ctx context.Context, js jetstream.JetStream, cred process.Acco
|
||||
func brokerSubject(subject string) bool {
|
||||
return strings.HasPrefix(subject, "mesh.seat."+meshbroker.Seat+".event.")
|
||||
}
|
||||
|
||||
// pipelineSubject hands the gate's `source-moved` and the runner's `pipeline-started` to the watchman (watch/pipelines.go,
|
||||
// design 47 G §12): the head of a watched branch and its pipeline. It says whether the subject was one of them.
|
||||
func pipelineSubject(w *watch.Watchman, subject string, body []byte) bool {
|
||||
switch {
|
||||
case strings.HasPrefix(subject, "mesh.seat."+meshdelivery.Seat+".event."+meshdelivery.EventSourceMoved+"."):
|
||||
w.SourceMoved(body)
|
||||
case strings.HasPrefix(subject, "mesh.seat."+meshpipelinerunner.Seat+".event."+meshpipelinerunner.EventPipelineStarted+"."):
|
||||
w.PipelineStarted(body)
|
||||
case strings.HasPrefix(subject, "mesh.seat."+meshpipelinerunner.Seat+".event."+meshpipelinerunner.EventStepStarted+"."),
|
||||
strings.HasPrefix(subject, "mesh.seat."+meshpipelinerunner.Seat+".event."+meshpipelinerunner.EventStepEnded+"."):
|
||||
w.StepEvent(body)
|
||||
default:
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// recordSubject hands the seats' `filling`, `full` and `room` of a store and the seat mesh-delivery's `record-opened` and
|
||||
// `record-ended` to the watchman (watch/records.go, design 47 G §148, §181). It says whether the subject was one of them.
|
||||
func recordSubject(w *watch.Watchman, subject string, body []byte) bool {
|
||||
parts := strings.Split(subject, ".")
|
||||
if len(parts) != 5 || parts[0] != "mesh" || parts[1] != "seat" || parts[3] != "event" {
|
||||
return false
|
||||
}
|
||||
seat, event := parts[2], parts[4]
|
||||
switch {
|
||||
case event == meshdelivery.EventFilling || event == meshdelivery.EventFull || event == meshdelivery.EventRoom:
|
||||
w.StoreFill(seat, event, body)
|
||||
case seat == meshdelivery.Seat && event == meshdelivery.EventRecordOpened:
|
||||
w.RecordOpen(body)
|
||||
case seat == meshdelivery.Seat && event == meshdelivery.EventRecordEnded:
|
||||
w.RecordEnded(body)
|
||||
default:
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
@@ -55,6 +55,23 @@ func main() {
|
||||
type kvOut struct {
|
||||
ctx context.Context
|
||||
presence, pulse, stuk, findings jetstream.KeyValue
|
||||
js jetstream.JetStream
|
||||
cred process.Account
|
||||
}
|
||||
|
||||
// Say publishes an event of the seat mesh-watchdogs, said under the watchman's own account (design 47 G §389, §436).
|
||||
func (o *kvOut) Say(subject string, body []byte, id string) error {
|
||||
msg := nats.NewMsg(subject)
|
||||
msg.Header.Set("x-event-id", id)
|
||||
msg.Header.Set("x-source", o.cred.Module)
|
||||
msg.Header.Set("x-node", o.cred.Node)
|
||||
msg.Header.Set("x-time", time.Now().UTC().Format(time.RFC3339Nano))
|
||||
msg.Header.Set("content-type", "application/json")
|
||||
msg.Data = body
|
||||
pub, cancel := context.WithTimeout(context.WithoutCancel(o.ctx), 10*time.Second)
|
||||
defer cancel()
|
||||
_, err := o.js.PublishMsg(pub, msg, jetstream.WithMsgID(id))
|
||||
return err
|
||||
}
|
||||
|
||||
func (o *kvOut) PutFinding(key string, value []byte) error {
|
||||
@@ -251,7 +268,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
return err
|
||||
}
|
||||
|
||||
w := watch.New(&kvOut{ctx: ctx, presence: kvs.presence, pulse: kvs.pulse, stuk: kvs.stuck, findings: kvs.findings}, time.Now)
|
||||
w := watch.New(&kvOut{ctx: ctx, presence: kvs.presence, pulse: kvs.pulse, stuk: kvs.stuck, findings: kvs.findings, js: js, cred: cred}, time.Now)
|
||||
w.Logf(logger.Printf)
|
||||
// What the wants and the modules hold is ended by their own late read (watch.ReadLate), never by the start's end.
|
||||
w.Awaits(watch.AssignerWantsBucket)
|
||||
|
||||
@@ -152,5 +152,6 @@ func (o *findingsOut) PutPresence(string, bool) error { return nil
|
||||
func (o *findingsOut) PutPulse(string, []byte, time.Duration) error { return nil }
|
||||
func (o *findingsOut) DeletePulse(string) error { o.pulseDeletes++; return nil }
|
||||
func (o *findingsOut) PutStuck(string, []byte) error { return nil }
|
||||
func (o *findingsOut) Say(string, []byte, string) error { return nil }
|
||||
func (o *findingsOut) PutFinding(key string, _ []byte) error { o.findings[key] = true; return nil }
|
||||
func (o *findingsOut) DeleteFinding(key string) error { delete(o.findings, key); return nil }
|
||||
|
||||
@@ -14,6 +14,8 @@ require (
|
||||
git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-delivery/contracts/go/clients/meshwantsclient v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-delivery/contracts/go/seats/meshpipelinerunner v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-delivery/modules/mesh-grants/busgrants v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-sdk/go v0.0.0-00010101000000-000000000000
|
||||
|
||||
@@ -0,0 +1,169 @@
|
||||
package watch
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery"
|
||||
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshpipelinerunner"
|
||||
)
|
||||
|
||||
// What the runner and the gate say of a watched branch's head, heard as events (design 47 G §12, §75, §112): a head the
|
||||
// gate says moved (`source-moved`) that starts no pipeline within five minutes is the finding `pipeline-stands`, naming the
|
||||
// branch and its head, raised by the condition keeper and cleared by the runner's `pipeline-started` for that head.
|
||||
|
||||
// KindPipelineStands is a watched branch's head, or a step, standing still (design 47 G §12, §75).
|
||||
const KindPipelineStands = "pipeline-stands"
|
||||
|
||||
// HeadBound is how long a watched branch's head may stand with no pipeline started (design 47 G §12, §112: the record's five
|
||||
// minutes).
|
||||
const HeadBound = 5 * time.Minute
|
||||
|
||||
// headRow is the signals table's row of a head with no pipeline.
|
||||
const headRow = "head-pipeline"
|
||||
|
||||
// head is a watched branch's head still waiting for its pipeline.
|
||||
type head struct{ revision int64 }
|
||||
|
||||
// SourceMoved hears the seat mesh-delivery's `source-moved`: the branch's head is a new source revision, and its pipeline
|
||||
// is due within HeadBound. A move made by a gathered group is a member governed by the take-in's own bound (ADR 0633 §5),
|
||||
// so it is not timed here (design 47 G §12). A later move of the same branch replaces the head that waits.
|
||||
func (w *Watchman) SourceMoved(body []byte) {
|
||||
var m meshdelivery.SourceMovedEvent
|
||||
if err := json.Unmarshal(body, &m); err != nil || m.Branch == "" || m.Revision == 0 {
|
||||
w.say("a source-moved does not read, left: %v", err)
|
||||
return
|
||||
}
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
id := headID(m.Branch)
|
||||
w.unsignal(headRow, id)
|
||||
w.unfind(FindingKey(ScopeDelivery, id, KindPipelineStands))
|
||||
if m.Group != "" {
|
||||
delete(w.heads, m.Branch)
|
||||
return
|
||||
}
|
||||
at := w.now()
|
||||
if m.At != nil && !m.At.IsZero() {
|
||||
at = *m.At
|
||||
}
|
||||
w.heads[m.Branch] = head{revision: m.Revision}
|
||||
f := Finding{Scope: ScopeDelivery, ID: id, Kind: KindPipelineStands, Severity: Warning, Source: headRow, Since: at,
|
||||
Summary: fmt.Sprintf("the head of %s (source revision %d) has started no pipeline since %s", m.Branch, m.Revision, stamp(at)),
|
||||
Said: fmt.Sprintf("no pipeline in %s", HeadBound),
|
||||
Headline: "A change has started no pipeline",
|
||||
Explanation: "The head of the watched branch " + m.Branch + " (source revision " + fmt.Sprint(m.Revision) + ") moved " +
|
||||
HeadBound.String() + " ago or more and the pipeline runner has said no pipeline started for it.",
|
||||
Needs: "the runner to start the pipeline (its pipeline_stands and list tools say where it is)"}
|
||||
left := HeadBound - w.now().Sub(at)
|
||||
if left > 0 {
|
||||
w.signal(headRow, id, left, signalDue{Do: dueFind, Finding: f})
|
||||
return
|
||||
}
|
||||
w.find(f)
|
||||
}
|
||||
|
||||
// PipelineStarted hears the runner's `pipeline-started`: it ends the wait of the head it is for (the branch's head at that
|
||||
// source revision or a later one). A run on request is no head's pipeline.
|
||||
func (w *Watchman) PipelineStarted(body []byte) {
|
||||
var p meshpipelinerunner.PipelineStartedEvent
|
||||
if err := json.Unmarshal(body, &p); err != nil || p.Branch == "" {
|
||||
w.say("a pipeline-started does not read, left: %v", err)
|
||||
return
|
||||
}
|
||||
if p.Run != "" {
|
||||
return
|
||||
}
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
h, ok := w.heads[p.Branch]
|
||||
if !ok || p.Source < h.revision {
|
||||
return
|
||||
}
|
||||
delete(w.heads, p.Branch)
|
||||
id := headID(p.Branch)
|
||||
w.unsignal(headRow, id)
|
||||
w.unfind(FindingKey(ScopeDelivery, id, KindPipelineStands))
|
||||
}
|
||||
|
||||
// headID is the finding's id of a branch's head.
|
||||
func headID(branch string) string { return "head-" + unsafeKey.ReplaceAllString(branch, "_") }
|
||||
|
||||
// KindBuildCapacityWaits is a build seat whose oldest queued step has waited the bound (design 47 G §86).
|
||||
const KindBuildCapacityWaits = "build-capacity-waits"
|
||||
|
||||
// CapacityBound is how long a step may wait queued on a build seat's stream before the seat is a finding (design 47 G §86).
|
||||
const CapacityBound = 60 * time.Minute
|
||||
|
||||
// capacityRow is the signals table's row of a build seat's queue.
|
||||
const capacityRow = "build-capacity"
|
||||
|
||||
// StepEvent hears the runner's `step-started` and `step-ended` (design 47 G §86): a step said waiting for a build node is
|
||||
// queued on its seat, owed no progress; it leaves the queue when it starts running or ends. A seat's finding is due when
|
||||
// its oldest queued step has waited CapacityBound, and ends when that seat has none that old.
|
||||
func (w *Watchman) StepEvent(body []byte) {
|
||||
var e meshpipelinerunner.StepEvent
|
||||
if err := json.Unmarshal(body, &e); err != nil || e.Pipeline == "" || e.Result.Step == "" {
|
||||
w.say("a step event does not read, left: %v", err)
|
||||
return
|
||||
}
|
||||
id := e.Pipeline + "/" + e.Result.Step
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
for seat, q := range w.queued {
|
||||
if _, ok := q[id]; ok && (e.Result.Outcome != meshpipelinerunner.StepResultOutcomeWaitingForABuildNode || e.Result.Seat != seat) {
|
||||
delete(q, id)
|
||||
w.capacity(seat)
|
||||
}
|
||||
}
|
||||
if e.Result.Outcome != meshpipelinerunner.StepResultOutcomeWaitingForABuildNode || e.Result.Seat == "" {
|
||||
return
|
||||
}
|
||||
since := w.now()
|
||||
switch {
|
||||
case e.Result.Started != nil:
|
||||
since = *e.Result.Started
|
||||
case e.At != nil:
|
||||
since = *e.At
|
||||
}
|
||||
seat := e.Result.Seat
|
||||
if w.queued[seat] == nil {
|
||||
w.queued[seat] = map[string]time.Time{}
|
||||
}
|
||||
if _, ok := w.queued[seat][id]; !ok {
|
||||
w.queued[seat][id] = since
|
||||
}
|
||||
w.capacity(seat)
|
||||
}
|
||||
|
||||
// capacity judges a build seat's queue again, w.mu held: one signal for its oldest queued step.
|
||||
func (w *Watchman) capacity(seat string) {
|
||||
w.unsignal(capacityRow, seat)
|
||||
key := FindingKey(ScopeDelivery, seat, KindBuildCapacityWaits)
|
||||
var oldest time.Time
|
||||
for _, t := range w.queued[seat] {
|
||||
if oldest.IsZero() || t.Before(oldest) {
|
||||
oldest = t
|
||||
}
|
||||
}
|
||||
if oldest.IsZero() {
|
||||
delete(w.queued, seat)
|
||||
w.unfind(key)
|
||||
return
|
||||
}
|
||||
f := Finding{Scope: ScopeDelivery, ID: seat, Kind: KindBuildCapacityWaits, Severity: Warning, Source: capacityRow, Since: oldest,
|
||||
Summary: fmt.Sprintf("the oldest step queued on %s has waited since %s", seat, stamp(oldest)),
|
||||
Said: "queued for " + CapacityBound.String(),
|
||||
Headline: "Builds wait for a build node",
|
||||
Explanation: "The oldest step queued on the build seat " + seat + " has waited " + CapacityBound.String() +
|
||||
" for a build node to take it.",
|
||||
Needs: "build capacity: a build node present and not paused, or fewer steps asked"}
|
||||
left := CapacityBound - w.now().Sub(oldest)
|
||||
if left > 0 {
|
||||
w.unfind(key)
|
||||
w.signal(capacityRow, seat, left, signalDue{Do: dueFind, Finding: f})
|
||||
return
|
||||
}
|
||||
w.find(f)
|
||||
}
|
||||
@@ -0,0 +1,205 @@
|
||||
package watch
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery"
|
||||
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshpipelinerunner"
|
||||
)
|
||||
|
||||
func moved(branch string, rev int64, group string, at time.Time) []byte {
|
||||
b, _ := json.Marshal(meshdelivery.SourceMovedEvent{Branch: branch, Revision: rev, Group: group, At: &at})
|
||||
return b
|
||||
}
|
||||
|
||||
func started(branch string, rev int64, run string) []byte {
|
||||
b, _ := json.Marshal(meshpipelinerunner.PipelineStartedEvent{Branch: branch, Source: rev, Run: run, Module: "m", Pipeline: "p"})
|
||||
return b
|
||||
}
|
||||
|
||||
// design 47 G §12, §79 (test 76): a head with no pipeline within five minutes is `pipeline-stands`, cleared by the
|
||||
// runner's `pipeline-started` for that head; a gathered group's member and a run on request do not.
|
||||
func TestAHeadWithNoPipelineStands(t *testing.T) {
|
||||
key := FindingKey(ScopeDelivery, headID("main"), KindPipelineStands)
|
||||
cases := []struct {
|
||||
name string
|
||||
run func(w *Watchman, o *fakeOut, c *clock)
|
||||
stand bool
|
||||
}{
|
||||
{"five minutes and no pipeline", func(w *Watchman, o *fakeOut, c *clock) {
|
||||
w.SourceMoved(moved("main", 7, "", c.t))
|
||||
if o.ttls[signalKey(headRow, headID("main"))] != HeadBound || len(o.findings) != 0 {
|
||||
t.Fatalf("within the bound: %v %v", o.ttls, o.findings)
|
||||
}
|
||||
expire(w, c, signalKey(headRow, headID("main")), HeadBound)
|
||||
}, true},
|
||||
{"its pipeline-started clears it", func(w *Watchman, o *fakeOut, c *clock) {
|
||||
w.SourceMoved(moved("main", 7, "", c.t))
|
||||
expire(w, c, signalKey(headRow, headID("main")), HeadBound)
|
||||
w.PipelineStarted(started("main", 7, ""))
|
||||
}, false},
|
||||
{"a pipeline before the bound raises none", func(w *Watchman, o *fakeOut, c *clock) {
|
||||
w.SourceMoved(moved("main", 7, "", c.t))
|
||||
w.PipelineStarted(started("main", 8, ""))
|
||||
if _, ok := o.pulses[signalKey(headRow, headID("main"))]; ok {
|
||||
t.Fatal("the pulse stands")
|
||||
}
|
||||
}, false},
|
||||
{"a pipeline of an older head does not clear a newer head", func(w *Watchman, o *fakeOut, c *clock) {
|
||||
w.SourceMoved(moved("main", 7, "", c.t))
|
||||
w.PipelineStarted(started("main", 6, ""))
|
||||
expire(w, c, signalKey(headRow, headID("main")), HeadBound)
|
||||
}, true},
|
||||
{"a gathered group's member is not timed", func(w *Watchman, o *fakeOut, c *clock) {
|
||||
w.SourceMoved(moved("main", 7, "grp", c.t))
|
||||
if len(o.pulses) != 0 {
|
||||
t.Fatalf("a pulse for a member: %v", o.pulses)
|
||||
}
|
||||
}, false},
|
||||
{"a run on request is no head's pipeline", func(w *Watchman, o *fakeOut, c *clock) {
|
||||
w.SourceMoved(moved("main", 7, "", c.t))
|
||||
w.PipelineStarted(started("main", 7, "run-1"))
|
||||
expire(w, c, signalKey(headRow, headID("main")), HeadBound)
|
||||
}, true},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
w, o, c := setup()
|
||||
tc.run(w, o, c)
|
||||
if _, ok := o.findings[key]; ok != tc.stand {
|
||||
t.Fatalf("the finding stands %v, want %v: %v", ok, tc.stand, o.findings)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func queuedStep(pipeline, step, seat, outcome string, started time.Time) []byte {
|
||||
b, _ := json.Marshal(meshpipelinerunner.StepEvent{Pipeline: pipeline, Branch: "main", Module: "m", Commit: "c",
|
||||
Result: meshpipelinerunner.StepResult{Step: step, Seat: seat, Outcome: outcome, Started: &started}})
|
||||
return b
|
||||
}
|
||||
|
||||
// design 47 G §86: a build seat whose oldest queued step has waited 60 minutes raises one `build-capacity-waits`, cleared
|
||||
// when that step starts.
|
||||
func TestABuildSeatsOldestQueuedStepWaitsAnHour(t *testing.T) {
|
||||
const seat = "node-build-go"
|
||||
key := FindingKey(ScopeDelivery, seat, KindBuildCapacityWaits)
|
||||
wait := meshpipelinerunner.StepResultOutcomeWaitingForABuildNode
|
||||
w, o, c := setup()
|
||||
w.StepEvent(queuedStep("p1", "unit", seat, wait, c.t))
|
||||
c.t = c.t.Add(10 * time.Minute)
|
||||
w.StepEvent(queuedStep("p2", "unit", seat, wait, c.t))
|
||||
pk := signalKey(capacityRow, seat)
|
||||
if o.ttls[pk] != 50*time.Minute || len(o.findings) != 0 {
|
||||
t.Fatalf("within the bound: %v %v", o.ttls, o.findings)
|
||||
}
|
||||
expire(w, c, pk, 50*time.Minute)
|
||||
if _, ok := o.findings[key]; !ok || len(o.findings) != 1 {
|
||||
t.Fatalf("an hour for the oldest: %v", o.findings)
|
||||
}
|
||||
w.StepEvent(queuedStep("p1", "unit", seat, meshpipelinerunner.StepResultOutcomeRunning, c.t))
|
||||
if len(o.findings) != 0 {
|
||||
t.Fatalf("the oldest started, the next has waited 50 minutes: %v", o.findings)
|
||||
}
|
||||
if o.ttls[pk] != 10*time.Minute {
|
||||
t.Fatalf("the next one's hour: %v", o.ttls[pk])
|
||||
}
|
||||
w.StepEvent(queuedStep("p2", "unit", seat, meshpipelinerunner.StepResultOutcomePassed, c.t))
|
||||
if _, ok := o.pulses[pk]; ok || len(o.findings) != 0 {
|
||||
t.Fatalf("the queue is empty: %v %v", o.pulses, o.findings)
|
||||
}
|
||||
}
|
||||
|
||||
// design 47 G §389, §436 (test 106's cause): a machine whose presence key expires says `machine-away`, once, and one
|
||||
// heard again says `machine-back`; a machine heard again that was never away says nothing.
|
||||
func TestAMachinesPresenceIsSaidAsEvents(t *testing.T) {
|
||||
w, o, c := setup()
|
||||
w.Connect("node.ace")
|
||||
w.Connect("node.ace") // heard again while never away
|
||||
w.Disconnect("node.ace")
|
||||
w.Disconnect("node.ace")
|
||||
if len(o.said) != 0 {
|
||||
t.Fatalf("said before the expiry: %v", o.said)
|
||||
}
|
||||
c.t = t0.Add(120 * time.Second)
|
||||
w.Pulse(pulseKeyOf(t, presenceNS, "ace"), Expired, c.t)
|
||||
w.Pulse(pulseKeyOf(t, presenceNS, "ace"), Expired, c.t) // the same expiry heard twice
|
||||
w.Connect("node.ace")
|
||||
w.Connect("node.ace")
|
||||
got := []string{}
|
||||
for _, s := range o.said {
|
||||
got = append(got, s.subject)
|
||||
}
|
||||
want := []string{"mesh.seat.mesh-watchdogs.event.machine-away.ace", "mesh.seat.mesh-watchdogs.event.machine-back.ace"}
|
||||
if len(got) != 2 || got[0] != want[0] || got[1] != want[1] {
|
||||
t.Fatalf("said %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func fillBody(store string, bytes int64) []byte {
|
||||
b, _ := json.Marshal(meshdelivery.FillingEvent{Store: store, Bytes: bytes, Cap: 100})
|
||||
return b
|
||||
}
|
||||
|
||||
// design 47 G §181, §263: a store past its share is `store-filling` naming the part and the store, full is urgent, `room` and a
|
||||
// fill under 70 % clear it.
|
||||
func TestAStoreFillingIsAFinding(t *testing.T) {
|
||||
w, o, _ := setup()
|
||||
key := FindingKey(ScopeDelivery, "mesh-delivery-records", KindStoreFilling)
|
||||
w.StoreFill("mesh-delivery", "filling", fillBody("records", 60))
|
||||
if len(o.findings) != 0 {
|
||||
t.Fatalf("under 70 %%: %v", o.findings)
|
||||
}
|
||||
w.StoreFill("mesh-delivery", "filling", fillBody("records", 85))
|
||||
if o.findings[key].Severity != Warning {
|
||||
t.Fatalf("filling: %v", o.findings)
|
||||
}
|
||||
w.StoreFill("mesh-delivery", "full", fillBody("records", 100))
|
||||
if o.findings[key].Severity != Urgent {
|
||||
t.Fatalf("full: %v", o.findings)
|
||||
}
|
||||
w.StoreFill("mesh-delivery", "room", fillBody("records", 60))
|
||||
if len(o.findings) != 0 {
|
||||
t.Fatalf("room: %v", o.findings)
|
||||
}
|
||||
}
|
||||
|
||||
func recordBody(id string, at time.Time) []byte {
|
||||
b, _ := json.Marshal(meshdelivery.RecordOpenedEvent{Record: id, Milestone: "47/G", At: &at})
|
||||
return b
|
||||
}
|
||||
|
||||
// design 47 G §148, §160: a record open 24 hours is `record-open`, cleared by `record-ended`; a record that ends in time
|
||||
// raises none; while a store is full, which already tells why, it raises none.
|
||||
func TestARecordOpenADayIsAFinding(t *testing.T) {
|
||||
w, o, c := setup()
|
||||
pk := signalKey(recordRow, "r1")
|
||||
w.RecordOpen(recordBody("r1", c.t))
|
||||
if o.ttls[pk] != RecordOpenBound || len(o.findings) != 0 {
|
||||
t.Fatalf("within the bound: %v %v", o.ttls, o.findings)
|
||||
}
|
||||
expire(w, c, pk, RecordOpenBound)
|
||||
key := FindingKey(ScopeDelivery, "r1", KindRecordOpen)
|
||||
if _, ok := o.findings[key]; !ok {
|
||||
t.Fatalf("past 24 hours: %v", o.findings)
|
||||
}
|
||||
w.StoreFill("mesh-delivery", "full", fillBody("records", 100))
|
||||
if _, ok := o.findings[key]; ok {
|
||||
t.Fatalf("a full store covers the record: %v", o.findings)
|
||||
}
|
||||
w.StoreFill("mesh-delivery", "room", fillBody("records", 10))
|
||||
if _, ok := o.findings[key]; !ok {
|
||||
t.Fatalf("room, and the record stands open again: %v", o.findings)
|
||||
}
|
||||
w.RecordEnded(recordBody("r1", c.t))
|
||||
if len(o.findings) != 0 {
|
||||
t.Fatalf("record-ended: %v", o.findings)
|
||||
}
|
||||
w.RecordOpen(recordBody("r2", c.t))
|
||||
w.RecordEnded(recordBody("r2", c.t))
|
||||
if _, ok := o.pulses[signalKey(recordRow, "r2")]; ok {
|
||||
t.Fatal("the pulse of an ended record stands")
|
||||
}
|
||||
}
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshwatchdogsclient"
|
||||
"git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshwatchdogs"
|
||||
)
|
||||
|
||||
// partPrefix is what the watchman keeps presence of among the module users: the parts of delivery.
|
||||
@@ -186,3 +187,39 @@ func (w *Watchman) Rebuild(ask func(body []byte) ([]byte, error)) error {
|
||||
w.Connections(users)
|
||||
return nil
|
||||
}
|
||||
|
||||
// machineAway says `machine-away` of the seat mesh-watchdogs when a machine's presence key has expired (design 47 G §389, §436):
|
||||
// the machine, when it was last heard and when this was said, once until the machine is heard again.
|
||||
func (w *Watchman) machineAway(node string, lastHeard time.Time) {
|
||||
if _, said := w.awaySaid[node]; said {
|
||||
return
|
||||
}
|
||||
at := w.now().UTC()
|
||||
body, err := json.Marshal(meshwatchdogs.MachineAwayEvent{V: 1, Machine: node, At: at, LastHeard: lastHeard.UTC()})
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if err := w.out.Say(meshwatchdogs.SubjectMachineAway(node), body, "machine-away."+node+"."+at.Format(time.RFC3339Nano)); err != nil {
|
||||
w.say("cannot say machine-away of %s: %v", node, err)
|
||||
return
|
||||
}
|
||||
w.awaySaid[node] = lastHeard.UTC()
|
||||
}
|
||||
|
||||
// machineBack says `machine-back` when a machine whose machine-away was said is heard again (design 47 G §389).
|
||||
func (w *Watchman) machineBack(node string) {
|
||||
lastHeard, said := w.awaySaid[node]
|
||||
if !said {
|
||||
return
|
||||
}
|
||||
at := w.now().UTC()
|
||||
body, err := json.Marshal(meshwatchdogs.MachineBackEvent{V: 1, Machine: node, At: at, LastHeard: lastHeard})
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if err := w.out.Say(meshwatchdogs.SubjectMachineBack(node), body, "machine-back."+node+"."+at.Format(time.RFC3339Nano)); err != nil {
|
||||
w.say("cannot say machine-back of %s: %v", node, err)
|
||||
return
|
||||
}
|
||||
delete(w.awaySaid, node)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,149 @@
|
||||
package watch
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery"
|
||||
)
|
||||
|
||||
// What the seats say of the delivery's records and stores, heard as the seats' events (design 47 G §148, §181, §218; ADR 0596
|
||||
// §2), never read from their state:
|
||||
// - `filling`, `full` and `room` of a store against its cap: `store-filling` while it stands past its cap's share, urgent
|
||||
// when it is full, cleared by `room` or by a `filling` under FillingFrom;
|
||||
// - `record-opened` and `record-ended` of a delivery record: `record-open` when it has stood open for 24 hours, unless
|
||||
// another condition already tells why (`acting-part-late`, or a store `full`).
|
||||
// Each is a row of the signals table: its pulse is its bound, written at the event, so a restart loses none.
|
||||
|
||||
// The kinds of these findings.
|
||||
const (
|
||||
KindActingPartLate = "acting-part-late"
|
||||
KindStoreFilling = "store-filling"
|
||||
KindRecordOpen = "record-open"
|
||||
)
|
||||
|
||||
// The rows of the signals table of these findings, which a restart keeps.
|
||||
const (
|
||||
storeRow = "store-fill"
|
||||
recordRow = "record-open"
|
||||
)
|
||||
|
||||
// uncovered is the While of a record-open signal: the finding is written at the bound only while no other condition covers
|
||||
// the record.
|
||||
const uncovered = "uncovered:record-open"
|
||||
|
||||
// RecordOpenBound is how long a delivery record may stay open before it is a finding (design 47 G §148).
|
||||
const RecordOpenBound = 24 * time.Hour
|
||||
|
||||
// FillingFrom is the share of a store's cap under which a `filling` clears its finding (design 47 G §181).
|
||||
const FillingFrom = 0.70
|
||||
|
||||
// StoreFill hears a seat's `filling`, `full` or `room` of one of its stores. A `filling` is a warning, naming the part and the
|
||||
// store, unless its share is under FillingFrom, which clears it; `full` is urgent, naming what it refuses; `room` clears either.
|
||||
func (w *Watchman) StoreFill(seat, event string, body []byte) {
|
||||
var f meshdelivery.FillingEvent
|
||||
if err := json.Unmarshal(body, &f); err != nil || f.Store == "" {
|
||||
w.say("the %s of the seat %s does not read, left: %v", event, seat, err)
|
||||
return
|
||||
}
|
||||
id := seat + "-" + f.Store
|
||||
key := FindingKey(ScopeDelivery, id, KindStoreFilling)
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
share := 0.0
|
||||
if f.Cap > 0 {
|
||||
share = float64(f.Bytes) / float64(f.Cap)
|
||||
}
|
||||
since := w.now()
|
||||
if f.At != nil && !f.At.IsZero() {
|
||||
since = *f.At
|
||||
}
|
||||
switch {
|
||||
case event == "room" || (event == "filling" && share < FillingFrom):
|
||||
w.unfind(key)
|
||||
case event == "full":
|
||||
w.find(Finding{Scope: ScopeDelivery, ID: id, Kind: KindStoreFilling, Severity: Urgent, Source: storeRow, Since: since,
|
||||
Summary: fmt.Sprintf("%s's store %s is full (%d of %d bytes)", seat, f.Store, f.Bytes, f.Cap),
|
||||
Said: "full", Headline: "A store of the delivery is full",
|
||||
Explanation: "The store " + f.Store + " of the seat " + seat + " is at its cap, so it refuses " + orWords(f.Refused, "what it is asked to keep") +
|
||||
" until room is made.",
|
||||
Needs: "room made in the store (a removal, or its cap raised)"})
|
||||
default:
|
||||
w.find(Finding{Scope: ScopeDelivery, ID: id, Kind: KindStoreFilling, Severity: Warning, Source: storeRow, Since: since,
|
||||
Summary: fmt.Sprintf("%s's store %s is at %.0f %% of its cap (%d of %d bytes)", seat, f.Store, share*100, f.Bytes, f.Cap),
|
||||
Said: "filling", Headline: "A store of the delivery is filling",
|
||||
Explanation: "The store " + f.Store + " of the seat " + seat + " is nearly full; once it is full it refuses new entries.",
|
||||
Needs: "room made before it is full"})
|
||||
}
|
||||
w.rejudgeRecords()
|
||||
}
|
||||
|
||||
// RecordOpen hears the seat mesh-delivery's `record-opened`: its finding is due when the record has stood open for 24 hours.
|
||||
func (w *Watchman) RecordOpen(body []byte) {
|
||||
var r meshdelivery.RecordOpenedEvent
|
||||
if err := json.Unmarshal(body, &r); err != nil || r.Record == "" {
|
||||
w.say("a record-opened does not read, left: %v", err)
|
||||
return
|
||||
}
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
since := w.now()
|
||||
if r.At != nil && !r.At.IsZero() {
|
||||
since = *r.At
|
||||
}
|
||||
f := Finding{Scope: ScopeDelivery, ID: r.Record, Kind: KindRecordOpen, Severity: Warning, Source: recordRow, Since: since,
|
||||
Summary: fmt.Sprintf("the delivery record of %s has stood open since %s", r.Milestone, stamp(since)),
|
||||
Said: "open for " + RecordOpenBound.String(), Headline: "A delivery is not on every machine yet",
|
||||
Explanation: "The delivery record of " + r.Milestone + " has stood open for " + RecordOpenBound.String() +
|
||||
": a node does not run what it changed, or a step never answered.",
|
||||
Needs: "what it waits for (gate_record names it), or the record ended by hand if a node is gone for good (gate_record_end)"}
|
||||
w.unsignal(recordRow, r.Record)
|
||||
w.unfind(f.Key())
|
||||
if left := RecordOpenBound - w.now().Sub(since); left > 0 {
|
||||
w.signal(recordRow, r.Record, left, signalDue{Do: dueFind, While: uncovered, Finding: f})
|
||||
return
|
||||
}
|
||||
w.opened[r.Record] = f
|
||||
w.rejudgeRecords()
|
||||
}
|
||||
|
||||
// RecordEnded hears the seat mesh-delivery's `record-ended`: its pulse and its finding end.
|
||||
func (w *Watchman) RecordEnded(body []byte) {
|
||||
var r meshdelivery.RecordEndedEvent
|
||||
if err := json.Unmarshal(body, &r); err != nil || r.Record == "" {
|
||||
w.say("a record-ended does not read, left: %v", err)
|
||||
return
|
||||
}
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
w.unsignal(recordRow, r.Record)
|
||||
delete(w.opened, r.Record)
|
||||
w.unfind(FindingKey(ScopeDelivery, r.Record, KindRecordOpen))
|
||||
}
|
||||
|
||||
// recordCovered says another condition tells the operator why a record is open: a wait past its bound or a full store.
|
||||
func (w *Watchman) recordCovered() bool {
|
||||
for _, v := range w.findings {
|
||||
f, err := DecodeFinding(v)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if f.Kind == KindActingPartLate || (f.Kind == KindStoreFilling && f.Severity == Urgent) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// rejudgeRecords writes the finding of each record past its bound that nothing covers, and ends it when something does.
|
||||
func (w *Watchman) rejudgeRecords() {
|
||||
for id, f := range w.opened {
|
||||
if w.recordCovered() {
|
||||
w.unfind(f.Key())
|
||||
continue
|
||||
}
|
||||
w.find(f)
|
||||
_ = id
|
||||
}
|
||||
}
|
||||
@@ -171,6 +171,11 @@ func (w *Watchman) signalExpired(pk string) {
|
||||
case dueEnd:
|
||||
w.unfind(due.Finding.Key())
|
||||
case dueFind:
|
||||
if due.While == uncovered {
|
||||
w.opened[due.Finding.ID] = due.Finding
|
||||
w.rejudgeRecords()
|
||||
return
|
||||
}
|
||||
if due.While != "" && !w.stillWrong(due.While, due.Finding.Kind) {
|
||||
return
|
||||
}
|
||||
@@ -181,6 +186,9 @@ func (w *Watchman) signalExpired(pk string) {
|
||||
// stillWrong says a have's key still says what a row's finding was waited on: failed for apply-failing, unhealthy for
|
||||
// unhealthy.
|
||||
func (w *Watchman) stillWrong(haveKey, kind string) bool {
|
||||
if haveKey == uncovered {
|
||||
return !w.recordCovered()
|
||||
}
|
||||
if node, ok := strings.CutPrefix(haveKey, "silent:"); ok {
|
||||
_, open := w.findings[FindingKey(ScopeMachine, node, KindSilent)]
|
||||
return open
|
||||
@@ -254,6 +262,11 @@ func (w *Watchman) stuckFinding(sk string, s meshwatchdogsclient.Stuck) {
|
||||
if s.Bucket == PresenceBucket {
|
||||
if node, part, _, err := meshwatchdogsclient.ParsePresenceKey(s.Key); err == nil && part == "" && node != "" {
|
||||
key := FindingKey(ScopeMachine, node, KindSilent)
|
||||
if s.Moved {
|
||||
w.machineBack(node)
|
||||
} else {
|
||||
w.machineAway(node, s.Since)
|
||||
}
|
||||
defer w.networks() // a silent machine is down on the record: what points at it is held there, said by its silence
|
||||
if s.Moved || w.away[node] {
|
||||
w.unfind(key)
|
||||
@@ -931,6 +944,8 @@ var SignalRows = []struct{ Row, Signal, Bound, Kind string }{
|
||||
{"S21", "a machine's have failed or its provisioner client's files not placed", ApplyEvery.String(), KindApplyFailing},
|
||||
{"health", "a have unhealthy in two writes in a row; urgent after " + UnhealthyUrgentAfter.String(), "two writes", KindUnhealthy},
|
||||
{"take-in", "a source revision or a delivery group the indexer holds back (ADR 0633 §5)", TakeInRevisionBound.String() + " a revision, " + TakeInGroupBound.String() + " a group", KindTakeInWaits},
|
||||
{storeRow, "a store of the delivery's parts filling or full (the seats' filling, full and room)", "filling from " + fmt.Sprintf("%.0f", FillingFrom*100) + " %, full urgent", KindStoreFilling},
|
||||
{recordRow, "a delivery record open (the seat mesh-delivery's record-opened, ended by record-ended)", RecordOpenBound.String(), KindRecordOpen},
|
||||
{"provisions", "a provision of mesh-provisioner in waits (ADR 0570 §7)", "at once", KindNeedsOperator},
|
||||
}
|
||||
|
||||
|
||||
@@ -36,6 +36,8 @@ type Out interface {
|
||||
// PutFinding writes one of the watchman's findings; DeleteFinding deletes one it no longer observes.
|
||||
PutFinding(key string, value []byte) error
|
||||
DeleteFinding(key string) error
|
||||
// Say publishes an event of the seat mesh-watchdogs on its subject (design 47 G §389), once under its id.
|
||||
Say(subject string, body []byte, id string) error
|
||||
}
|
||||
|
||||
type presence struct {
|
||||
@@ -112,6 +114,14 @@ type Watchman struct {
|
||||
// pulses, stuck keys and findings wait for its own end (ReadLate), so a restart while its owner is down neither drops
|
||||
// nor raises again what it holds.
|
||||
awaited map[string]bool
|
||||
// heads is, by watched branch, the head still waiting for its pipeline (pipelines.go).
|
||||
heads map[string]head
|
||||
// queued is, by build seat, each step queued on it for a build node and since when (pipelines.go).
|
||||
queued map[string]map[string]time.Time
|
||||
// awaySaid is each machine whose machine-away was said and not yet its machine-back, with when it was last heard (presence.go).
|
||||
awaySaid map[string]time.Time
|
||||
// opened is each record past its bound and not ended, with its finding (records.go).
|
||||
opened map[string]Finding
|
||||
}
|
||||
|
||||
// New is a watchman writing to out, reading the time from now.
|
||||
@@ -122,7 +132,7 @@ func New(out Out, now func() time.Time) *Watchman {
|
||||
haveState: map[string]nodeengine.HaveByNodeModuleValue{}, unhealthy: map[string]int{}, advised: map[string]int{}, advisedSince: map[string]time.Time{},
|
||||
control: map[string]bool{}, failedSince: map[string]time.Time{}, provFailed: map[string]map[string]ProvReport{}, closures: map[string]map[string]time.Time{}, waiting: map[string]map[string]string{},
|
||||
health: map[string]nodeengine.HealthByNodeValue{}, tunnels: map[string]string{},
|
||||
standing: map[string]map[string]bool{}, readWhole: map[string]bool{}, awaited: map[string]bool{}}
|
||||
standing: map[string]map[string]bool{}, readWhole: map[string]bool{}, awaited: map[string]bool{}, heads: map[string]head{}, queued: map[string]map[string]time.Time{}, awaySaid: map[string]time.Time{}, opened: map[string]Finding{}}
|
||||
}
|
||||
|
||||
// Logf sets where a write that failed is said; it is silent until set.
|
||||
|
||||
@@ -26,6 +26,14 @@ type fakeOut struct {
|
||||
stuckWrites int
|
||||
findings map[string]Finding
|
||||
findingWrites int
|
||||
said []said
|
||||
}
|
||||
|
||||
type said struct{ subject, id string }
|
||||
|
||||
func (o *fakeOut) Say(subject string, _ []byte, id string) error {
|
||||
o.said = append(o.said, said{subject, id})
|
||||
return nil
|
||||
}
|
||||
|
||||
func newOut() *fakeOut {
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"regexp"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
@@ -98,6 +99,10 @@ func TestManifestStateAndReads(t *testing.T) { // 1135/AC6
|
||||
"mesh-delivery-builder.builds", "mesh-host.health", "mesh-provisioner.provisions", "node-engine.have"}) {
|
||||
t.Errorf("reads = %v", reads)
|
||||
}
|
||||
// Design 47 G §368: the version is MAJOR.MINOR and nothing else.
|
||||
if !regexp.MustCompile(`^[0-9]+\.[0-9]+$`).MatchString(m.Version) {
|
||||
t.Errorf("version = %q, want MAJOR.MINOR", m.Version)
|
||||
}
|
||||
// DLV-M5 (ADR 0615 §1): the assigner's wants and the indexer's modules are their owners' states, asked by their read
|
||||
// tools and followed by their views families, never read.
|
||||
for _, r := range m.Reads {
|
||||
@@ -111,7 +116,7 @@ func TestManifestStateAndReads(t *testing.T) { // 1135/AC6
|
||||
t.Errorf("invokes = %v, want %v", m.Invokes, want)
|
||||
}
|
||||
// DLV-M5 its first views, 1585/T20 the indexer's take-in events, and DLV-W1 (ADR 0595 §4): the accepted verdicts and the bus module's created and refused, durably.
|
||||
if want := []string{"mesh-delivery-assigner.views.wants", "mesh-delivery-indexer.take-in.pending", "mesh-delivery-indexer.take-in.taken", "mesh-delivery-indexer.views.modules", "mesh-catalog.manifest-accepted", "mesh-broker.created", "mesh-broker.refused"}; !reflect.DeepEqual(m.Consumes, want) {
|
||||
if want := []string{"mesh-delivery-assigner.views.wants", "mesh-delivery-indexer.take-in.pending", "mesh-delivery-indexer.take-in.taken", "mesh-delivery-indexer.views.modules", "mesh-catalog.manifest-accepted", "mesh-delivery.source-moved", "mesh-delivery.filling", "mesh-delivery.full", "mesh-delivery.room", "mesh-delivery.record-opened", "mesh-delivery.record-ended", "mesh-catalog.filling", "mesh-catalog.full", "mesh-catalog.room", "mesh-wants.filling", "mesh-wants.full", "mesh-wants.room", "mesh-grants.filling", "mesh-grants.full", "mesh-grants.room", "mesh-pipeline-runner.pipeline-started", "mesh-pipeline-runner.step-started", "mesh-pipeline-runner.step-ended", "mesh-broker.created", "mesh-broker.refused"}; !reflect.DeepEqual(m.Consumes, want) {
|
||||
t.Errorf("consumes = %v, want %v", m.Consumes, want)
|
||||
}
|
||||
// 1952/WM1: the watchman holds the seat mesh-watchdogs at mesh scope, its first seat (novox/hq ADR 0564 §1, §4).
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"module": "mesh-delivery-watchman",
|
||||
"version": "1",
|
||||
"version": "0.1",
|
||||
"slug": "watchman",
|
||||
"claims": [
|
||||
{
|
||||
@@ -36,6 +36,24 @@
|
||||
"mesh-delivery-indexer.take-in.taken",
|
||||
"mesh-delivery-indexer.views.modules",
|
||||
"mesh-catalog.manifest-accepted",
|
||||
"mesh-delivery.source-moved",
|
||||
"mesh-delivery.filling",
|
||||
"mesh-delivery.full",
|
||||
"mesh-delivery.room",
|
||||
"mesh-delivery.record-opened",
|
||||
"mesh-delivery.record-ended",
|
||||
"mesh-catalog.filling",
|
||||
"mesh-catalog.full",
|
||||
"mesh-catalog.room",
|
||||
"mesh-wants.filling",
|
||||
"mesh-wants.full",
|
||||
"mesh-wants.room",
|
||||
"mesh-grants.filling",
|
||||
"mesh-grants.full",
|
||||
"mesh-grants.room",
|
||||
"mesh-pipeline-runner.pipeline-started",
|
||||
"mesh-pipeline-runner.step-started",
|
||||
"mesh-pipeline-runner.step-ended",
|
||||
"mesh-broker.created",
|
||||
"mesh-broker.refused"
|
||||
],
|
||||
|
||||
Reference in New Issue
Block a user