package main import ( "context" "encoding/json" "errors" "fmt" "io" "log" "os" "os/signal" "strconv" "strings" "syscall" "time" "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" "git.novox.be/novox/mesh-contracts/go/seats/meshbroker" "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient" occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient" oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel" catalog "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/process" "git.novox.be/novox/mesh-sdk/go/seatdef" "nats-tools/internal/bussnap" "nats-tools/internal/keeper" ) // The keeper (`nats-tools keep`): the bus module's process on its own account (own secret broker-keeper, ADR 0436 §1), // started by the node-engine as the manifest's process keeper. It reads the fill every minute from its start, asks the // indexer's first view once, then hears manifest-accepted and the operator's decided on its durable consumer, makes what // each verdict asks for, and acts on the persons' requests its tools put in its own state requests (requests.go) as they // are written or answered. It serves nothing (ADR 0436 §3). // AcceptedOf reads a manifest-accepted body: the seat mesh-catalog's typed event, with the indexer's revision of the // verdict when the event carries it (0 from an indexer that does not say it). func AcceptedOf(body []byte) (keeper.Accepted, error) { var a catalog.ManifestAcceptedEvent if err := json.Unmarshal(body, &a); err != nil { return keeper.Accepted{}, err } if a.Module == "" { return keeper.Accepted{}, fmt.Errorf("a manifest-accepted names no module") } var state []catalogue.BusStateEntry // the seat types an entry as the manifest wrote it; the keeper reads the catalogue's form if raw, err := json.Marshal(a.BusState); err != nil || json.Unmarshal(raw, &state) != nil { return keeper.Accepted{}, fmt.Errorf("the bus-state of the manifest-accepted of %s is not readable", a.Module) } if len(a.BusState) == 0 { state = nil } return keeper.Accepted{Module: a.Module, Digest: a.Digest, Revision: uint64(max(a.Revision, 0)), BusState: state}, nil } // AcceptedSubject is the seat mesh-catalog's manifest-accepted on the shared events stream. var AcceptedSubject = catalog.SubjectManifestAccepted // The router's records of asks, read under this module's own name (ADR 0259 §3), as the mail sender reads them. const asksBucket = "messenger_asks" // WarrantFrom reads the operator's warrant on one of the bus module's asks from the router's own record of it, under the // keeper's own account (read: process.Conn.Ask): the ask and its warrant, or nil while the operator has not answered. func WarrantFrom(read func(bucket, id string) (*stdio.StateEntry, error)) func(id string) (*oc.Ask, *oc.Warrant, error) { return func(id string) (*oc.Ask, *oc.Warrant, error) { e, err := read(asksBucket, id) if err != nil || e == nil { return nil, nil, err } var rec struct { Ask *oc.Ask `json:"ask"` Warrant *oc.Warrant `json:"warrant"` } if err := json.Unmarshal(e.Value, &rec); err != nil { return nil, nil, err } return rec.Ask, rec.Warrant, nil } } // processStore is the keeper's own state through its process connection. type processStore struct{ c *process.Conn } func (s processStore) Get(state, key string) ([]byte, uint64, bool, error) { e, err := s.c.State(state).Get(key) if err != nil || e == nil { return nil, 0, false, err } return e.Value, e.Revision, true, nil } func (s processStore) Put(state, key string, value []byte) (uint64, error) { return s.c.State(state).Put(key, json.RawMessage(value)) } func (s processStore) Delete(state, key string) error { return s.c.State(state).Delete(key) } func (s processStore) Keys(state string) ([]string, error) { return s.c.State(state).Keys() } func (s processStore) Create(state, key string, value []byte) (uint64, error) { rev, err := s.c.State(state).Create(key, json.RawMessage(value)) if errors.Is(err, stdio.ErrChanged) { return 0, keeper.ErrExists } return rev, err } // streamEmitter says an event of the seat mesh-broker, with the SDK's headers, under its one event id (the lead's ruling, // registry milestone, 13:0x; design 25 § 12.3): on the seat's own subject `mesh.seat.mesh-broker.event.`, which the // shared events stream keeps 7 days and every consumer of `mesh-broker.` hears, like any seat's event; then on the // bus module's own stream (its old `streams` section, 30 days, 256 MiB), which its tool `history` reads. The seat's subject // first: a reader hears it before the history holds it, never the other way round. It says it on s.Left (every subject // when nil) and leaves in s.Left what the bus did not take, so the keeper says that again with the same id and the bus // keeps it once (the two publishes are not one act: a failed second one is kept and said again, never silent). func streamEmitter(js jetstream.JetStream, a process.Account) func(*keeper.Saying) error { return func(s *keeper.Saying) error { data, err := json.Marshal(s.Body) if err != nil { return err } if s.ID == "" { s.ID = keeper.EventID() } at := s.At if at.IsZero() { at = time.Now().UTC() } subjects := s.Left if subjects == nil { subjects = EventSubjects(a, s.Event) } s.Said = nil for i, subject := range subjects { msg := nats.NewMsg(subject) msg.Header.Set("x-event-id", s.ID) msg.Header.Set("x-source", a.Module) msg.Header.Set("x-node", a.Node) msg.Header.Set("x-time", at.UTC().Format(time.RFC3339Nano)) msg.Header.Set("content-type", "application/json") msg.Data = data ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) _, err = js.PublishMsg(ctx, msg, jetstream.WithMsgID(s.ID)) cancel() if err != nil { s.Left = append([]string(nil), subjects[i:]...) return fmt.Errorf("saying %s on %s: %w", s.Event, subject, err) } s.Said = append(s.Said, subject) } s.Left = nil return nil } } // EventSubjects are where one event of the seat mesh-broker is said, in order: the seat's subject, then the bus module's // own stream. func EventSubjects(a process.Account, event string) []string { return []string{seatdef.EventSubject(meshbroker.Seat, event), a.StreamSubject(event)} } // sayer raises and clears the keeper's conditions through the condition keeper, as every module says them. func sayer(js jetstream.JetStream) func(oc.Condition) error { return func(c oc.Condition) error { subject, err := occ.ConditionSubject(keeper.Module) if err != nil { return err } data, err := occ.EncodeCondition(c) if err != nil { return err } ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() _, err = js.Publish(ctx, subject, data) return err } } // firstView asks the indexer's indexer_bus_state as the keeper's own account. func firstView(c *process.Conn) func() (keeper.FirstView, error) { return func() (keeper.FirstView, error) { raw, err := c.Call(keeper.FirstViewOwner+"."+keeper.FirstViewVerb, map[string]any{}) if err != nil { return keeper.FirstView{}, err } return FirstViewOf(raw) } } // FirstViewOf reads indexer_bus_state's answer, as the tool runner hands it back ({result, error} or bare). func FirstViewOf(raw []byte) (keeper.FirstView, error) { var wrapped struct { Result json.RawMessage `json:"result"` Error string `json:"error"` } if json.Unmarshal(raw, &wrapped) == nil && (wrapped.Result != nil || wrapped.Error != "") { if wrapped.Error != "" { return keeper.FirstView{}, fmt.Errorf("%s", wrapped.Error) } raw = wrapped.Result } var v keeper.FirstView if err := json.Unmarshal(raw, &v); err != nil { return keeper.FirstView{}, fmt.Errorf("the answer is not a first view: %w", err) } return v, nil } func percent(getenv func(string) string, name string, fallback int64) int64 { if n, err := strconv.ParseInt(strings.TrimSpace(getenv(name)), 10, 64); err == nil && n > 0 && n < 100 { return n } return fallback } // runKeeper is the keeper's process: it returns when ctx ends, or when it has no credential. func runKeeper(ctx context.Context, getenv func(string) string, logOut io.Writer) error { logger := log.New(logOut, "nats-keeper: ", log.LstdFlags) account, err := process.ReadBrokerAccount(getenv) if err != nil { return err } conn, err := process.Connect(account) if err != nil { return err } defer conn.Close() nc, js, err := account.Dial(nats.Name(account.Node + "." + account.Module + ".keeper")) if err != nil { return err } defer nc.Close() recorder := meshhandactsclient.Recorder{Ask: conn.Call} k := &keeper.Keeper{ Bus: keeper.JetStream{JS: js, Snap: bussnap.Into(nc, getenv("MESH_NATS_SNAPSHOTS_DIR"))}, Store: processStore{conn}, Emit: streamEmitter(js, account), Say: sayer(js), View: firstView(conn), Act: func(what, why, cause string) error { _, err := recorder.Record(meshhandactsclient.Request{What: what, Why: why, Cause: cause}) return err }, Warrant: WarrantFrom(conn.Ask), Logf: logger.Printf, Raise: percent(getenv, "MESH_NATS_FULL_RAISE", 75), Clear: percent(getenv, "MESH_NATS_FULL_CLEAR", 70), } logger.Printf("started on the account %s: asking %s.%s for the first view", account.User, keeper.FirstViewOwner, keeper.FirstViewVerb) return Keep(ctx, k, hearer(js, account, logger.Printf), requestsOf(conn), logger.Printf) } // hearer binds the keeper's durable consumer on EVENTS (_nats_keeper, filtered on manifest-accepted) and hands // each message to handle, which acknowledges it once applied. func hearer(js jetstream.JetStream, a process.Account, logf func(string, ...any)) func(context.Context, func(subject string, body []byte) error) error { return func(ctx context.Context, handle func(string, []byte) error) error { cons, err := js.Consumer(ctx, process.EventsStream, a.QueueConsumerName()) if err != nil { return fmt.Errorf("the keeper's consumer %s on %s: %w", a.QueueConsumerName(), process.EventsStream, err) } cc, err := process.HearEvents(cons, func(m jetstream.Msg) { if err := handle(m.Subject(), m.Data()); err != nil { logf("%v: delivered again", err) _ = m.NakWithDelay(30 * time.Second) return } _ = m.Ack() }, logf) if err != nil { return err } go func() { <-ctx.Done(); cc.Stop() }() return nil } } // FillEvery is how often the keeper's fill pass runs: a minute (a test's shorter bound beside). var FillEvery = time.Minute // DecidedSubject is where the operator channel says the operator's answer to one of the bus module's own asks (ADR // 0259 §6; ADR 0620 §2), heard on the keeper's durable consumer: the warrant a recreate or a drop waits for. var DecidedSubject = oc.SubjectDecided(keeper.Module) // Keep is the keeper's loop, apart from the bus so a test drives it: a fill pass at once and every minute, from its // start, so the store's one guard runs while the first view is still asked (design 25 § 12.3); the first view asked // until it answers, on its backoff; then hear, which applies each manifest-accepted and wakes the persons' requests on // the operator channel's decided for the bus module's own asks; the waiting entries made again on the same backoff; and // the persons' requests acted on as they are written or answered, never on a timer. It returns when ctx ends. func Keep(ctx context.Context, k *keeper.Keeper, hear func(context.Context, func(string, []byte) error) error, requests func(context.Context, func()) error, logf func(string, ...any)) error { tick := time.NewTicker(FillEvery) defer tick.Stop() k.FillPass() var wait time.Duration for { next, err := k.AskFirstView(wait) if err == nil { break } logf("the first view did not answer (%v): nothing is created or orphaned until it does; asked again in %s", err, next) wait = next again := time.After(next) waiting: for { select { case <-ctx.Done(): return ctx.Err() case <-tick.C: k.FillPass() case <-again: break waiting } } } logf("the first view answered at revision %d", k.Status().FirstView.Revision) wake := func(c chan struct{}) func() { return func() { select { case c <- struct{}{}: default: } } } heard, asked := make(chan struct{}, 1), make(chan struct{}, 1) for backoff := 10 * time.Second; ; backoff = min(backoff*2, 5*time.Minute) { err := hear(ctx, func(subject string, body []byte) error { if subject == DecidedSubject { // The operator answered one of the bus module's asks: the waiting requests are read again now. wake(asked)() return nil } if subject != AcceptedSubject && !strings.HasSuffix(subject, ".event."+catalog.EventManifestAccepted) { return nil } a, err := AcceptedOf(body) if err != nil { logf("a manifest-accepted cannot be read (%v): taken, and nothing made of it", err) return nil } k.Heard(a) wake(heard)() return nil }) if err == nil { break } logf("manifest-accepted cannot be heard yet (%v): asked again in %s", err, backoff) // The fill pass runs on while the consumer is bound again (review round 2, running F1). bound := time.After(backoff) binding: for { select { case <-ctx.Done(): return ctx.Err() case <-tick.C: k.FillPass() case <-bound: break binding } } } if requests != nil { if err := requests(ctx, wake(asked)); err != nil { logf("the persons' requests cannot be watched (%v): acted on as the operator answers", err) } } // What waited while the keeper was down, or before its first view, is acted on now. k.ActOnRequests() retry := 10 * time.Second var again <-chan time.Time for { if len(k.Waiting()) > 0 && again == nil { again = time.After(retry) } select { case <-ctx.Done(): return ctx.Err() case <-tick.C: k.FillPass() case <-asked: k.ActOnRequests() case <-heard: case <-again: again = nil k.CheckAll() if len(k.Waiting()) > 0 { retry = min(retry*2, 5*time.Minute) logf("%d entr(y/ies) wait to be made: %s; made again in %s", len(k.Waiting()), strings.Join(k.Waiting(), "; "), retry) } else { retry = 10 * time.Second } } } } // keepMain runs the keeper until a signal. func keepMain() { ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) defer stop() if err := runKeeper(ctx, os.Getenv, os.Stderr); err != nil && ctx.Err() == nil { fmt.Fprintln(os.Stderr, "nats-keeper:", err) os.Exit(1) } } // requestsOf watches the keeper's own state requests, waking act at each change. func requestsOf(c *process.Conn) func(context.Context, func()) error { return func(_ context.Context, act func()) error { return c.State(keeper.StateRequests).Watch("", func(stdio.StateChange) error { act() return nil }) } }