package inventory import ( "context" "time" ) // ProviderStanding is a consumer a provider says it keeps failing (novox/hq ADR 0224). type ProviderStanding struct { // Module is the provider's module, and ProviderNode the machine it runs on. Module string `json:"module"` ProviderNode string `json:"provider-node"` // Provision is the interface it provides, e.g. `oidc-client`. Provision string `json:"provision"` // Consumer is the identity the mesh derived for the consumer, ConsumerNode its machine. Consumer string `json:"consumer"` ConsumerNode string `json:"consumer-node"` // Class is what kind of failure: credentials-rejected, unreachable, secret-unreadable, refused. Class string `json:"class"` Error string `json:"error"` // Since is when the unbroken run of failures began; Attempts how many it has been. Since time.Time `json:"since"` Attempts int `json:"attempts"` // SaidAt is when the controller last heard it. A provider says it again every quarter of an hour // while it lasts, so an old one is a provider that stopped saying anything. SaidAt time.Time `json:"said-at"` } // SayAgainWithin is how long a failing standing stays current without being said again: twice the // quarter of an hour a provider repeats it at. Older, and status says the provider has gone quiet. const SayAgainWithin = 30 * time.Minute // Quiet says the provider has not repeated this standing for longer than it would while it lasts. func (s ProviderStanding) Quiet(now time.Time) bool { return now.Sub(s.SaidAt) > SayAgainWithin } // KeepStanding records a provider's newest word: failing keeps it, recovered removes it, and says // whether a recovery removed anything. func (i *Inventory) KeepStanding(ctx context.Context, failing bool, s ProviderStanding) (bool, error) { if !failing { tag, err := i.store.Pool().Exec(ctx, `delete from provider_standing where module = $1 and provider_node = $2 and consumer = $3`, s.Module, s.ProviderNode, s.Consumer) return err == nil && tag.RowsAffected() > 0, err } _, err := i.store.Pool().Exec(ctx, ` insert into provider_standing (module, provider_node, consumer, consumer_node, provision, class, error, since, attempts, said_at) values ($1, $2, $3, $4, $5, $6, $7, $8, $9, now()) on conflict (module, provider_node, consumer) do update set consumer_node = excluded.consumer_node, provision = excluded.provision, class = excluded.class, error = excluded.error, since = excluded.since, attempts = excluded.attempts, said_at = excluded.said_at`, s.Module, s.ProviderNode, s.Consumer, s.ConsumerNode, s.Provision, s.Class, s.Error, s.Since, s.Attempts) return false, err } // FailingProviders is every consumer a provider last said it keeps failing, oldest run first. func (i *Inventory) FailingProviders(ctx context.Context) ([]ProviderStanding, error) { rows, err := i.store.Pool().Query(ctx, ` select module, provider_node, provision, consumer, consumer_node, class, error, since, attempts, said_at from provider_standing order by since, module, consumer`) if err != nil { return nil, err } defer rows.Close() var out []ProviderStanding for rows.Next() { var s ProviderStanding if err := rows.Scan(&s.Module, &s.ProviderNode, &s.Provision, &s.Consumer, &s.ConsumerNode, &s.Class, &s.Error, &s.Since, &s.Attempts, &s.SaidAt); err != nil { return nil, err } out = append(out, s) } return out, rows.Err() }