jochen
8d9d33ae85
The identity provider failed every consumer for a day and status called the mesh well (hq issue 179). The controller now follows every provider's provisioner.failing/recovered, keeps the newest failing word per provider, machine and consumer (migration 0065), and status, its JSON and node show name it until it recovers. Every module that receives contributions is granted the two events, so no manifest can forget them.
77 lines
3.4 KiB
Go
77 lines
3.4 KiB
Go
package inventory
|
|
|
|
import (
|
|
"context"
|
|
"time"
|
|
)
|
|
|
|
// ProviderStanding is a consumer a provider says it keeps failing (novox/hq ADR 0224).
|
|
type ProviderStanding struct {
|
|
// Module is the provider's module, and ProviderNode the machine it runs on.
|
|
Module string `json:"module"`
|
|
ProviderNode string `json:"provider-node"`
|
|
// Provision is the interface it provides, e.g. `oidc-client`.
|
|
Provision string `json:"provision"`
|
|
// Consumer is the identity the mesh derived for the consumer, ConsumerNode its machine.
|
|
Consumer string `json:"consumer"`
|
|
ConsumerNode string `json:"consumer-node"`
|
|
// Class is what kind of failure: credentials-rejected, unreachable, secret-unreadable, refused.
|
|
Class string `json:"class"`
|
|
Error string `json:"error"`
|
|
// Since is when the unbroken run of failures began; Attempts how many it has been.
|
|
Since time.Time `json:"since"`
|
|
Attempts int `json:"attempts"`
|
|
// SaidAt is when the controller last heard it. A provider says it again every quarter of an hour
|
|
// while it lasts, so an old one is a provider that stopped saying anything.
|
|
SaidAt time.Time `json:"said-at"`
|
|
}
|
|
|
|
// SayAgainWithin is how long a failing standing stays current without being said again: twice the
|
|
// quarter of an hour a provider repeats it at. Older, and status says the provider has gone quiet.
|
|
const SayAgainWithin = 30 * time.Minute
|
|
|
|
// Quiet says the provider has not repeated this standing for longer than it would while it lasts.
|
|
func (s ProviderStanding) Quiet(now time.Time) bool { return now.Sub(s.SaidAt) > SayAgainWithin }
|
|
|
|
// KeepStanding records a provider's newest word: failing keeps it, recovered removes it, and says
|
|
// whether a recovery removed anything.
|
|
func (i *Inventory) KeepStanding(ctx context.Context, failing bool, s ProviderStanding) (bool, error) {
|
|
if !failing {
|
|
tag, err := i.store.Pool().Exec(ctx,
|
|
`delete from provider_standing where module = $1 and provider_node = $2 and consumer = $3`,
|
|
s.Module, s.ProviderNode, s.Consumer)
|
|
return err == nil && tag.RowsAffected() > 0, err
|
|
}
|
|
_, err := i.store.Pool().Exec(ctx, `
|
|
insert into provider_standing
|
|
(module, provider_node, consumer, consumer_node, provision, class, error, since, attempts, said_at)
|
|
values ($1, $2, $3, $4, $5, $6, $7, $8, $9, now())
|
|
on conflict (module, provider_node, consumer) do update set
|
|
consumer_node = excluded.consumer_node, provision = excluded.provision,
|
|
class = excluded.class, error = excluded.error, since = excluded.since,
|
|
attempts = excluded.attempts, said_at = excluded.said_at`,
|
|
s.Module, s.ProviderNode, s.Consumer, s.ConsumerNode, s.Provision, s.Class, s.Error, s.Since, s.Attempts)
|
|
return false, err
|
|
}
|
|
|
|
// FailingProviders is every consumer a provider last said it keeps failing, oldest run first.
|
|
func (i *Inventory) FailingProviders(ctx context.Context) ([]ProviderStanding, error) {
|
|
rows, err := i.store.Pool().Query(ctx, `
|
|
select module, provider_node, provision, consumer, consumer_node, class, error, since, attempts, said_at
|
|
from provider_standing order by since, module, consumer`)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer rows.Close()
|
|
var out []ProviderStanding
|
|
for rows.Next() {
|
|
var s ProviderStanding
|
|
if err := rows.Scan(&s.Module, &s.ProviderNode, &s.Provision, &s.Consumer, &s.ConsumerNode,
|
|
&s.Class, &s.Error, &s.Since, &s.Attempts, &s.SaidAt); err != nil {
|
|
return nil, err
|
|
}
|
|
out = append(out, s)
|
|
}
|
|
return out, rows.Err()
|
|
}
|