jochen
77fb1ecfb2
Twice the identity provider's admin kept an older password than the one the mesh minted (an adopted, then a moved database), and the provisioner failed every consumer until it was repaired by hand (hq issue 179). The module now checks the admin's login and repairs a refusal itself through the server's bootstrap command, verifies, brakes a failed repair and announces it, and stops asking the server while refused. Ported to Go to change it.
270 lines
8.2 KiB
Go
270 lines
8.2 KiB
Go
package main
|
|
|
|
// The guard (novox/hq issue 179): an admin refused with the mesh's password is repaired inside the
|
|
// container, without a secret on any command line, verified, and said; one it cannot repair is said
|
|
// loudly and braked; one it cannot reach is waited for; and while it is refused the provisioner does
|
|
// not ask Keycloak.
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
// fakeExec is the container: it records what it was asked, and — when it works — does what the
|
|
// script does, setting the fake server's admin password to the second line of its input.
|
|
type fakeExec struct {
|
|
f *fakeKeycloak
|
|
runs []execRun
|
|
fails bool
|
|
output string
|
|
noEffect bool
|
|
}
|
|
|
|
type execRun struct {
|
|
argv []string
|
|
stdin string
|
|
}
|
|
|
|
func (x *fakeExec) Run(_ context.Context, argv []string, stdin []byte) ([]byte, error) {
|
|
x.runs = append(x.runs, execRun{argv, string(stdin)})
|
|
if x.fails {
|
|
lines := strings.Split(string(stdin), "\n")
|
|
return []byte("mesh-repair-step: bootstrap-admin\nERROR: boom " + lines[0] + " " + lines[1] + "\nmesh-repair-left: x\n"), errors.New("exit status 1")
|
|
}
|
|
if !x.noEffect {
|
|
x.f.set(func() { x.f.password = strings.Split(string(stdin), "\n")[1] })
|
|
}
|
|
return []byte("mesh-repair-step: set-password\nmesh-repair-step: remove-temporary-admin\nmesh-repair-removed: x\nmesh-repair-done\n"), nil
|
|
}
|
|
|
|
type guardWorld struct {
|
|
f *fakeKeycloak
|
|
x *fakeExec
|
|
g *Guard
|
|
now time.Time
|
|
events []string
|
|
said []string
|
|
}
|
|
|
|
func newGuardWorld(t *testing.T) *guardWorld {
|
|
w := &guardWorld{now: time.Date(2026, 10, 5, 23, 55, 0, 0, time.UTC)}
|
|
w.f = newFakeKeycloak(t, "Novox", "old-password-from-2022")
|
|
w.x = &fakeExec{f: w.f}
|
|
w.g = &Guard{KC: w.f.client("the-mesh-minted-this"), Exec: w.x,
|
|
Now: func() time.Time { return w.now },
|
|
Log: func(f string, a ...any) { w.said = append(w.said, f) },
|
|
Announce: func(e string, _ map[string]any) { w.events = append(w.events, e) }}
|
|
return w
|
|
}
|
|
|
|
func TestARefusedAdminIsRepairedInsideTheContainerAndSaid(t *testing.T) {
|
|
w := newGuardWorld(t)
|
|
r := w.g.Ensure(ctx, true, false)
|
|
if !r.Repaired || r.State != AdminOK || w.f.password != "the-mesh-minted-this" {
|
|
t.Fatalf("%+v, server password %q", r, w.f.password)
|
|
}
|
|
if strings.Join(w.events, ",") != EventRepaired {
|
|
t.Fatal(w.events)
|
|
}
|
|
if !strings.Contains(strings.Join(w.said, "\n"), "REPAIRED the admin") {
|
|
t.Fatal(w.said)
|
|
}
|
|
run := w.x.runs[0]
|
|
if run.argv[0] != "docker" || run.argv[1] != "exec" || run.argv[2] != "-i" || !contains(run.argv, "keycloak") ||
|
|
!contains(run.argv, "ADMIN_USER=admin") {
|
|
t.Fatal(run.argv)
|
|
}
|
|
// Both passwords on standard input — the temporary one, then the mesh's — and on no command line.
|
|
lines := strings.Split(run.stdin, "\n")
|
|
if len(lines) != 3 || lines[1] != "the-mesh-minted-this" || len(lines[0]) < 40 {
|
|
t.Fatalf("stdin has %d lines", len(lines))
|
|
}
|
|
for _, a := range run.argv {
|
|
if strings.Contains(a, "the-mesh-minted-this") || strings.Contains(a, lines[0]) {
|
|
t.Fatalf("a password is on the command line: %q", a)
|
|
}
|
|
}
|
|
if w.g.Refused() {
|
|
t.Fatal("still refused after a repair")
|
|
}
|
|
}
|
|
|
|
func TestTheScriptIsKeycloaksOwnRecovery(t *testing.T) {
|
|
for _, want := range []string{
|
|
`kc.sh" bootstrap-admin user --username "$TMP_USER" --password:env TMP_PW --http-management-port="$MGMT_PORT"`,
|
|
`set-password -r master --config "$cfg" --username "$ADMIN_USER" --new-password "$NEW_PW"`,
|
|
`umask 077`, `trap cleanup EXIT`, `rm -f "$cfg"`, `delete "users/$tid"`,
|
|
} {
|
|
if !strings.Contains(repairScript, want) {
|
|
t.Errorf("the script lacks %s", want)
|
|
}
|
|
}
|
|
if strings.Contains(repairScript, "--cache") {
|
|
t.Error("bootstrap-admin takes no --cache")
|
|
}
|
|
}
|
|
|
|
func TestAnAdminThatLogsInIsLeftAlone(t *testing.T) {
|
|
w := newGuardWorld(t)
|
|
w.f.password = "the-mesh-minted-this"
|
|
if r := w.g.Ensure(ctx, true, false); r.State != AdminOK || r.Repaired || len(w.x.runs) != 0 {
|
|
t.Fatalf("%+v", r)
|
|
}
|
|
}
|
|
|
|
func TestAServerNotAnsweringIsWaitedForNeverRepaired(t *testing.T) {
|
|
w := newGuardWorld(t)
|
|
w.f.down = true
|
|
if r := w.g.Ensure(ctx, true, false); r.State != AdminUnreachable || len(w.x.runs) != 0 {
|
|
t.Fatalf("%+v", r)
|
|
}
|
|
w.f.srv.Close()
|
|
if r := w.g.Ensure(ctx, true, false); r.State != AdminUnreachable || len(w.x.runs) != 0 {
|
|
t.Fatalf("%+v", r)
|
|
}
|
|
}
|
|
|
|
func TestARepairThatFailsIsSaidLoudlyAndBraked(t *testing.T) {
|
|
w := newGuardWorld(t)
|
|
w.x.fails = true
|
|
r := w.g.Ensure(ctx, true, false)
|
|
if r.State != AdminRejected || r.LastRepair == nil || r.LastRepair.Step != "bootstrap-admin" || !r.LastRepair.TempLeft || r.BrakeUntil == "" {
|
|
t.Fatalf("%+v %+v", r, r.LastRepair)
|
|
}
|
|
// Neither password survives into what is said.
|
|
if strings.Contains(r.LastRepair.Error, "the-mesh-minted-this") || strings.Contains(r.LastRepair.Error, strings.Split(w.x.runs[0].stdin, "\n")[0]) {
|
|
t.Fatal(r.LastRepair.Error)
|
|
}
|
|
if strings.Join(w.events, ",") != EventUnrepaired || !strings.Contains(strings.Join(w.said, "\n"), "COULD NOT REPAIR") {
|
|
t.Fatal(w.events, w.said)
|
|
}
|
|
if !w.g.Refused() {
|
|
t.Fatal("not refused")
|
|
}
|
|
|
|
// Inside the brake: checked, not repaired.
|
|
w.now = w.now.Add(9 * time.Minute)
|
|
w.g.Ensure(ctx, true, false)
|
|
if len(w.x.runs) != 1 {
|
|
t.Fatalf("repaired inside the brake: %d runs", len(w.x.runs))
|
|
}
|
|
// Past it: tried again, and the next brake is twice as long.
|
|
w.now = w.now.Add(2 * time.Minute)
|
|
r = w.g.Ensure(ctx, true, false)
|
|
if len(w.x.runs) != 2 {
|
|
t.Fatalf("%d runs", len(w.x.runs))
|
|
}
|
|
if until, _ := time.Parse(time.RFC3339, r.BrakeUntil); until.Sub(w.now) != 20*time.Minute {
|
|
t.Fatal(r.BrakeUntil)
|
|
}
|
|
|
|
// An operator asking repairs now, brake or not.
|
|
w.x.fails = false
|
|
if r := w.g.Ensure(ctx, true, true); !r.Repaired || len(w.x.runs) != 3 || r.BrakeUntil != "" {
|
|
t.Fatalf("%+v", r)
|
|
}
|
|
}
|
|
|
|
func TestAScriptThatFinishesButChangesNothingIsNotARepair(t *testing.T) {
|
|
w := newGuardWorld(t)
|
|
w.x.noEffect = true
|
|
r := w.g.Ensure(ctx, true, false)
|
|
if r.Repaired || r.LastRepair.Step != "verify" || strings.Join(w.events, ",") != EventUnrepaired {
|
|
t.Fatalf("%+v %+v %v", r, r.LastRepair, w.events)
|
|
}
|
|
}
|
|
|
|
func TestOnlyCheckingRepairsNothing(t *testing.T) {
|
|
w := newGuardWorld(t)
|
|
if r := w.g.Ensure(ctx, false, true); r.State != AdminRejected || len(w.x.runs) != 0 {
|
|
t.Fatalf("%+v", r)
|
|
}
|
|
}
|
|
|
|
func TestWhileTheAdminIsRefusedTheProvisionerDoesNotAskKeycloak(t *testing.T) {
|
|
w := newGuardWorld(t)
|
|
w.x.fails = true
|
|
w.g.Ensure(ctx, true, false)
|
|
a := provisioner{clients: OidcClients{KC: w.g.KC, Realm: "Novox"}, guard: w.g,
|
|
announce: func(string, map[string]any) {}, log: func(string, ...any) {}}
|
|
logins := w.f.logins
|
|
err := a.Create(ctx, grafana("s", nil))
|
|
if !errors.Is(err, ErrRejected) || a.Class(err) != ClassCredentials {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := a.Holds(ctx, grafana("s", nil)); !errors.Is(err, ErrRejected) {
|
|
t.Fatal(err)
|
|
}
|
|
if w.f.logins != logins {
|
|
t.Fatal("Keycloak was asked while the admin is refused")
|
|
}
|
|
}
|
|
|
|
func TestARefusalSeenByTheAdminAPINudgesTheGuard(t *testing.T) {
|
|
w := newGuardWorld(t)
|
|
w.g.init()
|
|
w.g.KC.onRejected = w.g.Nudge
|
|
if _, err := w.g.KC.ListRealms(ctx); !errors.Is(err, ErrRejected) {
|
|
t.Fatal(err)
|
|
}
|
|
select {
|
|
case <-w.g.nudge:
|
|
default:
|
|
t.Fatal("not nudged")
|
|
}
|
|
// The guard's own check does not nudge it: that would be a guard checking in a loop.
|
|
w.g.Check(ctx)
|
|
select {
|
|
case <-w.g.nudge:
|
|
t.Fatal("the guard's own check nudged it")
|
|
default:
|
|
}
|
|
}
|
|
|
|
func TestTheGuardReadsTheMeshsSecretEachTime(t *testing.T) {
|
|
w := newGuardWorld(t)
|
|
secret := "first"
|
|
w.g.KC.password = func() (string, error) { return secret, nil }
|
|
w.f.password = "second"
|
|
if s, _ := w.g.Check(ctx); s != AdminRejected {
|
|
t.Fatal(s)
|
|
}
|
|
secret = "second"
|
|
if s, _ := w.g.Check(ctx); s != AdminOK {
|
|
t.Fatal(s)
|
|
}
|
|
}
|
|
|
|
func TestRunRepairsWhenNudged(t *testing.T) {
|
|
w := newGuardWorld(t)
|
|
w.f.password = "the-mesh-minted-this"
|
|
w.g.Every, w.g.Waiting = time.Hour, time.Hour
|
|
c, cancel := context.WithCancel(ctx)
|
|
defer cancel()
|
|
go w.g.Run(c)
|
|
deadline := time.Now().Add(5 * time.Second)
|
|
for w.g.State() != AdminOK {
|
|
if time.Now().After(deadline) {
|
|
t.Fatal("no first check")
|
|
}
|
|
time.Sleep(10 * time.Millisecond)
|
|
}
|
|
w.f.set(func() { w.f.password = "moved-database" })
|
|
w.g.Nudge()
|
|
for {
|
|
w.f.mu.Lock()
|
|
done := w.f.password == "the-mesh-minted-this"
|
|
w.f.mu.Unlock()
|
|
if done {
|
|
break
|
|
}
|
|
if time.Now().After(deadline) {
|
|
t.Fatal("not repaired when nudged")
|
|
}
|
|
time.Sleep(10 * time.Millisecond)
|
|
}
|
|
}
|