package main // The guard (novox/hq issue 179): an admin refused with the mesh's password is repaired inside the // container, without a secret on any command line, verified, and said; one it cannot repair is said // loudly and braked; one it cannot reach is waited for; and while it is refused the provisioner does // not ask Keycloak. import ( "context" "errors" "strings" "testing" "time" ) // fakeExec is the container: it records what it was asked, and — when it works — does what the // script does, setting the fake server's admin password to the second line of its input. type fakeExec struct { f *fakeKeycloak runs []execRun fails bool output string noEffect bool } type execRun struct { argv []string stdin string } func (x *fakeExec) Run(_ context.Context, argv []string, stdin []byte) ([]byte, error) { x.runs = append(x.runs, execRun{argv, string(stdin)}) if x.fails { lines := strings.Split(string(stdin), "\n") return []byte("mesh-repair-step: bootstrap-admin\nERROR: boom " + lines[0] + " " + lines[1] + "\nmesh-repair-left: x\n"), errors.New("exit status 1") } if !x.noEffect { x.f.set(func() { x.f.password = strings.Split(string(stdin), "\n")[1] }) } return []byte("mesh-repair-step: set-password\nmesh-repair-step: remove-temporary-admin\nmesh-repair-removed: x\nmesh-repair-done\n"), nil } type guardWorld struct { f *fakeKeycloak x *fakeExec g *Guard now time.Time events []string said []string } func newGuardWorld(t *testing.T) *guardWorld { w := &guardWorld{now: time.Date(2026, 10, 5, 23, 55, 0, 0, time.UTC)} w.f = newFakeKeycloak(t, "Novox", "old-password-from-2022") w.x = &fakeExec{f: w.f} w.g = &Guard{KC: w.f.client("the-mesh-minted-this"), Exec: w.x, Now: func() time.Time { return w.now }, Log: func(f string, a ...any) { w.said = append(w.said, f) }, Announce: func(e string, _ map[string]any) { w.events = append(w.events, e) }} return w } func TestARefusedAdminIsRepairedInsideTheContainerAndSaid(t *testing.T) { w := newGuardWorld(t) r := w.g.Ensure(ctx, true, false) if !r.Repaired || r.State != AdminOK || w.f.password != "the-mesh-minted-this" { t.Fatalf("%+v, server password %q", r, w.f.password) } if strings.Join(w.events, ",") != EventRepaired { t.Fatal(w.events) } if !strings.Contains(strings.Join(w.said, "\n"), "REPAIRED the admin") { t.Fatal(w.said) } run := w.x.runs[0] if run.argv[0] != "docker" || run.argv[1] != "exec" || run.argv[2] != "-i" || !contains(run.argv, "keycloak") || !contains(run.argv, "ADMIN_USER=admin") { t.Fatal(run.argv) } // Both passwords on standard input — the temporary one, then the mesh's — and on no command line. lines := strings.Split(run.stdin, "\n") if len(lines) != 3 || lines[1] != "the-mesh-minted-this" || len(lines[0]) < 40 { t.Fatalf("stdin has %d lines", len(lines)) } for _, a := range run.argv { if strings.Contains(a, "the-mesh-minted-this") || strings.Contains(a, lines[0]) { t.Fatalf("a password is on the command line: %q", a) } } if w.g.Refused() { t.Fatal("still refused after a repair") } } func TestTheScriptIsKeycloaksOwnRecovery(t *testing.T) { for _, want := range []string{ `kc.sh" bootstrap-admin user --username "$TMP_USER" --password:env TMP_PW --http-management-port="$MGMT_PORT"`, `KC_CLI_PASSWORD="$NEW_PW" "$bin/kcadm.sh" set-password -r master --config "$cfg" --username "$ADMIN_USER"`, `umask 077`, `trap cleanup EXIT`, `rm -f "$cfg"`, `delete "users/$tid"`, } { if !strings.Contains(repairScript, want) { t.Errorf("the script lacks %s", want) } } if strings.Contains(repairScript, "--cache") { t.Error("bootstrap-admin takes no --cache") } // Nothing the script runs is given a password as an argument (novox/hq issue 282). for _, never := range []string{`--password "$`, `--new-password`, `-p "$`} { if strings.Contains(repairScript, never) { t.Errorf("the script passes a password as an argument: %s", never) } } } func TestAnAdminThatLogsInIsLeftAlone(t *testing.T) { w := newGuardWorld(t) w.f.password = "the-mesh-minted-this" if r := w.g.Ensure(ctx, true, false); r.State != AdminOK || r.Repaired || len(w.x.runs) != 0 { t.Fatalf("%+v", r) } } func TestAServerNotAnsweringIsWaitedForNeverRepaired(t *testing.T) { w := newGuardWorld(t) w.f.down = true if r := w.g.Ensure(ctx, true, false); r.State != AdminUnreachable || len(w.x.runs) != 0 { t.Fatalf("%+v", r) } w.f.srv.Close() if r := w.g.Ensure(ctx, true, false); r.State != AdminUnreachable || len(w.x.runs) != 0 { t.Fatalf("%+v", r) } } func TestARepairThatFailsIsSaidLoudlyAndBraked(t *testing.T) { w := newGuardWorld(t) w.x.fails = true r := w.g.Ensure(ctx, true, false) if r.State != AdminRejected || r.LastRepair == nil || r.LastRepair.Step != "bootstrap-admin" || !r.LastRepair.TempLeft || r.BrakeUntil == "" { t.Fatalf("%+v %+v", r, r.LastRepair) } // Neither password survives into what is said. if strings.Contains(r.LastRepair.Error, "the-mesh-minted-this") || strings.Contains(r.LastRepair.Error, strings.Split(w.x.runs[0].stdin, "\n")[0]) { t.Fatal(r.LastRepair.Error) } if strings.Join(w.events, ",") != EventUnrepaired || !strings.Contains(strings.Join(w.said, "\n"), "COULD NOT REPAIR") { t.Fatal(w.events, w.said) } if !w.g.Refused() { t.Fatal("not refused") } // Inside the brake: checked, not repaired. w.now = w.now.Add(9 * time.Minute) w.g.Ensure(ctx, true, false) if len(w.x.runs) != 1 { t.Fatalf("repaired inside the brake: %d runs", len(w.x.runs)) } // Past it: tried again, and the next brake is twice as long. w.now = w.now.Add(2 * time.Minute) r = w.g.Ensure(ctx, true, false) if len(w.x.runs) != 2 { t.Fatalf("%d runs", len(w.x.runs)) } if until, _ := time.Parse(time.RFC3339, r.BrakeUntil); until.Sub(w.now) != 20*time.Minute { t.Fatal(r.BrakeUntil) } // An operator asking repairs now, brake or not. w.x.fails = false if r := w.g.Ensure(ctx, true, true); !r.Repaired || len(w.x.runs) != 3 || r.BrakeUntil != "" { t.Fatalf("%+v", r) } } func TestAScriptThatFinishesButChangesNothingIsNotARepair(t *testing.T) { w := newGuardWorld(t) w.x.noEffect = true r := w.g.Ensure(ctx, true, false) if r.Repaired || r.LastRepair.Step != "verify" || strings.Join(w.events, ",") != EventUnrepaired { t.Fatalf("%+v %+v %v", r, r.LastRepair, w.events) } } func TestOnlyCheckingRepairsNothing(t *testing.T) { w := newGuardWorld(t) if r := w.g.Ensure(ctx, false, true); r.State != AdminRejected || len(w.x.runs) != 0 { t.Fatalf("%+v", r) } } func TestWhileTheAdminIsRefusedTheProvisionerDoesNotAskKeycloak(t *testing.T) { w := newGuardWorld(t) w.x.fails = true w.g.Ensure(ctx, true, false) a := provisioner{clients: OidcClients{KC: w.g.KC, Realm: "Novox"}, guard: w.g, announce: func(string, map[string]any) {}, log: func(string, ...any) {}} logins := w.f.logins err := a.Create(ctx, grafana("s", nil)) if !errors.Is(err, ErrRejected) || a.Class(err) != ClassCredentials { t.Fatal(err) } if _, err := a.Holds(ctx, grafana("s", nil)); !errors.Is(err, ErrRejected) { t.Fatal(err) } if w.f.logins != logins { t.Fatal("Keycloak was asked while the admin is refused") } } func TestARefusalSeenByTheAdminAPINudgesTheGuard(t *testing.T) { w := newGuardWorld(t) w.g.init() w.g.KC.onRejected = w.g.Nudge if _, err := w.g.KC.ListRealms(ctx); !errors.Is(err, ErrRejected) { t.Fatal(err) } select { case <-w.g.nudge: default: t.Fatal("not nudged") } // The guard's own check does not nudge it: that would be a guard checking in a loop. w.g.Check(ctx) select { case <-w.g.nudge: t.Fatal("the guard's own check nudged it") default: } } func TestTheGuardReadsTheMeshsSecretEachTime(t *testing.T) { w := newGuardWorld(t) secret := "first" w.g.KC.password = func() (string, error) { return secret, nil } w.f.password = "second" if s, _ := w.g.Check(ctx); s != AdminRejected { t.Fatal(s) } secret = "second" if s, _ := w.g.Check(ctx); s != AdminOK { t.Fatal(s) } } func TestRunRepairsWhenNudged(t *testing.T) { w := newGuardWorld(t) w.f.password = "the-mesh-minted-this" w.g.Every, w.g.Waiting = time.Hour, time.Hour c, cancel := context.WithCancel(ctx) defer cancel() go w.g.Run(c) deadline := time.Now().Add(5 * time.Second) for w.g.State() != AdminOK { if time.Now().After(deadline) { t.Fatal("no first check") } time.Sleep(10 * time.Millisecond) } w.f.set(func() { w.f.password = "moved-database" }) w.g.Nudge() for { w.f.mu.Lock() done := w.f.password == "the-mesh-minted-this" w.f.mu.Unlock() if done { break } if time.Now().After(deadline) { t.Fatal("not repaired when nudged") } time.Sleep(10 * time.Millisecond) } }