package main // The admin guard: Keycloak's admin must log in with the password the mesh minted, and when it does // not, the module makes it — itself, inside the container, and says so (novox/hq issue 179). // // **Why it is needed.** The manifest mints an `admin` own-secret and renders it into the server's // environment, and Keycloak applies that environment only when it creates its master realm. A // database that was adopted, restored or moved already has a master realm, whose `admin` keeps the // password it had. Every admin call then fails with *401 invalid_grant*. It happened twice: an // adopted database on 2026-10-01, and a moved one on 2026-10-05, when the provisioner failed every // five seconds for twenty-three hours — about 31,000 times — and nothing but the journal said so. // Both times the fix was the same by hand. This is that fix, run by the module. // // **Where it runs, and why here.** In the module's own bundle, which already holds the two things the // repair needs: the mesh's admin secret (the file the provisioner reads) and the container runtime // (the `container-runtime` capability; the nats and nextcloud bundles reach their containers the // same way). A declared host step would have to be handed the secret a second time and could not tell // the provisioner to stop; the bundle can, and it is the process that sees the 401 first. // // **What it does.** Checks the admin's login at start, every five minutes, and at once when the // admin API refuses the credentials. On a refusal — and only a refusal: an unreachable server is // waited for, never repaired — it runs Keycloak's own recovery inside the container: a temporary // admin through `kc.sh bootstrap-admin`, which sets the mesh's admin password (creating or enabling // that admin if it must), and is removed again. Then it checks again. Both passwords travel on the // exec's standard input, never on a command line, and neither is ever printed. // // **When it cannot.** It says so loudly (`admin.unrepaired`, and the provisioner's standing names the // consumers it fails), and it brakes: the next automatic attempt is ten minutes on, doubling to six // hours. While the admin is refused, the provisioner does not call Keycloak at all — each attempt // would be one more failed login against the admin, and enough of those lock it out. import ( "bufio" "bytes" "context" "crypto/rand" "encoding/base64" "encoding/hex" "errors" "fmt" "math/big" "net" "os/exec" "strings" "sync" "sync/atomic" "time" ) // AdminState is what the last check found. type AdminState string const ( AdminUnknown AdminState = "unknown" AdminOK AdminState = "ok" AdminRejected AdminState = "rejected" AdminUnreachable AdminState = "unreachable" // AdminUnchecked is a check that could not be made for a reason of the module's own — the // mesh's secret unreadable, say. Nothing to repair in Keycloak. AdminUnchecked AdminState = "unchecked" ) // Events the guard emits. const ( EventRepaired = "admin.repaired" EventUnrepaired = "admin.unrepaired" ) // ErrAdminRejected is what the provisioner is answered while the admin is refused: fast, and without // asking Keycloak. var ErrAdminRejected = fmt.Errorf("the admin is refused by Keycloak and not yet repaired (%w); not asking it again until it is", ErrRejected) // Executor runs one command with something on its standard input, answering its combined output. type Executor interface { Run(ctx context.Context, argv []string, stdin []byte) ([]byte, error) } // DockerExec runs commands on this machine. type DockerExec struct{} func (DockerExec) Run(ctx context.Context, argv []string, stdin []byte) ([]byte, error) { cmd := exec.CommandContext(ctx, argv[0], argv[1:]...) cmd.Stdin = bytes.NewReader(stdin) return cmd.CombinedOutput() } // Repair is what one repair did. type Repair struct { At time.Time `json:"at"` Outcome string `json:"outcome"` // "repaired" | "unrepaired" Step string `json:"step,omitempty"` Error string `json:"error,omitempty"` TempUser string `json:"temporaryAdmin,omitempty"` // TempLeft says the temporary admin may still be in the master realm, for a person to delete. TempLeft bool `json:"temporaryAdminLeft,omitempty"` } // Guard keeps the admin's login true. type Guard struct { KC *Client Exec Executor Container string Every time.Duration // 5m Waiting time.Duration // 15s: how often to look for a server not yet seen BrakeFrom time.Duration // 10m BrakeMax time.Duration // 6h Timeout time.Duration // 5m: the whole repair Now func() time.Time Log func(format string, args ...any) Announce func(event string, body map[string]any) once sync.Once mu sync.Mutex // one check-and-repair at a time: the background's or an operator's state atomic.Value last *Repair brakeTill time.Time brakeWait time.Duration nudge chan struct{} } func (g *Guard) init() { g.once.Do(func() { if g.Every == 0 { g.Every = 5 * time.Minute } if g.Waiting == 0 { g.Waiting = 15 * time.Second } if g.BrakeFrom == 0 { g.BrakeFrom = 10 * time.Minute } if g.BrakeMax == 0 { g.BrakeMax = 6 * time.Hour } if g.Timeout == 0 { g.Timeout = 5 * time.Minute } if g.Now == nil { g.Now = time.Now } if g.Log == nil { g.Log = func(string, ...any) {} } if g.Container == "" { g.Container = "keycloak" } if g.Exec == nil { g.Exec = DockerExec{} } g.nudge = make(chan struct{}, 1) g.state.Store(AdminUnknown) }) } // State is what the last check found. func (g *Guard) State() AdminState { g.init() return g.state.Load().(AdminState) } // Refused says the admin was refused at the last check and has not been repaired since: what the // provisioner asks before calling Keycloak. func (g *Guard) Refused() bool { return g.State() == AdminRejected } // Nudge asks for a check now; never blocks. func (g *Guard) Nudge() { g.init() select { case g.nudge <- struct{}{}: default: } } // Check asks Keycloak for a token as the admin, with the mesh's secret as it is now. func (g *Guard) Check(ctx context.Context) (AdminState, error) { g.init() cctx, cancel := context.WithTimeout(ctx, 30*time.Second) defer cancel() _, _, err := g.KC.Login(cctx) state := classifyLogin(err) g.state.Store(state) return state, err } func classifyLogin(err error) AdminState { if err == nil { return AdminOK } if errors.Is(err, ErrRejected) { return AdminRejected } var terr *TokenError if errors.As(err, &terr) { if terr.Status >= 500 || terr.Status == 404 { return AdminUnreachable // starting, or not Keycloak yet } return AdminUnchecked } var nerr net.Error if errors.As(err, &nerr) || errors.Is(err, context.DeadlineExceeded) || strings.Contains(err.Error(), "connection refused") || strings.Contains(err.Error(), "EOF") { return AdminUnreachable } return AdminUnchecked } // Report is the guard's account of itself, for the tool. type Report struct { State AdminState `json:"state"` Error string `json:"error,omitempty"` Repaired bool `json:"repaired,omitempty"` LastRepair *Repair `json:"lastRepair,omitempty"` BrakeUntil string `json:"brakeUntil,omitempty"` Note string `json:"note,omitempty"` } // Ensure checks the admin and repairs a refused one. operator is a person asking: the brake is // theirs to override. Without repair, it only checks. func (g *Guard) Ensure(ctx context.Context, repair, operator bool) Report { g.init() g.mu.Lock() defer g.mu.Unlock() state, err := g.Check(ctx) r := g.report(state, err) if state != AdminRejected || !repair { if state == AdminOK { g.brakeTill, g.brakeWait = time.Time{}, 0 r.BrakeUntil = "" } return r } if !operator && g.Now().Before(g.brakeTill) { r.Note = "the last repair failed; the next automatic attempt is at brakeUntil (keycloak_admin_check with repair: true tries now)" return r } g.Log("[keycloak] the admin is refused by Keycloak (invalid_grant) with the mesh's password; repairing it inside %s", g.Container) done := g.repair(ctx) state, err = g.Check(ctx) if state == AdminOK && done.Outcome == "repaired" { g.brakeTill, g.brakeWait = time.Time{}, 0 g.last = &done g.Log("[keycloak] REPAIRED the admin: the realm's admin did not take the mesh's password (invalid_grant) — " + "the database was adopted or moved and kept an older one; set to the mesh's through a temporary " + "bootstrap admin, which was removed (novox/hq issue 179)") if done.TempLeft { g.Log("[keycloak] the temporary admin %s could not be removed: delete it from the master realm", done.TempUser) } g.announce(EventRepaired, map[string]any{ "cause": "the master realm's admin kept a password older than the mesh's — an adopted, restored or moved database", "at": done.At.UTC().Format(time.RFC3339), "temporaryAdminLeft": done.TempLeft, }) r = g.report(state, err) r.Repaired = true return r } if done.Outcome == "repaired" { // The script finished and the login still fails: say what the check found. done.Outcome, done.Step = "unrepaired", "verify" done.Error = fmt.Sprintf("after the repair the admin still cannot log in: %s", errText(err)) } g.last = &done if g.brakeWait == 0 { g.brakeWait = g.BrakeFrom } else if g.brakeWait *= 2; g.brakeWait > g.BrakeMax { g.brakeWait = g.BrakeMax } g.brakeTill = g.Now().Add(g.brakeWait) left := "" if done.TempLeft { left = fmt.Sprintf(" A temporary admin %s may be left in the master realm: delete it.", done.TempUser) } g.Log("[keycloak] COULD NOT REPAIR the admin at step %s: %s. Every consumer's client is unmanaged until it is; "+ "the next automatic attempt is in %s. By hand: novox/hq issue 179.%s", done.Step, done.Error, g.brakeWait, left) g.announce(EventUnrepaired, map[string]any{ "step": done.Step, "error": done.Error, "temporaryAdminLeft": done.TempLeft, "next": g.brakeTill.UTC().Format(time.RFC3339), }) r = g.report(state, err) return r } func (g *Guard) report(state AdminState, err error) Report { r := Report{State: state, LastRepair: g.last} if err != nil { r.Error = errText(err) } if g.Now().Before(g.brakeTill) { r.BrakeUntil = g.brakeTill.UTC().Format(time.RFC3339) } return r } func errText(err error) string { if err == nil { return "" } return err.Error() } func (g *Guard) announce(event string, body map[string]any) { if g.Announce != nil { g.Announce(event, body) } } // Run checks until ctx ends: often until the server has been seen, then every Every, and at once // when nudged. func (g *Guard) Run(ctx context.Context) { g.init() seen := false for { r := g.Ensure(ctx, true, false) if r.State != AdminUnreachable && r.State != AdminUnknown { seen = true } wait := g.Every if !seen { wait = g.Waiting } select { case <-ctx.Done(): return case <-g.nudge: case <-time.After(wait): } } } // repairScript is Keycloak's own recovery, run inside its container (Keycloak 26). It reads the // temporary admin's password and then the mesh's admin password from standard input; nothing secret // is on its command line or in its environment as docker sees it. Nor on the command line of anything // it runs (novox/hq issue 282): kcadm takes a password from KC_CLI_PASSWORD, set for the one command // that needs it, and a command's environment is readable by its own user only, where its argv is // readable by every user of the machine for as long as the JVM runs (verified against Keycloak 26.0.8). Every step announces itself on // stderr, so a failure names the step it failed at. const repairScript = `set -eu umask 077 IFS= read -r TMP_PW IFS= read -r NEW_PW export TMP_PW bin=/opt/keycloak/bin cfg=/tmp/mesh-kcadm.$$.config bootstrapped= logged_in= step() { echo "mesh-repair-step: $1" >&2; } user_id() { "$bin/kcadm.sh" get users -r master --config "$cfg" -q username="$1" -q exact=true --fields id --format csv --noquotes } cleanup() { rc=$? set +e if [ -z "$logged_in" ] && [ -n "$bootstrapped" ]; then # The bootstrap may have made the temporary admin and failed after (it did once, on a held port): # log in as it anyway, so it is removed rather than left behind. KC_CLI_PASSWORD="$TMP_PW" "$bin/kcadm.sh" config credentials --config "$cfg" --server http://localhost:8080 \ --realm master --user "$TMP_USER" >/dev/null 2>&1 && logged_in=1 fi if [ -n "$logged_in" ]; then tid=$(user_id "$TMP_USER" 2>/dev/null) if [ -n "$tid" ] && "$bin/kcadm.sh" delete "users/$tid" -r master --config "$cfg" >&2; then echo "mesh-repair-removed: $TMP_USER" >&2 else echo "mesh-repair-left: $TMP_USER" >&2 fi elif [ -n "$bootstrapped" ]; then echo "mesh-repair-left: $TMP_USER" >&2 fi rm -f "$cfg" exit $rc } trap cleanup EXIT step bootstrap-admin bootstrapped=1 "$bin/kc.sh" bootstrap-admin user --username "$TMP_USER" --password:env TMP_PW --http-management-port="$MGMT_PORT" >&2 step login KC_CLI_PASSWORD="$TMP_PW" "$bin/kcadm.sh" config credentials --config "$cfg" --server http://localhost:8080 --realm master --user "$TMP_USER" >&2 logged_in=1 step find-admin id=$(user_id "$ADMIN_USER") if [ -z "$id" ]; then step create-admin "$bin/kcadm.sh" create users -r master --config "$cfg" -s username="$ADMIN_USER" -s enabled=true >&2 "$bin/kcadm.sh" add-roles -r master --config "$cfg" --uusername "$ADMIN_USER" --rolename admin >&2 id=$(user_id "$ADMIN_USER") fi step enable-admin "$bin/kcadm.sh" update "users/$id" -r master --config "$cfg" -s enabled=true >&2 step set-password KC_CLI_PASSWORD="$NEW_PW" "$bin/kcadm.sh" set-password -r master --config "$cfg" --username "$ADMIN_USER" >&2 step remove-temporary-admin echo "mesh-repair-done" >&2 ` // repair runs the script once. func (g *Guard) repair(ctx context.Context) Repair { done := Repair{At: g.Now(), Outcome: "unrepaired", Step: "prepare"} meshPW, err := g.KC.Password() if err != nil { done.Error = "the mesh's admin password cannot be read: " + err.Error() return done } if strings.ContainsAny(meshPW, "\n\r") { done.Error = "the mesh's admin password spans lines and cannot be handed over on one" return done } tmpPW, user, port, err := temporaries() if err != nil { done.Error = err.Error() return done } done.TempUser = user argv := []string{"docker", "exec", "-i", "-e", "TMP_USER=" + user, "-e", "MGMT_PORT=" + port, "-e", "ADMIN_USER=" + g.KC.AdminUser, g.Container, "bash", "-c", repairScript} rctx, cancel := context.WithTimeout(ctx, g.Timeout) defer cancel() out, runErr := g.Exec.Run(rctx, argv, []byte(tmpPW+"\n"+meshPW+"\n")) text := scrubAll(string(out), tmpPW, meshPW) sc := bufio.NewScanner(strings.NewReader(text)) finished := false for sc.Scan() { line := sc.Text() switch { case strings.HasPrefix(line, "mesh-repair-step: "): done.Step = strings.TrimPrefix(line, "mesh-repair-step: ") case strings.HasPrefix(line, "mesh-repair-left: "): done.TempLeft = true case line == "mesh-repair-done": finished = true } } if runErr == nil && finished { done.Outcome, done.Step = "repaired", "" return done } why := "the script did not finish" if runErr != nil { why = scrubAll(runErr.Error(), tmpPW, meshPW) } done.Error = why + ": " + tail(text, 20) return done } // temporaries are the temporary admin's password and name, and a management port for the second // server bootstrap-admin starts (the running server holds the default one). func temporaries() (pw, user, port string, err error) { raw := make([]byte, 32) if _, err = rand.Read(raw); err != nil { return "", "", "", fmt.Errorf("no randomness for a temporary password: %w", err) } id := make([]byte, 4) if _, err = rand.Read(id); err != nil { return "", "", "", err } n, err := rand.Int(rand.Reader, big.NewInt(1000)) if err != nil { return "", "", "", err } return base64.RawURLEncoding.EncodeToString(raw), "mesh-repair-" + hex.EncodeToString(id), fmt.Sprint(19000 + n.Int64()), nil } func scrubAll(text string, secrets ...string) string { for _, s := range secrets { if s != "" { text = strings.ReplaceAll(text, s, "***") } } return text } // tail is the last n non-empty lines of text, on one line each joined by " | ". func tail(text string, n int) string { var lines []string for _, l := range strings.Split(text, "\n") { if l = strings.TrimSpace(l); l != "" { lines = append(lines, l) } } if len(lines) > n { lines = lines[len(lines)-n:] } return strings.Join(lines, " | ") }