mesh-catalog/modules/keycloak/cmd/keycloak-provider/admin.go

package main

// The admin guard: Keycloak's admin must log in with the password the mesh minted, and when it does
// not, the module makes it — itself, inside the container, and says so (novox/hq issue 179).
//
// **Why it is needed.** The manifest mints an `admin` own-secret and renders it into the server's
// environment, and Keycloak applies that environment only when it creates its master realm. A
// database that was adopted, restored or moved already has a master realm, whose `admin` keeps the
// password it had. Every admin call then fails with *401 invalid_grant*. It happened twice: an
// adopted database on 2026-10-01, and a moved one on 2026-10-05, when the provisioner failed every
// five seconds for twenty-three hours — about 31,000 times — and nothing but the journal said so.
// Both times the fix was the same by hand. This is that fix, run by the module.
//
// **Where it runs, and why here.** In the module's own bundle, which already holds the two things the
// repair needs: the mesh's admin secret (the file the provisioner reads) and the container runtime
// (the `container-runtime` capability; the nats and nextcloud bundles reach their containers the
// same way). A declared host step would have to be handed the secret a second time and could not tell
// the provisioner to stop; the bundle can, and it is the process that sees the 401 first.
//
// **What it does.** Checks the admin's login at start, every five minutes, and at once when the
// admin API refuses the credentials. On a refusal — and only a refusal: an unreachable server is
// waited for, never repaired — it runs Keycloak's own recovery inside the container: a temporary
// admin through `kc.sh bootstrap-admin`, which sets the mesh's admin password (creating or enabling
// that admin if it must), and is removed again. Then it checks again. Both passwords travel on the
// exec's standard input, never on a command line, and neither is ever printed.
//
// **When it cannot.** It says so loudly (`admin.unrepaired`, and the provisioner's standing names the
// consumers it fails), and it brakes: the next automatic attempt is ten minutes on, doubling to six
// hours. While the admin is refused, the provisioner does not call Keycloak at all — each attempt
// would be one more failed login against the admin, and enough of those lock it out.

import (
	"bufio"
	"bytes"
	"context"
	"crypto/rand"
	"encoding/base64"
	"encoding/hex"
	"errors"
	"fmt"
	"math/big"
	"net"
	"os/exec"
	"strings"
	"sync"
	"sync/atomic"
	"time"
)

// AdminState is what the last check found.
type AdminState string

const (
	AdminUnknown     AdminState = "unknown"
	AdminOK          AdminState = "ok"
	AdminRejected    AdminState = "rejected"
	AdminUnreachable AdminState = "unreachable"
	// AdminUnchecked is a check that could not be made for a reason of the module's own — the
	// mesh's secret unreadable, say. Nothing to repair in Keycloak.
	AdminUnchecked AdminState = "unchecked"
)

// Events the guard emits.
const (
	EventRepaired   = "admin.repaired"
	EventUnrepaired = "admin.unrepaired"
)

// ErrAdminRejected is what the provisioner is answered while the admin is refused: fast, and without
// asking Keycloak.
var ErrAdminRejected = fmt.Errorf("the admin is refused by Keycloak and not yet repaired (%w); not asking it again until it is", ErrRejected)

// Executor runs one command with something on its standard input, answering its combined output.
type Executor interface {
	Run(ctx context.Context, argv []string, stdin []byte) ([]byte, error)
}

// DockerExec runs commands on this machine.
type DockerExec struct{}

func (DockerExec) Run(ctx context.Context, argv []string, stdin []byte) ([]byte, error) {
	cmd := exec.CommandContext(ctx, argv[0], argv[1:]...)
	cmd.Stdin = bytes.NewReader(stdin)
	return cmd.CombinedOutput()
}

// Repair is what one repair did.
type Repair struct {
	At       time.Time `json:"at"`
	Outcome  string    `json:"outcome"` // "repaired" | "unrepaired"
	Step     string    `json:"step,omitempty"`
	Error    string    `json:"error,omitempty"`
	TempUser string    `json:"temporaryAdmin,omitempty"`
	// TempLeft says the temporary admin may still be in the master realm, for a person to delete.
	TempLeft bool `json:"temporaryAdminLeft,omitempty"`
}

// Guard keeps the admin's login true.
type Guard struct {
	KC        *Client
	Exec      Executor
	Container string
	Every     time.Duration // 5m
	Waiting   time.Duration // 15s: how often to look for a server not yet seen
	BrakeFrom time.Duration // 10m
	BrakeMax  time.Duration // 6h
	Timeout   time.Duration // 5m: the whole repair
	Now       func() time.Time
	Log       func(format string, args ...any)
	Announce  func(event string, body map[string]any)

	once      sync.Once
	mu        sync.Mutex // one check-and-repair at a time: the background's or an operator's
	state     atomic.Value
	last      *Repair
	brakeTill time.Time
	brakeWait time.Duration
	nudge     chan struct{}
}

func (g *Guard) init() {
	g.once.Do(func() {
		if g.Every == 0 {
			g.Every = 5 * time.Minute
		}
		if g.Waiting == 0 {
			g.Waiting = 15 * time.Second
		}
		if g.BrakeFrom == 0 {
			g.BrakeFrom = 10 * time.Minute
		}
		if g.BrakeMax == 0 {
			g.BrakeMax = 6 * time.Hour
		}
		if g.Timeout == 0 {
			g.Timeout = 5 * time.Minute
		}
		if g.Now == nil {
			g.Now = time.Now
		}
		if g.Log == nil {
			g.Log = func(string, ...any) {}
		}
		if g.Container == "" {
			g.Container = "keycloak"
		}
		if g.Exec == nil {
			g.Exec = DockerExec{}
		}
		g.nudge = make(chan struct{}, 1)
		g.state.Store(AdminUnknown)
	})
}

// State is what the last check found.
func (g *Guard) State() AdminState {
	g.init()
	return g.state.Load().(AdminState)
}

// Refused says the admin was refused at the last check and has not been repaired since: what the
// provisioner asks before calling Keycloak.
func (g *Guard) Refused() bool { return g.State() == AdminRejected }

// Nudge asks for a check now; never blocks.
func (g *Guard) Nudge() {
	g.init()
	select {
	case g.nudge <- struct{}{}:
	default:
	}
}

// Check asks Keycloak for a token as the admin, with the mesh's secret as it is now.
func (g *Guard) Check(ctx context.Context) (AdminState, error) {
	g.init()
	cctx, cancel := context.WithTimeout(ctx, 30*time.Second)
	defer cancel()
	_, _, err := g.KC.Login(cctx)
	state := classifyLogin(err)
	g.state.Store(state)
	return state, err
}

func classifyLogin(err error) AdminState {
	if err == nil {
		return AdminOK
	}
	if errors.Is(err, ErrRejected) {
		return AdminRejected
	}
	var terr *TokenError
	if errors.As(err, &terr) {
		if terr.Status >= 500 || terr.Status == 404 {
			return AdminUnreachable // starting, or not Keycloak yet
		}
		return AdminUnchecked
	}
	var nerr net.Error
	if errors.As(err, &nerr) || errors.Is(err, context.DeadlineExceeded) ||
		strings.Contains(err.Error(), "connection refused") || strings.Contains(err.Error(), "EOF") {
		return AdminUnreachable
	}
	return AdminUnchecked
}

// Report is the guard's account of itself, for the tool.
type Report struct {
	State      AdminState `json:"state"`
	Error      string     `json:"error,omitempty"`
	Repaired   bool       `json:"repaired,omitempty"`
	LastRepair *Repair    `json:"lastRepair,omitempty"`
	BrakeUntil string     `json:"brakeUntil,omitempty"`
	Note       string     `json:"note,omitempty"`
}

// Ensure checks the admin and repairs a refused one. operator is a person asking: the brake is
// theirs to override. Without repair, it only checks.
func (g *Guard) Ensure(ctx context.Context, repair, operator bool) Report {
	g.init()
	g.mu.Lock()
	defer g.mu.Unlock()

	state, err := g.Check(ctx)
	r := g.report(state, err)
	if state != AdminRejected || !repair {
		if state == AdminOK {
			g.brakeTill, g.brakeWait = time.Time{}, 0
			r.BrakeUntil = ""
		}
		return r
	}
	if !operator && g.Now().Before(g.brakeTill) {
		r.Note = "the last repair failed; the next automatic attempt is at brakeUntil (keycloak_admin_check with repair: true tries now)"
		return r
	}

	g.Log("[keycloak] the admin is refused by Keycloak (invalid_grant) with the mesh's password; repairing it inside %s", g.Container)
	done := g.repair(ctx)
	state, err = g.Check(ctx)
	if state == AdminOK && done.Outcome == "repaired" {
		g.brakeTill, g.brakeWait = time.Time{}, 0
		g.last = &done
		g.Log("[keycloak] REPAIRED the admin: the realm's admin did not take the mesh's password (invalid_grant) — " +
			"the database was adopted or moved and kept an older one; set to the mesh's through a temporary " +
			"bootstrap admin, which was removed (novox/hq issue 179)")
		if done.TempLeft {
			g.Log("[keycloak] the temporary admin %s could not be removed: delete it from the master realm", done.TempUser)
		}
		g.announce(EventRepaired, map[string]any{
			"cause": "the master realm's admin kept a password older than the mesh's — an adopted, restored or moved database",
			"at":    done.At.UTC().Format(time.RFC3339), "temporaryAdminLeft": done.TempLeft,
		})
		r = g.report(state, err)
		r.Repaired = true
		return r
	}
	if done.Outcome == "repaired" {
		// The script finished and the login still fails: say what the check found.
		done.Outcome, done.Step = "unrepaired", "verify"
		done.Error = fmt.Sprintf("after the repair the admin still cannot log in: %s", errText(err))
	}
	g.last = &done
	if g.brakeWait == 0 {
		g.brakeWait = g.BrakeFrom
	} else if g.brakeWait *= 2; g.brakeWait > g.BrakeMax {
		g.brakeWait = g.BrakeMax
	}
	g.brakeTill = g.Now().Add(g.brakeWait)
	left := ""
	if done.TempLeft {
		left = fmt.Sprintf(" A temporary admin %s may be left in the master realm: delete it.", done.TempUser)
	}
	g.Log("[keycloak] COULD NOT REPAIR the admin at step %s: %s. Every consumer's client is unmanaged until it is; "+
		"the next automatic attempt is in %s. By hand: novox/hq issue 179.%s",
		done.Step, done.Error, g.brakeWait, left)
	g.announce(EventUnrepaired, map[string]any{
		"step": done.Step, "error": done.Error, "temporaryAdminLeft": done.TempLeft,
		"next": g.brakeTill.UTC().Format(time.RFC3339),
	})
	r = g.report(state, err)
	return r
}

func (g *Guard) report(state AdminState, err error) Report {
	r := Report{State: state, LastRepair: g.last}
	if err != nil {
		r.Error = errText(err)
	}
	if g.Now().Before(g.brakeTill) {
		r.BrakeUntil = g.brakeTill.UTC().Format(time.RFC3339)
	}
	return r
}

func errText(err error) string {
	if err == nil {
		return ""
	}
	return err.Error()
}

func (g *Guard) announce(event string, body map[string]any) {
	if g.Announce != nil {
		g.Announce(event, body)
	}
}

// Run checks until ctx ends: often until the server has been seen, then every Every, and at once
// when nudged.
func (g *Guard) Run(ctx context.Context) {
	g.init()
	seen := false
	for {
		r := g.Ensure(ctx, true, false)
		if r.State != AdminUnreachable && r.State != AdminUnknown {
			seen = true
		}
		wait := g.Every
		if !seen {
			wait = g.Waiting
		}
		select {
		case <-ctx.Done():
			return
		case <-g.nudge:
		case <-time.After(wait):
		}
	}
}

// repairScript is Keycloak's own recovery, run inside its container (Keycloak 26). It reads the
// temporary admin's password and then the mesh's admin password from standard input; nothing secret
// is on its command line or in its environment as docker sees it. Every step announces itself on
// stderr, so a failure names the step it failed at.
const repairScript = `set -eu
umask 077
IFS= read -r TMP_PW
IFS= read -r NEW_PW
export TMP_PW
bin=/opt/keycloak/bin
cfg=/tmp/mesh-kcadm.$$.config
bootstrapped=
logged_in=
step() { echo "mesh-repair-step: $1" >&2; }
user_id() {
  "$bin/kcadm.sh" get users -r master --config "$cfg" -q username="$1" -q exact=true --fields id --format csv --noquotes
}
cleanup() {
  rc=$?
  set +e
  if [ -z "$logged_in" ] && [ -n "$bootstrapped" ]; then
    # The bootstrap may have made the temporary admin and failed after (it did once, on a held port):
    # log in as it anyway, so it is removed rather than left behind.
    "$bin/kcadm.sh" config credentials --config "$cfg" --server http://localhost:8080 --realm master \
      --user "$TMP_USER" --password "$TMP_PW" >/dev/null 2>&1 && logged_in=1
  fi
  if [ -n "$logged_in" ]; then
    tid=$(user_id "$TMP_USER" 2>/dev/null)
    if [ -n "$tid" ] && "$bin/kcadm.sh" delete "users/$tid" -r master --config "$cfg" >&2; then
      echo "mesh-repair-removed: $TMP_USER" >&2
    else
      echo "mesh-repair-left: $TMP_USER" >&2
    fi
  elif [ -n "$bootstrapped" ]; then
    echo "mesh-repair-left: $TMP_USER" >&2
  fi
  rm -f "$cfg"
  exit $rc
}
trap cleanup EXIT
step bootstrap-admin
bootstrapped=1
"$bin/kc.sh" bootstrap-admin user --username "$TMP_USER" --password:env TMP_PW --http-management-port="$MGMT_PORT" >&2
step login
"$bin/kcadm.sh" config credentials --config "$cfg" --server http://localhost:8080 --realm master --user "$TMP_USER" --password "$TMP_PW" >&2
logged_in=1
step find-admin
id=$(user_id "$ADMIN_USER")
if [ -z "$id" ]; then
  step create-admin
  "$bin/kcadm.sh" create users -r master --config "$cfg" -s username="$ADMIN_USER" -s enabled=true >&2
  "$bin/kcadm.sh" add-roles -r master --config "$cfg" --uusername "$ADMIN_USER" --rolename admin >&2
  id=$(user_id "$ADMIN_USER")
fi
step enable-admin
"$bin/kcadm.sh" update "users/$id" -r master --config "$cfg" -s enabled=true >&2
step set-password
"$bin/kcadm.sh" set-password -r master --config "$cfg" --username "$ADMIN_USER" --new-password "$NEW_PW" >&2
step remove-temporary-admin
echo "mesh-repair-done" >&2
`

// repair runs the script once.
func (g *Guard) repair(ctx context.Context) Repair {
	done := Repair{At: g.Now(), Outcome: "unrepaired", Step: "prepare"}
	meshPW, err := g.KC.Password()
	if err != nil {
		done.Error = "the mesh's admin password cannot be read: " + err.Error()
		return done
	}
	if strings.ContainsAny(meshPW, "\n\r") {
		done.Error = "the mesh's admin password spans lines and cannot be handed over on one"
		return done
	}
	tmpPW, user, port, err := temporaries()
	if err != nil {
		done.Error = err.Error()
		return done
	}
	done.TempUser = user
	argv := []string{"docker", "exec", "-i",
		"-e", "TMP_USER=" + user, "-e", "MGMT_PORT=" + port, "-e", "ADMIN_USER=" + g.KC.AdminUser,
		g.Container, "bash", "-c", repairScript}
	rctx, cancel := context.WithTimeout(ctx, g.Timeout)
	defer cancel()
	out, runErr := g.Exec.Run(rctx, argv, []byte(tmpPW+"\n"+meshPW+"\n"))
	text := scrubAll(string(out), tmpPW, meshPW)

	sc := bufio.NewScanner(strings.NewReader(text))
	finished := false
	for sc.Scan() {
		line := sc.Text()
		switch {
		case strings.HasPrefix(line, "mesh-repair-step: "):
			done.Step = strings.TrimPrefix(line, "mesh-repair-step: ")
		case strings.HasPrefix(line, "mesh-repair-left: "):
			done.TempLeft = true
		case line == "mesh-repair-done":
			finished = true
		}
	}
	if runErr == nil && finished {
		done.Outcome, done.Step = "repaired", ""
		return done
	}
	why := "the script did not finish"
	if runErr != nil {
		why = scrubAll(runErr.Error(), tmpPW, meshPW)
	}
	done.Error = why + ": " + tail(text, 20)
	return done
}

// temporaries are the temporary admin's password and name, and a management port for the second
// server bootstrap-admin starts (the running server holds the default one).
func temporaries() (pw, user, port string, err error) {
	raw := make([]byte, 32)
	if _, err = rand.Read(raw); err != nil {
		return "", "", "", fmt.Errorf("no randomness for a temporary password: %w", err)
	}
	id := make([]byte, 4)
	if _, err = rand.Read(id); err != nil {
		return "", "", "", err
	}
	n, err := rand.Int(rand.Reader, big.NewInt(1000))
	if err != nil {
		return "", "", "", err
	}
	return base64.RawURLEncoding.EncodeToString(raw), "mesh-repair-" + hex.EncodeToString(id),
		fmt.Sprint(19000 + n.Int64()), nil
}

func scrubAll(text string, secrets ...string) string {
	for _, s := range secrets {
		if s != "" {
			text = strings.ReplaceAll(text, s, "***")
		}
	}
	return text
}

// tail is the last n non-empty lines of text, on one line each joined by " | ".
func tail(text string, n int) string {
	var lines []string
	for _, l := range strings.Split(text, "\n") {
		if l = strings.TrimSpace(l); l != "" {
			lines = append(lines, l)
		}
	}
	if len(lines) > n {
		lines = lines[len(lines)-n:]
	}
	return strings.Join(lines, " | ")
}