Refuse local development behind a TLS terminator and document sign-in

cmd/dispatcher/profile_test.go

@@ -58,6 +58,7 @@ func TestConfigurationRefusesTheProfileOutsideALocalEnvironment(t *testing.T) {
 		{name: "TLS listener enabled", change: func(c *config.DispatcherConfig) { c.TLS.Disable = false }, names: "tls.disable"},
 		{name: "payments enabled", change: func(c *config.DispatcherConfig) { c.Sui.Disabled = false }, names: "sui.disabled"},
 		{name: "every interface", change: func(c *config.DispatcherConfig) { c.Server.BindHost = "" }, names: "server.bind_host"},
+		{name: "behind a TLS terminator", change: func(c *config.DispatcherConfig) { c.Server.BehindTLSTerminator = true }, names: "server.behind_tls_terminator"},
 	} {
 		t.Run(tc.name, func(t *testing.T) {
 			cfg := dispatcherConfig(filepath.Join(t.TempDir(), "dispatcher.sqlite"))

docs/operations/authentication.md

@@ -67,11 +67,43 @@ CILogon discovery refreshes hourly; signing keys refresh when an unknown key is
 encountered. Failed discovery, exchange or verification cannot create a session.
 The console distinguishes cancellation, expired sign-in and temporary provider
 failure. Restart the dispatcher after rotating client secrets; verify sign-in in
-staging before enabling production. Disabling a provider prevents new sign-ins
+staging before enabling production. [Provider registrations](#provider-registrations-ownership-renewal-and-callback-changes)
+describes rotation with the deployment playbooks. Disabling a provider prevents new sign-ins
 through it but does not by itself revoke existing Debuglet sessions: revoke them
 explicitly during an incident. Preserve another enabled sign-in or recovery path
 before disabling the last provider.
 
+## Provider registrations: ownership, renewal and callback changes
+
+The deployment's operator holds the GitHub OAuth app and the CILogon client
+registration of each environment and records each client ID and registered
+callback with the deployment's private configuration. Register exactly
+`https://<dispatcher_base_url>/api/auth/<provider>/callback`, with `github` or
+`cilogon` as the provider: `deploy/ansible/group_vars/dispatcher.yml` renders
+these as `dispatcher_github_oauth_callback_url` and
+`dispatcher_cilogon_oidc_callback_url`, and the dispatcher sends the configured
+value as the redirect URI. `dispatcher_cilogon_oidc_issuer` must be
+`https://cilogon.org` or `https://test.cilogon.org`; a client approved on
+`cilogon.org` does not work against `test.cilogon.org`.
+
+To change a callback, register the new URL with the provider first, deploy the
+configuration with `make deploy-update-config DEPLOY_ENV=<env>`, verify sign-in
+through that provider, then remove the old URL. If the provider accepts only one
+callback URL, sign-in through it can fail until the registration and the
+deployed configuration match. Each callback must share its origin with its
+success URL and, when it is set, with `authentication.public_url`.
+
+To renew or re-register a client, write the new values into the owner-only
+`secrets/<env>/github-oauth.env` or `secrets/<env>/cilogon_oidc.env` beside the
+playbooks, created from `github-oauth.env.example` or
+`cilogon_oidc.env.example`. The dispatcher reads `GITHUB_OAUTH_CLIENT_ID`,
+`GITHUB_OAUTH_CLIENT_SECRET`, `CILOGON_CLIENT_ID` and `CILOGON_CLIENT_SECRET`
+only when it starts. `make deploy-update-config` installs a changed credential
+file with mode `0600`, refreshes the service unit's list of environment files
+and restarts the dispatcher, which then reads the new values. Verify sign-in,
+then revoke the old secret at the provider. A GitHub identity is the GitHub
+account's numeric ID, so a new GitHub OAuth app keeps existing identities.
+
 ## Approve a CLI or headless host
 
 ```sh
@@ -132,6 +164,35 @@ authenticated requests; it does not cancel already admitted measurements.
 Handle active work separately and verify its observed outcome before claiming
 traffic has stopped.
 
+## Provider-side incidents
+
+**Leaked client secret.** Create a new secret at the provider, revoke the old
+one and install the new value as described in
+[provider registrations](#provider-registrations-ownership-renewal-and-callback-changes),
+then verify sign-in. Debuglet sessions do not record the provider that created
+them, so they cannot be revoked by provider. Revocation is per account: each
+affected account holder revokes all credentials on the **Credentials** page or,
+where the account has one, uses its recovery code, which revokes every session
+of the account. Otherwise
+sessions and API credentials expire twelve hours after issue.
+
+**Compromised or rotated CILogon signing key.** The dispatcher fetches new
+signing keys when a token names an unknown key and refreshes discovery hourly,
+so a routine rotation needs no action. If every CILogon sign-in fails
+validation, no session is created; check the configured issuer and that the
+dispatcher reaches the provider. A restart discards cached discovery metadata.
+If the provider reports a key compromise, set
+`dispatcher_cilogon_oidc_enabled: false`, deploy the configuration and revoke
+the affected accounts' credentials as above. GitHub sign-in does not verify a signed token: the dispatcher reads the
+account from GitHub's API over HTTPS with the exchanged access token.
+
+**Provider outage.** Existing Debuglet sessions keep working, while new sign-ins
+through that provider fail without creating a session; see
+[configure a dispatcher](#configure-a-dispatcher).
+
+**Compromised provider account.** Follow
+[respond to a copied credential](#respond-to-a-copied-credential).
+
 ## Authentication request limits
 
 Authentication issuance and polling have bounded per-client rate limits and a

docs/operations/remote-deployment.md

@@ -1,6 +1,6 @@
 # Run Debuglet across hosts
 
-A networked deployment has one dispatcher and one or more executors on Linux amd64 hosts. Every role runs the same Debuglet release. Use TLS, authenticate users, and keep the local-development profile disabled.
+A networked deployment has one dispatcher and one or more executors on Linux amd64 hosts. Every role runs the same Debuglet release. Use TLS, authenticate users, and keep the local-development profile disabled. Running a second dispatcher, against the same database or a separate one, is unsupported, and the dispatcher does not detect one at startup. To recover from a dispatcher outage, restart the dispatcher as described in [dispatcher outage](dispatcher-outage.md); [backup and restore](backup-restore.md) covers generated local TEST state.
 
 Use the full release bundle for the bundled samples. The native daemons accept
 operator-supplied configuration; `dbl up`, `dbl executor up`, and `dbl service

docs/security.md

@@ -43,6 +43,77 @@ Keep it off in a managed deployment. Authentication does not provide a general
 registration, login or workload abuse prevention service; restrict exposure to
 the intended users.
 
+### Accounts, sessions and provider sign-in
+
+The dispatcher issues and verifies every Debuglet credential. After a provider
+sign-in, a browser holds its session only as an `HttpOnly`, `SameSite=Strict`
+cookie, marked `Secure` when the dispatcher serves TLS or is configured behind
+a TLS terminator, and repeats the session's CSRF token in the `X-Debuglet-CSRF`
+header of every cookie-authenticated change. `dbl` and other API clients present
+a bearer credential. GitHub OAuth and CILogon OIDC only identify a person: their
+authorization codes and tokens are used during the callback exchange and never
+become or are stored as Debuglet credentials. Account keys, recovery codes,
+sessions and API credentials are stored only as SHA-256 digests of their secret
+part.
+
+Sessions and API credentials expire twelve hours after issue and are not
+extended by use. A provider sign-in attempt and a device-login request expire
+after ten minutes. Creating, listing or revoking API credentials, approving a
+device and changing sign-in methods require a browser session authenticated in
+the last ten minutes. An API credential carries only the approved scopes among
+`account:read`, `measurements:read`, `measurements:write`, `executors:read` and
+`executors:write`, is accepted only for the configured
+`authentication.public_url` audience and never from a cookie, and cannot manage
+credentials or sign-in methods. Logout revokes the presenting session;
+`dbl logout` also forgets the stored copy.
+
+Protections:
+
+- Login CSRF and code interception: every sign-in uses PKCE (`S256`) and a
+  random state, stored server-side only as a hash, consumed once and matched
+  against short-lived `HttpOnly` cookies of the browser that started it.
+- Forged CILogon tokens: the ID token's signature, issuer, audience, expiry,
+  nonce, authorized party and access-token hash are checked; any failure
+  creates no session.
+- Provider mix-up: each provider has its own callback and cookies, and an
+  attempt is consumed only by the provider that started it. GitHub identities
+  use the fixed issuer `https://github.com`; the CILogon issuer must be
+  `https://cilogon.org` or `https://test.cilogon.org`.
+- Open redirect: the configured callback is the redirect URI, and the callback
+  returns only to the configured success URL; both are absolute HTTPS URLs on
+  one origin without query or fragment.
+- Session fixation: the server mints a new session at sign-in and revokes the
+  browser's previous session.
+- Device-login phishing: approval needs a recent browser session, the requested
+  audience must be this dispatcher, the label and scopes can be inspected first,
+  and the approval must be confirmed explicitly. A request is consumed once;
+  revoking the approving session cancels an unconsumed approval.
+
+An external identity is its issuer and subject. Names never match accounts and
+email is not read, so a first sign-in creates a new ordinary account. Linking requires a recent session, a fresh provider sign-in started
+from it and explicit confirmation; an identity owned by another account is
+refused. Removing a method is refused when no other enabled method and no
+recovery code would remain. Linking or removing revokes all of the account's
+sessions and API credentials and issues a new browser session. A recovery code
+names only its own account: using it replaces the account key and recovery
+code and revokes every session of that account. No route claims an account by
+UUID or email.
+
+The dispatcher logs sign-in method changes, device-login decisions and
+completions, and API credential issuance and revocation with the account ID and,
+where one applies, the credential identifier, never the secret. Successful provider sign-ins are not
+logged as separate events. The request log records the matched route, never the
+raw URL or query, so authorization codes and state values are not logged.
+Private error diagnostics replace the request's `Authorization` header, bearer
+token and cookie values with `[redacted]`.
+
+`server.local_development` serves requests that present no credential as the
+local operator. It requires `tls.disable = true`, `sui.disabled = true`,
+`server.behind_tls_terminator = false` and a loopback `server.bind_host`, and
+the dispatcher refuses to start unless both listeners are bound to loopback. A
+presented credential is still verified. The deployment templates do not enable
+it.
+
 ### Dispatcher and executor control
 
 Both control paths use the negotiated binding, secret session token and a live

internal/dispatcher/config/config.go

@@ -225,6 +225,8 @@ func (cfg *DispatcherConfig) Validate() error {
 			return errors.New("server.local_development requires sui.disabled = true; it serves unauthenticated requests as an operator and is only for a local environment")
 		case cfg.Server.BindHost == "":
 			return errors.New("server.local_development requires server.bind_host to be a loopback address; it serves unauthenticated requests as an operator and must not listen on every interface")
+		case cfg.Server.BehindTLSTerminator:
+			return errors.New("server.local_development requires server.behind_tls_terminator = false; it serves unauthenticated requests as an operator and must not be published through a TLS terminator")
 		}
 	}
 	if err := configcheck.LogLevel("logging.log_level", cfg.Logging.LogLevel); err != nil {

internal/dispatcher/transport/api/disclosure_test.go

@@ -529,4 +529,27 @@ func TestPrivateHTTPDiagnosticsRedactPresentedCredentials(t *testing.T) {
 	if len(private) != 1 || !strings.Contains(dcRender(private[0]), "private SQL diagnostic") || strings.Contains(fmt.Sprint(private[0].ContextMap()["error"]), "\n") {
 		t.Fatal("operator diagnostic lost or unbounded")
 	}
+
+	// A sign-in callback carries its authorization code and state in the query,
+	// and a state-changing browser request its CSRF token in a header.
+	core, logs = observer.New(zapcore.DebugLevel)
+	request = httptest.NewRequest(http.MethodGet, "/auth/github/callback?code=code-sentinel&state=state-sentinel", nil)
+	request.Header.Set(csrfHeaderName, "csrf-sentinel")
+	c = echo.New().NewContext(request, httptest.NewRecorder())
+	c.SetPath("/auth/github/callback")
+	h = NewHandler(nil, nil, zap.New(core))
+	h.errorHandler(apiErrorFrom(http.StatusInternalServerError, CodeInternal, "operation failed", errors.New("exchange code-sentinel for state-sentinel with csrf-sentinel failed")), c)
+	private = logs.FilterMessage("Private request diagnostic").All()
+	if len(private) != 1 {
+		t.Fatal("callback diagnostic lost")
+	}
+	rendered := dcRender(private[0])
+	for _, secret := range []string{"code-sentinel", "state-sentinel", "csrf-sentinel"} {
+		if strings.Contains(rendered, secret) {
+			t.Errorf("private diagnostic retained %s", secret)
+		}
+	}
+	if !strings.Contains(rendered, "exchange [redacted] for [redacted] with [redacted] failed") {
+		t.Error("private diagnostic lost the redacted cause")
+	}
 }

internal/dispatcher/transport/api/errors.go

@@ -175,7 +175,9 @@ func requestDiagnostic(c echo.Context, cause error, additional ...string) string
 		return ""
 	}
 	detail := cause.Error()
-	secrets := append(additional, c.Request().Header.Get("Authorization"), c.Request().Header.Get("X-CSRF-Token"))
+	// An authorization code and state arrive in the query of a sign-in callback.
+	secrets := append(additional, c.Request().Header.Get("Authorization"), c.Request().Header.Get(csrfHeaderName),
+		c.QueryParam("code"), c.QueryParam("state"))
 	if scheme, token, ok := strings.Cut(c.Request().Header.Get("Authorization"), " "); ok && strings.EqualFold(scheme, "Bearer") {
 		secrets = append(secrets, token)
 	}

internal/dispatcher/transport/api/github_oauth_test.go

@@ -8,6 +8,7 @@ import (
 	"net/http/httptest"
 	"net/url"
 	"strings"
+	"sync/atomic"
 	"testing"
 )
 
@@ -129,3 +130,103 @@ func TestGitHubOAuthRejectsMismatchedState(t *testing.T) {
 		t.Fatalf("status = %d", response.StatusCode)
 	}
 }
+
+// stubGitHub serves the GitHub authorization, token and user endpoints from
+// handler for the rest of the test.
+func stubGitHub(t *testing.T, handler http.HandlerFunc) {
+	t.Helper()
+	provider := httptest.NewServer(handler)
+	oldAuthorize, oldToken, oldUser, oldClient := githubAuthorizeURL, githubTokenURL, githubUserURL, githubOAuthHTTPClient
+	githubAuthorizeURL, githubTokenURL, githubUserURL, githubOAuthHTTPClient = provider.URL+"/authorize", provider.URL+"/token", provider.URL+"/user", provider.Client()
+	t.Cleanup(func() {
+		githubAuthorizeURL, githubTokenURL, githubUserURL, githubOAuthHTTPClient = oldAuthorize, oldToken, oldUser, oldClient
+		provider.Close()
+	})
+}
+
+func githubFixture(t *testing.T, options ...Option) *ccFixture {
+	t.Helper()
+	return ccNewFixtureWith(t, append([]Option{CookieSecure(true), GitHubOAuth(GitHubOAuthConfig{Enabled: true, ClientID: "test-id", ClientSecret: "test-secret", CallbackURL: "https://example.test/api/auth/github/callback", SuccessURL: "https://example.test/console/"})}, options...)...)
+}
+
+// githubCallback starts a GitHub sign-in and returns the provider's callback
+// for it, carrying the issued state, the login cookies and query.
+func githubCallback(t *testing.T, f *ccFixture, query url.Values) *http.Request {
+	t.Helper()
+	start, err := http.NewRequest(http.MethodGet, f.root.URL+"/auth/github", nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	response := cilogonResponse(t, start)
+	location, err := url.Parse(response.Header.Get("Location"))
+	if err != nil || response.StatusCode != http.StatusFound || location.Query().Get("state") == "" {
+		t.Fatalf("start = %d %s", response.StatusCode, response.Header.Get("Location"))
+	}
+	query.Set("state", location.Query().Get("state"))
+	callback, err := http.NewRequest(http.MethodGet, f.root.URL+"/auth/github/callback?"+query.Encode(), nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, cookie := range response.Cookies() {
+		callback.AddCookie(cookie)
+	}
+	return callback
+}
+
+// githubSignInFailed asserts the callback ended at the sign-in failure state
+// without an account, a linked identity or a session.
+func githubSignInFailed(t *testing.T, f *ccFixture, response *http.Response, code string) {
+	t.Helper()
+	if response.StatusCode != http.StatusFound || response.Header.Get("Location") != "https://example.test/console/?auth_error="+code {
+		t.Fatalf("callback = %d %s, want auth_error=%s", response.StatusCode, response.Header.Get("Location"), code)
+	}
+	for _, cookie := range response.Cookies() {
+		if (cookie.Name == sessionCookieName || cookie.Name == csrfCookieName) && cookie.Value != "" {
+			t.Fatalf("failed sign-in set %s", cookie.Name)
+		}
+	}
+	var users, identities int
+	if err := f.db.QueryRow(`SELECT COUNT(*) FROM users`).Scan(&users); err != nil {
+		t.Fatal(err)
+	}
+	if err := f.db.QueryRow(`SELECT COUNT(*) FROM oauth_identities`).Scan(&identities); err != nil {
+		t.Fatal(err)
+	}
+	if users != 0 || identities != 0 {
+		t.Fatalf("failed sign-in stored users=%d identities=%d", users, identities)
+	}
+}
+
+func TestGitHubOAuthProviderOutageIsATemporarySignInFailure(t *testing.T) {
+	var exchanges, lookups atomic.Int32
+	stubGitHub(t, func(w http.ResponseWriter, r *http.Request) {
+		switch r.URL.Path {
+		case "/token":
+			exchanges.Add(1)
+			http.Error(w, "service unavailable", http.StatusServiceUnavailable)
+		case "/user":
+			lookups.Add(1)
+			http.Error(w, "unexpected", http.StatusInternalServerError)
+		default:
+			http.NotFound(w, r)
+		}
+	})
+	f := githubFixture(t)
+	githubSignInFailed(t, f, cilogonResponse(t, githubCallback(t, f, url.Values{"code": {"one-time"}})), "provider_unavailable")
+	if exchanges.Load() != 1 || lookups.Load() != 0 {
+		t.Fatalf("token exchanges=%d user lookups=%d", exchanges.Load(), lookups.Load())
+	}
+}
+
+func TestGitHubOAuthCancelledSignInCreatesNoAccount(t *testing.T) {
+	var requests atomic.Int32
+	stubGitHub(t, func(w http.ResponseWriter, r *http.Request) {
+		requests.Add(1)
+		http.NotFound(w, r)
+	})
+	f := githubFixture(t)
+	githubSignInFailed(t, f, cilogonResponse(t, githubCallback(t, f, url.Values{"error": {"access_denied"}})), "cancelled")
+	if requests.Load() != 0 {
+		t.Fatalf("a cancelled sign-in contacted the provider %d times", requests.Load())
+	}
+}