Reduce database load from dashboard polling and correct activity reporting

ARCHITECTURE.md

@@ -202,6 +202,7 @@ details intentionally omit users, environment UUIDs, query names, and payloads.
 - Per-environment access lives in `user_permissions`.
 - The user manager's `CheckPermissions(...)` method grants access by environment UUID/string plus level (`user`, `query`, `carve`, `admin`).
 - Full admins bypass per-environment checks.
+- HTTP handlers use `CheckPermissionsContext` with a request-scoped cache. Authentication seeds the user row; each environment's permission rows are loaded once per request. Streams create a fresh snapshot for every batch and heartbeat, preserving database-authoritative revocation across replicas and CLI changes. No permission snapshot is reused between requests or stream batches.
 
 ## Settings Subsystem
 
@@ -285,7 +286,10 @@ Handled by `pkg/auditlog`.
 
 - Model: `AuditLog`
 - GORM table: `audit_logs`
-- Created from API and hosted MCP actions such as login, node/query/carve actions, settings changes, and MCP tool calls. TLS also records failed enrollment and consumed service-control events.
+- Created from API and hosted MCP actions such as login, node/query/carve mutations, explicit downloads, settings changes, and MCP tool calls. Routine successful REST reads and polling do not create audit rows. TLS records successful and failed enrollment and consumed service-control events.
+- Successful enrollment/re-enrollment uses `LogTypeEnroll` (12), after node persistence succeeds. The audit-backed environment activity endpoint counts only that type as enrollment; generic node events, including historical viewed rows, are excluded. Query actions use `LogTypeQuery`. Earlier successful enrollments were not audited, so historical enrollment counts are not reconstructed from generic node events.
+- The API seeds the global `api/audit_log_retention_days` integer setting (90 days; valid range 1–36500). It can be edited through the settings API. If older audit history must be retained on upgrade, preconfigure its global `setting_values` row before starting the upgraded API. At startup and hourly, the API physically deletes expired rows from all services, including soft-deleted rows, in batches of 1,000, with a 100 ms pause and five-minute sweep limit. Pruning runs even if new audit writes are disabled. Invalid settings or setting-read failures stop cleanup. API replicas may safely race; no cleanup worker runs when the API is stopped.
+- DB log startup builds `(uuid, created_at)` indexes before removing the redundant status/result UUID indexes. PostgreSQL replacement indexes must be valid and not building before removal; drops use `CONCURRENTLY`. The UUID column retains its original `varchar(191)` type on MySQL and `text` type on PostgreSQL/SQLite, avoiding a column rewrite. The query-log UUID index stays because `(name, created_at)` does not cover it. Complete rollout of the new logger code before relying on removal: an older process can recreate the old indexes at startup.
 
 ## Request Flow
 

cmd/api/auth.go

@@ -10,6 +10,7 @@ import (
 	"github.com/jmpsec/osctrl/cmd/api/handlers"
 	"github.com/jmpsec/osctrl/pkg/config"
 	"github.com/jmpsec/osctrl/pkg/types"
+	"github.com/jmpsec/osctrl/pkg/users"
 	"github.com/jmpsec/osctrl/pkg/utils"
 	"github.com/rs/zerolog/log"
 )
@@ -102,7 +103,7 @@ func handlerAuthCheck(h http.Handler, auth, jwtSecret string) http.Handler {
 			// Set middleware values
 			s := make(handlers.ContextValue)
 			s["user"] = "admin"
-			ctx := context.WithValue(r.Context(), handlers.ContextKey(contextAPI), s)
+			ctx := users.WithPermissionCache(context.WithValue(r.Context(), handlers.ContextKey(contextAPI), s))
 			// Access granted
 			h.ServeHTTP(w, r.WithContext(ctx))
 		case config.AuthJWT:
@@ -163,7 +164,8 @@ func handlerAuthCheck(h http.Handler, auth, jwtSecret string) http.Handler {
 			// Set middleware values
 			s := make(handlers.ContextValue)
 			s["user"] = claims.Username
-			ctx := context.WithValue(r.Context(), handlers.ContextKey(contextAPI), s)
+			ctx := users.WithPermissionCache(context.WithValue(r.Context(), handlers.ContextKey(contextAPI), s))
+			users.CacheAuthenticatedUser(ctx, user)
 			// Access granted
 			h.ServeHTTP(w, r.WithContext(ctx))
 		}
@@ -186,5 +188,9 @@ func eventSessionValid(r *http.Request, auth, secret string) bool {
 	}
 	user, err := apiUsers.Get(claims.Username)
 	ctx, ok := r.Context().Value(handlers.ContextKey(contextAPI)).(handlers.ContextValue)
-	return ok && ctx["user"] == claims.Username && err == nil && user.APIToken != "" && subtle.ConstantTimeCompare([]byte(user.APIToken), []byte(token)) == 1
+	valid = ok && ctx["user"] == claims.Username && err == nil && user.APIToken != "" && subtle.ConstantTimeCompare([]byte(user.APIToken), []byte(token)) == 1
+	if valid {
+		users.CacheAuthenticatedUser(r.Context(), user)
+	}
+	return valid
 }

cmd/api/events_auth_test.go

@@ -50,3 +50,32 @@ func TestEventSessionRechecksStoredTokenAndIdentity(t *testing.T) {
 	r.Header.Set("Cookie", cookieNameToken+"="+expired)
 	require.False(t, eventSessionValid(r, config.AuthJWT, "event-test-secret-at-least-thirty-two-bytes"))
 }
+
+func TestEventPermissionSnapshotReusesAuthenticatedUser(t *testing.T) {
+	db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
+	require.NoError(t, err)
+	previous := apiUsers
+	t.Cleanup(func() { apiUsers = previous })
+	secret := "event-test-secret-at-least-thirty-two-bytes"
+	apiUsers = users.CreateUserManager(db).WithJWT(&config.YAMLConfigurationJWT{JWTSecret: secret, HoursToExpire: 1})
+	require.NoError(t, apiUsers.Create(users.AdminUser{Username: "alice"}))
+	require.NoError(t, apiUsers.CreatePermission(users.UserPermission{Username: "alice", Environment: "env", AccessType: int(users.AdminLevel), AccessValue: true}))
+	token, expiry, err := apiUsers.CreateToken("alice", "api", 1)
+	require.NoError(t, err)
+	require.NoError(t, apiUsers.UpdateToken("alice", token, expiry))
+	r := httptest.NewRequest("GET", "/api/v1/events", nil)
+	r.Header.Set("Authorization", "Bearer "+token)
+	r = r.WithContext(context.WithValue(r.Context(), handlers.ContextKey(contextAPI), handlers.ContextValue{"user": "alice"}))
+	queries := 0
+	require.NoError(t, db.Callback().Query().Before("gorm:query").Register("count_auth_reads", func(*gorm.DB) { queries++ }))
+	pass := r.WithContext(users.WithPermissionCache(r.Context()))
+	require.True(t, eventSessionValid(pass, config.AuthJWT, secret))
+	require.True(t, apiUsers.CheckPermissionsContext(pass.Context(), "alice", users.QueryLevel, "env"))
+	require.True(t, apiUsers.CheckPermissionsContext(pass.Context(), "alice", users.CarveLevel, "env"))
+	require.Equal(t, 2, queries, "one token/user read plus one permissions read for both topics")
+	require.NoError(t, db.Model(&users.UserPermission{}).Where("username = ?", "alice").Update("access_value", false).Error)
+	pass = r.WithContext(users.WithPermissionCache(r.Context()))
+	require.True(t, eventSessionValid(pass, config.AuthJWT, secret))
+	require.False(t, apiUsers.CheckPermissionsContext(pass.Context(), "alice", users.QueryLevel, "env"))
+	require.Equal(t, 4, queries, "the next batch must refresh both rows")
+}

cmd/api/handlers/alerts.go

@@ -10,7 +10,6 @@ import (
 	"time"
 
 	"github.com/jmpsec/osctrl/pkg/alerts"
-	"github.com/jmpsec/osctrl/pkg/auditlog"
 	"github.com/jmpsec/osctrl/pkg/config"
 	"github.com/jmpsec/osctrl/pkg/servicecommands"
 	"github.com/jmpsec/osctrl/pkg/types"
@@ -100,7 +99,7 @@ func toAlertChannelDTO(c alerts.AlertChannel, reveal bool) alertChannelDTO {
 
 func (h *HandlersApi) requireAlertsAdmin(w http.ResponseWriter, r *http.Request) (string, bool) {
 	ctx := r.Context().Value(ContextKey(contextAPI)).(ContextValue)
-	if !h.Users.CheckPermissions(ctx[ctxUser], users.AdminLevel, users.NoEnvironment) {
+	if !h.Users.CheckPermissionsContext(r.Context(), ctx[ctxUser], users.AdminLevel, users.NoEnvironment) {
 		apiErrorResponse(w, "no access", http.StatusForbidden, fmt.Errorf("attempt to use alerts API by user %s", ctx[ctxUser]))
 		return "", false
 	}
@@ -134,7 +133,7 @@ func (h *HandlersApi) AlertRulesListHandler(w http.ResponseWriter, r *http.Reque
 	if h.DebugHTTPConfig != nil && h.DebugHTTPConfig.EnableHTTP {
 		utils.DebugHTTPDump(h.DebugHTTP, r, false)
 	}
-	user, ok := h.requireAlertsAdmin(w, r)
+	_, ok := h.requireAlertsAdmin(w, r)
 	if !ok {
 		return
 	}
@@ -156,7 +155,6 @@ func (h *HandlersApi) AlertRulesListHandler(w http.ResponseWriter, r *http.Reque
 	for _, row := range rows {
 		out = append(out, toAlertRuleDTO(row))
 	}
-	h.AuditLog.Visit(user, r.URL.Path, strings.Split(r.RemoteAddr, ":")[0], auditlog.NoEnvironment)
 	utils.HTTPResponse(w, utils.JSONApplicationUTF8, http.StatusOK, out)
 }
 
@@ -179,7 +177,7 @@ func (h *HandlersApi) AlertRuleGetHandler(w http.ResponseWriter, r *http.Request
 	if h.DebugHTTPConfig != nil && h.DebugHTTPConfig.EnableHTTP {
 		utils.DebugHTTPDump(h.DebugHTTP, r, false)
 	}
-	user, ok := h.requireAlertsAdmin(w, r)
+	_, ok := h.requireAlertsAdmin(w, r)
 	if !ok {
 		return
 	}
@@ -197,7 +195,6 @@ func (h *HandlersApi) AlertRuleGetHandler(w http.ResponseWriter, r *http.Request
 		respondAlertsErr(w, err)
 		return
 	}
-	h.AuditLog.Visit(user, r.URL.Path, strings.Split(r.RemoteAddr, ":")[0], auditlog.NoEnvironment)
 	utils.HTTPResponse(w, utils.JSONApplicationUTF8, http.StatusOK, toAlertRuleDTO(row))
 }
 
@@ -377,7 +374,7 @@ func (h *HandlersApi) AlertChannelsListHandler(w http.ResponseWriter, r *http.Re
 	if h.DebugHTTPConfig != nil && h.DebugHTTPConfig.EnableHTTP {
 		utils.DebugHTTPDump(h.DebugHTTP, r, false)
 	}
-	user, ok := h.requireAlertsAdmin(w, r)
+	_, ok := h.requireAlertsAdmin(w, r)
 	if !ok {
 		return
 	}
@@ -400,7 +397,6 @@ func (h *HandlersApi) AlertChannelsListHandler(w http.ResponseWriter, r *http.Re
 	for _, row := range rows {
 		out = append(out, toAlertChannelDTO(row, reveal))
 	}
-	h.AuditLog.Visit(user, r.URL.Path, strings.Split(r.RemoteAddr, ":")[0], auditlog.NoEnvironment)
 	utils.HTTPResponse(w, utils.JSONApplicationUTF8, http.StatusOK, out)
 }
 
@@ -468,7 +464,7 @@ func (h *HandlersApi) AlertChannelsGetHandler(w http.ResponseWriter, r *http.Req
 	if h.DebugHTTPConfig != nil && h.DebugHTTPConfig.EnableHTTP {
 		utils.DebugHTTPDump(h.DebugHTTP, r, false)
 	}
-	user, ok := h.requireAlertsAdmin(w, r)
+	_, ok := h.requireAlertsAdmin(w, r)
 	if !ok {
 		return
 	}
@@ -486,7 +482,6 @@ func (h *HandlersApi) AlertChannelsGetHandler(w http.ResponseWriter, r *http.Req
 		respondAlertsErr(w, err)
 		return
 	}
-	h.AuditLog.Visit(user, r.URL.Path, strings.Split(r.RemoteAddr, ":")[0], auditlog.NoEnvironment)
 	utils.HTTPResponse(w, utils.JSONApplicationUTF8, http.StatusOK, toAlertChannelDTO(row, parseReveal(r)))
 }
 
@@ -737,7 +732,7 @@ func (h *HandlersApi) AlertHistoryHandler(w http.ResponseWriter, r *http.Request
 	if h.DebugHTTPConfig != nil && h.DebugHTTPConfig.EnableHTTP {
 		utils.DebugHTTPDump(h.DebugHTTP, r, false)
 	}
-	user, ok := h.requireAlertsAdmin(w, r)
+	_, ok := h.requireAlertsAdmin(w, r)
 	if !ok {
 		return
 	}
@@ -751,7 +746,6 @@ func (h *HandlersApi) AlertHistoryHandler(w http.ResponseWriter, r *http.Request
 		apiErrorResponse(w, "error listing alert history", http.StatusInternalServerError, err)
 		return
 	}
-	h.AuditLog.Visit(user, r.URL.Path, strings.Split(r.RemoteAddr, ":")[0], auditlog.NoEnvironment)
 	utils.HTTPResponse(w, utils.JSONApplicationUTF8, http.StatusOK, rows)
 }
 

cmd/api/handlers/audit.go

@@ -18,15 +18,14 @@ import (
 //
 //	?service=...       exact match on service name
 //	?username=...      case-insensitive partial match on username
-//	?type=...          log type integer (1..11), see pkg/auditlog.LogType*
+//	?type=...          log type integer (1..12), see pkg/auditlog.LogType*
 //	?env_uuid=...      filter to one environment (resolved to internal ID)
 //	?since=RFC3339     created_at >= since
 //	?until=RFC3339     created_at <= until
 //	?page=N            1-indexed page; default 1
 //	?page_size=N       default 50, max 500
 //
-// Returns the SPA-canonical paginated envelope. The handler audit-logs the
-// visit on success.
+// Returns the SPA-canonical paginated envelope without auditing reads.
 // @Summary List audit logs
 // @Description Returns paginated API audit log entries.
 // @Tags audit

cmd/api/handlers/audit_growth_test.go

@@ -0,0 +1,61 @@
+package handlers
+
+import (
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/jmpsec/osctrl/pkg/auditlog"
+	"github.com/jmpsec/osctrl/pkg/config"
+	"github.com/stretchr/testify/require"
+)
+
+func TestPollingDoesNotCreateAuditRows(t *testing.T) {
+	db, h, env, _ := setupConsoleHandlers(t)
+	h.DebugHTTPConfig = &config.YAMLConfigurationDebug{}
+	var err error
+	h.AuditLog, err = auditlog.CreateAuditLogManager(db, config.ServiceAPI, true)
+	require.NoError(t, err)
+	for _, handler := range []http.HandlerFunc{h.NodesPagedHandler, h.QueryListHandler, h.CarveListHandler} {
+		for i := 0; i < 3; i++ {
+			r := consoleRequest(http.MethodGet, "/poll", nil, "alice")
+			r.SetPathValue("env", env.Name)
+			r.SetPathValue("target", "all")
+			w := httptest.NewRecorder()
+			handler(w, r)
+			require.Equal(t, http.StatusOK, w.Code, w.Body.String())
+		}
+	}
+	var count int64
+	require.NoError(t, db.Model(&auditlog.AuditLog{}).Count(&count).Error)
+	require.Zero(t, count)
+}
+
+func TestEnvActivityCountsOnlySuccessfulEnrollments(t *testing.T) {
+	db, h, env, _ := setupConsoleHandlers(t)
+	var err error
+	h.AuditLog, err = auditlog.CreateAuditLogManager(db, config.ServiceAPI, true)
+	require.NoError(t, err)
+	h.AuditLog.NodeAction("alice", "viewed paged nodes", "", env.ID)
+	h.AuditLog.NodeAction("alice", "deleted node", "", env.ID)
+	h.AuditLog.FailedEnroll("", env.Name, "invalid secret", env.ID)
+	h.AuditLog.Denied("alice", "/nodes", "", "forbidden", auditlog.LogTypeNode, env.ID)
+	h.AuditLog.QueryAction("alice", "complete query", "", env.ID)
+	h.AuditLog.NewEnroll("NODE", "", env.ID)
+	h.AuditLog.NewEnroll("OTHER-ENV", "", env.ID+1)
+	r := consoleRequest(http.MethodGet, "/activity", nil, "alice")
+	r.SetPathValue("env", env.Name)
+	w := httptest.NewRecorder()
+	h.EnvActivityHandler(w, r)
+	require.Equal(t, http.StatusOK, w.Code, w.Body.String())
+	var buckets []ActivityBucket
+	require.NoError(t, json.Unmarshal(w.Body.Bytes(), &buckets))
+	var enrolls, queries int
+	for _, bucket := range buckets {
+		enrolls += bucket.Enroll
+		queries += bucket.Query
+	}
+	require.Equal(t, 1, enrolls)
+	require.Equal(t, 1, queries)
+}

cmd/api/handlers/auth_providers.go

@@ -12,7 +12,6 @@ import (
 	"strings"
 	"time"
 
-	"github.com/jmpsec/osctrl/pkg/auditlog"
 	"github.com/jmpsec/osctrl/pkg/authproviders"
 	"github.com/jmpsec/osctrl/pkg/config"
 	"github.com/jmpsec/osctrl/pkg/servicecommands"
@@ -57,7 +56,7 @@ func toAuthProviderDTO(p authproviders.AuthProvider, reveal bool) authProviderDT
 
 func (h *HandlersApi) requireAuthProvidersAdmin(w http.ResponseWriter, r *http.Request) (string, bool) {
 	ctx := r.Context().Value(ContextKey(contextAPI)).(ContextValue)
-	if !h.Users.CheckPermissions(ctx[ctxUser], users.AdminLevel, users.NoEnvironment) {
+	if !h.Users.CheckPermissionsContext(r.Context(), ctx[ctxUser], users.AdminLevel, users.NoEnvironment) {
 		apiErrorResponse(w, "no access", http.StatusForbidden, fmt.Errorf("attempt to use auth-providers API by user %s", ctx[ctxUser]))
 		return "", false
 	}
@@ -69,7 +68,7 @@ func (h *HandlersApi) AuthProvidersListHandler(w http.ResponseWriter, r *http.Re
 	if h.DebugHTTPConfig != nil && h.DebugHTTPConfig.EnableHTTP {
 		utils.DebugHTTPDump(h.DebugHTTP, r, false)
 	}
-	user, ok := h.requireAuthProvidersAdmin(w, r)
+	_, ok := h.requireAuthProvidersAdmin(w, r)
 	if !ok {
 		return
 	}
@@ -87,7 +86,6 @@ func (h *HandlersApi) AuthProvidersListHandler(w http.ResponseWriter, r *http.Re
 	for _, row := range rows {
 		out = append(out, toAuthProviderDTO(row, reveal))
 	}
-	h.AuditLog.Visit(user, r.URL.Path, strings.Split(r.RemoteAddr, ":")[0], auditlog.NoEnvironment)
 	utils.HTTPResponse(w, utils.JSONApplicationUTF8, http.StatusOK, out)
 }
 
@@ -125,7 +123,7 @@ func (h *HandlersApi) AuthProvidersGetHandler(w http.ResponseWriter, r *http.Req
 	if h.DebugHTTPConfig != nil && h.DebugHTTPConfig.EnableHTTP {
 		utils.DebugHTTPDump(h.DebugHTTP, r, false)
 	}
-	user, ok := h.requireAuthProvidersAdmin(w, r)
+	_, ok := h.requireAuthProvidersAdmin(w, r)
 	if !ok {
 		return
 	}
@@ -147,7 +145,6 @@ func (h *HandlersApi) AuthProvidersGetHandler(w http.ResponseWriter, r *http.Req
 		apiErrorResponse(w, "error getting auth provider", http.StatusInternalServerError, err)
 		return
 	}
-	h.AuditLog.Visit(user, r.URL.Path, strings.Split(r.RemoteAddr, ":")[0], auditlog.NoEnvironment)
 	utils.HTTPResponse(w, utils.JSONApplicationUTF8, http.StatusOK, toAuthProviderDTO(row, r.URL.Query().Get("reveal") == "1"))
 }