diff --git a/.github/workflows/hourly-jobs.yml b/.github/workflows/hourly-jobs.yml new file mode 100644 index 000000000..9f109ac7d --- /dev/null +++ b/.github/workflows/hourly-jobs.yml @@ -0,0 +1,41 @@ +name: Hourly jobs + +# Vercel Hobby only runs crons once a day, so hourly jobs are triggered from here. +# Requires repository secrets APP_URL (e.g. https://devtrack.example.com) and CRON_SECRET. +on: + schedule: + - cron: "5 * * * *" + workflow_dispatch: + +permissions: {} + +concurrency: + group: hourly-jobs + cancel-in-progress: false + +jobs: + trigger: + if: github.repository == 'Priyanshu-byte-coder/devtrack' + runs-on: ubuntu-latest + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + path: + - /api/cron/goals-sync + - /api/notifications/weekly + steps: + - name: Call ${{ matrix.path }} + env: + APP_URL: ${{ secrets.APP_URL }} + CRON_SECRET: ${{ secrets.CRON_SECRET }} + run: | + if [ -z "$APP_URL" ] || [ -z "$CRON_SECRET" ]; then + echo "::notice::APP_URL or CRON_SECRET is not configured; skipping." + exit 0 + fi + status=$(curl -sS -o response.json -w '%{http_code}' --max-time 310 \ + -H "Authorization: Bearer $CRON_SECRET" "${APP_URL%/}${{ matrix.path }}") + echo "HTTP $status" + head -c 500 response.json; echo + test "$status" -lt 400 diff --git a/src/app/api/metrics/coding-activity-insights/route.ts b/src/app/api/metrics/coding-activity-insights/route.ts index f239b6f99..9b2d59889 100644 --- a/src/app/api/metrics/coding-activity-insights/route.ts +++ b/src/app/api/metrics/coding-activity-insights/route.ts @@ -1,13 +1,9 @@ import { githubRateLimitResponse } from "@/lib/github-rate-limit"; -import { githubRequest } from "@/lib/github-request"; +import { collectAuthorCommits, requireUnthrottled } from "@/lib/github-search"; import { getSessionWithToken } from "@/lib/get-session-token"; import { NextRequest } from "next/server"; -import { - getAccountToken, - getAllAccounts, -} from "@/lib/github-accounts"; -import { GITHUB_API } from "@/lib/github"; -import { GitHubAuthError, githubAuthErrorResponse } from "@/lib/github-fetch"; +import { getAccountToken, getAllAccounts } from "@/lib/github-accounts"; +import { githubAuthErrorResponse } from "@/lib/github-fetch"; import { isMetricsCacheBypassed, METRICS_CACHE_TTL_SECONDS, @@ -68,58 +64,19 @@ async function fetchCommitTimestampsForAccount( since.setDate(since.getDate() - SEARCH_WINDOW_DAYS); const sinceStr = since.toISOString().slice(0, 10); - const timestamps: string[] = []; - let totalCount = 0; - let page = 1; - - while (page <= 10) { - const response = await githubRequest( - `${GITHUB_API}/search/commits?q=author:${githubLogin}+author-date:>=${sinceStr}&per_page=100&page=${page}&sort=author-date&order=desc`, + const { items } = requireUnthrottled( + await collectAuthorCommits( + githubLogin, + sinceStr, + token, { - headers: { - Authorization: `Bearer ${token}`, - Accept: "application/vnd.github+json", - }, - cache: "no-store", - } - ); - - if (!response.ok) { - if (response.status === 401) throw new GitHubAuthError(); - if ((response.status === 403 || response.status === 429) && timestamps.length > 0) { - break; + fresh: cacheContext.bypass, } - - throw new Error("GitHub API error"); - } - - const payload = (await response.json()) as { - total_count: number; - items: CommitSearchItem[]; - }; - - if (page === 1) { - totalCount = payload.total_count; - } - - timestamps.push( - ...payload.items - .map((item) => item.commit?.author?.date) - .filter((date): date is string => Boolean(date)) - ); - - if (payload.items.length < 100) { - break; - } - - if (timestamps.length >= 1000 || timestamps.length >= totalCount) { - break; - } - - page += 1; - } - - return timestamps; + ) + ); + return items + .map((item) => item.commit?.author?.date) + .filter((date): date is string => Boolean(date)); } ); } @@ -142,7 +99,11 @@ async function buildInsightsForAccount( export async function GET(req: NextRequest) { const sessionData = await getSessionWithToken(); - if (!sessionData || !sessionData.session.githubLogin || !sessionData.session.githubId) { + if ( + !sessionData || + !sessionData.session.githubLogin || + !sessionData.session.githubId + ) { return Response.json({ error: "Unauthorized" }, { status: 401 }); } @@ -244,10 +205,15 @@ export async function GET(req: NextRequest) { return Response.json({ error: "Account not found" }, { status: 404 }); } - const data = await buildInsightsForAccount(token, accountRow.github_login, timeZone, { - bypass, - userId: accountId, - }); + const data = await buildInsightsForAccount( + token, + accountRow.github_login, + timeZone, + { + bypass, + userId: accountId, + } + ); return Response.json(data); } catch (e) { @@ -256,4 +222,4 @@ export async function GET(req: NextRequest) { return Response.json({ error: "GitHub API error" }, { status: 502 }); } -} \ No newline at end of file +} diff --git a/src/app/api/metrics/commit-times/route.ts b/src/app/api/metrics/commit-times/route.ts index 22514cc02..68a1a4f17 100644 --- a/src/app/api/metrics/commit-times/route.ts +++ b/src/app/api/metrics/commit-times/route.ts @@ -1,9 +1,8 @@ import { githubRateLimitResponse } from "@/lib/github-rate-limit"; -import { githubRequest } from "@/lib/github-request"; +import { collectAuthorCommits, requireUnthrottled } from "@/lib/github-search"; import { getServerSession } from "@/lib/server-session"; import { NextRequest } from "next/server"; import { authOptions } from "@/lib/auth"; -import { GITHUB_API } from "@/lib/github"; import { isMetricsCacheBypassed, METRICS_CACHE_TTL_SECONDS, @@ -39,28 +38,19 @@ export async function GET(req: NextRequest) { new Array(24).fill(0) ); - let page = 1; - while (true) { - const res = await githubRequest( - `${GITHUB_API}/search/commits?q=author:${session.githubLogin}+author-date:>=${sinceStr}&per_page=100&page=${page}&sort=author-date&order=desc`, + const { items } = requireUnthrottled( + await collectAuthorCommits( + session.githubLogin!, + sinceStr, + session.accessToken, { - headers: { - Authorization: `Bearer ${session.accessToken}`, - Accept: "application/vnd.github+json", - }, - cache: "no-store", + fresh: bypass, } - ); - if (!res.ok) throw new Error("GitHub API error"); - const data = (await res.json()) as { - items: Array<{ commit: { author: { date: string } } }>; - }; - for (const item of data.items) { - const d = new Date(item.commit.author.date); - matrix[d.getDay()][d.getHours()]++; - } - if (data.items.length < 100 || page >= 10) break; - page++; + ) + ); + for (const item of items) { + const d = new Date(item.commit.author.date); + matrix[d.getDay()][d.getHours()]++; } // Find peak slot @@ -83,8 +73,8 @@ export async function GET(req: NextRequest) { return Response.json(result); } catch (error) { - const githubLimit = githubRateLimitResponse(error); - if (githubLimit) return githubLimit; + const githubLimit = githubRateLimitResponse(error); + if (githubLimit) return githubLimit; return Response.json({ error: "GitHub API error" }, { status: 502 }); } -} \ No newline at end of file +} diff --git a/src/app/api/metrics/consistency-score/route.ts b/src/app/api/metrics/consistency-score/route.ts index 9cf8439f8..9f0b4faad 100644 --- a/src/app/api/metrics/consistency-score/route.ts +++ b/src/app/api/metrics/consistency-score/route.ts @@ -1,10 +1,9 @@ import { githubRateLimitResponse } from "@/lib/github-rate-limit"; -import { githubRequest } from "@/lib/github-request"; +import { collectAuthorCommits, requireUnthrottled } from "@/lib/github-search"; import { getServerSession } from "@/lib/server-session"; import { NextRequest } from "next/server"; import { authOptions } from "@/lib/auth"; import { getAccountToken, getAllAccounts } from "@/lib/github-accounts"; -import { GITHUB_API } from "@/lib/github"; import { isMetricsCacheBypassed, METRICS_CACHE_TTL_SECONDS, @@ -44,46 +43,26 @@ async function fetchActiveDates( const sinceStr = since.toISOString().slice(0, 10); const activeDates = new Set(); - let page = 1; - - while (true) { - const searchRes = await githubRequest( - `${GITHUB_API}/search/commits?q=author:${githubLogin}+author-date:>=${sinceStr}&per_page=100&page=${page}&sort=author-date&order=desc`, - { - headers: { - Authorization: `Bearer ${token}`, - Accept: "application/vnd.github+json", - }, - cache: "no-store", - } - ); - - if (!searchRes.ok) { - throw new Error("GitHub API error"); - } + const { items } = requireUnthrottled( + await collectAuthorCommits(githubLogin, sinceStr, token, { + fresh: cacheContext.bypass, + }) + ); - const data = (await searchRes.json()) as { - items: Array<{ commit: { author: { date: string } } }>; - }; - - for (const item of data.items) { - const commitDate = new Date(item.commit.author.date); - const parts = new Intl.DateTimeFormat("en", { - timeZone, - year: "numeric", - month: "2-digit", - day: "2-digit", - }).formatToParts(commitDate); - const year = parts.find((p) => p.type === "year")?.value; - const month = parts.find((p) => p.type === "month")?.value; - const day = parts.find((p) => p.type === "day")?.value; - if (year && month && day) { - activeDates.add(`${year}-${month}-${day}`); - } + for (const item of items) { + const commitDate = new Date(item.commit.author.date); + const parts = new Intl.DateTimeFormat("en", { + timeZone, + year: "numeric", + month: "2-digit", + day: "2-digit", + }).formatToParts(commitDate); + const year = parts.find((p) => p.type === "year")?.value; + const month = parts.find((p) => p.type === "month")?.value; + const day = parts.find((p) => p.type === "day")?.value; + if (year && month && day) { + activeDates.add(`${year}-${month}-${day}`); } - - if (data.items.length < 100 || page >= 10) break; - page += 1; } return Array.from(activeDates); @@ -240,8 +219,8 @@ export async function GET(req: NextRequest) { }); return Response.json(result); } catch (error) { - const githubLimit = githubRateLimitResponse(error); - if (githubLimit) return githubLimit; + const githubLimit = githubRateLimitResponse(error); + if (githubLimit) return githubLimit; return Response.json({ error: "GitHub API error" }, { status: 502 }); } } diff --git a/src/app/api/metrics/contributions/hourly/route.ts b/src/app/api/metrics/contributions/hourly/route.ts index 36f74a6c6..8ccf052a4 100644 --- a/src/app/api/metrics/contributions/hourly/route.ts +++ b/src/app/api/metrics/contributions/hourly/route.ts @@ -1,9 +1,8 @@ import { githubRateLimitResponse } from "@/lib/github-rate-limit"; -import { githubRequest } from "@/lib/github-request"; +import { collectAuthorCommits, requireUnthrottled } from "@/lib/github-search"; import { getServerSession } from "@/lib/server-session"; import { NextRequest } from "next/server"; import { authOptions } from "@/lib/auth"; -import { GITHUB_API } from "@/lib/github"; import { isMetricsCacheBypassed, METRICS_CACHE_TTL_SECONDS, @@ -21,7 +20,9 @@ export async function GET(req: NextRequest) { const daysParam = req.nextUrl.searchParams.get("days"); const parsedDays = daysParam ? parseInt(daysParam, 10) : NaN; - const days = Number.isNaN(parsedDays) ? 30 : Math.max(1, Math.min(365, parsedDays)); + const days = Number.isNaN(parsedDays) + ? 30 + : Math.max(1, Math.min(365, parsedDays)); const bypass = isMetricsCacheBypassed(req); const key = metricsCacheKey( session.githubId ?? session.githubLogin, @@ -37,32 +38,20 @@ export async function GET(req: NextRequest) { ttlSeconds: METRICS_CACHE_TTL_SECONDS.contributions, }, async () => { - const since = new Date(); + const since = new Date(); since.setDate(since.getDate() - days); const sinceStr = since.toISOString().slice(0, 10); - const allItems: Array<{ commit: { author: { date: string } } }> = []; - let page = 1; - - while (true) { - const searchRes = await githubRequest( - `${GITHUB_API}/search/commits?q=author:${session.githubLogin}+author-date:>=${sinceStr}&per_page=100&page=${page}&sort=author-date&order=desc`, + const { items: allItems } = requireUnthrottled( + await collectAuthorCommits( + session.githubLogin!, + sinceStr, + session.accessToken, { - headers: { - Authorization: `Bearer ${session.accessToken}`, - Accept: "application/vnd.github+json", - }, - cache: "no-store", + fresh: bypass, } - ); - if (!searchRes.ok) throw new Error("GitHub API error"); - const data = (await searchRes.json()) as { - items: Array<{ commit: { author: { date: string } } }>; - }; - allItems.push(...data.items); - if (data.items.length < 100 || page >= 10) break; - page++; - } + ) + ); // Initialize all 24 hours to 0 const hourMap: Record = {}; @@ -74,7 +63,6 @@ export async function GET(req: NextRequest) { hourMap[hour]++; } - const hours = Array.from({ length: 24 }, (_, i) => ({ hour: i, commits: hourMap[i], diff --git a/src/app/api/metrics/contributions/route.ts b/src/app/api/metrics/contributions/route.ts index 2bb22615d..4d95383aa 100644 --- a/src/app/api/metrics/contributions/route.ts +++ b/src/app/api/metrics/contributions/route.ts @@ -1,5 +1,5 @@ import { extractValidRepoFromGoal } from "@/lib/goals-sync-utils"; -import { collectGitHubSearch } from "@/lib/github-search"; +import { collectAuthorCommits, scopeCommits } from "@/lib/github-search"; import { githubRequest } from "@/lib/github-request"; import { githubRateLimitResponse, @@ -130,8 +130,6 @@ async function fetchContributionsForAccount( excludedOrgs: string[] = [], toDate?: string ): Promise { - const repoFilter = repo ? ` repo:${repo}` : ""; - const key = metricsCacheKey(cacheContext.userId, "contributions", { days, githubLogin, @@ -158,21 +156,31 @@ async function fetchContributionsForAccount( const commitItems: CommitItem[] = []; let totalCount = 0; - let q = `author:${githubLogin} author-date:${toDate ? `${sinceStr}..${toDate}` : `>=${sinceStr}`}${repoFilter}`; - if (orgName) { - q += ` org:${orgName}`; - } else if (excludedOrgs.length > 0) { - q += excludedOrgs.map((org) => ` -org:${org}`).join(""); - } - - const search = await collectGitHubSearch('commits',q,token); + // Every view is a slice of the shared commit timeline, so filters cost no searches. + const search = await collectAuthorCommits( + githubLogin, + sinceStr, + token, + { fresh: cacheContext.bypass } + ); + search.items = scopeCommits(search.items, { + repo, + org: orgName, + excludedOrgs: orgName ? [] : excludedOrgs, + until: toDate, + }); + search.total = search.items.length; allItems = search.items; totalCount = search.total; const commitsByDay: Record = {}; - const timeBlocks: TimeBlocks = { morning: 0, afternoon: 0, evening: 0, night: 0 }; + const timeBlocks: TimeBlocks = { + morning: 0, + afternoon: 0, + evening: 0, + night: 0, + }; for (const item of allItems) { - const date = getDateInTimezone(item.commit.author.date, timezone); commitsByDay[date] = (commitsByDay[date] ?? 0) + 1; commitItems.push({ @@ -190,7 +198,16 @@ async function fetchContributionsForAccount( else timeBlocks.night++; } - return { days, total: totalCount, data: commitsByDay, commits: commitItems, timeBlocks, complete: search.complete, sampledCommits: search.items.length, partialReason: search.partialReason }; + return { + days, + total: totalCount, + data: commitsByDay, + commits: commitItems, + timeBlocks, + complete: search.complete, + sampledCommits: search.items.length, + partialReason: search.partialReason, + }; } ); } @@ -324,25 +341,44 @@ export async function GET(req: NextRequest) { if (fromParam && toParam) { fromDate = fromParam; const msPerDay = 1000 * 60 * 60 * 24; - days = Math.ceil( - (new Date(toParam).getTime() - new Date(fromParam).getTime()) / msPerDay - ) + 1; + days = + Math.ceil( + (new Date(toParam).getTime() - new Date(fromParam).getTime()) / msPerDay + ) + 1; } else { const daysParam = req.nextUrl.searchParams.get("days"); const parsedDays = daysParam ? parseInt(daysParam, 10) : NaN; - days = Number.isNaN(parsedDays) ? 30 : Math.max(1, Math.min(365, parsedDays)); + days = Number.isNaN(parsedDays) + ? 30 + : Math.max(1, Math.min(365, parsedDays)); } - try { new Intl.DateTimeFormat('en', {timeZone:timezone}).format(); } - catch { return Response.json({error:'Invalid timezone'}, {status:400}); } - if (!Number.isFinite(days) || days < 1 || days > 365 || (fromParam && !/^\d{4}-\d{2}-\d{2}$/.test(fromParam)) || (toParam && !/^\d{4}-\d{2}-\d{2}$/.test(toParam)) || Boolean(fromParam)!==Boolean(toParam)) { - return Response.json({error:'Invalid date range'}, {status:400}); + try { + new Intl.DateTimeFormat("en", { timeZone: timezone }).format(); + } catch { + return Response.json({ error: "Invalid timezone" }, { status: 400 }); + } + if ( + !Number.isFinite(days) || + days < 1 || + days > 365 || + (fromParam && !/^\d{4}-\d{2}-\d{2}$/.test(fromParam)) || + (toParam && !/^\d{4}-\d{2}-\d{2}$/.test(toParam)) || + Boolean(fromParam) !== Boolean(toParam) + ) { + return Response.json({ error: "Invalid date range" }, { status: 400 }); } - if(repoParam && !extractValidRepoFromGoal({id:'',unit:'commits',repo:repoParam})) return Response.json({error:'Invalid repository'},{status:400}); + if ( + repoParam && + !extractValidRepoFromGoal({ id: "", unit: "commits", repo: repoParam }) + ) + return Response.json({ error: "Invalid repository" }, { status: 400 }); const accountId = req.nextUrl.searchParams.get("accountId"); const usernameParam = req.nextUrl.searchParams.get("username"); - const username = usernameParam ? normalizeGitHubUsername(usernameParam) : null; + const username = usernameParam + ? normalizeGitHubUsername(usernameParam) + : null; const bypass = isMetricsCacheBypassed(req); const gitlabToken = typeof session.gitlabToken === "string" ? session.gitlabToken : undefined; @@ -360,7 +396,10 @@ export async function GET(req: NextRequest) { targetAccountId = parts[1]; orgName = parts[2]; if (!targetAccountId || !orgName) { - return Response.json({ error: "Invalid organization account ID" }, { status: 400 }); + return Response.json( + { error: "Invalid organization account ID" }, + { status: 400 } + ); } } @@ -374,7 +413,10 @@ export async function GET(req: NextRequest) { .eq("github_id", session.githubId) .single(); - const orgsConfig = (dbUser?.organizations_config || {}) as Record; + const orgsConfig = (dbUser?.organizations_config || {}) as Record< + string, + boolean + >; excludedOrgs = Object.entries(orgsConfig) .filter(([_, enabled]) => enabled === false) .map(([org]) => org); @@ -402,9 +444,9 @@ export async function GET(req: NextRequest) { toParam ?? undefined ); return Response.json(result); - } catch (error) { - return githubApiErrorResponse(error); - } + } catch (error) { + return githubApiErrorResponse(error); + } } if (!targetAccountId) { @@ -469,26 +511,27 @@ export async function GET(req: NextRequest) { repoParam, orgName, excludedOrgs, - toParam ?? undefined + toParam ?? undefined ) ) ); - const rateLimitedResult = results.find( - (result): result is PromiseRejectedResult => - result.status === "rejected" && - githubRateLimitResponse(result.reason) !== null -); + (result): result is PromiseRejectedResult => + result.status === "rejected" && + githubRateLimitResponse(result.reason) !== null + ); -if (rateLimitedResult) { - return githubApiErrorResponse(rateLimitedResult.reason); -} + if (rateLimitedResult) { + return githubApiErrorResponse(rateLimitedResult.reason); + } const merged = mergeMetrics(results, (a, b) => ({ days: a.days, complete: a.complete !== false && b.complete !== false, - sampledCommits: (a.sampledCommits ?? a.commits.length) + (b.sampledCommits ?? b.commits.length), + sampledCommits: + (a.sampledCommits ?? a.commits.length) + + (b.sampledCommits ?? b.commits.length), partialReason: a.partialReason ?? b.partialReason, total: a.total + b.total, data: mergeContributionDays(a.data, b.data), @@ -585,4 +628,4 @@ if (rateLimitedResult) { return githubApiErrorResponse(error); } -} \ No newline at end of file +} diff --git a/src/app/api/metrics/languages/route.ts b/src/app/api/metrics/languages/route.ts index 86bcb5ec2..f6bd21558 100644 --- a/src/app/api/metrics/languages/route.ts +++ b/src/app/api/metrics/languages/route.ts @@ -1,9 +1,15 @@ import { githubRateLimitResponse } from "@/lib/github-rate-limit"; import { githubRequest } from "@/lib/github-request"; +import { collectAuthorCommits, requireUnthrottled } from "@/lib/github-search"; import { getServerSession } from "@/lib/server-session"; import { NextRequest } from "next/server"; import { authOptions } from "@/lib/auth"; -import { isMetricsCacheBypassed, metricsCacheKey, withMetricsCache, METRICS_CACHE_TTL_SECONDS} from "@/lib/metrics-cache"; +import { + isMetricsCacheBypassed, + metricsCacheKey, + withMetricsCache, + METRICS_CACHE_TTL_SECONDS, +} from "@/lib/metrics-cache"; import { getAccountToken } from "@/lib/github-accounts"; import { supabaseAdmin } from "@/lib/supabase"; import { resolveAppUser } from "@/lib/resolve-user"; @@ -13,7 +19,8 @@ const GITHUB_API = "https://api.github.com"; export async function GET(req: NextRequest) { const session = await getServerSession(authOptions); - if (!session?.accessToken || !session.githubLogin) return Response.json({ error: "Unauthorized" }, { status: 401 }); + if (!session?.accessToken || !session.githubLogin) + return Response.json({ error: "Unauthorized" }, { status: 401 }); const accountId = req.nextUrl.searchParams.get("accountId"); const bypass = isMetricsCacheBypassed(req); @@ -49,39 +56,53 @@ export async function GET(req: NextRequest) { } const rawDays = Number(req.nextUrl.searchParams.get("days") ?? "90"); - const days = Number.isFinite(rawDays) && rawDays > 0 ? Math.min(Math.floor(rawDays), 365) : 90; - const key = metricsCacheKey( - userId, - "languages" as any, - { accountId: accountId || undefined, days } - ); + const days = + Number.isFinite(rawDays) && rawDays > 0 + ? Math.min(Math.floor(rawDays), 365) + : 90; + const key = metricsCacheKey(userId, "languages" as any, { + accountId: accountId || undefined, + days, + }); try { - const data = await withMetricsCache({ bypass, key, ttlSeconds: METRICS_CACHE_TTL_SECONDS.languages }, async () => { - const headers = { Authorization: `Bearer ${token}`, Accept: "application/vnd.github+json" }; - const since = new Date(); - since.setUTCDate(since.getUTCDate() - days); - - const searchRes = await githubRequest( - `${GITHUB_API}/search/commits?q=author:${githubLogin}+author-date:>=${since.toISOString().slice(0, 10)}&per_page=100&sort=author-date&order=desc`, - { headers, cache: "no-store" } - ); - if (!searchRes.ok) throw new Error("API Error"); - - const raw = await searchRes.json(); - const repoNames = Array.from(new Set(raw.items.map((i: any) => i.repository.full_name))); - const topRepoNames = repoNames.slice(0, 20); - const langTotals: Record = {}; - const failedRepos: Array<{ name: string; statusCode?: number; error: string }> = []; - - await Promise.all( - topRepoNames.map(async (repoName) => { - try { + const data = await withMetricsCache( + { bypass, key, ttlSeconds: METRICS_CACHE_TTL_SECONDS.languages }, + async () => { + const headers = { + Authorization: `Bearer ${token}`, + Accept: "application/vnd.github+json", + }; + const since = new Date(); + since.setUTCDate(since.getUTCDate() - days); + + const { items } = requireUnthrottled( + await collectAuthorCommits<{ + commit: { author: { date: string } }; + repository: { full_name: string }; + }>(githubLogin, since.toISOString().slice(0, 10), token, { + fresh: bypass, + }) + ); + const repoNames = Array.from( + new Set(items.map((i) => i.repository.full_name)) + ); + const topRepoNames = repoNames.slice(0, 20); + const langTotals: Record = {}; + const failedRepos: Array<{ + name: string; + statusCode?: number; + error: string; + }> = []; + + await Promise.all( + topRepoNames.map(async (repoName) => { + try { const repoCacheKey = metricsCacheKey( userId, "repo_languages" as any, { repoName } - ); + ); const langs = await withMetricsCache( { @@ -92,7 +113,7 @@ export async function GET(req: NextRequest) { async () => { const res = await githubRequest( `${GITHUB_API}/repos/${repoName}/languages`, - { headers, cache: "no-store" }, + { headers, cache: "no-store" } ); if (!res.ok) { @@ -105,40 +126,49 @@ export async function GET(req: NextRequest) { } return await res.json(); - }, + } + ); + for (const [lang, bytes] of Object.entries(langs)) { + langTotals[lang] = (langTotals[lang] ?? 0) + (bytes as number); + } + } catch (e) { + const errorMessage = + e instanceof Error ? e.message : "Unknown error"; + failedRepos.push({ + name: repoName, + error: errorMessage, + }); + console.warn( + `[METRICS] Failed to fetch languages for ${repoName}: ${errorMessage}`, + { userId, githubLogin } ); - for (const [lang, bytes] of Object.entries(langs)) { - langTotals[lang] = (langTotals[lang] ?? 0) + (bytes as number); } - } catch (e) { - const errorMessage = e instanceof Error ? e.message : "Unknown error"; - failedRepos.push({ - name: repoName, - error: errorMessage, - }); - console.warn( - `[METRICS] Failed to fetch languages for ${repoName}: ${errorMessage}`, - { userId, githubLogin } - ); - } - }) - ); - - const totalBytes = Object.values(langTotals).reduce((s, b) => s + b, 0); - const languages = Object.entries(langTotals) - .map(([name, bytes]) => ({ name, bytes, percentage: totalBytes > 0 ? Math.round((bytes / totalBytes) * 1000) / 10 : 0 })) - .sort((a, b) => b.percentage - a.percentage) - .slice(0, 6); - - const isComplete = failedRepos.length === 0; - - return { - languages, - isComplete, - failedRepositoriesCount: failedRepos.length, - ...(process.env.NODE_ENV === "development" && { failedRepositories: failedRepos }), - }; - }); + }) + ); + + const totalBytes = Object.values(langTotals).reduce((s, b) => s + b, 0); + const languages = Object.entries(langTotals) + .map(([name, bytes]) => ({ + name, + bytes, + percentage: + totalBytes > 0 ? Math.round((bytes / totalBytes) * 1000) / 10 : 0, + })) + .sort((a, b) => b.percentage - a.percentage) + .slice(0, 6); + + const isComplete = failedRepos.length === 0; + + return { + languages, + isComplete, + failedRepositoriesCount: failedRepos.length, + ...(process.env.NODE_ENV === "development" && { + failedRepositories: failedRepos, + }), + }; + } + ); return Response.json(data); } catch (e) { const githubLimit = githubRateLimitResponse(e); @@ -149,6 +179,9 @@ export async function GET(req: NextRequest) { userId: session.githubId ?? session.githubLogin, error: errorMessage, }); - return Response.json({ error: "GitHub API error", isComplete: false }, { status: 502 }); + return Response.json( + { error: "GitHub API error", isComplete: false }, + { status: 502 } + ); } } diff --git a/src/app/api/metrics/productive-hours/route.ts b/src/app/api/metrics/productive-hours/route.ts index 0e1b79009..8004c0c1e 100644 --- a/src/app/api/metrics/productive-hours/route.ts +++ b/src/app/api/metrics/productive-hours/route.ts @@ -1,5 +1,5 @@ import { githubRateLimitResponse } from "@/lib/github-rate-limit"; -import { githubRequest } from "@/lib/github-request"; +import { collectAuthorCommits, requireUnthrottled } from "@/lib/github-search"; import { getServerSession } from "@/lib/server-session"; import { NextRequest } from "next/server"; import { authOptions } from "@/lib/auth"; @@ -8,7 +8,7 @@ import { getAllAccounts, mergeMetrics, } from "@/lib/github-accounts"; -import { GITHUB_API, GitHubCommitSearchItem, GitHubAuthError } from "@/lib/github"; +import { GitHubCommitSearchItem, GitHubAuthError } from "@/lib/github"; import { githubAuthErrorResponse } from "@/lib/github-fetch"; import { isMetricsCacheBypassed, @@ -45,8 +45,6 @@ async function fetchProductiveHoursForAccount( fromDate?: string, repo?: string | null ): Promise { - const repoFilter = repo ? ` repo:${repo}` : ""; - const key = metricsCacheKey(cacheContext.userId, "productive-hours", { days, githubLogin, @@ -66,64 +64,23 @@ async function fetchProductiveHoursForAccount( since.setDate(since.getDate() - days); const sinceStr = fromDate ?? toLocalDateStr(since); - let allItems: GitHubCommitSearchItem[] = []; - let totalCount = 0; - let page = 1; - - // Paginate GitHub commit search — mirrors contributions/route.ts exactly. - // Up to 10 pages × 100 items = 1 000 commits max. - while (page <= 10) { - const searchUrl = new URL(`${GITHUB_API}/search/commits`); - searchUrl.searchParams.set( - "q", - `author:${githubLogin} author-date:>=${sinceStr}${repoFilter}` - ); - searchUrl.searchParams.set("per_page", "100"); - searchUrl.searchParams.set("page", String(page)); - searchUrl.searchParams.set("sort", "author-date"); - searchUrl.searchParams.set("order", "desc"); - - const searchRes = await githubRequest(searchUrl.toString(), { - headers: { - Authorization: `Bearer ${token}`, - Accept: "application/vnd.github+json", - }, - cache: "no-store", - }); - - if (!searchRes.ok) { - if (searchRes.status === 401) throw new GitHubAuthError(); - // Graceful degradation on rate-limit — return partial data already collected. - if (searchRes.status === 429 || searchRes.status === 403) { - if (allItems.length === 0) { - throw new Error(`GitHub API error: ${searchRes.status}`); - } - break; + const result = requireUnthrottled( + await collectAuthorCommits( + githubLogin, + sinceStr, + token, + { + fresh: cacheContext.bypass, } - throw new Error("GitHub API error"); - } - - const data = (await searchRes.json()) as { - total_count: number; - items: GitHubCommitSearchItem[]; - }; - - if (page === 1) { - totalCount = data.total_count; - } - - allItems = allItems.concat(data.items); - - if ( - data.items.length < 100 || - allItems.length >= 1000 || - allItems.length >= totalCount - ) { - break; - } - - page += 1; - } + ) + ); + const repoName = repo?.toLowerCase(); + const allItems = repoName + ? result.items.filter( + (item) => item.repository?.full_name?.toLowerCase() === repoName + ) + : result.items; + const totalCount = repoName ? allItems.length : result.total; const counts: Record = {}; @@ -134,7 +91,7 @@ async function fetchProductiveHoursForAccount( utcDate.toLocaleString("en-US", { timeZone: timezone }) ); - const day = localDate.getDay(); // 0 Sun … 6 Sat + const day = localDate.getDay(); // 0 Sun … 6 Sat const hour = localDate.getHours(); // 0–23 const k = `${day}-${hour}`; counts[k] = (counts[k] ?? 0) + 1; @@ -225,7 +182,9 @@ export async function GET(req: NextRequest) { } else { const daysParam = searchParams.get("days"); const parsedDays = daysParam ? parseInt(daysParam, 10) : NaN; - days = Number.isNaN(parsedDays) ? 90 : Math.max(1, Math.min(365, parsedDays)); + days = Number.isNaN(parsedDays) + ? 90 + : Math.max(1, Math.min(365, parsedDays)); } const accountId = searchParams.get("accountId"); @@ -358,4 +317,4 @@ function sanitizeTimezone(tz: string): string { } catch { return "UTC"; } -} \ No newline at end of file +} diff --git a/src/app/api/metrics/repo-explorer/route.ts b/src/app/api/metrics/repo-explorer/route.ts index 32259fb82..591c9823b 100644 --- a/src/app/api/metrics/repo-explorer/route.ts +++ b/src/app/api/metrics/repo-explorer/route.ts @@ -1,13 +1,16 @@ import { githubRateLimitResponse } from "@/lib/github-rate-limit"; -import { githubRequest } from "@/lib/github-request"; +import { collectAuthorCommits, requireUnthrottled } from "@/lib/github-search"; import { getSessionWithToken } from "@/lib/get-session-token"; import { fetchUserRepos } from "@/lib/github"; import { NextRequest } from "next/server"; -import { isMetricsCacheBypassed, metricsCacheKey, withMetricsCache } from "@/lib/metrics-cache"; +import { + isMetricsCacheBypassed, + metricsCacheKey, + withMetricsCache, +} from "@/lib/metrics-cache"; import { ExplorerRepoCardData } from "@/lib/repo-analytics-types"; export const dynamic = "force-dynamic"; -const GITHUB_API = "https://api.github.com"; export async function GET(req: NextRequest) { const sessionData = await getSessionWithToken(); @@ -19,79 +22,87 @@ export async function GET(req: NextRequest) { const accessToken = sessionData.accessToken; const bypass = isMetricsCacheBypassed(req); - const key = metricsCacheKey(session.githubId ?? session.githubLogin!, "repo-explorer-v2" as any, { days: 7 }); + const key = metricsCacheKey( + session.githubId ?? session.githubLogin!, + "repo-explorer-v2" as any, + { days: 7 } + ); try { - const data = await withMetricsCache({ bypass, key, ttlSeconds: 30 * 60 }, async () => { - // Paginate through all pages (up to 1000 repos) so users with more - // than 100 repositories see their complete list — fixes #2843. - const repos = await fetchUserRepos(accessToken, { perPage: 100, maxPages: 10 }); - const since = new Date(); - since.setDate(since.getDate() - 30); - const sinceStr = since.toISOString().slice(0, 10); - - const searchRes = await githubRequest(`${GITHUB_API}/search/commits?q=author:${session.githubLogin}+author-date:>=${sinceStr}&per_page=100&sort=author-date&order=desc`, { - headers: { Authorization: `Bearer ${accessToken}`, Accept: "application/vnd.github+json" }, - cache: "no-store", - }); - - const repoCommits: Record = {}; - - if (searchRes.ok) { - const searchData = await searchRes.json(); - const items = searchData.items || []; + const data = await withMetricsCache( + { bypass, key, ttlSeconds: 30 * 60 }, + async () => { + // Paginate through all pages (up to 1000 repos) so users with more + // than 100 repositories see their complete list — fixes #2843. + const repos = await fetchUserRepos(accessToken, { + perPage: 100, + maxPages: 10, + }); + const since = new Date(); + since.setDate(since.getDate() - 30); + const sinceStr = since.toISOString().slice(0, 10); + + const { items } = requireUnthrottled( + await collectAuthorCommits<{ + commit: { author: { date: string } }; + repository: { full_name: string }; + }>(session.githubLogin!, sinceStr, accessToken, { fresh: bypass }) + ); + + const repoCommits: Record = {}; for (const item of items) { const repoName = item.repository.full_name; - if (!repoCommits[repoName]) { - repoCommits[repoName] = []; - } - repoCommits[repoName].push(item.commit.author.date); + (repoCommits[repoName] ??= []).push(item.commit.author.date); } - } - const result: ExplorerRepoCardData[] = []; + const result: ExplorerRepoCardData[] = []; - for (const repo of repos) { - const commitDates = repoCommits[repo.full_name] || []; - const commitCount = commitDates.length; + for (const repo of repos) { + const commitDates = repoCommits[repo.full_name] || []; + const commitCount = commitDates.length; - const dayMap: Record = {}; - for (let i = 6; i >= 0; i--) { - const d = new Date(); - d.setDate(d.getDate() - i); - dayMap[d.toISOString().slice(0, 10)] = 0; - } + const dayMap: Record = {}; + for (let i = 6; i >= 0; i--) { + const d = new Date(); + d.setDate(d.getDate() - i); + dayMap[d.toISOString().slice(0, 10)] = 0; + } - for (const dateStr of commitDates) { - const dStr = dateStr.slice(0, 10); - if (dayMap[dStr] !== undefined) { - dayMap[dStr]++; + for (const dateStr of commitDates) { + const dStr = dateStr.slice(0, 10); + if (dayMap[dStr] !== undefined) { + dayMap[dStr]++; + } } + + const activity7d = Object.entries(dayMap).map(([date, count]) => { + const d = new Date(date); + const dayName = d.toLocaleDateString("en-US", { weekday: "short" }); + return { day: dayName, commits: count }; + }); + + result.push({ + id: String(repo.id), + name: repo.name, + fullName: repo.full_name, + commitCount, + createdAt: repo.created_at, + updatedAt: repo.updated_at, + primaryLanguage: repo.language ?? undefined, + htmlUrl: repo.html_url, + activity7d, + }); } - const activity7d = Object.entries(dayMap).map(([date, count]) => { - const d = new Date(date); - const dayName = d.toLocaleDateString('en-US', { weekday: 'short' }); - return { day: dayName, commits: count }; - }); + result.sort( + (a, b) => + b.commitCount - a.commitCount || + new Date(b.updatedAt).getTime() - new Date(a.updatedAt).getTime() + ); - result.push({ - id: String(repo.id), - name: repo.name, - fullName: repo.full_name, - commitCount, - createdAt: repo.created_at, - updatedAt: repo.updated_at, - primaryLanguage: repo.language ?? undefined, - htmlUrl: repo.html_url, - activity7d, - }); + return { repos: result }; } - - result.sort((a, b) => b.commitCount - a.commitCount || new Date(b.updatedAt).getTime() - new Date(a.updatedAt).getTime()); - - return { repos: result }; - }); + ); return Response.json(data); } catch (error) { const githubLimit = githubRateLimitResponse(error); diff --git a/src/app/api/metrics/repo-health/route.ts b/src/app/api/metrics/repo-health/route.ts index 484ef664e..32a21916e 100644 --- a/src/app/api/metrics/repo-health/route.ts +++ b/src/app/api/metrics/repo-health/route.ts @@ -1,52 +1,54 @@ import { githubRateLimitResponse } from "@/lib/github-rate-limit"; -import { githubRequest } from "@/lib/github-request"; +import { githubGraphQL } from "@/lib/github-fetch"; +import { collectAuthorCommits, requireUnthrottled } from "@/lib/github-search"; import { getServerSession } from "@/lib/server-session"; import { NextRequest } from "next/server"; import { authOptions } from "@/lib/auth"; import { computeHealthScore } from "@/lib/repo-health"; -import { isMetricsCacheBypassed, metricsCacheKey, withMetricsCache } from "@/lib/metrics-cache"; -import type { RepoHealthResponse, RepoHealthSignals, RepoHealthScore } from "@/types/repo-health"; +import { + isMetricsCacheBypassed, + metricsCacheKey, + withMetricsCache, +} from "@/lib/metrics-cache"; +import type { + RepoHealthResponse, + RepoHealthSignals, + RepoHealthScore, +} from "@/types/repo-health"; export const dynamic = "force-dynamic"; -const GITHUB_API = "https://api.github.com"; - -interface RepoSummary { name: string; commits: number; url: string; } -interface RepoListResponse { repos: RepoSummary[]; days: number; } - -async function fetchReposForAccount(token: string, githubLogin: string, days: number): Promise { - const since = new Date(Date.now() - days * 86400000).toISOString().slice(0, 10); - - // GitHub Commit Search API — finds recent commits to identify the user's active repos. - // Rate limits: - // • Authenticated (OAuth token / PAT): 30 requests/minute - // • Unauthenticated: 10 requests/minute - // This single request costs 1 of the 30 req/min quota. - // The outer withMetricsCache (ttlSeconds: 10 * 60) in the GET handler prevents - // re-fetching on every page load — this is only called on a cache miss. - const searchRes = await githubRequest( - `${GITHUB_API}/search/commits?q=author:${githubLogin}+author-date:>=${since}&per_page=100&sort=author-date&order=desc`, - { - headers: { - // OAuth token / PAT: raises the Search API limit from 10 → 30 req/min. - // Without this, a single repo-health load could exhaust the unauthenticated - // 10 req/min quota when combined with other concurrent Search API calls. - Authorization: `Bearer ${token}`, - // Mandatory for the Commit Search endpoint — omitting returns HTTP 415. - Accept: "application/vnd.github+json", - }, - cache: "no-store", - } - ); - // HTTP 403 = Search API rate limit exceeded. HTTP 422 = malformed query. - // Throws here so the GET handler catches it and returns HTTP 502 to the client. - if (!searchRes.ok) throw new Error("API error"); +interface RepoSummary { + name: string; + commits: number; + url: string; +} +interface RepoListResponse { + repos: RepoSummary[]; + days: number; +} - const data = await searchRes.json(); +async function fetchReposForAccount( + token: string, + githubLogin: string, + days: number +): Promise { + const since = new Date(Date.now() - days * 86400000) + .toISOString() + .slice(0, 10); + + // Active repos come from the shared commit timeline, so this costs no extra searches. + const data = requireUnthrottled( + await collectAuthorCommits<{ + commit: { author: { date: string } }; + repository: { full_name: string; html_url: string }; + }>(githubLogin, since, token) + ); const repoMap: Record = {}; for (const item of data.items) { const name = item.repository.full_name; - if (!repoMap[name]) repoMap[name] = { commits: 0, url: item.repository.html_url }; + if (!repoMap[name]) + repoMap[name] = { commits: 0, url: item.repository.html_url }; repoMap[name].commits++; } // Slice to top 6 repos to cap the number of fetchSignalsForRepo calls below. @@ -59,84 +61,95 @@ async function fetchReposForAccount(token: string, githubLogin: string, days: nu }; } -function hoursBetween(a: string, b: string): number { return (new Date(b).getTime() - new Date(a).getTime()) / 3600000; } -function daysSince(isoDate: string): number { return Math.max(0, Math.floor((Date.now() - new Date(isoDate).getTime()) / 86400000)); } - -// Shared fetch helper for all GitHub API calls inside fetchSignalsForRepo. -// Throws on any non-ok response so the per-repo try/catch in the GET handler -// can skip a single failing repo without aborting the entire health check. -async function fetchJson(url: string, token: string, accept?: string): Promise { - const res = await githubRequest(url, { - headers: { - // OAuth token / PAT used for every request — keeps all calls in the - // authenticated tier (5,000/hr REST, 30/min Search API). - Authorization: `Bearer ${token}`, - Accept: accept ?? "application/vnd.github+json", - }, - cache: "no-store", - }); - // Non-ok responses include HTTP 403 (rate limit exceeded) and 404 (repo not found). - if (!res.ok) throw new Error("API error"); - return await res.json(); +function hoursBetween(a: string, b: string): number { + return (new Date(b).getTime() - new Date(a).getTime()) / 3600000; } - -async function fetchSignalsForRepo(token: string, repoFullName: string, days: number): Promise { - const since = new Date(Date.now() - days * 86400000).toISOString().split("T")[0]; - - // This function makes 5 sequential GitHub API calls per repo: - // 1. Commit Search → Search API (30 req/min quota) - // 2. Opened PRs → Search API (30 req/min quota) - // 3. Merged PRs → Search API (30 req/min quota) - // 4. Open Issues → Search API (30 req/min quota) - // 5. Latest commit → REST API (5,000 req/hr quota) - // - // With up to 6 repos (from fetchReposForAccount), one full repo-health load - // can consume up to 24 Search API requests (4 × 6) in a single minute. - // The 10-minute withMetricsCache TTL in the GET handler is critical here — - // without it, repeated loads would quickly exhaust the 30 req/min Search limit. - // - // Calls 1–4 use the Search API (30 req/min authenticated limit). - const commitSearch = await fetchJson( - `${GITHUB_API}/search/commits?q=repo:${repoFullName}+committer-date:>${since}&per_page=100&sort=committer-date&order=desc`, - token, - "application/vnd.github+json" // required Accept header for Commit Search - ); - const openedPrs = await fetchJson( - `${GITHUB_API}/search/issues?q=repo:${repoFullName}+type:pr+created:>${since}&per_page=100&sort=created&order=desc`, - token - ); - const mergedPrs = await fetchJson( - `${GITHUB_API}/search/issues?q=repo:${repoFullName}+type:pr+is:merged+merged:>${since}&per_page=100&sort=updated&order=desc`, - token +function daysSince(isoDate: string): number { + return Math.max( + 0, + Math.floor((Date.now() - new Date(isoDate).getTime()) / 86400000) ); +} + +const REPO_SIGNALS_QUERY = ` + query RepoSignals($owner: String!, $name: String!, $since: GitTimestamp!) { + repository(owner: $owner, name: $name) { + issues(states: OPEN) { totalCount } + recentPrs: pullRequests(first: 100, orderBy: { field: CREATED_AT, direction: DESC }) { + nodes { createdAt closedAt } + } + mergedPrs: pullRequests(first: 100, states: MERGED, orderBy: { field: UPDATED_AT, direction: DESC }) { + nodes { mergedAt } + } + defaultBranchRef { + target { + ... on Commit { + window: history(since: $since) { totalCount } + latest: history(first: 1) { nodes { committedDate } } + } + } + } + } + } +`; + +interface RepoSignalsData { + repository: { + issues: { totalCount: number }; + recentPrs: { nodes: Array<{ createdAt: string; closedAt: string | null }> }; + mergedPrs: { nodes: Array<{ mergedAt: string | null }> }; + defaultBranchRef: { + target: { + window?: { totalCount: number }; + latest?: { nodes: Array<{ committedDate: string }> }; + }; + } | null; + } | null; +} - const openedCount = openedPrs.total_count || 0; - const mergedCount = mergedPrs.total_count || 0; - const closedItems = (openedPrs.items ?? []).filter((i: any) => i.closed_at); - const avgPrOpenTimeHours = closedItems.length > 0 - ? closedItems.reduce((sum: number, pr: any) => sum + hoursBetween(pr.created_at, pr.closed_at!), 0) / closedItems.length - : 0; - - // Call 4 — open issue count via Search API (counts toward the 30 req/min limit). - // per_page=1 minimises response size since we only need total_count. - const openIssues = await fetchJson( - `${GITHUB_API}/search/issues?q=repo:${repoFullName}+type:issue+state:open&per_page=1`, - token +// One GraphQL request per repo replaces four Search API calls: six repos used to +// spend 24 of the 30 searches/minute shared by the whole dashboard. +async function fetchSignalsForRepo( + token: string, + repoFullName: string, + days: number +): Promise { + const sinceMs = Date.now() - days * 86400000; + const [owner, name] = repoFullName.split("/"); + const { repository } = await githubGraphQL( + REPO_SIGNALS_QUERY, + token, + { + owner, + name, + since: new Date(sinceMs).toISOString(), + } ); + if (!repository) throw new Error("Repository not found"); - // Call 5 — latest commit date via REST API (5,000 req/hr, separate from Search quota). - // per_page=1 fetches only the most recent commit to minimise payload size. - const commits = await fetchJson( - `${GITHUB_API}/repos/${repoFullName}/commits?per_page=1`, - token + const opened = repository.recentPrs.nodes.filter( + (pr) => Date.parse(pr.createdAt) > sinceMs ); - const lastCommitDate = commits?.[0]?.commit?.committer?.date ?? null; + const mergedCount = repository.mergedPrs.nodes.filter( + (pr) => pr.mergedAt && Date.parse(pr.mergedAt) > sinceMs + ).length; + const closedItems = opened.filter((pr) => pr.closedAt); + const avgPrOpenTimeHours = + closedItems.length > 0 + ? closedItems.reduce( + (sum, pr) => sum + hoursBetween(pr.createdAt, pr.closedAt!), + 0 + ) / closedItems.length + : 0; + + const target = repository.defaultBranchRef?.target; + const lastCommitDate = target?.latest?.nodes[0]?.committedDate ?? null; return { - commitFrequency: Array.isArray(commitSearch.items) ? commitSearch.items.length : 0, - prMergeRate: openedCount > 0 ? mergedCount / openedCount : 0, + commitFrequency: target?.window?.totalCount ?? 0, + prMergeRate: opened.length > 0 ? mergedCount / opened.length : 0, avgPrOpenTimeHours, - openIssuesCount: openIssues.total_count || 0, + openIssuesCount: repository.issues.totalCount, // 9999 signals "no commits found" — treated as maximum staleness by computeHealthScore. daysSinceLastCommit: lastCommitDate ? daysSince(lastCommitDate) : 9999, contributorCount: 0, @@ -152,44 +165,65 @@ export async function GET(req: NextRequest) { return Response.json({ error: "Unauthorized" }, { status: 401 }); } - const requestedDays = parseInt(req.nextUrl.searchParams.get("days") ?? "30", 10); + const requestedDays = parseInt( + req.nextUrl.searchParams.get("days") ?? "30", + 10 + ); // Only allow 7, 30, or 90 day windows — other values default to 30. - const days = requestedDays === 7 || requestedDays === 30 || requestedDays === 90 ? requestedDays : 30; + const days = + requestedDays === 7 || requestedDays === 30 || requestedDays === 90 + ? requestedDays + : 30; const bypass = isMetricsCacheBypassed(req); - const key = metricsCacheKey(session.githubId ?? session.githubLogin, "repo-health" as any, { days }); + const key = metricsCacheKey( + session.githubId ?? session.githubLogin, + "repo-health" as any, + { days } + ); try { - // Cache TTL of 10 minutes (600 seconds) — longer than most other metrics because - // repo health signals (commit frequency, PR merge rate, open issues) change slowly - // and fetchSignalsForRepo is expensive: up to 24 Search API calls for 6 repos. - // Without this cache, a single user refreshing the page repeatedly could exhaust - // the 30 req/min Search API quota in under 2 minutes. - const data = await withMetricsCache({ bypass, key, ttlSeconds: 10 * 60 }, async () => { - // Step 1: identify the top 6 repos via Commit Search (1 Search API request). - const topRepos = (await fetchReposForAccount(session.accessToken!, session.githubLogin!, days)).repos; - - const scores: RepoHealthScore[] = []; - for (const repo of topRepos) { - try { - // Step 2: fetch health signals for each repo (up to 4 Search + 1 REST per repo). - // Individual repo failures are silently skipped — a rate limit on one repo - // should not prevent health scores for the remaining repos from loading. - const signals = await fetchSignalsForRepo(session.accessToken!, repo.name, days); - scores.push(computeHealthScore(repo.name, signals)); - } catch (e) { - // Swallow per-repo errors (rate limit, private repo, network blip). - // The repo is simply omitted from the scores array rather than failing the request. + // Cache TTL of 10 minutes (600 seconds) — repo health signals (commit frequency, + // PR merge rate, open issues) change slowly and each load costs one GraphQL + // request per repo. + const data = await withMetricsCache( + { bypass, key, ttlSeconds: 10 * 60 }, + async () => { + // Step 1: identify the top 6 repos from the shared commit timeline. + const topRepos = ( + await fetchReposForAccount( + session.accessToken!, + session.githubLogin!, + days + ) + ).repos; + + const scores: RepoHealthScore[] = []; + for (const repo of topRepos) { + try { + // Step 2: fetch health signals for each repo (one GraphQL request per repo). + // Individual repo failures are silently skipped — a rate limit on one repo + // should not prevent health scores for the remaining repos from loading. + const signals = await fetchSignalsForRepo( + session.accessToken!, + repo.name, + days + ); + scores.push(computeHealthScore(repo.name, signals)); + } catch (e) { + // Swallow per-repo errors (rate limit, private repo, network blip). + // The repo is simply omitted from the scores array rather than failing the request. + } } + return { repos: scores }; } - return { repos: scores }; - }); + ); return Response.json(data); } catch (e) { const githubLimit = githubRateLimitResponse(e); if (githubLimit) return githubLimit; - // Catches errors from fetchReposForAccount (the initial Search API call). + // Catches errors from fetchReposForAccount (the shared commit timeline). // Returns 502 so the client shows an error state rather than an empty health widget. return Response.json({ error: "GitHub API error" }, { status: 502 }); } diff --git a/src/app/api/metrics/repos/route.ts b/src/app/api/metrics/repos/route.ts index d70ec00c1..622e15dc5 100644 --- a/src/app/api/metrics/repos/route.ts +++ b/src/app/api/metrics/repos/route.ts @@ -1,4 +1,4 @@ -import { collectGitHubSearch } from "@/lib/github-search"; +import { collectAuthorCommits, scopeCommits } from "@/lib/github-search"; import { githubRateLimitResponse } from "@/lib/github-rate-limit"; import { githubRequest } from "@/lib/github-request"; import { getServerSession } from "@/lib/server-session"; @@ -50,7 +50,15 @@ function mergeRepoCommits( ): Array { // Merges repo commit counts across multiple GitHub accounts for the "combined" view. // If both accounts committed to the same repo, their counts are summed. - const map = new Map(); + const map = new Map< + string, + { + commits: number; + description: string | null; + url: string; + languages?: RepoLanguage[]; + } + >(); for (const repo of [...a, ...b]) { const existing = map.get(repo.name); map.set(repo.name, { @@ -100,7 +108,10 @@ async function fetchRepoLanguages( } const langs = (await res.json()) as Record; - const totalBytes = Object.values(langs).reduce((sum, bytes) => sum + bytes, 0); + const totalBytes = Object.values(langs).reduce( + (sum, bytes) => sum + bytes, + 0 + ); if (totalBytes <= 0) { return []; @@ -147,21 +158,34 @@ async function fetchReposForAccount( since.setDate(since.getDate() - days); const sinceStr = since.toISOString().slice(0, 10); // "YYYY-MM-DD" - let q = `author:${githubLogin}`; - if (orgName) { - q += `+org:${orgName}`; - } else if (excludedOrgs.length > 0) { - q += excludedOrgs.map((org) => `+-org:${org}`).join(""); - } - q += `+author-date:>=${sinceStr}`; - - const search = await collectGitHubSearch<{repository: {full_name:string; html_url:string; description:string|null}; commit:{author:{date:string}}}>('commits',q.replace(/\+/g,' '),token); - const data = {items:search.items}; + type RepoCommit = { + repository: { + full_name: string; + html_url: string; + description: string | null; + }; + commit: { author: { date: string } }; + }; + // Org views are slices of the shared commit timeline, so they cost no searches. + const search = await collectAuthorCommits( + githubLogin, + sinceStr, + token, + { fresh: cacheContext.bypass } + ); + search.items = scopeCommits(search.items, { + org: orgName, + excludedOrgs: orgName ? [] : excludedOrgs, + }); + const data = { items: search.items }; // Group commits by repository and count them. // Each item in the Search API response represents one commit, so we tally // per repo to find the most actively committed-to repositories. - const repoMap: Record = {}; + const repoMap: Record< + string, + { commits: number; description: string | null; url: string } + > = {}; for (const item of data.items) { const name = item.repository.full_name; repoMap[name] = { @@ -174,7 +198,12 @@ async function fetchReposForAccount( // Take the top 6 repos by commit count to keep the dashboard widget compact // and to limit the number of subsequent language API calls (6 REST requests max). const repos = Object.entries(repoMap) - .map(([name, { commits, description, url }]) => ({ name, commits, description, url })) + .map(([name, { commits, description, url }]) => ({ + name, + commits, + description, + url, + })) .sort((a, b) => b.commits - a.commits) .slice(0, 6); @@ -187,7 +216,13 @@ async function fetchReposForAccount( }) ); - return { repos: reposWithLanguages, days, complete: search.complete, sampledCommits: search.items.length, partialReason: search.partialReason }; + return { + repos: reposWithLanguages, + days, + complete: search.complete, + sampledCommits: search.items.length, + partialReason: search.partialReason, + }; } ); } @@ -207,7 +242,9 @@ export async function GET(req: NextRequest) { const daysParam = req.nextUrl.searchParams.get("days"); const parsedDays = daysParam ? parseInt(daysParam, 10) : NaN; // Clamp days between 1 and 365 — GitHub Commit Search supports up to ~1 year lookback. - const days = Number.isNaN(parsedDays) ? 30 : Math.max(1, Math.min(365, parsedDays)); + const days = Number.isNaN(parsedDays) + ? 30 + : Math.max(1, Math.min(365, parsedDays)); const accountId = req.nextUrl.searchParams.get("accountId"); const bypass = isMetricsCacheBypassed(req); @@ -230,7 +267,10 @@ export async function GET(req: NextRequest) { .eq("github_id", session.githubId) .single(); - const orgsConfig = (dbUser?.organizations_config || {}) as Record; + const orgsConfig = (dbUser?.organizations_config || {}) as Record< + string, + boolean + >; excludedOrgs = Object.entries(orgsConfig) .filter(([_, enabled]) => enabled === false) .map(([org]) => org); diff --git a/src/app/api/metrics/weekly-summary/route.ts b/src/app/api/metrics/weekly-summary/route.ts index f9b0a60b3..e69c939d4 100644 --- a/src/app/api/metrics/weekly-summary/route.ts +++ b/src/app/api/metrics/weekly-summary/route.ts @@ -1,11 +1,16 @@ import { githubRateLimitResponse } from "@/lib/github-rate-limit"; import { githubRequest } from "@/lib/github-request"; +import { collectAuthorCommits, requireUnthrottled } from "@/lib/github-search"; import { getServerSession } from "@/lib/server-session"; import { NextRequest } from "next/server"; import { authOptions } from "@/lib/auth"; import { GITHUB_API } from "@/lib/github"; import { GitHubAuthError, githubAuthErrorResponse } from "@/lib/github-fetch"; -import { isMetricsCacheBypassed, metricsCacheKey, withMetricsCache } from "@/lib/metrics-cache"; +import { + isMetricsCacheBypassed, + metricsCacheKey, + withMetricsCache, +} from "@/lib/metrics-cache"; import { getAccountToken, getAllAccounts } from "@/lib/github-accounts"; import { supabaseAdmin } from "@/lib/supabase"; import { resolveAppUser } from "@/lib/resolve-user"; @@ -34,61 +39,25 @@ function calculateCurrentStreak(activeDates: Set): number { return currentStreak; } -async function fetchActiveDates(githubLogin: string, token: string): Promise> { - // Look back 90 days — the maximum window the GitHub Commit Search API supports. +async function fetchActiveDates( + githubLogin: string, + token: string, + bypass: boolean +): Promise> { const since = new Date(); since.setDate(since.getDate() - 90); - const sinceStr = since.toISOString().slice(0, 10); // "YYYY-MM-DD" - - const activeDates = new Set(); - let page = 1; - - // GitHub Commit Search API rate limits: - // • Authenticated (OAuth token / PAT): 30 requests/minute - // • Unauthenticated: 10 requests/minute - // - // This loop pages up to 10 pages (1,000 commits max) to cover the 90-day window. - // Each page = 1 request against the 30 req/min quota. - // NOTE: this function is called INSIDE withMetricsCache in the GET handler, - // so it only runs on a cache miss — repeated page loads reuse cached dates. - while (true) { - const searchRes = await githubRequest( - `${GITHUB_API}/search/commits?q=author:${githubLogin}+author-date:>=${sinceStr}&per_page=100&page=${page}&sort=author-date&order=desc`, + const { items } = requireUnthrottled( + await collectAuthorCommits( + githubLogin, + since.toISOString().slice(0, 10), + token, { - headers: { - // OAuth token / PAT: raises the Search API limit from 10 → 30 req/min. - // Without this, the streak pagination loop could exhaust the unauthenticated - // quota on its own, blocking all other Search API calls on the same IP. - Authorization: `Bearer ${token}`, - // Mandatory Accept header for the Commit Search endpoint. - // Omitting it causes GitHub to return HTTP 415 (Unsupported Media Type). - Accept: "application/vnd.github+json", - }, - cache: "no-store", + fresh: bypass, } - ); - - // HTTP 401 = token revoked/invalid. HTTP 403 = rate limit. - // Both throw so the outer GET handler can distinguish auth failures. - if (!searchRes.ok) { - if (searchRes.status === 401) throw new GitHubAuthError(); - throw new Error("GitHub API error"); - } - - const data = (await searchRes.json()) as { items: Array<{ commit: { author: { date: string } } }> }; - - // Extract just the "YYYY-MM-DD" date from each commit timestamp. - // Set deduplicates — multiple commits on the same day count as one active day. - for (const item of data.items) { - activeDates.add(item.commit.author.date.slice(0, 10)); - } - - // Stop when GitHub returns fewer than 100 items (last page) or the 10-page cap is hit. - if (data.items.length < 100 || page >= 10) break; - page++; - } - - return activeDates; + ) + ); + // Set deduplicates — multiple commits on the same day count as one active day. + return new Set(items.map((item) => item.commit.author.date.slice(0, 10))); } interface WeeklySummaryData { @@ -133,27 +102,13 @@ async function fetchWeeklySummaryForAccount( // are covered with one Search API request instead of two. const fourteenDaysAgoStr = toDateStr(new Date(Date.now() - 14 * 86400000)); - // Search API call 1 of 3 — fetches commits for the past 14 days. - const commitsRes = await githubRequest( - `${GITHUB_API}/search/commits?q=author:${githubLogin}+author-date:>=${fourteenDaysAgoStr}&per_page=100`, - { - headers: { - Authorization: `Bearer ${token}`, - Accept: "application/vnd.github+json", - }, - cache: "no-store", - } - ); - - if (!commitsRes.ok && commitsRes.status === 401) throw new GitHubAuthError(); - const commitsData: { - items: Array<{ + // Both weeks come from the shared commit timeline, not a search of their own. + const commitsData = requireUnthrottled( + await collectAuthorCommits<{ commit: { author: { date: string } }; repository: { full_name: string }; - }>; - } = commitsRes.ok - ? await commitsRes.json() - : { items: [] }; + }>(githubLogin, fourteenDaysAgoStr, token, { fresh: bypass }) + ); let commitsThisWeek = 0; let commitsPrevWeek = 0; @@ -161,7 +116,7 @@ async function fetchWeeklySummaryForAccount( const activeDaysLastWeek = new Set(); const repoCounts = new Map(); const recent7DaysMap = new Map(); - + const today = new Date(); const sevenDaysAgo = new Date(today); sevenDaysAgo.setUTCDate(sevenDaysAgo.getUTCDate() - 6); @@ -245,8 +200,7 @@ async function fetchWeeklySummaryForAccount( } } - // Search API calls 3+ — fetchActiveDates pages through to build the 90-day commit date set. - const streakDates = await fetchActiveDates(githubLogin, token); + const streakDates = await fetchActiveDates(githubLogin, token, bypass); const commitDelta = commitsThisWeek - commitsPrevWeek; // Search API call 4 - fetches Issues opened in the past 14 days @@ -282,8 +236,9 @@ async function fetchWeeklySummaryForAccount( for (const item of issuesData.items) { const createdAt = new Date(item.created_at); if (Number.isNaN(createdAt.getTime())) continue; - - const isClosed = item.state === "closed" && item.state_reason === "completed"; + + const isClosed = + item.state === "closed" && item.state_reason === "completed"; if (createdAt >= currentWeekStart) { issuesOpenedThisWeek++; @@ -327,8 +282,14 @@ async function fetchWeeklySummaryForAccount( lastWeek: { opened: prsOpenedLastWeek, merged: prsMergedLastWeek }, }, issues: { - thisWeek: { opened: issuesOpenedThisWeek, closed: issuesClosedThisWeek }, - lastWeek: { opened: issuesOpenedLastWeek, closed: issuesClosedLastWeek }, + thisWeek: { + opened: issuesOpenedThisWeek, + closed: issuesClosedThisWeek, + }, + lastWeek: { + opened: issuesOpenedLastWeek, + closed: issuesClosedLastWeek, + }, }, activeDays: { thisWeek: activeDaysThisWeek.size, @@ -368,123 +329,194 @@ export async function GET(req: NextRequest) { } try { - const combinedKey = metricsCacheKey(userRow.id, "weekly-summary", { accountId: "combined" }); - - const data = await withMetricsCache({ bypass, key: combinedKey, ttlSeconds: 5 * 60 }, async () => { - const accounts = await getAllAccounts( - { - token: session.accessToken!, - githubId: session.githubId!, - githubLogin: session.githubLogin!, - }, - userRow.id - ); - - const summaryPromises = accounts.map(async (acc) => { - const token = acc.githubId === session.githubId - ? session.accessToken - : await getAccountToken(userRow.id, acc.githubId); - if (!token) return null; - return fetchWeeklySummaryForAccount(token, acc.githubLogin, acc.githubId, bypass); - }); - - const resultsRaw = await Promise.allSettled(summaryPromises); - const results = resultsRaw - .filter((r): r is PromiseFulfilledResult => r.status === "fulfilled" && r.value !== null) - .map((r) => r.value); - - if (results.length === 0) { - throw new Error("No account weekly summaries were successfully fetched"); - } + const combinedKey = metricsCacheKey(userRow.id, "weekly-summary", { + accountId: "combined", + }); - // Merge the summaries - const commitsCurrent = results.reduce((sum, r) => sum + r.commits.current, 0); - const commitsPrevious = results.reduce((sum, r) => sum + r.commits.previous, 0); - const commitsDelta = commitsCurrent - commitsPrevious; - - const prsThisWeekOpened = results.reduce((sum, r) => sum + r.prs.thisWeek.opened, 0); - const prsThisWeekMerged = results.reduce((sum, r) => sum + r.prs.thisWeek.merged, 0); - const prsLastWeekOpened = results.reduce((sum, r) => sum + r.prs.lastWeek.opened, 0); - const prsLastWeekMerged = results.reduce((sum, r) => sum + r.prs.lastWeek.merged, 0); - - const issuesThisWeekOpened = results.reduce((sum, r) => sum + r.issues.thisWeek.opened, 0); - const issuesThisWeekClosed = results.reduce((sum, r) => sum + r.issues.thisWeek.closed, 0); - const issuesLastWeekOpened = results.reduce((sum, r) => sum + r.issues.lastWeek.opened, 0); - const issuesLastWeekClosed = results.reduce((sum, r) => sum + r.issues.lastWeek.closed, 0); - - const activeDaysThisWeek = Math.min(7, results.reduce((sum, r) => sum + r.activeDays.thisWeek, 0)); - const activeDaysLastWeek = Math.min(7, results.reduce((sum, r) => sum + r.activeDays.lastWeek, 0)); - - const maxStreak = Math.max(...results.map((r) => r.streak)); - - // Find the top repository by commits count - let topRepo: string | null = null; - let highestCommits = -1; - for (const res of results) { - if (res.topRepo && res.commits.current > highestCommits) { - topRepo = res.topRepo; - highestCommits = res.commits.current; + const data = await withMetricsCache( + { bypass, key: combinedKey, ttlSeconds: 5 * 60 }, + async () => { + const accounts = await getAllAccounts( + { + token: session.accessToken!, + githubId: session.githubId!, + githubLogin: session.githubLogin!, + }, + userRow.id + ); + + const summaryPromises = accounts.map(async (acc) => { + const token = + acc.githubId === session.githubId + ? session.accessToken + : await getAccountToken(userRow.id, acc.githubId); + if (!token) return null; + return fetchWeeklySummaryForAccount( + token, + acc.githubLogin, + acc.githubId, + bypass + ); + }); + + const resultsRaw = await Promise.allSettled(summaryPromises); + const results = resultsRaw + .filter( + (r): r is PromiseFulfilledResult => + r.status === "fulfilled" && r.value !== null + ) + .map((r) => r.value); + + if (results.length === 0) { + throw new Error( + "No account weekly summaries were successfully fetched" + ); } - } - const repoBreakdownMap = new Map(); - for (const res of results) { - for (const repo of res.repoBreakdown) { - repoBreakdownMap.set(repo.repoName, (repoBreakdownMap.get(repo.repoName) ?? 0) + repo.commits); + // Merge the summaries + const commitsCurrent = results.reduce( + (sum, r) => sum + r.commits.current, + 0 + ); + const commitsPrevious = results.reduce( + (sum, r) => sum + r.commits.previous, + 0 + ); + const commitsDelta = commitsCurrent - commitsPrevious; + + const prsThisWeekOpened = results.reduce( + (sum, r) => sum + r.prs.thisWeek.opened, + 0 + ); + const prsThisWeekMerged = results.reduce( + (sum, r) => sum + r.prs.thisWeek.merged, + 0 + ); + const prsLastWeekOpened = results.reduce( + (sum, r) => sum + r.prs.lastWeek.opened, + 0 + ); + const prsLastWeekMerged = results.reduce( + (sum, r) => sum + r.prs.lastWeek.merged, + 0 + ); + + const issuesThisWeekOpened = results.reduce( + (sum, r) => sum + r.issues.thisWeek.opened, + 0 + ); + const issuesThisWeekClosed = results.reduce( + (sum, r) => sum + r.issues.thisWeek.closed, + 0 + ); + const issuesLastWeekOpened = results.reduce( + (sum, r) => sum + r.issues.lastWeek.opened, + 0 + ); + const issuesLastWeekClosed = results.reduce( + (sum, r) => sum + r.issues.lastWeek.closed, + 0 + ); + + const activeDaysThisWeek = Math.min( + 7, + results.reduce((sum, r) => sum + r.activeDays.thisWeek, 0) + ); + const activeDaysLastWeek = Math.min( + 7, + results.reduce((sum, r) => sum + r.activeDays.lastWeek, 0) + ); + + const maxStreak = Math.max(...results.map((r) => r.streak)); + + // Find the top repository by commits count + let topRepo: string | null = null; + let highestCommits = -1; + for (const res of results) { + if (res.topRepo && res.commits.current > highestCommits) { + topRepo = res.topRepo; + highestCommits = res.commits.current; + } } - } - const combinedRepoBreakdown = Array.from(repoBreakdownMap.entries()) - .map(([repoName, commits]) => ({ repoName, commits })) - .sort((a, b) => b.commits - a.commits); - - const dailyCommitsMap = new Map(); - for (const res of results) { - for (const day of res.dailyCommits) { - dailyCommitsMap.set(day.date, (dailyCommitsMap.get(day.date) ?? 0) + day.commits); + + const repoBreakdownMap = new Map(); + for (const res of results) { + for (const repo of res.repoBreakdown) { + repoBreakdownMap.set( + repo.repoName, + (repoBreakdownMap.get(repo.repoName) ?? 0) + repo.commits + ); + } } - } - - const combinedDailyCommits: { date: string; commits: number }[] = []; - let mostActiveDay: string | null = null; - let maxCommitsForDay = -1; - for (let i = 6; i >= 0; i--) { - const d = new Date(); - d.setUTCDate(d.getUTCDate() - i); - const dateStr = d.toISOString().slice(0, 10); - const commits = dailyCommitsMap.get(dateStr) ?? 0; - combinedDailyCommits.push({ date: dateStr, commits }); - if (commits > maxCommitsForDay) { - maxCommitsForDay = commits; - if (commits > 0) mostActiveDay = dateStr; + const combinedRepoBreakdown = Array.from(repoBreakdownMap.entries()) + .map(([repoName, commits]) => ({ repoName, commits })) + .sort((a, b) => b.commits - a.commits); + + const dailyCommitsMap = new Map(); + for (const res of results) { + for (const day of res.dailyCommits) { + dailyCommitsMap.set( + day.date, + (dailyCommitsMap.get(day.date) ?? 0) + day.commits + ); + } } - } - return { - commits: { - current: commitsCurrent, - previous: commitsPrevious, - delta: commitsDelta, - trend: commitsDelta > 0 ? "up" : commitsDelta < 0 ? "down" : "same", - }, - prs: { - thisWeek: { opened: prsThisWeekOpened, merged: prsThisWeekMerged }, - lastWeek: { opened: prsLastWeekOpened, merged: prsLastWeekMerged }, - }, - issues: { - thisWeek: { opened: issuesThisWeekOpened, closed: issuesThisWeekClosed }, - lastWeek: { opened: issuesLastWeekOpened, closed: issuesLastWeekClosed }, - }, - activeDays: { - thisWeek: activeDaysThisWeek, - lastWeek: activeDaysLastWeek, - }, - streak: maxStreak, - topRepo, - repoBreakdown: combinedRepoBreakdown, - dailyCommits: combinedDailyCommits, - mostActiveDay, - }; - }); + const combinedDailyCommits: { date: string; commits: number }[] = []; + let mostActiveDay: string | null = null; + let maxCommitsForDay = -1; + for (let i = 6; i >= 0; i--) { + const d = new Date(); + d.setUTCDate(d.getUTCDate() - i); + const dateStr = d.toISOString().slice(0, 10); + const commits = dailyCommitsMap.get(dateStr) ?? 0; + combinedDailyCommits.push({ date: dateStr, commits }); + if (commits > maxCommitsForDay) { + maxCommitsForDay = commits; + if (commits > 0) mostActiveDay = dateStr; + } + } + + return { + commits: { + current: commitsCurrent, + previous: commitsPrevious, + delta: commitsDelta, + trend: + commitsDelta > 0 ? "up" : commitsDelta < 0 ? "down" : "same", + }, + prs: { + thisWeek: { + opened: prsThisWeekOpened, + merged: prsThisWeekMerged, + }, + lastWeek: { + opened: prsLastWeekOpened, + merged: prsLastWeekMerged, + }, + }, + issues: { + thisWeek: { + opened: issuesThisWeekOpened, + closed: issuesThisWeekClosed, + }, + lastWeek: { + opened: issuesLastWeekOpened, + closed: issuesLastWeekClosed, + }, + }, + activeDays: { + thisWeek: activeDaysThisWeek, + lastWeek: activeDaysLastWeek, + }, + streak: maxStreak, + topRepo, + repoBreakdown: combinedRepoBreakdown, + dailyCommits: combinedDailyCommits, + mostActiveDay, + }; + } + ); return Response.json(data); } catch (e) { @@ -526,7 +558,12 @@ export async function GET(req: NextRequest) { } try { - const data = await fetchWeeklySummaryForAccount(token, githubLogin, userId, bypass); + const data = await fetchWeeklySummaryForAccount( + token, + githubLogin, + userId, + bypass + ); return Response.json(data); } catch (e) { const githubLimit = githubRateLimitResponse(e); diff --git a/src/lib/ci-analytics.ts b/src/lib/ci-analytics.ts index a17862a8f..c70c0f3cd 100644 --- a/src/lib/ci-analytics.ts +++ b/src/lib/ci-analytics.ts @@ -1,56 +1,148 @@ import { githubRequest } from "@/lib/github-request"; import { GITHUB_API } from "@/lib/github"; +import { collectAuthorCommits, requireUnthrottled } from "@/lib/github-search"; -interface TopRepo { name: string; commits: number; } -interface WorkflowRun { conclusion: string | null; created_at: string; name: string | null; updated_at: string; } -export interface CIAnalyticsResponse { successRate: number; averageDurationMinutes: number; flakiestWorkflow: string | null; totalRuns: number; reposChecked: number; failedRepos?: string[]; } +interface TopRepo { + name: string; + commits: number; +} +interface WorkflowRun { + conclusion: string | null; + created_at: string; + name: string | null; + updated_at: string; +} +export interface CIAnalyticsResponse { + successRate: number; + averageDurationMinutes: number; + flakiestWorkflow: string | null; + totalRuns: number; + reposChecked: number; + failedRepos?: string[]; +} -function toIsoDate(daysAgo: number): string { const d = new Date(); d.setDate(d.getDate() - daysAgo); return d.toISOString().slice(0, 10); } -function getRunDurationMinutes(run: WorkflowRun): number { const c = new Date(run.created_at).getTime(), u = new Date(run.updated_at).getTime(); return (isNaN(c) || isNaN(u) || u < c) ? 0 : (u - c) / 60000; } +function toIsoDate(daysAgo: number): string { + const d = new Date(); + d.setDate(d.getDate() - daysAgo); + return d.toISOString().slice(0, 10); +} +function getRunDurationMinutes(run: WorkflowRun): number { + const c = new Date(run.created_at).getTime(), + u = new Date(run.updated_at).getTime(); + return isNaN(c) || isNaN(u) || u < c ? 0 : (u - c) / 60000; +} -export function mergeCIAnalytics(a: CIAnalyticsResponse, b: CIAnalyticsResponse): CIAnalyticsResponse { +export function mergeCIAnalytics( + a: CIAnalyticsResponse, + b: CIAnalyticsResponse +): CIAnalyticsResponse { const totalRuns = a.totalRuns + b.totalRuns; - const weightedDuration = totalRuns === 0 ? 0 : (a.averageDurationMinutes * a.totalRuns + b.averageDurationMinutes * b.totalRuns) / totalRuns; - const successes = Math.round((a.successRate / 100) * a.totalRuns) + Math.round((b.successRate / 100) * b.totalRuns); - return { successRate: totalRuns === 0 ? 0 : Math.round((successes / totalRuns) * 100), averageDurationMinutes: Math.round(weightedDuration * 10) / 10, flakiestWorkflow: a.flakiestWorkflow ?? b.flakiestWorkflow, totalRuns, reposChecked: a.reposChecked + b.reposChecked, failedRepos: [...(a.failedRepos ?? []), ...(b.failedRepos ?? [])] }; + const weightedDuration = + totalRuns === 0 + ? 0 + : (a.averageDurationMinutes * a.totalRuns + + b.averageDurationMinutes * b.totalRuns) / + totalRuns; + const successes = + Math.round((a.successRate / 100) * a.totalRuns) + + Math.round((b.successRate / 100) * b.totalRuns); + return { + successRate: + totalRuns === 0 ? 0 : Math.round((successes / totalRuns) * 100), + averageDurationMinutes: Math.round(weightedDuration * 10) / 10, + flakiestWorkflow: a.flakiestWorkflow ?? b.flakiestWorkflow, + totalRuns, + reposChecked: a.reposChecked + b.reposChecked, + failedRepos: [...(a.failedRepos ?? []), ...(b.failedRepos ?? [])], + }; } -export async function fetchCIAnalyticsForAccount(token: string, githubLogin: string): Promise { - const searchRes = await githubRequest(`${GITHUB_API}/search/commits?q=author:${githubLogin}+author-date:>=${toIsoDate(30)}&per_page=100&sort=author-date&order=desc`, { headers: { Authorization: `Bearer ${token}`, Accept: "application/vnd.github+json" }, cache: "no-store" }); - if (!searchRes.ok) throw new Error("API error"); - const data = await searchRes.json(); - +export async function fetchCIAnalyticsForAccount( + token: string, + githubLogin: string +): Promise { + const data = requireUnthrottled( + await collectAuthorCommits<{ + commit: { author: { date: string } }; + repository: { full_name: string }; + }>(githubLogin, toIsoDate(30), token) + ); + const repoMap = new Map(); - for (const item of data.items) { const n = item.repository.full_name; repoMap.set(n, (repoMap.get(n) ?? 0) + 1); } - const repos = Array.from(repoMap.entries()).map(([name, commits]) => ({ name, commits })).sort((a, b) => b.commits - a.commits).slice(0, 3); + for (const item of data.items) { + const n = item.repository.full_name; + repoMap.set(n, (repoMap.get(n) ?? 0) + 1); + } + const repos = Array.from(repoMap.entries()) + .map(([name, commits]) => ({ name, commits })) + .sort((a, b) => b.commits - a.commits) + .slice(0, 3); const failedRepos: string[] = []; - const runsByRepo = await Promise.all(repos.map(async (repo) => { - try { - const res = await githubRequest(`${GITHUB_API}/repos/${repo.name}/actions/runs?per_page=100&created=>=${toIsoDate(30)}`, { headers: { Authorization: `Bearer ${token}`, Accept: "application/vnd.github+json" }, cache: "no-store" }); - if (res.status === 404 || res.status === 403) return []; - if (!res.ok) throw new Error("API error"); - const d = await res.json(); return d.workflow_runs ?? []; - } catch (err) { - console.error(`Failed to fetch signals for repo ${repo.name}:`, err); - failedRepos.push(repo.name); - return []; - } - })); + const runsByRepo = await Promise.all( + repos.map(async (repo) => { + try { + const res = await githubRequest( + `${GITHUB_API}/repos/${repo.name}/actions/runs?per_page=100&created=>=${toIsoDate(30)}`, + { + headers: { + Authorization: `Bearer ${token}`, + Accept: "application/vnd.github+json", + }, + cache: "no-store", + } + ); + if (res.status === 404 || res.status === 403) return []; + if (!res.ok) throw new Error("API error"); + const d = await res.json(); + return d.workflow_runs ?? []; + } catch (err) { + console.error(`Failed to fetch signals for repo ${repo.name}:`, err); + failedRepos.push(repo.name); + return []; + } + }) + ); const runs = runsByRepo.flat().filter((r: WorkflowRun) => r.conclusion); - const successfulRuns = runs.filter((r: WorkflowRun) => r.conclusion === "success"); - const workflowStats = new Map(); + const successfulRuns = runs.filter( + (r: WorkflowRun) => r.conclusion === "success" + ); + const workflowStats = new Map(); for (const run of runs) { const name = run.name ?? "Unnamed workflow"; const stats = workflowStats.get(name) ?? { failures: 0, total: 0 }; - stats.total += 1; if (run.conclusion !== "success") stats.failures += 1; + stats.total += 1; + if (run.conclusion !== "success") stats.failures += 1; workflowStats.set(name, stats); } - const flakiestWorkflow = Array.from(workflowStats.entries()).filter(([, s]) => s.failures > 0).sort((a, b) => (b[1].failures / b[1].total) - (a[1].failures / a[1].total) || b[1].failures - a[1].failures)[0]?.[0] ?? null; - const totalDuration = runs.reduce((sum: number, run: any) => sum + getRunDurationMinutes(run), 0); + const flakiestWorkflow = + Array.from(workflowStats.entries()) + .filter(([, s]) => s.failures > 0) + .sort( + (a, b) => + b[1].failures / b[1].total - a[1].failures / a[1].total || + b[1].failures - a[1].failures + )[0]?.[0] ?? null; + const totalDuration = runs.reduce( + (sum: number, run: any) => sum + getRunDurationMinutes(run), + 0 + ); - return { successRate: runs.length === 0 ? 0 : Math.round((successfulRuns.length / runs.length) * 100), averageDurationMinutes: runs.length === 0 ? 0 : Math.round((totalDuration / runs.length) * 10) / 10, flakiestWorkflow, totalRuns: runs.length, reposChecked: repos.length, failedRepos }; + return { + successRate: + runs.length === 0 + ? 0 + : Math.round((successfulRuns.length / runs.length) * 100), + averageDurationMinutes: + runs.length === 0 + ? 0 + : Math.round((totalDuration / runs.length) * 10) / 10, + flakiestWorkflow, + totalRuns: runs.length, + reposChecked: repos.length, + failedRepos, + }; } diff --git a/src/lib/github-request.ts b/src/lib/github-request.ts index f0b5cea01..d6e5cf7d7 100644 --- a/src/lib/github-request.ts +++ b/src/lib/github-request.ts @@ -11,11 +11,60 @@ type StoredResponse = { headers: [string, string][]; savedAt: number; }; +type Resource = "core" | "graphql" | "search"; const flights = new Map>(); -const queues = new Map>(); +const slots = new Map void> }>(); const cooldowns = new Map(); const REQUEST_TIMEOUT_MS = 12_000; -const FRESH_MS = 30_000; +// Search allows 30 requests/minute; its data also changes slowly, so it is +// cached longest. Callers force revalidation with `cache: "no-cache"`. +const FRESH_MS: Record = { + core: 30_000, + graphql: 120_000, + search: 300_000, +}; +// A forced refresh fans out to every dashboard widget at once; widgets sharing +// a query reuse one revalidation instead of each spending a search. +const REVALIDATED_MS = 15_000; +// Bounded per-credential concurrency keeps well under GitHub's secondary +// limits without letting cheap REST calls queue behind slow search pages. +const CONCURRENCY: Record = { + core: 4, + graphql: 2, + search: 1, +}; + +function acquireSlot(key: string, limit: number, timeoutMs: number) { + let slot = slots.get(key); + if (!slot) { + slot = { active: 0, waiting: [] }; + slots.set(key, slot); + } + const held = slot; + const release = () => { + held.active--; + const next = held.waiting.shift(); + if (next) next(); + else if (held.active === 0 && slots.get(key) === held) slots.delete(key); + }; + if (held.active < limit) { + held.active++; + return Promise.resolve(release); + } + return new Promise<() => void>((resolve, reject) => { + const grant = () => { + clearTimeout(timer); + held.active++; + resolve(release); + }; + const timer = setTimeout(() => { + const index = held.waiting.indexOf(grant); + if (index >= 0) held.waiting.splice(index, 1); + reject(new Error("GitHub request queue timed out")); + }, timeoutMs); + held.waiting.push(grant); + }); +} async function hash(value: string): Promise { const bytes = await crypto.subtle.digest( @@ -116,7 +165,7 @@ export async function githubRequest( if (!headers.has("accept")) headers.set("accept", "application/vnd.github+json"); const credential = await hash(headers.get("authorization") ?? "anonymous"); - const resource = url.pathname.startsWith("/search/") + const resource: Resource = url.pathname.startsWith("/search/") ? "search" : url.pathname === "/graphql" ? "graphql" @@ -141,41 +190,24 @@ export async function githubRequest( )); if (init.signal?.aborted) throw init.signal.reason; let cached: StoredResponse | null = null; + const revalidate = init.cache === "no-cache" || init.cache === "reload"; if (cacheable) { cached = await cacheGet(key); - if (cached && Date.now() - cached.savedAt < FRESH_MS) + const freshFor = revalidate ? REVALIDATED_MS : FRESH_MS[resource]; + if (cached && Date.now() - cached.savedAt < freshFor) return asResponse(cached); const existing = flights.get(key); if (existing) return asResponse(await abortable(existing, init.signal)); } - const previous = queues.get(credential) ?? Promise.resolve(); - let release!: () => void; - const turn = new Promise((resolve) => { - release = resolve; - }); - const chained = previous.then(() => turn); - queues.set(credential, chained); - // A queued caller can expire before its predecessor finishes. Retain the - // chain until it settles so new callers cannot bypass active work. - void chained.then(() => { - if (queues.get(credential) === chained) queues.delete(credential); - }); const startedAt = Date.now(); const work = (async () => { - // Serialize outbound requests per credential, as recommended by GitHub. // Deadline includes queue time, so busy accounts cannot occupy a request indefinitely. - let timer: ReturnType | undefined; + const release = await acquireSlot( + bucket, + CONCURRENCY[resource], + REQUEST_TIMEOUT_MS + ); try { - await Promise.race([ - previous, - new Promise((_, reject) => { - timer = setTimeout( - () => reject(new Error("GitHub request queue timed out")), - REQUEST_TIMEOUT_MS - ); - }), - ]); - clearTimeout(timer); const until = Math.max( cooldowns.get(bucket) ?? 0, cooldowns.get(secondaryBucket) ?? 0, @@ -243,12 +275,15 @@ export async function githubRequest( : retrySeconds(response, body); if (retry !== null) { const reset = Date.now() + retry * 1000; + // GitHub answers an exhausted primary quota with 403 or 429 and + // remaining=0; only that resource waits. Anything else is a secondary + // limit, which applies to the whole credential. const secondary = - (graphLimited && - response.headers.get("x-ratelimit-remaining") !== "0") || - response.status === 429 || - response.headers.has("retry-after") || - /secondary rate limit|exceeded a secondary/i.test(body); + response.headers.get("x-ratelimit-remaining") !== "0" && + (graphLimited || + response.status === 429 || + response.headers.has("retry-after") || + /secondary rate limit|exceeded a secondary/i.test(body)); cooldowns.set(secondary ? secondaryBucket : bucket, reset); await cacheSet(secondary ? secondaryKey : cooldownKey, reset, retry); return limited(reset); @@ -279,7 +314,6 @@ export async function githubRequest( await cacheSet(key, stored, 86400); return stored; } finally { - clearTimeout(timer); release(); for (const [key, until] of cooldowns) if (until <= Date.now()) cooldowns.delete(key); @@ -299,6 +333,6 @@ export async function githubRequest( /** Test isolation without weakening production behavior. */ export function resetGitHubRequestState() { flights.clear(); - queues.clear(); + slots.clear(); cooldowns.clear(); } diff --git a/src/lib/github-search.ts b/src/lib/github-search.ts index 1880dae48..0a10b7ee8 100644 --- a/src/lib/github-search.ts +++ b/src/lib/github-search.ts @@ -9,12 +9,359 @@ export type SearchResult = { partialReason?: string; retryAfter?: number; }; +export type AuthorCommitItem = { commit: { author: { date: string } } }; + +/** Results cut short by a rate limit must surface as one, never be cached as complete. */ +export function requireUnthrottled( + result: SearchResult +): SearchResult { + if (result.retryAfter !== undefined) + throw new GitHubRateLimitError( + new Date(Date.now() + result.retryAfter * 1000), + result.retryAfter + ); + return result; +} + +const COMMIT_TIMELINE_DAYS = 366; +const HISTORY_BATCH = 20; +const HISTORY_ROUNDS = 20; + +type TimelineCommit = { + sha: string; + html_url: string; + commit: { author: { date: string }; message: string }; + repository: { + full_name: string; + html_url: string; + description: string | null; + }; +}; + +type RepoHistory = { + nameWithOwner: string; + url: string; + description: string | null; + defaultBranchRef: { + target: { + history?: { + pageInfo: { hasNextPage: boolean; endCursor: string | null }; + nodes: Array<{ + oid: string; + messageHeadline: string; + url: string; + authoredDate: string; + }>; + }; + }; + } | null; +}; + +type CommitRepos = { + commitContributionsByRepository: Array<{ + repository: { nameWithOwner: string }; + }>; +}; + +const COMMIT_REPOS_QUERY = ` + query CommitRepos($login: String!, $year: DateTime!, $recent: DateTime!) { + user(login: $login) { + id + year: contributionsCollection(from: $year) { + commitContributionsByRepository(maxRepositories: 100) { repository { nameWithOwner } } + } + recent: contributionsCollection(from: $recent) { + commitContributionsByRepository(maxRepositories: 100) { repository { nameWithOwner } } + } + } + } +`; + +function commitHistoryQuery(count: number): string { + const params = Array.from( + { length: count }, + (_, i) => `$o${i}: String!, $n${i}: String!, $a${i}: String` + ).join(", "); + const repos = Array.from( + { length: count }, + (_, i) => `r${i}: repository(owner: $o${i}, name: $n${i}) { + nameWithOwner url description + defaultBranchRef { target { ... on Commit { + history(first: 100, since: $since, author: { id: $author }, after: $a${i}) { + pageInfo { hasNextPage endCursor } + nodes { oid messageHeadline url authoredDate } + } + } } } + }` + ).join("\n"); + return `query CommitHistory($since: GitTimestamp!, $author: ID!, ${params}) {\n${repos}\n}`; +} + +/** GraphQL that tolerates per-repository errors (deleted or inaccessible repos resolve to null). */ +async function graphqlData( + query: string, + variables: Record, + token: string, + fresh?: boolean +): Promise { + const response = await githubRequest("https://api.github.com/graphql", { + method: "POST", + headers: { + Authorization: `Bearer ${token}`, + "Content-Type": "application/json", + }, + body: JSON.stringify({ query, variables }), + cache: fresh ? "no-cache" : "no-store", + }); + if (response.status === 401) throw new GitHubAuthError(); + if (!response.ok) throw new Error("GitHub GraphQL unavailable"); + const json = (await response.json()) as { data?: T | null }; + if (!json.data) throw new Error("GitHub GraphQL error"); + return json.data; +} + +/** + * A year of the user's commits, newest first, read through GraphQL: the + * repositories they committed to, then each one's default-branch history + * filtered to their author id. This matches what commit search indexes but + * spends no Search API quota; paging a year of commit search trips GitHub's + * secondary (anti-scraping) limit, which then blocks every API for the user. + */ +async function fetchCommitTimeline( + login: string, + start: string, + token: string, + fresh?: boolean +): Promise> { + const today = new Date(); + today.setUTCHours(0, 0, 0, 0); + // Day-aligned bounds keep the request bodies, and so the response cache, stable all day. + const recent = new Date(today.getTime() - 30 * 86_400_000).toISOString(); + const { user } = await graphqlData<{ + user: { id: string; year: CommitRepos; recent: CommitRepos } | null; + }>( + COMMIT_REPOS_QUERY, + { login, year: `${start}T00:00:00Z`, recent }, + token, + fresh + ); + if (!user) throw new Error("GitHub user not found"); + const names = new Set( + [ + ...user.year.commitContributionsByRepository, + ...user.recent.commitContributionsByRepository, + ].map((entry) => entry.repository.nameWithOwner) + ); + + const items: TimelineCommit[] = []; + let pending = [...names].map((name) => ({ + name, + after: null as string | null, + })); + for (let round = 0; pending.length > 0 && round < HISTORY_ROUNDS; round++) { + const next: typeof pending = []; + for (let i = 0; i < pending.length; i += HISTORY_BATCH) { + const batch = pending.slice(i, i + HISTORY_BATCH); + const variables: Record = { + since: `${start}T00:00:00Z`, + author: user.id, + }; + batch.forEach((repo, j) => { + const [owner, name] = repo.name.split("/"); + variables[`o${j}`] = owner; + variables[`n${j}`] = name; + variables[`a${j}`] = repo.after; + }); + const data = await graphqlData>( + commitHistoryQuery(batch.length), + variables, + token, + fresh + ); + batch.forEach((pendingRepo, j) => { + const repo = data[`r${j}`]; + const history = repo?.defaultBranchRef?.target.history; + if (!repo || !history) return; + for (const node of history.nodes) + items.push({ + sha: node.oid, + html_url: node.url, + commit: { + author: { date: node.authoredDate }, + message: node.messageHeadline, + }, + repository: { + full_name: repo.nameWithOwner, + html_url: repo.url, + description: repo.description, + }, + }); + if (history.pageInfo.hasNextPage && history.pageInfo.endCursor) + next.push({ + name: pendingRepo.name, + after: history.pageInfo.endCursor, + }); + }); + } + pending = next; + } + items.sort( + (a, b) => + Date.parse(b.commit.author.date) - Date.parse(a.commit.author.date) + ); + return pending.length === 0 + ? { items, total: items.length, complete: true } + : { + items, + total: items.length, + complete: false, + partialReason: "Commit history exceeds the per-request page budget", + }; +} + +const timelines = new Map< + string, + { at: number; result: Promise> } +>(); +const TIMELINE_FRESH_MS = 5 * 60_000; +// A forced refresh reaches every widget within about a minute; they share one rebuild. +const TIMELINE_REFRESHED_MS = 60_000; + +function sharedCommitTimeline( + login: string, + start: string, + token: string, + fresh?: boolean +): Promise> { + const key = `${token}:${login}:${start}`; + const entry = timelines.get(key); + if ( + entry && + Date.now() - entry.at < (fresh ? TIMELINE_REFRESHED_MS : TIMELINE_FRESH_MS) + ) + return entry.result; + for (const [k, v] of timelines) + if (Date.now() - v.at >= TIMELINE_FRESH_MS) timelines.delete(k); + const result = fetchCommitTimeline(login, start, token, fresh); + timelines.set(key, { at: Date.now(), result }); + result.catch(() => { + if (timelines.get(key)?.result === result) timelines.delete(key); + }); + return result; +} + +/** Test isolation. */ +export function resetCommitTimelines() { + timelines.clear(); +} + +/** + * Commits authored by `login` on or after `since` (YYYY-MM-DD, UTC). + * + * Every window within the last year is derived from one shared timeline + * instead of a search per window: the Search API allows only 30 + * requests/minute, and per-window queries multiplied one dashboard load into + * ~100 searches. + */ +export async function collectAuthorCommits( + login: string, + since: string, + token?: string, + options: { fresh?: boolean } = {} +): Promise> { + const start = new Date(Date.now() - COMMIT_TIMELINE_DAYS * 86_400_000) + .toISOString() + .slice(0, 10); + const author = login.toLowerCase(); + if (since < start) + return collectGitHubSearch( + "commits", + `author:${author} author-date:>=${since}`, + token, + { + fresh: options.fresh, + } + ); + + const timeline = token + ? ((await sharedCommitTimeline( + author, + start, + token, + options.fresh + )) as unknown as SearchResult) + : await collectGitHubSearch( + "commits", + `author:${author} author-date:>=${start}`, + token, + { + timeoutMs: 25_000, + fresh: options.fresh, + } + ); + const sinceMs = Date.parse(`${since}T00:00:00Z`); + const items = timeline.items.filter( + (item) => Date.parse(item.commit.author.date) >= sinceMs + ); + const oldest = timeline.items.at(-1); + // Newest-first: once the fetched tail is older than the window, the window is whole. + const covered = + timeline.complete || + (oldest !== undefined && Date.parse(oldest.commit.author.date) < sinceMs); + return covered + ? { items, total: items.length, complete: true } + : { + items, + total: items.length, + complete: false, + partialReason: timeline.partialReason, + ...(timeline.retryAfter ? { retryAfter: timeline.retryAfter } : {}), + }; +} + +/** Narrows timeline commits the way the search qualifiers repo:, org: and -org: would. */ +export function scopeCommits< + T extends AuthorCommitItem & { repository?: { full_name: string } }, +>( + items: T[], + scope: { + repo?: string | null; + org?: string | null; + excludedOrgs?: string[]; + until?: string; + } +): T[] { + const repo = scope.repo?.toLowerCase(); + const org = scope.org?.toLowerCase(); + const excluded = new Set( + (scope.excludedOrgs ?? []).map((o) => o.toLowerCase()) + ); + const untilMs = scope.until + ? Date.parse(`${scope.until}T23:59:59.999Z`) + : Infinity; + return items.filter((item) => { + const name = item.repository?.full_name.toLowerCase() ?? ""; + const owner = name.split("/")[0]; + return ( + (!repo || name === repo) && + (!org || owner === org) && + !excluded.has(owner) && + Date.parse(item.commit.author.date) <= untilMs + ); + }); +} + /** GitHub Search exposes at most 1,000 items. A bounded sample is explicitly labelled. */ export async function collectGitHubSearch( path: "commits" | "issues", query: string, token?: string, - options: { sort?: string; order?: string; timeoutMs?: number } = {} + options: { + sort?: string; + order?: string; + timeoutMs?: number; + fresh?: boolean; + } = {} ): Promise> { const deadline = AbortSignal.timeout(options.timeoutMs ?? 10_000); const items: T[] = []; @@ -34,6 +381,7 @@ export async function collectGitHubSearch( { headers: token ? { Authorization: `Bearer ${token}` } : {}, signal: deadline, + ...(options.fresh ? { cache: "no-cache" as const } : {}), } ); if (!response.ok) { diff --git a/src/lib/streak-data.ts b/src/lib/streak-data.ts index 96137a9f5..f7b64c25d 100644 --- a/src/lib/streak-data.ts +++ b/src/lib/streak-data.ts @@ -1,5 +1,5 @@ import "server-only"; -import { collectGitHubSearch } from "@/lib/github-search"; +import { collectAuthorCommits, collectGitHubSearch } from "@/lib/github-search"; import { metricsCacheKey, withMetricsCache, @@ -31,13 +31,17 @@ export async function fetchCommitStreakDates( async () => { const since = new Date(); since.setUTCDate(since.getUTCDate() - 365); - const search = await collectGitHubSearch<{ - commit: { author: { date: string } }; - }>( - "commits", - `author:${login} author-date:>=${since.toISOString().slice(0, 10)}${visibility === "public" ? " is:public" : ""}`, - token - ); + const sinceStr = since.toISOString().slice(0, 10); + const search = + visibility === "public" + ? await collectGitHubSearch<{ commit: { author: { date: string } } }>( + "commits", + `author:${login} author-date:>=${sinceStr} is:public`, + token + ) + : await collectAuthorCommits(login, sinceStr, token, { + fresh: context.bypass, + }); const dates = new Set(); const formatter = new Intl.DateTimeFormat("en-CA", { timeZone, diff --git a/test/ci-metrics.test.ts b/test/ci-metrics.test.ts index 0e2d5268f..a2f5cc8c8 100644 --- a/test/ci-metrics.test.ts +++ b/test/ci-metrics.test.ts @@ -1,5 +1,9 @@ import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; -import { fetchCIAnalyticsForAccount, mergeCIAnalytics, CIAnalyticsResponse } from "../src/lib/ci-analytics"; +import { + fetchCIAnalyticsForAccount, + mergeCIAnalytics, + CIAnalyticsResponse, +} from "../src/lib/ci-analytics"; // Mock the route's dependencies to prevent resolution failures during import vi.mock("@/lib/supabase", () => ({ supabaseAdmin: {} as any })); @@ -38,13 +42,23 @@ describe("CI Metrics Route Helpers", () => { ok: true, json: async () => ({ items: [ - { repository: { full_name: "user/repo-a" } }, - { repository: { full_name: "user/repo-b" } }, + { + repository: { full_name: "user/repo-a" }, + commit: { author: { date: new Date().toISOString() } }, + }, + { + repository: { full_name: "user/repo-b" }, + commit: { author: { date: new Date().toISOString() } }, + }, ], + total_count: 2, }), }); } - if (url.includes("/repos/user/repo-a/actions/runs") || url.includes("/repos/user/repo-b/actions/runs")) { + if ( + url.includes("/repos/user/repo-a/actions/runs") || + url.includes("/repos/user/repo-b/actions/runs") + ) { return Promise.resolve({ ok: true, json: async () => ({ @@ -79,9 +93,16 @@ describe("CI Metrics Route Helpers", () => { ok: true, json: async () => ({ items: [ - { repository: { full_name: "user/repo-a" } }, - { repository: { full_name: "user/repo-b" } }, + { + repository: { full_name: "user/repo-a" }, + commit: { author: { date: new Date().toISOString() } }, + }, + { + repository: { full_name: "user/repo-b" }, + commit: { author: { date: new Date().toISOString() } }, + }, ], + total_count: 2, }), }); } @@ -108,7 +129,7 @@ describe("CI Metrics Route Helpers", () => { vi.stubGlobal("fetch", mockFetch); const result = await fetchCIAnalyticsForAccount("fake-token", "user"); - + // Should aggregate only the successful one (repo-a) expect(result.successRate).toBe(100); expect(result.averageDurationMinutes).toBe(5); @@ -116,7 +137,9 @@ describe("CI Metrics Route Helpers", () => { expect(result.reposChecked).toBe(2); // Should log the error expect(consoleErrorSpy).toHaveBeenCalled(); - expect(consoleErrorSpy.mock.calls[0][0]).toContain("Failed to fetch signals for repo user/repo-b:"); + expect(consoleErrorSpy.mock.calls[0][0]).toContain( + "Failed to fetch signals for repo user/repo-b:" + ); // Should track the failed repository in the failedRepos metadata field expect(result.failedRepos).toEqual(["user/repo-b"]); }); @@ -128,8 +151,12 @@ describe("CI Metrics Route Helpers", () => { ok: true, json: async () => ({ items: [ - { repository: { full_name: "user/repo-a" } }, + { + repository: { full_name: "user/repo-a" }, + commit: { author: { date: new Date().toISOString() } }, + }, ], + total_count: 2, }), }); } @@ -145,7 +172,7 @@ describe("CI Metrics Route Helpers", () => { vi.stubGlobal("fetch", mockFetch); const result = await fetchCIAnalyticsForAccount("fake-token", "user"); - + expect(result.successRate).toBe(0); expect(result.totalRuns).toBe(0); expect(result.reposChecked).toBe(1); @@ -160,9 +187,16 @@ describe("CI Metrics Route Helpers", () => { ok: true, json: async () => ({ items: [ - { repository: { full_name: "user/repo-a" } }, - { repository: { full_name: "user/repo-b" } }, + { + repository: { full_name: "user/repo-a" }, + commit: { author: { date: new Date().toISOString() } }, + }, + { + repository: { full_name: "user/repo-b" }, + commit: { author: { date: new Date().toISOString() } }, + }, ], + total_count: 2, }), }); } @@ -193,7 +227,7 @@ describe("CI Metrics Route Helpers", () => { vi.stubGlobal("fetch", mockFetch); const result = await fetchCIAnalyticsForAccount("fake-token", "user"); - + expect(result.successRate).toBe(100); expect(result.totalRuns).toBe(1); expect(result.reposChecked).toBe(2); @@ -229,7 +263,11 @@ describe("CI Metrics Route Helpers", () => { expect(merged.flakiestWorkflow).toBe("workflow-a"); expect(merged.totalRuns).toBe(30); expect(merged.reposChecked).toBe(5); - expect(merged.failedRepos).toEqual(["user/repo-c", "user/repo-d", "user/repo-e"]); + expect(merged.failedRepos).toEqual([ + "user/repo-c", + "user/repo-d", + "user/repo-e", + ]); }); it("should handle missing failedRepos fields", () => { diff --git a/test/commit-timeline-shim.ts b/test/commit-timeline-shim.ts new file mode 100644 index 000000000..2a5a3f521 --- /dev/null +++ b/test/commit-timeline-shim.ts @@ -0,0 +1,97 @@ +/** + * Route tests describe a user's commits as commit-search responses. The app + * reads the commit timeline through GraphQL, so this shim answers those + * GraphQL queries from whatever the test's fetch mock returns for the + * equivalent commit search. + */ +type SearchCommit = { + sha?: string; + html_url?: string; + commit: { author: { date: string }; message?: string }; + repository?: { + full_name: string; + html_url?: string; + description?: string | null; + }; +}; + +const commitsByAuthor = new Map(); + +function json(data: unknown): Response { + return new Response(JSON.stringify({ data }), { + status: 200, + headers: { "content-type": "application/json" }, + }); +} + +export async function commitTimelineShim( + input: string | URL | Request, + init?: RequestInit +): Promise { + const body = typeof init?.body === "string" ? init.body : ""; + if ( + !String(input).endsWith("/graphql") || + !/query Commit(Repos|History)\b/.test(body) + ) + return fetch(input, init); + const { variables } = JSON.parse(body) as { + variables: Record; + }; + + if (body.includes("query CommitRepos")) { + const login = variables.login!; + const q = `author:${login} author-date:>=${variables.year!.slice(0, 10)}`; + const res = await fetch( + `https://api.github.com/search/commits?q=${encodeURIComponent(q)}&per_page=100&page=1`, + { headers: init?.headers } + ); + if (!res.ok) return res; + const items = + ((await res.json()) as { items?: SearchCommit[] }).items ?? []; + commitsByAuthor.set(`U_${login}`, items); + const repos = [ + ...new Set( + items.map((item) => item.repository?.full_name ?? "test/repo") + ), + ]; + return json({ + user: { + id: `U_${login}`, + year: { + commitContributionsByRepository: repos.map((nameWithOwner) => ({ + repository: { nameWithOwner }, + })), + }, + recent: { commitContributionsByRepository: [] }, + }, + }); + } + + const items = commitsByAuthor.get(variables.author!) ?? []; + const data: Record = {}; + for (let i = 0; variables[`o${i}`] !== undefined; i++) { + const name = `${variables[`o${i}`]}/${variables[`n${i}`]}`; + const own = items.filter( + (item) => (item.repository?.full_name ?? "test/repo") === name + ); + data[`r${i}`] = { + nameWithOwner: name, + url: own[0]?.repository?.html_url ?? `https://github.com/${name}`, + description: own[0]?.repository?.description ?? null, + defaultBranchRef: { + target: { + history: { + pageInfo: { hasNextPage: false, endCursor: null }, + nodes: own.map((item, n) => ({ + oid: item.sha ?? `sha-${n}`, + messageHeadline: item.commit.message ?? "", + url: item.html_url ?? "", + authoredDate: item.commit.author.date, + })), + }, + }, + }, + }; + } + return json(data); +} diff --git a/test/contributions-hourly.test.ts b/test/contributions-hourly.test.ts index 8fc59bebb..4eb6bba3a 100644 --- a/test/contributions-hourly.test.ts +++ b/test/contributions-hourly.test.ts @@ -10,7 +10,9 @@ const mocks = vi.hoisted(() => ({ fetch: vi.fn(), })); -vi.mock("@/lib/server-session", () => ({ getServerSession: mocks.getServerSession })); +vi.mock("@/lib/server-session", () => ({ + getServerSession: mocks.getServerSession, +})); vi.mock("@/lib/auth", () => ({ authOptions: {} })); vi.mock("@/lib/metrics-cache", () => ({ isMetricsCacheBypassed: mocks.isMetricsCacheBypassed, @@ -44,7 +46,7 @@ describe("GET /api/metrics/contributions/hourly — days validation", () => { mocks.withMetricsCache.mockImplementation(async (_opts, fn) => fn()); mocks.fetch.mockResolvedValue({ ok: true, - json: async () => ({ items: [] }), + json: async () => ({ items: [], total_count: 0 }), }); }); @@ -73,7 +75,9 @@ describe("GET /api/metrics/contributions/hourly — days validation", () => { const res = await GET(makeRequest("Infinity")); expect(res.status).toBe(200); - const fetchUrl = String(mocks.fetch.mock.calls[0]?.[0] ?? ""); + const fetchUrl = decodeURIComponent( + String(mocks.fetch.mock.calls[0]?.[0] ?? "") + ); const match = fetchUrl.match(/author-date:>=(\d{4}-\d{2}-\d{2})/); expect(match).not.toBeNull(); expect(new Date(match![1]).toString()).not.toBe("Invalid Date"); diff --git a/test/github-commit-timeline.test.ts b/test/github-commit-timeline.test.ts new file mode 100644 index 000000000..3d7cbb308 --- /dev/null +++ b/test/github-commit-timeline.test.ts @@ -0,0 +1,151 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +vi.mock("@/lib/github-request", () => ({ + githubRequest: (input: string | URL | Request, init?: RequestInit) => + fetch(input, init), +})); + +import { + collectAuthorCommits, + resetCommitTimelines, + scopeCommits, +} from "@/lib/github-search"; + +const daysAgo = (n: number) => + new Date(Date.now() - n * 86_400_000).toISOString(); +const dateOnly = (n: number) => daysAgo(n).slice(0, 10); + +function respond(data: unknown) { + return new Response(JSON.stringify({ data }), { status: 200 }); +} + +function history(nodes: string[], next: string | null = null) { + return { + defaultBranchRef: { + target: { + history: { + pageInfo: { hasNextPage: next !== null, endCursor: next }, + nodes: nodes.map((date, i) => ({ + oid: `${date}-${i}`, + messageHeadline: "msg", + url: "u", + authoredDate: date, + })), + }, + }, + }, + }; +} + +describe("collectAuthorCommits GraphQL timeline", () => { + const fetchMock = vi.fn(); + beforeEach(() => { + vi.stubGlobal("fetch", fetchMock); + fetchMock.mockReset(); + resetCommitTimelines(); + }); + + function serve( + repos: string[], + pages: Record | null>> + ) { + fetchMock.mockImplementation(async (_url: string, init: RequestInit) => { + const { query, variables } = JSON.parse(String(init.body)); + if (query.includes("query CommitRepos")) + return respond({ + user: { + id: "U_1", + year: { + commitContributionsByRepository: repos.map((nameWithOwner) => ({ + repository: { nameWithOwner }, + })), + }, + recent: { + commitContributionsByRepository: [ + { repository: { nameWithOwner: repos[0] } }, + ], + }, + }, + }); + const data: Record = {}; + for (let i = 0; variables[`o${i}`] !== undefined; i++) { + const name = `${variables[`o${i}`]}/${variables[`n${i}`]}`; + const page = variables[`a${i}`] ? 1 : 0; + const entry = pages[name][page]; + data[`r${i}`] = entry && { + nameWithOwner: name, + url: `https://github.com/${name}`, + description: null, + ...entry, + }; + } + return respond(data); + }); + } + + it("follows history cursors, skips missing repos and filters to the window", async () => { + serve(["a/one", "b/two", "c/gone"], { + "a/one": [ + history([daysAgo(1), daysAgo(40)], "cursor"), + history([daysAgo(100)]), + ], + "b/two": [history([daysAgo(5)])], + "c/gone": [null], + }); + + const result = await collectAuthorCommits("Alice", dateOnly(60), "tok"); + + expect(result.complete).toBe(true); + expect(result.items.map((item) => item.commit.author.date)).toEqual( + [daysAgo(1), daysAgo(5), daysAgo(40)].map((d) => + expect.stringContaining(d.slice(0, 10)) + ) + ); + // 1 repo discovery + first history batch + one follow-up page for a/one + expect(fetchMock).toHaveBeenCalledTimes(3); + const discovery = JSON.parse(String(fetchMock.mock.calls[0][1].body)); + expect(discovery.variables.login).toBe("alice"); + }); + + it("serves every window from one timeline", async () => { + serve(["a/one"], { "a/one": [history([daysAgo(1), daysAgo(200)])] }); + + const year = await collectAuthorCommits("alice", dateOnly(365), "tok"); + const month = await collectAuthorCommits("alice", dateOnly(30), "tok"); + + expect(year.items).toHaveLength(2); + expect(month.items).toHaveLength(1); + expect(fetchMock).toHaveBeenCalledTimes(2); + }); + + it("does not use search quota when a token is available", async () => { + serve(["a/one"], { "a/one": [history([daysAgo(1)])] }); + await collectAuthorCommits("alice", dateOnly(90), "tok"); + for (const [url] of fetchMock.mock.calls) + expect(String(url)).toBe("https://api.github.com/graphql"); + }); +}); + +describe("scopeCommits", () => { + const at = (date: string, full_name: string) => ({ + commit: { author: { date } }, + repository: { full_name }, + }); + const items = [ + at("2026-03-02T10:00:00Z", "Acme/api"), + at("2026-03-01T10:00:00Z", "acme/web"), + at("2026-02-01T10:00:00Z", "alice/dotfiles"), + ]; + + it("matches repo:, org: and -org: case-insensitively", () => { + expect(scopeCommits(items, { repo: "acme/API" })).toHaveLength(1); + expect(scopeCommits(items, { org: "ACME" })).toHaveLength(2); + expect(scopeCommits(items, { excludedOrgs: ["acme"] })).toEqual([items[2]]); + }); + + it("includes the whole `until` day", () => { + expect(scopeCommits(items, { until: "2026-03-01" })).toEqual( + items.slice(1) + ); + }); +}); diff --git a/test/github-orgs.test.ts b/test/github-orgs.test.ts index ad741c0e6..39b6e8ab9 100644 --- a/test/github-orgs.test.ts +++ b/test/github-orgs.test.ts @@ -23,7 +23,9 @@ const mocks = vi.hoisted(() => ({ mergeMetrics: vi.fn(), })); -vi.mock("@/lib/server-session", () => ({ getServerSession: mocks.getServerSession })); +vi.mock("@/lib/server-session", () => ({ + getServerSession: mocks.getServerSession, +})); vi.mock("@/lib/auth", () => ({ authOptions: {} })); vi.mock("@/lib/resolve-user", () => ({ resolveAppUser: mocks.resolveAppUser, @@ -48,11 +50,7 @@ const SESSION = { }; const USER = { id: "user-uuid-100" }; -function makeRequest( - method: string, - url: string, - body?: unknown -): NextRequest { +function makeRequest(method: string, url: string, body?: unknown): NextRequest { return new NextRequest(url, { method, headers: body ? { "Content-Type": "application/json" } : {}, @@ -184,7 +182,10 @@ describe("GET /api/user/github-orgs", () => { const { GET } = await import("@/app/api/user/github-orgs/route"); const res = await GET(); expect(res.status).toBe(200); - const body = await res.json() as { orgs: unknown[]; hasReadOrgScope: boolean }; + const body = (await res.json()) as { + orgs: unknown[]; + hasReadOrgScope: boolean; + }; expect(body.orgs).toHaveLength(1); expect(body.hasReadOrgScope).toBe(true); }); @@ -203,7 +204,7 @@ describe("GET /api/user/github-orgs", () => { const { GET } = await import("@/app/api/user/github-orgs/route"); const res = await GET(); expect(res.status).toBe(200); - const body = await res.json() as { orgs: unknown[] }; + const body = (await res.json()) as { orgs: unknown[] }; expect(body.orgs).toHaveLength(0); }); @@ -262,7 +263,7 @@ describe("PATCH /api/user/github-orgs", () => { }) ); expect(res.status).toBe(200); - const body = await res.json() as { ok: boolean }; + const body = (await res.json()) as { ok: boolean }; expect(body.ok).toBe(true); }); @@ -326,17 +327,36 @@ describe("GET /api/metrics/contributions — org: accountId", () => { mocks.resolveAppUser.mockResolvedValue(USER); }); - it("passes org filter in search query for org: accountId", async () => { + it("limits org: accountId views to that org's repositories", async () => { // Stub the commit search request — first call is the GitHub Search API + const now = new Date().toISOString(); mocks.fetchFn.mockResolvedValueOnce({ ok: true, - json: async () => ({ total_count: 0, items: [] }), + json: async () => ({ + total_count: 2, + items: [ + { + sha: "a", + html_url: "", + commit: { author: { date: now }, message: "in org" }, + repository: { full_name: "acme-corp/api" }, + }, + { + sha: "b", + html_url: "", + commit: { author: { date: now }, message: "personal" }, + repository: { full_name: "alice/dotfiles" }, + }, + ], + }), }); mocks.getAccountToken.mockResolvedValue("tok-alice"); // Stub metrics cache (supabase) — supports infinite chain-mocking - const cacheSingle = vi.fn().mockResolvedValue({ data: { github_login: "alice" }, error: null }); + const cacheSingle = vi + .fn() + .mockResolvedValue({ data: { github_login: "alice" }, error: null }); const createChain = () => { const obj = { eq: () => obj, @@ -351,11 +371,13 @@ describe("GET /api/metrics/contributions — org: accountId", () => { "GET", "http://localhost/api/metrics/contributions?days=30&accountId=org:12345:acme-corp" ); - await GET(req); + const res = await GET(req); + const body = await res.json(); - // The GitHub Search API must have been called with org:acme-corp in the query - const calledUrl = (mocks.fetchFn.mock.calls[0]?.[0] as string) ?? ""; - expect(calledUrl).toContain("org%3Aacme-corp"); + expect(body.total).toBe(1); + expect(body.commits.map((c: { repo: string }) => c.repo)).toEqual([ + "acme-corp/api", + ]); }); it("returns 400 for empty org login (org: with no name)", async () => { diff --git a/test/github-request.test.ts b/test/github-request.test.ts index 9718d6d58..4de3a0f9e 100644 --- a/test/github-request.test.ts +++ b/test/github-request.test.ts @@ -67,22 +67,47 @@ describe("GitHub transport coordination", () => { }); expect(fetch).toHaveBeenCalledTimes(3); }); - it("serializes independent URLs using one credential", async () => { - let active = 0, - maximum = 0; - vi.mocked(fetch).mockImplementation(async () => { - active++; - maximum = Math.max(maximum, active); + it("bounds concurrency per credential and resource", async () => { + const active = { core: 0, search: 0 }; + const maximum = { core: 0, search: 0 }; + vi.mocked(fetch).mockImplementation(async (input) => { + const kind = String(input).includes("/search/") ? "search" : "core"; + active[kind]++; + maximum[kind] = Math.max(maximum[kind], active[kind]); await new Promise((resolve) => setTimeout(resolve, 5)); - active--; - return json({}); + active[kind]--; + return json({ items: [], total_count: 0 }); }); - await Promise.all( - ["/user", "/user/repos", "/user/orgs"].map((path) => - githubRequest("https://api.github.com" + path, { headers: token() }) - ) + await Promise.all([ + ...Array.from({ length: 8 }, (_, i) => + githubRequest(`https://api.github.com/repos/o/r${i}`, { + headers: token(), + }) + ), + ...Array.from({ length: 3 }, (_, i) => + githubRequest(`https://api.github.com/search/commits?q=x&page=${i}`, { + headers: token(), + }) + ), + ]); + expect(maximum).toEqual({ core: 4, search: 1 }); + }); + it("serves search from cache for minutes but revalidates when asked", async () => { + vi.mocked(fetch).mockImplementation(async () => + json({ items: [] }, { etag: '"s"' }) ); - expect(maximum).toBe(1); + const url = "https://api.github.com/search/commits?q=author:a"; + await githubRequest(url, { headers: token() }); + for (const value of cache.values()) + if (value && typeof value === "object" && "savedAt" in value) + (value as { savedAt: number }).savedAt -= 120_000; + await githubRequest(url, { headers: token() }); + expect(fetch).toHaveBeenCalledTimes(1); + await githubRequest(url, { headers: token(), cache: "no-cache" }); + expect(fetch).toHaveBeenCalledTimes(2); + // Sibling widgets refreshing in the same burst share that revalidation. + await githubRequest(url, { headers: token(), cache: "no-cache" }); + expect(fetch).toHaveBeenCalledTimes(2); }); it("conditionally refreshes stale responses with ETag", async () => { vi.mocked(fetch) @@ -131,6 +156,26 @@ describe("GitHub transport coordination", () => { }); expect(fetch).toHaveBeenCalledTimes(3); }); + it("treats a 429 with remaining=0 as primary and keeps other resources open", async () => { + vi.mocked(fetch) + .mockResolvedValueOnce( + new Response('{"message":"API rate limit exceeded"}', { + status: 429, + headers: { + "x-ratelimit-remaining": "0", + "x-ratelimit-reset": String(Math.ceil(Date.now() / 1000) + 40), + }, + }) + ) + .mockImplementation(async () => json({})); + await expect( + githubRequest("https://api.github.com/search/commits?q=x", { + headers: token(), + }) + ).rejects.toBeInstanceOf(GitHubRateLimitError); + await githubRequest("https://api.github.com/user", { headers: token() }); + expect(fetch).toHaveBeenCalledTimes(2); + }); it("applies secondary cooldowns to all resources and propagates Retry-After", async () => { vi.mocked(fetch).mockResolvedValue( new Response('{"message":"secondary rate limit"}', { @@ -253,7 +298,7 @@ it("rejects redirects to another host without forwarding credentials", async () expect(fetch).toHaveBeenCalledTimes(1); }); -it("keeps an active predecessor serialized after a queued caller times out", async () => { +it("keeps a full slot held after a queued caller times out", async () => { const originalTimer = globalThis.setTimeout; const timerSpy = vi .spyOn(globalThis, "setTimeout") @@ -276,18 +321,20 @@ it("keeps an active predecessor serialized after a queued caller times out", asy }) ) .mockImplementation(async () => json({})); - const active = githubRequest("https://api.github.com/user", { + const active = githubRequest("https://api.github.com/search/commits?q=a", { headers: token(), }); for (let i = 0; i < 100 && !complete; i++) await new Promise((resolve) => originalTimer(resolve, 1)); expect(complete).toBeDefined(); - const queued = githubRequest("https://api.github.com/user/repos", { + const queued = githubRequest("https://api.github.com/search/commits?q=b", { headers: token(), }).catch((error) => error); expect(await queued).toBeInstanceOf(Error); await expect( - githubRequest("https://api.github.com/user/orgs", { headers: token() }) + githubRequest("https://api.github.com/search/commits?q=c", { + headers: token(), + }) ).rejects.toThrow("queue timed out"); expect(fetch).toHaveBeenCalledTimes(1); complete(json({ login: "alice" })); diff --git a/test/metrics-languages.test.ts b/test/metrics-languages.test.ts index e4749dd1d..a8979e4d8 100644 --- a/test/metrics-languages.test.ts +++ b/test/metrics-languages.test.ts @@ -1,9 +1,9 @@ -import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; -import { GET } from '@/app/api/metrics/languages/route'; -import { NextRequest } from 'next/server'; +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; +import { GET } from "@/app/api/metrics/languages/route"; +import { NextRequest } from "next/server"; import { getServerSession } from "@/lib/server-session"; -import { resolveAppUser } from '@/lib/resolve-user'; -import { getAccountToken } from '@/lib/github-accounts'; +import { resolveAppUser } from "@/lib/resolve-user"; +import { getAccountToken } from "@/lib/github-accounts"; // Mock next-auth vi.mock("@/lib/server-session", () => ({ @@ -11,12 +11,12 @@ vi.mock("@/lib/server-session", () => ({ })); // Mock resolve-user -vi.mock('@/lib/resolve-user', () => ({ +vi.mock("@/lib/resolve-user", () => ({ resolveAppUser: vi.fn(), })); // Mock github-accounts -vi.mock('@/lib/github-accounts', () => ({ +vi.mock("@/lib/github-accounts", () => ({ getAccountToken: vi.fn(), })); @@ -29,21 +29,23 @@ const mockFrom = vi.fn().mockImplementation((table: string) => { }; }); -vi.mock('@/lib/supabase', () => ({ +vi.mock("@/lib/supabase", () => ({ supabaseAdmin: { from: (table: string) => mockFrom(table), }, })); // Mock metrics cache -vi.mock('@/lib/metrics-cache', () => ({ +vi.mock("@/lib/metrics-cache", () => ({ isMetricsCacheBypassed: vi.fn(() => false), metricsCacheKey: vi.fn((userId: string, endpoint: string, params: any) => { - const paramStr = params ? Object.entries(params) - .sort() - .map(([k, v]) => `${k}=${v}`) - .join('&') : ''; - return `metrics:${userId}:${endpoint}${paramStr ? ':' + paramStr : ''}`; + const paramStr = params + ? Object.entries(params) + .sort() + .map(([k, v]) => `${k}=${v}`) + .join("&") + : ""; + return `metrics:${userId}:${endpoint}${paramStr ? ":" + paramStr : ""}`; }), withMetricsCache: vi.fn(async (_config, callback) => callback()), METRICS_CACHE_TTL_SECONDS: { @@ -55,7 +57,7 @@ vi.mock('@/lib/metrics-cache', () => ({ const originalFetch = global.fetch; let fetchMock: any; -describe('Language Metrics Endpoint', () => { +describe("Language Metrics Endpoint", () => { beforeEach(() => { vi.clearAllMocks(); fetchMock = vi.fn(); @@ -63,9 +65,9 @@ describe('Language Metrics Endpoint', () => { // Default successful session (getServerSession as any).mockResolvedValue({ - accessToken: 'test-token', - githubLogin: 'test-user', - githubId: 'user-123', + accessToken: "test-token", + githubLogin: "test-user", + githubId: "user-123", }); }); @@ -73,45 +75,48 @@ describe('Language Metrics Endpoint', () => { global.fetch = originalFetch; }); - it('returns 401 when no session', async () => { + it("returns 401 when no session", async () => { (getServerSession as any).mockResolvedValue(null); - const req = new NextRequest('http://localhost/api/metrics/languages'); + const req = new NextRequest("http://localhost/api/metrics/languages"); const res = await GET(req); const data = await res.json(); expect(res.status).toBe(401); - expect(data.error).toBe('Unauthorized'); + expect(data.error).toBe("Unauthorized"); }); - it('returns 401 when no accessToken', async () => { + it("returns 401 when no accessToken", async () => { (getServerSession as any).mockResolvedValue({ - githubLogin: 'test-user', + githubLogin: "test-user", }); - const req = new NextRequest('http://localhost/api/metrics/languages'); + const req = new NextRequest("http://localhost/api/metrics/languages"); const res = await GET(req); const data = await res.json(); expect(res.status).toBe(401); - expect(data.error).toBe('Unauthorized'); + expect(data.error).toBe("Unauthorized"); }); - it('returns complete language data when all repositories succeed', async () => { + it("returns complete language data when all repositories succeed", async () => { fetchMock .mockResolvedValueOnce({ ok: true, json: async () => ({ + total_count: 1, items: [ { repository: { - full_name: 'user/repo1', + full_name: "user/repo1", }, + commit: { author: { date: new Date().toISOString() } }, }, { repository: { - full_name: 'user/repo2', + full_name: "user/repo2", }, + commit: { author: { date: new Date().toISOString() } }, }, ], }), @@ -131,7 +136,7 @@ describe('Language Metrics Endpoint', () => { }), }); - const req = new NextRequest('http://localhost/api/metrics/languages'); + const req = new NextRequest("http://localhost/api/metrics/languages"); const res = await GET(req); const data = await res.json(); @@ -143,21 +148,24 @@ describe('Language Metrics Endpoint', () => { expect(data.languages.length).toBeGreaterThan(0); }); - it('tracks failed repository with 403 status', async () => { + it("tracks failed repository with 403 status", async () => { fetchMock .mockResolvedValueOnce({ ok: true, json: async () => ({ + total_count: 1, items: [ { repository: { - full_name: 'user/repo1', + full_name: "user/repo1", }, + commit: { author: { date: new Date().toISOString() } }, }, { repository: { - full_name: 'user/repo2', + full_name: "user/repo2", }, + commit: { author: { date: new Date().toISOString() } }, }, ], }), @@ -173,7 +181,7 @@ describe('Language Metrics Endpoint', () => { status: 403, }); - const req = new NextRequest('http://localhost/api/metrics/languages'); + const req = new NextRequest("http://localhost/api/metrics/languages"); const res = await GET(req); const data = await res.json(); @@ -183,21 +191,24 @@ describe('Language Metrics Endpoint', () => { expect(data.languages).toBeDefined(); }); - it('tracks failed repository with 404 status', async () => { + it("tracks failed repository with 404 status", async () => { fetchMock .mockResolvedValueOnce({ ok: true, json: async () => ({ + total_count: 1, items: [ { repository: { - full_name: 'user/repo1', + full_name: "user/repo1", }, + commit: { author: { date: new Date().toISOString() } }, }, { repository: { - full_name: 'user/repo-deleted', + full_name: "user/repo-deleted", }, + commit: { author: { date: new Date().toISOString() } }, }, ], }), @@ -213,7 +224,7 @@ describe('Language Metrics Endpoint', () => { status: 404, }); - const req = new NextRequest('http://localhost/api/metrics/languages'); + const req = new NextRequest("http://localhost/api/metrics/languages"); const res = await GET(req); const data = await res.json(); @@ -222,16 +233,29 @@ describe('Language Metrics Endpoint', () => { expect(data.failedRepositoriesCount).toBe(1); }); - it('tracks multiple failed repositories', async () => { + it("tracks multiple failed repositories", async () => { fetchMock .mockResolvedValueOnce({ ok: true, json: async () => ({ + total_count: 1, items: [ - { repository: { full_name: 'user/repo1' } }, - { repository: { full_name: 'user/repo2' } }, - { repository: { full_name: 'user/repo3' } }, - { repository: { full_name: 'user/repo4' } }, + { + repository: { full_name: "user/repo1" }, + commit: { author: { date: new Date().toISOString() } }, + }, + { + repository: { full_name: "user/repo2" }, + commit: { author: { date: new Date().toISOString() } }, + }, + { + repository: { full_name: "user/repo3" }, + commit: { author: { date: new Date().toISOString() } }, + }, + { + repository: { full_name: "user/repo4" }, + commit: { author: { date: new Date().toISOString() } }, + }, ], }), }) @@ -252,7 +276,7 @@ describe('Language Metrics Endpoint', () => { status: 500, }); - const req = new NextRequest('http://localhost/api/metrics/languages'); + const req = new NextRequest("http://localhost/api/metrics/languages"); const res = await GET(req); const data = await res.json(); @@ -262,14 +286,21 @@ describe('Language Metrics Endpoint', () => { expect(data.languages).toBeDefined(); }); - it('handles fetch exception for repository language request', async () => { + it("handles fetch exception for repository language request", async () => { fetchMock .mockResolvedValueOnce({ ok: true, json: async () => ({ + total_count: 1, items: [ - { repository: { full_name: 'user/repo1' } }, - { repository: { full_name: 'user/repo2' } }, + { + repository: { full_name: "user/repo1" }, + commit: { author: { date: new Date().toISOString() } }, + }, + { + repository: { full_name: "user/repo2" }, + commit: { author: { date: new Date().toISOString() } }, + }, ], }), }) @@ -277,9 +308,9 @@ describe('Language Metrics Endpoint', () => { ok: true, json: async () => ({ TypeScript: 50000 }), }) - .mockRejectedValueOnce(new Error('Network timeout')); + .mockRejectedValueOnce(new Error("Network timeout")); - const req = new NextRequest('http://localhost/api/metrics/languages'); + const req = new NextRequest("http://localhost/api/metrics/languages"); const res = await GET(req); const data = await res.json(); @@ -288,33 +319,39 @@ describe('Language Metrics Endpoint', () => { expect(data.failedRepositoriesCount).toBe(1); }); - it('returns 502 when search API fails', async () => { + it("returns 502 when search API fails", async () => { fetchMock.mockResolvedValueOnce({ ok: false, status: 403, }); - const req = new NextRequest('http://localhost/api/metrics/languages'); + const req = new NextRequest("http://localhost/api/metrics/languages"); const res = await GET(req); const data = await res.json(); expect(res.status).toBe(502); - expect(data.error).toBe('GitHub API error'); + expect(data.error).toBe("GitHub API error"); expect(data.isComplete).toBe(false); }); - it('includes failed repositories in development mode', async () => { - + it("includes failed repositories in development mode", async () => { const originalEnv = process.env.NODE_ENV; - (process.env as Record).NODE_ENV = 'development'; + (process.env as Record).NODE_ENV = "development"; fetchMock .mockResolvedValueOnce({ ok: true, json: async () => ({ + total_count: 1, items: [ - { repository: { full_name: 'user/repo1' } }, - { repository: { full_name: 'user/repo-broken' } }, + { + repository: { full_name: "user/repo1" }, + commit: { author: { date: new Date().toISOString() } }, + }, + { + repository: { full_name: "user/repo-broken" }, + commit: { author: { date: new Date().toISOString() } }, + }, ], }), }) @@ -327,31 +364,41 @@ describe('Language Metrics Endpoint', () => { status: 403, }); - const req = new NextRequest('http://localhost/api/metrics/languages'); + const req = new NextRequest("http://localhost/api/metrics/languages"); const res = await GET(req); const data = await res.json(); expect(data.failedRepositories).toBeDefined(); expect(Array.isArray(data.failedRepositories)).toBe(true); expect(data.failedRepositories.length).toBe(1); - expect(data.failedRepositories[0]).toHaveProperty('name', 'user/repo-broken'); - expect(data.failedRepositories[0]).toHaveProperty('statusCode', 403); - expect(data.failedRepositories[0]).toHaveProperty('error'); - - (process.env as Record).NODE_ENV = originalEnv ?? ''; + expect(data.failedRepositories[0]).toHaveProperty( + "name", + "user/repo-broken" + ); + expect(data.failedRepositories[0]).toHaveProperty("statusCode", 403); + expect(data.failedRepositories[0]).toHaveProperty("error"); + + (process.env as Record).NODE_ENV = originalEnv ?? ""; }); - it('does not include failed repositories in production mode', async () => { + it("does not include failed repositories in production mode", async () => { const originalEnv = process.env.NODE_ENV; - (process.env as Record).NODE_ENV = 'production'; + (process.env as Record).NODE_ENV = "production"; fetchMock .mockResolvedValueOnce({ ok: true, json: async () => ({ + total_count: 1, items: [ - { repository: { full_name: 'user/repo1' } }, - { repository: { full_name: 'user/repo-broken' } }, + { + repository: { full_name: "user/repo1" }, + commit: { author: { date: new Date().toISOString() } }, + }, + { + repository: { full_name: "user/repo-broken" }, + commit: { author: { date: new Date().toISOString() } }, + }, ], }), }) @@ -364,24 +411,24 @@ describe('Language Metrics Endpoint', () => { status: 403, }); - const req = new NextRequest('http://localhost/api/metrics/languages'); + const req = new NextRequest("http://localhost/api/metrics/languages"); const res = await GET(req); const data = await res.json(); expect(data.failedRepositories).toBeUndefined(); expect(data.failedRepositoriesCount).toBe(1); - (process.env as Record).NODE_ENV = originalEnv ?? ''; + (process.env as Record).NODE_ENV = originalEnv ?? ""; }); - it('handles linked account request', async () => { - (resolveAppUser as any).mockResolvedValue({ id: 'app-user-123' }); - (getAccountToken as any).mockResolvedValue('linked-token'); + it("handles linked account request", async () => { + (resolveAppUser as any).mockResolvedValue({ id: "app-user-123" }); + (getAccountToken as any).mockResolvedValue("linked-token"); mockSelect.mockReturnValue({ eq: vi.fn().mockReturnValue({ eq: vi.fn().mockReturnValue({ single: vi.fn().mockResolvedValue({ - data: { github_login: 'linked-user' }, + data: { github_login: "linked-user" }, error: null, }), }), @@ -392,7 +439,13 @@ describe('Language Metrics Endpoint', () => { .mockResolvedValueOnce({ ok: true, json: async () => ({ - items: [{ repository: { full_name: 'linked-user/repo1' } }], + total_count: 1, + items: [ + { + repository: { full_name: "linked-user/repo1" }, + commit: { author: { date: new Date().toISOString() } }, + }, + ], }), }) .mockResolvedValueOnce({ @@ -400,7 +453,9 @@ describe('Language Metrics Endpoint', () => { json: async () => ({ TypeScript: 50000 }), }); - const req = new NextRequest('http://localhost/api/metrics/languages?accountId=linked-user-456'); + const req = new NextRequest( + "http://localhost/api/metrics/languages?accountId=linked-user-456" + ); const res = await GET(req); const data = await res.json(); @@ -408,15 +463,17 @@ describe('Language Metrics Endpoint', () => { expect(data.isComplete).toBe(true); }); - it('returns 404 when linked account not found', async () => { - (resolveAppUser as any).mockResolvedValue({ id: 'app-user-123' }); + it("returns 404 when linked account not found", async () => { + (resolveAppUser as any).mockResolvedValue({ id: "app-user-123" }); (getAccountToken as any).mockResolvedValue(null); - const req = new NextRequest('http://localhost/api/metrics/languages?accountId=non-existent'); + const req = new NextRequest( + "http://localhost/api/metrics/languages?accountId=non-existent" + ); const res = await GET(req); const data = await res.json(); expect(res.status).toBe(404); - expect(data.error).toBe('Account not found'); + expect(data.error).toBe("Account not found"); }); }); diff --git a/test/setup.ts b/test/setup.ts index 8afee81ec..c56a48a40 100644 --- a/test/setup.ts +++ b/test/setup.ts @@ -1,20 +1,27 @@ -process.env.TZ = 'UTC'; -process.env.NEXT_PUBLIC_SUPABASE_URL = 'https://test.supabase.co'; -process.env.SUPABASE_SERVICE_ROLE_KEY = 'test-key'; -process.env.NEXTAUTH_SECRET = 'test-secret'; -process.env.GITHUB_ID = 'test-github-id'; -process.env.GITHUB_SECRET = 'test-github-secret'; +process.env.TZ = "UTC"; +process.env.NEXT_PUBLIC_SUPABASE_URL = "https://test.supabase.co"; +process.env.SUPABASE_SERVICE_ROLE_KEY = "test-key"; +process.env.NEXTAUTH_SECRET = "test-secret"; +process.env.GITHUB_ID = "test-github-id"; +process.env.GITHUB_SECRET = "test-github-secret"; -import { vi } from "vitest"; +import { beforeEach, vi } from "vitest"; vi.mock("server-only", () => ({})); - // Route unit tests stub the transport boundary. Dedicated transport tests unmock it. -vi.mock("@/lib/github-request", () => ({ - githubRequest: (input: string | URL | Request, init?: RequestInit) => fetch(input, init), -})); +vi.mock("@/lib/github-request", async () => { + const { commitTimelineShim } = await import("./commit-timeline-shim"); + return { githubRequest: commitTimelineShim }; +}); + +// The commit timeline is memoized per process; each test starts from its own mocks. +beforeEach(async () => { + const search = await import("@/lib/github-search").catch(() => null); + if (search && typeof search.resetCommitTimelines === "function") + search.resetCommitTimelines(); +}); vi.mock("@/lib/safe-webhook-fetch", async (importOriginal) => ({ - ...await importOriginal(), + ...(await importOriginal()), safeWebhookFetch: (input: string, init?: RequestInit) => fetch(input, init), })); diff --git a/test/weekly-summary-combined.test.ts b/test/weekly-summary-combined.test.ts index cf7afda68..ab2c99fd7 100644 --- a/test/weekly-summary-combined.test.ts +++ b/test/weekly-summary-combined.test.ts @@ -16,7 +16,9 @@ vi.mock("@/lib/metrics-cache", () => ({ metricsCacheKey: vi.fn().mockImplementation((userId, endpoint, params) => { return `metrics:${userId}:${endpoint}:${JSON.stringify(params || {})}`; }), - withMetricsCache: vi.fn().mockImplementation((_opts: unknown, fn: () => unknown) => fn()), + withMetricsCache: vi + .fn() + .mockImplementation((_opts: unknown, fn: () => unknown) => fn()), })); vi.mock("@/lib/resolve-user", () => ({ @@ -81,58 +83,51 @@ describe("GET /api/metrics/weekly-summary?accountId=combined", () => { }); // 4. Mock GitHub API Responses for both accounts - mockFetch.mockImplementation(async (url: string) => { - // Commits search for primary-login - if (url.includes("search/commits") && url.includes("author:primary-login")) { - if (url.includes("&page=")) { - // fetchActiveDates (streak) call - return { - ok: true, - status: 200, - json: async () => ({ - items: [ - { commit: { author: { date: "2026-06-14T12:00:00Z" } } }, // Sunday - { commit: { author: { date: "2026-06-13T12:00:00Z" } } }, // Saturday - ], - }), - } as any; - } - // 14-day commits call + mockFetch.mockImplementation(async (rawUrl: string) => { + const url = decodeURIComponent(String(rawUrl)); + // Commit timeline for primary-login (one shared search serves both weeks and the streak) + if ( + url.includes("search/commits") && + url.includes("author:primary-login") + ) { return { ok: true, status: 200, json: async () => ({ + total_count: 2, items: [ - { commit: { author: { date: "2026-06-15T08:00:00Z" } }, repository: { full_name: "org/repo-a" } }, // Monday (this week) - { commit: { author: { date: "2026-06-13T12:00:00Z" } }, repository: { full_name: "org/repo-a" } }, // Saturday (last week) + { + commit: { author: { date: "2026-06-15T08:00:00Z" } }, + repository: { full_name: "org/repo-a" }, + }, // Monday (this week) + { + commit: { author: { date: "2026-06-13T12:00:00Z" } }, + repository: { full_name: "org/repo-a" }, + }, // Saturday (last week) ], }), } as any; } - // Commits search for linked-login - if (url.includes("search/commits") && url.includes("author:linked-login")) { - if (url.includes("&page=")) { - // fetchActiveDates (streak) call - return { - ok: true, - status: 200, - json: async () => ({ - items: [ - { commit: { author: { date: "2026-06-14T12:00:00Z" } } }, - { commit: { author: { date: "2026-06-12T12:00:00Z" } } }, - ], - }), - } as any; - } - // 14-day commits call + // Commit timeline for linked-login + if ( + url.includes("search/commits") && + url.includes("author:linked-login") + ) { return { ok: true, status: 200, json: async () => ({ + total_count: 2, items: [ - { commit: { author: { date: "2026-06-15T09:00:00Z" } }, repository: { full_name: "org/repo-b" } }, // Monday (this week) - { commit: { author: { date: "2026-06-12T12:00:00Z" } }, repository: { full_name: "org/repo-b" } }, // Friday (last week) + { + commit: { author: { date: "2026-06-15T09:00:00Z" } }, + repository: { full_name: "org/repo-b" }, + }, // Monday (this week) + { + commit: { author: { date: "2026-06-12T12:00:00Z" } }, + repository: { full_name: "org/repo-b" }, + }, // Friday (last week) ], }), } as any; @@ -145,7 +140,11 @@ describe("GET /api/metrics/weekly-summary?accountId=combined", () => { status: 200, json: async () => ({ items: [ - { created_at: "2026-06-15T10:00:00Z", state: "closed", pull_request: { merged_at: "2026-06-15T11:00:00Z" } }, + { + created_at: "2026-06-15T10:00:00Z", + state: "closed", + pull_request: { merged_at: "2026-06-15T11:00:00Z" }, + }, ], }), } as any; @@ -159,12 +158,14 @@ describe("GET /api/metrics/weekly-summary?accountId=combined", () => { }); const { GET } = await import("@/app/api/metrics/weekly-summary/route"); - const req = new NextRequest("http://localhost/api/metrics/weekly-summary?accountId=combined"); + const req = new NextRequest( + "http://localhost/api/metrics/weekly-summary?accountId=combined" + ); const res = await GET(req); expect(res.status).toBe(200); const data = await res.json(); - + expect(data.commits.current).toBe(2); expect(data.commits.previous).toBe(2); expect(data.commits.delta).toBe(0); diff --git a/vercel.json b/vercel.json index 168f9c928..17bac9e6f 100644 --- a/vercel.json +++ b/vercel.json @@ -8,10 +8,6 @@ "path": "/api/wakatime/sync", "schedule": "0 0 * * *" }, - { - "path": "/api/cron/goals-sync", - "schedule": "0 * * * *" - }, { "path": "/api/sponsors/sync", "schedule": "0 0 * * *" @@ -19,10 +15,6 @@ { "path": "/api/notifications/discord-sync", "schedule": "0 20 * * *" - }, - { - "path": "/api/notifications/weekly", - "schedule": "0 * * * *" } ] }