feat: declare the rate-limit policy of the HubSpot, Salesforce and Pipedrive modules

packages/v1-ready/hubspot/README.md

@@ -113,3 +113,23 @@ colliding.
 - **`HUBSPOT_WEBHOOK`** — queue event (worker, hydrated integration): default no-op;
   override via `binding.handlers.HUBSPOT_WEBHOOK` to run your per-event logic with the
   correct per-account context.
+
+## Rate limits
+
+`Api.rateLimit` tells the Frigg Requester how HubSpot limits calls. The numbers come
+from the [API usage guidelines and limits](https://developers.hubspot.com/docs/developer-tooling/platform/usage-guidelines).
+A `@friggframework/core` release without rate-limit awareness ignores the declaration.
+
+- **Burst:** each account that installs a marketplace OAuth app is limited to 110
+  requests every 10 seconds. A throttled call returns `429` with `policyName`
+  `TEN_SECONDLY_ROLLING`. The Requester waits for `Retry-After` when the response has
+  one, and for the 10 second window when it has none.
+- **Daily:** a `429` with `policyName` `DAILY`. HubSpot resets the daily limit at
+  midnight in the account's time zone, which the module does not know, so it does not
+  compute a reset time. It reports a one hour probe interval: the Requester throws a
+  `RateLimitError` with `reason: 'daily'` and a `retryAt` one hour ahead. It does not
+  sleep for an hour in process.
+- **Any other status** is not a limit and keeps its normal handling.
+
+HubSpot documents no `Retry-After` or reset header for these responses, and the search
+endpoints send no `X-HubSpot-RateLimit-*` headers at all.

packages/v1-ready/hubspot/api.js

@@ -1,6 +1,37 @@
 const {OAuth2Requester, get} = require('@friggframework/core');
 
+const DAILY_PROBE_INTERVAL_MS = 3_600_000;
+
 class Api extends OAuth2Requester {
+    static rateLimit = {
+        scope: 'entity',
+        windows: [{ name: 'burst', limit: 110, perMs: 10_000 }],
+        maxConcurrency: 10,
+        parsers: ['retryAfter', 'resetHeaders'],
+        classify({ status, body }) {
+            if (status !== 429) return null;
+            if (body?.policyName === 'DAILY') {
+                return {
+                    reason: 'daily',
+                    policy: 'DAILY',
+                    waitMs: DAILY_PROBE_INTERVAL_MS,
+                    source: 'static',
+                };
+            }
+            return { reason: 'burst', policy: body?.policyName };
+        },
+        userHints: {
+            daily: {
+                links: [
+                    {
+                        label: 'API usage guidelines and limits',
+                        url: 'https://developers.hubspot.com/docs/developer-tooling/platform/usage-guidelines',
+                    },
+                ],
+            },
+        },
+    };
+
     constructor(params) {
         super(params);
         // The majority of the properties for OAuth are default loaded by OAuth2Requester.

packages/v1-ready/hubspot/tests/rateLimit.test.js

@@ -0,0 +1,231 @@
+const { FetchError, RateLimitError } = require('@friggframework/core');
+const { Api } = require('../api');
+
+const ONE_HOUR_MS = 3_600_000;
+
+const burstLimitBody = {
+    status: 'error',
+    message: 'You have reached your ten_secondly_rolling limit.',
+    errorType: 'RATE_LIMIT',
+    correlationId: '00000000-0000-4000-8000-000000000000',
+    policyName: 'TEN_SECONDLY_ROLLING',
+};
+
+const dailyLimitBody = {
+    status: 'error',
+    message: 'You have reached your daily limit.',
+    errorType: 'RATE_LIMIT',
+    correlationId: '00000000-0000-4000-8000-000000000000',
+    policyName: 'DAILY',
+};
+
+const burstHeaders = {
+    'X-HubSpot-RateLimit-Max': '110',
+    'X-HubSpot-RateLimit-Remaining': '0',
+    'X-HubSpot-RateLimit-Interval-Milliseconds': '10000',
+};
+
+function throttledFetch(responses) {
+    const queue = [...responses];
+    return jest.fn(async () => {
+        const next = queue.shift();
+        if (!next) throw new Error('unexpected fetch call');
+        const body = next.body ?? {};
+        return {
+            status: next.status,
+            bodyUsed: false,
+            headers: new Map(
+                Object.entries({
+                    'Content-Type': next.contentType ?? 'application/json',
+                    ...next.headers,
+                })
+            ),
+            json: async () => body,
+            text: jest.fn(async () => next.text ?? JSON.stringify(body)),
+        };
+    });
+}
+
+const limited = (body, headers) => ({ status: 429, body, headers });
+const ok = { status: 200, body: { results: [] } };
+
+describe('HubSpot rate limits', () => {
+    let delays;
+
+    beforeEach(() => {
+        delays = [];
+        const realSetTimeout = global.setTimeout;
+        jest.spyOn(global, 'setTimeout').mockImplementation(
+            (fn, delay, ...args) => {
+                delays.push(delay);
+                return realSetTimeout(fn, 0, ...args);
+            }
+        );
+        jest.spyOn(Math, 'random').mockReturnValue(0);
+    });
+
+    afterEach(() => jest.restoreAllMocks());
+
+    const makeApi = (fetch, ApiClass = Api) =>
+        new ApiClass({
+            fetch,
+            requestTimeoutMs: 0,
+            delegate: { name: 'hubspot', entity: { id: 7 } },
+        });
+
+    describe('a burst 429', () => {
+        it('waits as long as Retry-After says, then retries', async () => {
+            const fetch = throttledFetch([
+                limited(burstLimitBody, {
+                    ...burstHeaders,
+                    'Retry-After': '2',
+                }),
+                ok,
+            ]);
+
+            const result = await makeApi(fetch).listCompanies();
+
+            expect(result).toEqual({ results: [] });
+            expect(delays).toEqual([2_000]);
+            expect(fetch).toHaveBeenCalledTimes(2);
+        });
+
+        it('waits the 10 second burst window when the response has no Retry-After', async () => {
+            const fetch = throttledFetch([
+                limited(burstLimitBody, burstHeaders),
+                ok,
+            ]);
+
+            const result = await makeApi(fetch).listCompanies();
+
+            expect(result).toEqual({ results: [] });
+            expect(delays).toEqual([10_000]);
+            expect(fetch).toHaveBeenCalledTimes(2);
+        });
+
+        it('keeps the burst reason and the policy name when a long Retry-After makes it throw', async () => {
+            const fetch = throttledFetch([
+                limited(burstLimitBody, { 'Retry-After': '400' }),
+                ok,
+            ]);
+
+            const error = await makeApi(fetch)
+                .listCompanies()
+                .catch((e) => e);
+
+            expect(error).toBeInstanceOf(RateLimitError);
+            expect(error).toMatchObject({
+                reason: 'burst',
+                policy: 'TEN_SECONDLY_ROLLING',
+                source: 'header',
+                waitMs: 400_000,
+            });
+            expect(fetch).toHaveBeenCalledTimes(1);
+        });
+
+        it('waits the burst window when the body is not JSON', async () => {
+            const fetch = throttledFetch([
+                {
+                    status: 429,
+                    contentType: 'text/html',
+                    text: '<html>Too Many Requests</html>',
+                },
+                ok,
+            ]);
+
+            await makeApi(fetch).listCompanies();
+
+            expect(delays).toEqual([10_000]);
+        });
+    });
+
+    describe('a DAILY 429', () => {
+        it('throws a RateLimitError that probes again in one hour', async () => {
+            const fetch = throttledFetch([limited(dailyLimitBody), ok]);
+            const before = Date.now();
+
+            const error = await makeApi(fetch)
+                .listCompanies()
+                .catch((e) => e);
+
+            expect(error).toBeInstanceOf(RateLimitError);
+            expect(error).toBeInstanceOf(FetchError);
+            expect(error).toMatchObject({
+                isRateLimited: true,
+                statusCode: 429,
+                reason: 'daily',
+                policy: 'DAILY',
+                source: 'static',
+                waitMs: ONE_HOUR_MS,
+                module: 'hubspot',
+                scopeKey: 'hubspot:entity:7',
+            });
+            expect(error.retryAt.getTime()).toBeGreaterThanOrEqual(
+                before + ONE_HOUR_MS
+            );
+            expect(error.retryAt.getTime()).toBeLessThanOrEqual(
+                Date.now() + ONE_HOUR_MS
+            );
+            expect(fetch).toHaveBeenCalledTimes(1);
+            expect(delays).toEqual([]);
+        });
+
+        it('sleeps the probe interval and retries when the in-process budget allows it', async () => {
+            class PatientApi extends Api {
+                static rateLimit = {
+                    ...Api.rateLimit,
+                    maxInProcessWaitMs: ONE_HOUR_MS + 60_000,
+                };
+            }
+            const fetch = throttledFetch([limited(dailyLimitBody), ok]);
+
+            const result = await makeApi(fetch, PatientApi).listCompanies();
+
+            expect(result).toEqual({ results: [] });
+            expect(delays).toEqual([ONE_HOUR_MS]);
+            expect(fetch).toHaveBeenCalledTimes(2);
+        });
+
+        it('links the daily reason to the HubSpot usage guidelines', () => {
+            const { links } = Api.rateLimit.userHints.daily;
+
+            expect(links).toEqual([
+                {
+                    label: expect.any(String),
+                    url: 'https://developers.hubspot.com/docs/developer-tooling/platform/usage-guidelines',
+                },
+            ]);
+        });
+    });
+
+    describe('other failures', () => {
+        it('does not treat a 403 as a limit', async () => {
+            const fetch = throttledFetch([
+                {
+                    status: 403,
+                    body: { status: 'error', category: 'MISSING_SCOPES' },
+                },
+                ok,
+            ]);
+
+            const error = await makeApi(fetch)
+                .listCompanies()
+                .catch((e) => e);
+
+            expect(error).toBeInstanceOf(FetchError);
+            expect(error).not.toBeInstanceOf(RateLimitError);
+            expect(error.statusCode).toBe(403);
+            expect(fetch).toHaveBeenCalledTimes(1);
+            expect(delays).toEqual([]);
+        });
+    });
+
+    it('declares its limits', () => {
+        expect(Api.rateLimit).toMatchObject({
+            scope: 'entity',
+            windows: [{ name: 'burst', limit: 110, perMs: 10_000 }],
+            maxConcurrency: 10,
+            parsers: ['retryAfter', 'resetHeaders'],
+        });
+    });
+});

packages/v1-ready/pipedrive/README.md

@@ -3,3 +3,11 @@
 This is the API Module for pipedrive that allows the [Frigg](https://friggframework.org) code to talk to the pipedrive API.
 
 Read more on the [Frigg documentation site](https://docs.friggframework.org/api-modules/list/pipedrive
+
+## Rate limits
+
+`Api.rateLimit` tells the Frigg Requester how Pipedrive limits calls. The numbers come from [Pipedrive's rate limiting guide](https://pipedrive.readme.io/docs/core-api-concepts-rate-limiting).
+
+- **Burst:** a rolling 2 second window, counted per token. An OAuth app gets 80 requests in the window on the Lite plan, 160 on Growth, 400 on Premium and 480 on Ultimate. The module declares 80, the lowest. Pipedrive reports the real figure in `x-ratelimit-limit`.
+- **A throttled call** returns `429`. The Requester waits for `Retry-After` when the response has one, and otherwise for `x-ratelimit-reset` (in seconds), then retries.
+- **Daily token budget:** each company has a budget of 30,000 tokens times a plan multiplier times the number of seats. It resets at midnight in Pipedrive's server time zone. When it is spent, every call gets a `429`. Pipedrive documents no wait time for it, so the module declares none. Unless the response carries `Retry-After` or `x-ratelimit-reset`, such a `429` keeps the fixed backoff ladder, and then the queue redelivers the message.

packages/v1-ready/pipedrive/frigg-core.d.ts

@@ -39,6 +39,54 @@ declare module '@friggframework/core' {
     headers?: Record<string, string>;
   }
 
+  export type RateLimitScope =
+    | "credential"
+    | "entity"
+    | "app"
+    | ((requester: OAuth2Requester) => string | number | null | undefined);
+
+  export type RateLimitSignal = {
+    status?: number;
+    headers?: { get(name: string): string | null } | object;
+    body?: unknown;
+  };
+
+  export type RateLimitWindow = {
+    name: string;
+    limit?: number;
+    perMs?: number;
+    rollingMs?: number;
+    resets?: { at: string; tz: string };
+  };
+
+  export type RateLimitHint = {
+    retryAt: Date;
+    waitMs: number;
+    reason: "burst" | "daily" | "monthly" | "concurrency" | "unknown";
+    policy?: string;
+    remaining?: number;
+    source: "header" | "body" | "static" | "backoff";
+  };
+
+  export type RateLimitPolicy = {
+    scope?: RateLimitScope;
+    windows?: RateLimitWindow[];
+    maxConcurrency?: number;
+    minRetryAfterMs?: number;
+    maxInProcessWaitMs?: number;
+    parsers?: Array<"retryAfter" | "resetHeaders" | "ietf">;
+    classify?(signal: RateLimitSignal):
+      | (Partial<Omit<RateLimitHint, "source">> & {
+          source?: "header" | "body" | "static";
+        })
+      | null
+      | undefined;
+    userHints?: Record<
+      string,
+      { links?: Array<{ label: string; url: string }> }
+    >;
+  };
+
   export class OAuth2Requester {
     access_token: string | null;
     refresh_token: string | null;

packages/v1-ready/pipedrive/src/api.ts

@@ -1,4 +1,9 @@
-import { get, OAuth2Requester, RequestOptions } from "@friggframework/core";
+import {
+  get,
+  OAuth2Requester,
+  RateLimitPolicy,
+  RequestOptions,
+} from "@friggframework/core";
 import {
   OAuth2RequesterOptions,
   ActivityParams,
@@ -23,6 +28,12 @@ import {
 } from "./types";
 
 export class Api extends OAuth2Requester {
+  static rateLimit: RateLimitPolicy = {
+    scope: "entity",
+    windows: [{ name: "burst", limit: 80, perMs: 2_000 }],
+    parsers: ["retryAfter", "resetHeaders"],
+  };
+
   companyDomain: string | null;
   URLs: {
     activities: string;