Throttle and retry mechanism

keeperapi/src/browser/platform.ts

@@ -989,38 +989,53 @@ export const browserPlatform: Platform = class {
         return new Uint8Array(digest)
     }
 
-    static async get(url: string, headers: any): Promise<KeeperHttpResponse> {
-        let resp = await fetch(url, {
-            method: 'GET',
-            headers: Object.entries(headers),
-        })
-        let body = await resp.arrayBuffer()
-        return {
-            statusCode: resp.status,
-            headers: resp.headers,
-            data: new Uint8Array(body),
+    static async get(url: string, headers: any, timeoutMs = 300000): Promise<KeeperHttpResponse> {
+        const controller = new AbortController()
+        const timeout = setTimeout(() => controller.abort(), timeoutMs)
+        try {
+            let resp = await fetch(url, {
+                method: 'GET',
+                headers: Object.entries(headers),
+                signal: controller.signal,
+            })
+            let body = await resp.arrayBuffer()
+            return {
+                statusCode: resp.status,
+                headers: resp.headers,
+                data: new Uint8Array(body),
+            }
+        } finally {
+            clearTimeout(timeout)
         }
     }
 
     static async post(
         url: string,
         request: Uint8Array | string,
-        headers?: { [key: string]: string }
+        headers?: { [key: string]: string },
+        timeoutMs = 300000
     ): Promise<KeeperHttpResponse> {
-        let resp = await fetch(url, {
-            method: 'POST',
-            headers: new Headers({
-                'Content-Type': 'application/octet-stream',
-                'Content-Length': String(request.length),
-                ...headers,
-            }),
-            body: request,
-        })
-        let body = await resp.arrayBuffer()
-        return {
-            statusCode: resp.status,
-            headers: resp.headers,
-            data: new Uint8Array(body),
+        const controller = new AbortController()
+        const timeout = setTimeout(() => controller.abort(), timeoutMs)
+        try {
+            let resp = await fetch(url, {
+                method: 'POST',
+                headers: new Headers({
+                    'Content-Type': 'application/octet-stream',
+                    'Content-Length': String(request.length),
+                    ...headers,
+                }),
+                body: request,
+                signal: controller.signal,
+            })
+            let body = await resp.arrayBuffer()
+            return {
+                statusCode: resp.status,
+                headers: resp.headers,
+                data: new Uint8Array(body),
+            }
+        } finally {
+            clearTimeout(timeout)
         }
     }
 

keeperapi/src/configuration.ts

@@ -5,6 +5,7 @@ import TwoFactorPushType = Authentication.TwoFactorPushType
 import TwoFactorChannelType = Authentication.TwoFactorChannelType
 import { KeyWrapper } from './platform'
 import { SessionParams } from './auth'
+import { DEFAULT_REQUEST_TIMEOUT_MS, MAX_THROTTLE_RETRIES } from './retry'
 
 export type KeeperHost = KeeperEnvironment | string
 
@@ -31,11 +32,18 @@ export interface ClientConfiguration {
     // (`wss://connect.<host>/api/user/client`) once a session token is available,
     // in addition to the KeeperApp push socket. See `Auth.connectToRouter`.
     connectToRouter?: boolean
+    requestTimeoutMs?: number
+    failOnThrottle?: boolean
+    maxThrottleRetries?: number
+    maxKeyRetries?: number
 }
 export interface ClientConfigurationInternal extends ClientConfiguration {
     deviceConfig: DeviceConfig // v15+ device config
 }
 
+export const defaultRequestTimeoutMs = DEFAULT_REQUEST_TIMEOUT_MS
+export const defaultMaxThrottleRetries = MAX_THROTTLE_RETRIES
+
 export type KeeperError = {
     additional_info?: string
     error?: string

keeperapi/src/endpoint.ts

@@ -33,6 +33,14 @@ import WssConnectionRequest = Push.WssConnectionRequest
 import SsoCloudResponse = SsoCloud.SsoCloudResponse
 import { KeeperHttpResponse, RestCommand } from './commands'
 import { AllowedEcKeyIds, AllowedMlKemKeyIds, isAllowedEcKeyId, isAllowedMlKemKeyId } from './transmissionKeys'
+import {
+    delay,
+    isThrottleResponse,
+    parseErrorResponse,
+    parseThrottleWaitSeconds,
+    throttleBackoffSeconds,
+} from './retry'
+import { defaultMaxThrottleRetries, defaultRequestTimeoutMs } from './configuration'
 
 export type ExecuteRestOptions = {
     skipRegionRedirect?: boolean
@@ -61,6 +69,32 @@ export class KeeperEndpoint {
         }
     }
 
+    private async retryThrottle(response: KeeperHttpResponse, path: string, retryCount: number): Promise<number> {
+        const parsed = parseErrorResponse(response.data, (data) => platform.bytesToString(data))
+        const errorCode = parsed.body?.error
+        if (!isThrottleResponse(response.statusCode, errorCode)) return -1
+        if (this.options.failOnThrottle) {
+            if (parsed.body) throw parsed.body
+            throw new Error(parsed.message)
+        }
+
+        const nextRetry = retryCount + 1
+        const maxRetries = this.options.maxThrottleRetries ?? defaultMaxThrottleRetries
+        if (nextRetry > maxRetries) {
+            if (parsed.body) throw parsed.body
+            throw new Error(parsed.message)
+        }
+        const wait = throttleBackoffSeconds(
+            nextRetry,
+            parseThrottleWaitSeconds(parsed.body?.message || parsed.message, response.headers)
+        )
+        logger.warn(
+            `Throttled ${path} (attempt ${nextRetry}/${maxRetries}), retrying in ${wait} seconds: ${parsed.body?.message || parsed.message}`
+        )
+        await delay(wait * 1000)
+        return nextRetry
+    }
+
     async getTransmissionKey(): Promise<TransmissionKey> {
         const DEFAULT_PROD_EC_KEY_ID = 10
         const DEFAULT_PROD_ML_KEM_KEY_ID = 136
@@ -215,7 +249,24 @@ export class KeeperEndpoint {
             logger.debug(...formatProto(`→ ${url}`, (message as { data?: unknown }).data))
         }
         const startTime = Date.now()
-        const response = await platform.post(url, encryptedPayload, headers)
+        let throttleRetries = 0
+        let response = await platform.post(
+            url,
+            encryptedPayload,
+            headers,
+            this.options.requestTimeoutMs ?? defaultRequestTimeoutMs
+        )
+        while (true) {
+            const nextRetry = await this.retryThrottle(response, message.path, throttleRetries)
+            if (nextRetry < 0) break
+            throttleRetries = nextRetry
+            response = await platform.post(
+                url,
+                encryptedPayload,
+                headers,
+                this.options.requestTimeoutMs ?? defaultRequestTimeoutMs
+            )
+        }
         if (!response.data || response.data.length === 0) {
             if ('fromBytes' in message) throw new Error(`Empty response from router for ${message.path}`)
             return
@@ -254,6 +305,8 @@ export class KeeperEndpoint {
         options?: ExecuteRestOptions
     ): Promise<TOut | void> {
         this._transmissionKey = await this.getTransmissionKey()
+        let throttleRetries = 0
+        let keyRetries = 0
         while (true) {
             const payload = 'toBytes' in message ? message.toBytes() : new Uint8Array()
             const apiVersion = message.apiVersion || 0
@@ -272,7 +325,17 @@ export class KeeperEndpoint {
                 logger.debug(...formatProto(`→ ${url}`, (message as { data?: unknown }).data))
             }
             const startTime = Date.now()
-            const response = await platform.post(url, request)
+            const response = await platform.post(
+                url,
+                request,
+                undefined,
+                this.options.requestTimeoutMs ?? defaultRequestTimeoutMs
+            )
+            const nextThrottleRetry = await this.retryThrottle(response, message.path, throttleRetries)
+            if (nextThrottleRetry >= 0) {
+                throttleRetries = nextThrottleRetry
+                continue
+            }
             if (!response.data || (response.data.length === 0 && response.statusCode === 200)) {
                 if ('fromBytes' in message) {
                     throw Error(`Missing expected a response for ${message.path}`)
@@ -301,6 +364,12 @@ export class KeeperEndpoint {
                     const errorObj: KeeperError = JSON.parse(errorMessage)
                     switch (errorObj.error) {
                         case 'key':
+                            keyRetries += 1
+                            if (keyRetries > (this.options.maxKeyRetries ?? 3)) {
+                                throw new Error(
+                                    `Transmission key refresh failed after ${keyRetries - 1} retries: ${errorMessage}`
+                                )
+                            }
                             let newEcKeyId: AllowedEcKeyIds
                             let newMlKemKeyId: AllowedMlKemKeyIds
                             let disableHpke = false
@@ -374,7 +443,25 @@ export class KeeperEndpoint {
             ...command.request,
         }
         const requestBytes = await this.prepareRequest(payload)
-        const response = await platform.post(this.getUrl('vault/execute_v2_command'), requestBytes)
+        const url = this.getUrl('vault/execute_v2_command')
+        let throttleRetries = 0
+        let response = await platform.post(
+            url,
+            requestBytes,
+            undefined,
+            this.options.requestTimeoutMs ?? defaultRequestTimeoutMs
+        )
+        while (true) {
+            const nextRetry = await this.retryThrottle(response, 'vault/execute_v2_command', throttleRetries)
+            if (nextRetry < 0) break
+            throttleRetries = nextRetry
+            response = await platform.post(
+                url,
+                requestBytes,
+                undefined,
+                this.options.requestTimeoutMs ?? defaultRequestTimeoutMs
+            )
+        }
         let decrypted
         try {
             decrypted = await platform.aesGcmDecrypt(response.data, this._transmissionKey.key)
@@ -390,7 +477,7 @@ export class KeeperEndpoint {
     }
 
     async get(path: string): Promise<KeeperHttpResponse> {
-        return platform.get(this.getUrl(path), {})
+        return platform.get(this.getUrl(path), {}, this.options.requestTimeoutMs ?? defaultRequestTimeoutMs)
     }
 
     public async updateTransmissionKey(ecKeyId: AllowedEcKeyIds, mlKemKeyId: AllowedMlKemKeyIds) {

keeperapi/src/node/platform.ts

@@ -383,7 +383,7 @@ export const nodePlatform: Platform = class {
         return Promise.resolve(crypto.createHash('SHA256').update(data).digest())
     }
 
-    static get(url: string, headers?: { [key: string]: string }): Promise<KeeperHttpResponse> {
+    static get(url: string, headers?: { [key: string]: string }, timeoutMs = 300000): Promise<KeeperHttpResponse> {
         return new Promise<KeeperHttpResponse>((resolve, reject) => {
             let get = https.request(
                 url,
@@ -393,20 +393,23 @@ export const nodePlatform: Platform = class {
                         'User-Agent': `Node/${process.version}`,
                         ...headers,
                     },
+                    timeout: timeoutMs,
                 },
                 (res) => {
                     this.fetchData(res, resolve)
                 }
             )
             get.on('error', reject)
+            get.on('timeout', () => get.destroy(new Error(`Request timed out after ${timeoutMs}ms`)))
             get.end()
         })
     }
 
     static post(
         url: string,
         request: Uint8Array | string,
-        headers?: { [key: string]: string }
+        headers?: { [key: string]: string },
+        timeoutMs = 300000
     ): Promise<KeeperHttpResponse> {
         return new Promise<KeeperHttpResponse>((resolve, reject) => {
             let post = https.request(
@@ -419,12 +422,14 @@ export const nodePlatform: Platform = class {
                         'User-Agent': `Node/${process.version}`,
                         ...headers,
                     },
+                    timeout: timeoutMs,
                 },
                 (res) => {
                     this.fetchData(res, resolve)
                 }
             )
             post.on('error', reject)
+            post.on('timeout', () => post.destroy(new Error(`Request timed out after ${timeoutMs}ms`)))
             post.write(request)
             post.end()
         })

keeperapi/src/platform.ts

@@ -126,9 +126,9 @@ export interface Platform {
 
     sha256(data: Uint8Array): Promise<Uint8Array>
 
-    get(url: string, headers: any): Promise<KeeperHttpResponse>
+    get(url: string, headers: any, timeoutMs?: number): Promise<KeeperHttpResponse>
 
-    post(url: string, request: Uint8Array, headers?: any): Promise<KeeperHttpResponse>
+    post(url: string, request: Uint8Array, headers?: any, timeoutMs?: number): Promise<KeeperHttpResponse>
 
     fileUpload(url: string, uploadParameters: any, data: Uint8Array | Blob): Promise<any>
 

keeperapi/src/retry.ts

@@ -0,0 +1,66 @@
+import type { KeeperError } from './configuration'
+
+export const DEFAULT_REQUEST_TIMEOUT_MS = 300_000
+export const MAX_THROTTLE_RETRIES = 3
+export const MAX_THROTTLE_WAIT_SECONDS = 300
+export const DEFAULT_THROTTLE_WAIT_SECONDS = 60
+
+export type ResponseHeaders = {
+    get?: (name: string) => string | null
+    [key: string]: unknown
+}
+
+export function getHeader(headers: unknown, name: string): string | undefined {
+    if (!headers || typeof headers !== 'object') return undefined
+    const value = headers as ResponseHeaders
+    if (typeof value.get === 'function') return value.get(name) || value.get(name.toLowerCase()) || undefined
+    const key = Object.keys(value).find((key) => key.toLowerCase() === name.toLowerCase())
+    const header = key ? value[key] : undefined
+    return Array.isArray(header) ? header[0] : typeof header === 'string' ? header : undefined
+}
+
+export function parseRetryAfter(value: string | undefined, now = Date.now()): number | undefined {
+    if (!value) return undefined
+    const seconds = Number(value.trim())
+    if (Number.isFinite(seconds) && seconds >= 0) return Math.ceil(seconds)
+    const date = Date.parse(value)
+    if (Number.isNaN(date)) return undefined
+    return Math.max(0, Math.ceil((date - now) / 1000))
+}
+
+export function parseThrottleWaitSeconds(message: string, headers: unknown): number {
+    const retryAfter = parseRetryAfter(getHeader(headers, 'retry-after'))
+    if (retryAfter !== undefined) return Math.min(retryAfter, MAX_THROTTLE_WAIT_SECONDS)
+
+    const match = message.match(/(\d+)\s*(second|seconds|minute|minutes)/i)
+    if (!match) return DEFAULT_THROTTLE_WAIT_SECONDS
+    const value = Number(match[1]) * (/minute/i.test(match[2]) ? 60 : 1)
+    return Math.min(value, MAX_THROTTLE_WAIT_SECONDS)
+}
+
+export function throttleBackoffSeconds(retryNumber: number, waitSeconds: number): number {
+    return Math.max(waitSeconds, 30 * 2 ** Math.max(0, retryNumber - 1))
+}
+
+export function parseErrorResponse(
+    data: Uint8Array,
+    bytesToString: (data: Uint8Array) => string
+): {
+    body?: KeeperError
+    message: string
+} {
+    const message = bytesToString(data)
+    try {
+        return { body: JSON.parse(message) as KeeperError, message }
+    } catch {
+        return { message }
+    }
+}
+
+export function isThrottleResponse(statusCode: number, error?: string): boolean {
+    return statusCode === 429 || (statusCode === 403 && error?.toLowerCase() === 'throttled')
+}
+
+export function delay(milliseconds: number): Promise<void> {
+    return new Promise((resolve) => setTimeout(resolve, milliseconds))
+}