feat(azblob): add azure blob storage provider

.github/workflows/ci-cd.yml

@@ -45,7 +45,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        storage-driver: [filesystem, s3, gcs]
+        storage-driver: [filesystem, s3, gcs, azblob]
         db-driver: [postgres, mysql, sqlite]
     steps:
       - name: pnpm install

install/kubernetes/github-actions-cache-server/templates/_helpers.tpl

@@ -195,6 +195,21 @@ Generate environment variables from config values.
   value: {{ .endpoint | quote }}
 {{- end }}
 {{- end }}
+{{- else if eq .Values.config.storage.driver "azblob" }}
+{{- with .Values.config.storage.azblob }}
+{{- if .account }}
+- name: STORAGE_AZBLOB_ACCOUNT
+  value: {{ .account | quote }}
+{{- end }}
+{{- if .container }}
+- name: STORAGE_AZBLOB_CONTAINER
+  value: {{ .container | quote }}
+{{- end }}
+{{- if .endpoint }}
+- name: STORAGE_AZBLOB_ENDPOINT
+  value: {{ .endpoint | quote }}
+{{- end }}
+{{- end }}
 {{- end }}
 {{/* Database driver */}}
 - name: DB_DRIVER

install/kubernetes/github-actions-cache-server/values.yaml

@@ -73,7 +73,7 @@ config:
   # -- Storage driver configuration
   # See https://gha-cache-server.falcondev.io/storage-drivers
   storage:
-    # -- Storage driver to use: "filesystem", "s3", or "gcs"
+    # -- Storage driver to use: "filesystem", "s3", "gcs", or "azblob"
     driver: filesystem
 
     # -- Filesystem storage driver settings
@@ -108,6 +108,18 @@ config:
       # -- Custom GCS API endpoint
       # endpoint: ''
 
+    # -- Azure Blob Storage driver settings
+    # Auth is either a connection string (STORAGE_AZBLOB_CONNECTION_STRING via existingSecret)
+    # or `account` + workload identity (DefaultAzureCredential). The connection string carries
+    # its own account and endpoint, so `account` and `endpoint` are ignored in that mode.
+    azblob:
+      # -- Azure Blob container name (required when driver is "azblob")
+      # container: ''
+      # -- Azure Storage account name (required only when using DefaultAzureCredential; ignored when a connection string is set)
+      # account: ''
+      # -- Custom Azure Blob Storage endpoint URL (optional, defaults to https://<account>.blob.core.windows.net; ignored when a connection string is set)
+      # endpoint: ''
+
   # -- Database driver configuration
   # See https://gha-cache-server.falcondev.io/database-drivers
   db:
@@ -153,7 +165,7 @@ config:
 
 # -- Name of an existing Kubernetes Secret containing sensitive environment variables.
 # Use this to avoid storing secrets in values.yaml. The secret keys should match
-# the env var names (e.g., AWS_SECRET_ACCESS_KEY, DB_POSTGRES_PASSWORD, DB_POSTGRES_URL, etc.)
+# the env var names (e.g., AWS_SECRET_ACCESS_KEY, DB_POSTGRES_PASSWORD, DB_POSTGRES_URL, STORAGE_AZBLOB_CONNECTION_STRING  etc.)
 existingSecret: ''
 
 serviceAccount:

lib/schemas.ts

@@ -20,6 +20,20 @@ export const envStorageDriverSchema = type.or(
     'STORAGE_GCS_SERVICE_ACCOUNT_KEY?': 'string',
     'STORAGE_GCS_ENDPOINT?': 'string.url',
   },
+  {
+    'STORAGE_DRIVER': type.unit('azblob'),
+    'STORAGE_AZBLOB_CONTAINER': 'string',
+    'STORAGE_AZBLOB_CONNECTION_STRING': 'string',
+    'STORAGE_AZBLOB_ACCOUNT?': 'undefined',
+    'STORAGE_AZBLOB_ENDPOINT?': 'undefined',
+  },
+  {
+    'STORAGE_DRIVER': type.unit('azblob'),
+    'STORAGE_AZBLOB_CONTAINER': 'string',
+    'STORAGE_AZBLOB_ACCOUNT': 'string',
+    'STORAGE_AZBLOB_ENDPOINT?': 'string.url',
+    'STORAGE_AZBLOB_CONNECTION_STRING?': 'undefined',
+  },
 )
 export const envDbDriverSchema = type.or(
   type.or(

lib/storage.ts

@@ -26,6 +26,12 @@ import {
 } from '@aws-sdk/client-s3'
 import { Upload as S3Upload } from '@aws-sdk/lib-storage'
 import { getSignedUrl } from '@aws-sdk/s3-request-presigner'
+import { DefaultAzureCredential } from '@azure/identity'
+import {
+  BlobSASPermissions,
+  BlobServiceClient,
+  generateBlobSASQueryParameters,
+} from '@azure/storage-blob'
 import { Storage as GcsClient } from '@google-cloud/storage'
 import { NodeHttpHandler } from '@smithy/node-http-handler'
 import { sql } from 'kysely'
@@ -80,6 +86,7 @@ export class Storage {
       .with({ STORAGE_DRIVER: 's3' }, S3Adapter.fromEnv)
       .with({ STORAGE_DRIVER: 'filesystem' }, FileSystemAdapter.fromEnv)
       .with({ STORAGE_DRIVER: 'gcs' }, GcsAdapter.fromEnv)
+      .with({ STORAGE_DRIVER: 'azblob' }, AzBlobAdapter.fromEnv)
       .exhaustive()
   }
 
@@ -1405,3 +1412,211 @@ class GcsAdapter implements StorageAdapter {
       .then((res) => res[0])
   }
 }
+
+class AzBlobAdapter implements StorageAdapter {
+  static async fromEnv(env: Extract<Env, { STORAGE_DRIVER: 'azblob' }>) {
+    const container = env.STORAGE_AZBLOB_CONTAINER
+
+    const usesSharedKey = Boolean(env.STORAGE_AZBLOB_CONNECTION_STRING)
+    const client = env.STORAGE_AZBLOB_CONNECTION_STRING
+      ? BlobServiceClient.fromConnectionString(env.STORAGE_AZBLOB_CONNECTION_STRING)
+      : new BlobServiceClient(
+          env.STORAGE_AZBLOB_ENDPOINT ??
+            `https://${env.STORAGE_AZBLOB_ACCOUNT}.blob.core.windows.net`,
+          new DefaultAzureCredential(),
+        )
+
+    const containerClient = client.getContainerClient(container)
+    await containerClient.createIfNotExists()
+
+    return new AzBlobAdapter({
+      client,
+      account: client.accountName,
+      container,
+      usesSharedKey,
+    })
+  }
+
+  private client
+  private account
+  private container
+  private usesSharedKey
+  private keyPrefix = 'gh-actions-cache'
+
+  // Put Block From URL, service version 2020-04-08+: 4000 MiB per block, 50000 blocks per blob.
+  // https://learn.microsoft.com/en-us/rest/api/storageservices/put-block-from-url
+  composeParts = {
+    minPartBytes: 0,
+    maxPartBytes: 4000 * 1024 ** 2,
+    maxParts: 50_000,
+    run: async (folderName: string, partCount: number) => {
+      const mergedBlobClient = this.containerClient.getBlockBlobClient(
+        this.blobKey(`${folderName}/merged`),
+      )
+      // stageBlockFromURL needs an authenticated URL to each source part.
+      const expiresAt = Date.now() + DIRECT_DOWNLOAD_LEASE_DURATION_MS
+      const blockIds: string[] = []
+      for (let index = 0; index < partCount; index++) {
+        const blockId = Buffer.from(String(index).padStart(10, '0')).toString('base64')
+        const sourceUrl = await this.createDownloadUrl(`${folderName}/parts/${index}`, expiresAt)
+        try {
+          await mergedBlobClient.stageBlockFromURL(blockId, sourceUrl)
+        } catch (err) {
+          logger.error('AzBlob stageBlockFromURL failed', {
+            error: err,
+            sourceUrl,
+            blockId,
+          })
+          throw err
+        }
+        blockIds.push(blockId)
+      }
+      await mergedBlobClient.commitBlockList(blockIds)
+    },
+  }
+
+  constructor({
+    client,
+    account,
+    container,
+    usesSharedKey,
+  }: {
+    client: BlobServiceClient
+    account: string
+    container: string
+    usesSharedKey: boolean
+  }) {
+    this.client = client
+    this.account = account
+    this.container = container
+    this.usesSharedKey = usesSharedKey
+  }
+
+  private get containerClient() {
+    return this.client.getContainerClient(this.container)
+  }
+
+  private blobKey(objectName: string) {
+    return `${this.keyPrefix}/${objectName}`
+  }
+
+  async createDownloadStream(objectName: string): Promise<Readable> {
+    const blockBlobClient = this.containerClient.getBlockBlobClient(this.blobKey(objectName))
+    const response = await blockBlobClient.download()
+    if (!response.readableStreamBody) throw new Error(`No stream for blob "${objectName}"`)
+    return Readable.from(response.readableStreamBody)
+  }
+
+  async uploadStream(objectName: string, stream: AsyncIterable<Uint8Array>): Promise<void> {
+    const blockBlobClient = this.containerClient.getBlockBlobClient(this.blobKey(objectName))
+    // TODO: consider blockSize / concurrency tuning similar to S3Upload options
+    await blockBlobClient.uploadStream(Readable.from(stream))
+  }
+
+  async objectExists(objectName: string): Promise<boolean> {
+    return this.containerClient.getBlobClient(this.blobKey(objectName)).exists()
+  }
+
+  async deleteByPrefix(prefix: string): Promise<StorageDeletion> {
+    // Azure caps a batch at 256 subrequests - align LIST paging with that so each page = one batch.
+    const BATCH_SIZE = 256
+    const deleted = { objects: 0, bytes: 0 }
+    const batchClient = this.containerClient.getBlobBatchClient()
+
+    const pages = this.containerClient.listBlobsFlat({ prefix }).byPage({ maxPageSize: BATCH_SIZE })
+
+    for await (const page of pages) {
+      const blobs = page.segment.blobItems
+      if (blobs.length === 0) continue
+
+      const clients = blobs.map((blob) => {
+        deleted.objects += 1
+        deleted.bytes += blob.properties.contentLength ?? 0
+        return this.containerClient.getBlobClient(blob.name)
+      })
+
+      await batchClient.deleteBlobs(clients)
+    }
+
+    return deleted
+  }
+
+  async deleteFolder(folderName: string): Promise<StorageDeletion> {
+    return this.deleteByPrefix(`${this.blobKey(folderName)}/`)
+  }
+
+  async clear(): Promise<void> {
+    await this.deleteByPrefix(this.blobKey(''))
+  }
+
+  async countFilesInFolder(folderName: string): Promise<number> {
+    let count = 0
+    const blobs = this.containerClient.listBlobsFlat({
+      prefix: `${this.blobKey(folderName)}/`,
+    })
+    for await (const _ of blobs) {
+      count++
+    }
+    return count
+  }
+
+  async listFolder(folderName: string): Promise<StorageObject[]> {
+    const prefix = `${this.blobKey(folderName)}/`
+    const objects: StorageObject[] = []
+    const blobs = this.containerClient.listBlobsFlat({ prefix })
+    for await (const blob of blobs) {
+      objects.push({
+        name: blob.name.slice(prefix.length),
+        bytes: blob.properties.contentLength ?? 0,
+      })
+    }
+    return objects
+  }
+
+  async listStorageFolders(): Promise<StorageFolder[]> {
+    const folders = new Map<string, StorageFolder>()
+    const prefix = this.blobKey('')
+
+    const blobs = this.containerClient.listBlobsFlat({ prefix })
+    for await (const blob of blobs) {
+      const relativeName = blob.name.slice(prefix.length)
+      const folderName = relativeName.split('/', 1)[0]
+      if (!folderName) continue
+
+      const size = blob.properties.contentLength ?? 0
+      const updatedAt = blob.properties.lastModified?.getTime() ?? 0
+      accumulateFolder(folders, folderName, size, updatedAt)
+    }
+
+    return [...folders.values()]
+  }
+
+  async createDownloadUrl(objectName: string, expiresAt: number): Promise<string> {
+    // Backdate start by 5 minutes so clock skew doesn't cause "not yet valid" rejections.
+    const CLOCK_SKEW_MS = 5 * 60 * 1000
+    const startsOn = new Date(Date.now() - CLOCK_SKEW_MS)
+    const expiresOn = new Date(expiresAt)
+    const permissions = BlobSASPermissions.parse('r')
+    const blobClient = this.containerClient.getBlobClient(this.blobKey(objectName))
+
+    // Shared-key connection strings can't request a user delegation key
+    // (that requires Entra ID), so sign directly against the shared key.
+    if (this.usesSharedKey) {
+      return blobClient.generateSasUrl({ permissions, startsOn, expiresOn })
+    }
+
+    const delegationKey = await this.client.getUserDelegationKey(startsOn, expiresOn)
+    const sasParams = generateBlobSASQueryParameters(
+      {
+        containerName: this.container,
+        blobName: this.blobKey(objectName),
+        permissions,
+        startsOn,
+        expiresOn,
+      },
+      delegationKey,
+      this.account,
+    )
+    return `${blobClient.url}?${sasParams.toString()}`
+  }
+}

package.json

@@ -15,7 +15,7 @@
     "lint:fix": "eslint --fix --cache . && prettier --write --cache .",
     "type-check": "tsc --noEmit",
     "test:run": "DEBUG=true vitest run",
-    "test:matrix": "for db in sqlite postgres mysql; do for st in filesystem s3 gcs; do echo \"### $db + $st\"; VITEST_DB_DRIVER=$db VITEST_STORAGE_DRIVER=$st pnpm run test:run || exit 1; done; done"
+    "test:matrix": "for db in sqlite postgres mysql; do for st in filesystem s3 gcs azblob; do echo \"### $db + $st\"; VITEST_DB_DRIVER=$db VITEST_STORAGE_DRIVER=$st pnpm run test:run || exit 1; done; done"
   },
   "changelogithub": {
     "extends": "gh:falcondev-it/configs/changelogithub"
@@ -25,6 +25,8 @@
     "@aws-sdk/client-s3": "^3.1085.0",
     "@aws-sdk/lib-storage": "^3.1085.0",
     "@aws-sdk/s3-request-presigner": "^3.1085.0",
+    "@azure/identity": "^4.13.1",
+    "@azure/storage-blob": "^12.33.0",
     "@google-cloud/storage": "^7.21.0",
     "@orpc/client": "^1.14.7",
     "@orpc/json-schema": "^1.14.7",