WIP: add blobstore performance benchmarking

.github/scripts/benchmark/setup.sh

@@ -0,0 +1,139 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+script_dir="$( cd "$(dirname "${0}")" && pwd )"
+
+
+: "${benchmark_config_file:?}"
+: "${provider:?provider is required (s3|gcs|azurebs|alioss)}"
+CONFIG_FILE="${benchmark_config_file}"
+
+case "${provider}" in
+
+  s3)
+    : "${aws_access_key_id:?}"
+    : "${aws_secret_access_key:?}"
+    : "${aws_region:=us-east-1}"
+    : "${aws_stack_name:=s3cli-public-bucket}"
+
+    # translate to names the s3 setup-aws-infrastructure.sh script expects
+    export access_key_id="${aws_access_key_id}"
+    export secret_access_key="${aws_secret_access_key}"
+    export region_name="${aws_region}"
+    export stack_name="${aws_stack_name}"
+
+    bash "${script_dir}/../s3/setup-aws-infrastructure.sh"
+
+    # source utils to use get_stack_info functions
+    # re-export AWS vars since subprocess env does not propagate back
+    export AWS_ACCESS_KEY_ID="${aws_access_key_id}"
+    export AWS_SECRET_ACCESS_KEY="${aws_secret_access_key}"
+    export AWS_DEFAULT_REGION="${aws_region}"
+    source "${script_dir}/../s3/utils.sh"
+    stack_info=$(get_stack_info "${aws_stack_name}")
+    bucket_name=$(get_stack_info_of "${stack_info}" "BucketName")
+    : "${bucket_name:?failed to read bucket name from stack outputs}"
+
+    jq -n \
+      --arg access_key_id     "${aws_access_key_id}" \
+      --arg secret_access_key "${aws_secret_access_key}" \
+      --arg bucket_name       "${bucket_name}" \
+      --arg region            "${aws_region}" \
+      '{
+        access_key_id:     $access_key_id,
+        secret_access_key: $secret_access_key,
+        bucket_name:       $bucket_name,
+        region:            $region
+      }' > "${CONFIG_FILE}"
+    ;;
+
+  gcs)
+    : "${google_json_key_data:?}"
+
+    # login via gcloud
+    source "${script_dir}/../gcs/utils.sh"
+    gcloud_login
+
+    # create the regional bucket
+    make -C "${script_dir}/../gcs" regional-bucket
+
+    # read bucket name from the lock file
+    bucket_name="$(read_bucket_name "regional")"
+    : "${bucket_name:?failed to read bucket name from gcs lock file}"
+
+    jq -n \
+      --arg json_key    "${google_json_key_data}" \
+      --arg bucket_name "${bucket_name}" \
+      '{
+        provider:                    "Google",
+        credentials_source:          "static",
+        json_key:                    $json_key,
+        bucket_name:                 $bucket_name,
+        uniform_bucket_level_access: true
+      }' > "${CONFIG_FILE}"
+    ;;
+
+  azurebs)
+    : "${azure_storage_account:?}"
+    : "${azure_storage_key:?}"
+    export environment="AzureCloud"
+
+    # create the container
+    bash "${script_dir}/../azurebs/setup.sh"
+
+    # read the container name from the lock file
+    source "${script_dir}/../azurebs/utils.sh"
+    container_name="$(read_container_name_from_file "${environment}")"
+    : "${container_name:?failed to read container name from lock file}"
+
+    jq -n \
+      --arg account_name   "${azure_storage_account}" \
+      --arg account_key    "${azure_storage_key}" \
+      --arg container_name "${container_name}" \
+      '{
+        account_name:   $account_name,
+        account_key:    $account_key,
+        container_name: $container_name
+      }' > "${CONFIG_FILE}"
+    ;;
+
+  alioss)
+    : "${alioss_access_key_id:?}"
+    : "${alioss_access_key_secret:?}"
+    : "${alioss_region:=eu-central-1}"
+
+    # translate to names the alioss script expects
+    export access_key_id="${alioss_access_key_id}"
+    export access_key_secret="${alioss_access_key_secret}"
+    export region="${alioss_region}"
+    export test_name="benchmark"
+
+    # create the bucket
+    bash "${script_dir}/../alioss/setup.sh"
+
+    # read the bucket name from the lock file
+    source "${script_dir}/../alioss/utils.sh"
+    bucket_name="$(read_bucket_name_from_file "${test_name}")"
+    : "${bucket_name:?failed to read bucket name from alioss lock file}"
+
+    jq -n \
+      --arg access_key_id     "${alioss_access_key_id}" \
+      --arg access_key_secret "${alioss_access_key_secret}" \
+      --arg bucket_name       "${bucket_name}" \
+      --arg endpoint          "oss-${alioss_region}.aliyuncs.com" \
+      '{
+        access_key_id:     $access_key_id,
+        access_key_secret: $access_key_secret,
+        bucket_name:       $bucket_name,
+        endpoint:          $endpoint
+      }' > "${CONFIG_FILE}"
+    ;;
+
+  *)
+    echo "Unknown provider: ${provider}. Must be one of: s3, gcs, azurebs, alioss"
+    exit 1
+    ;;
+esac
+
+echo "Setup complete. Config written to ${CONFIG_FILE}"
+chmod 600 "${CONFIG_FILE}"

.github/scripts/benchmark/teardown.sh

@@ -0,0 +1,73 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+script_dir="$( cd "$(dirname "${0}")" && pwd )"
+
+: "${benchmark_config_file:?}"
+: "${provider:?provider is required (s3|gcs|azurebs|alioss)}"
+CONFIG_FILE="${benchmark_config_file}"
+
+# always remove the config file even if teardown fails
+trap 'rm -f "${CONFIG_FILE}"' EXIT
+
+case "${provider}" in
+
+  s3)
+    : "${aws_access_key_id:?}"
+    : "${aws_secret_access_key:?}"
+    : "${aws_region:=us-east-1}"
+    : "${aws_stack_name:=s3cli-public-bucket}"
+
+    # translate to names the s3 teardown script expects
+    export access_key_id="${aws_access_key_id}"
+    export secret_access_key="${aws_secret_access_key}"
+    export region_name="${aws_region}"
+    export stack_name="${aws_stack_name}"
+
+    bash "${script_dir}/../s3/teardown-infrastructure.sh"
+    ;;
+
+  gcs)
+    : "${google_json_key_data:?}"
+
+    source "${script_dir}/../gcs/utils.sh"
+    gcloud_login
+
+    # read bucket name and delete — cannot use make clean-gcs since it
+    # requires all three lock files (regional, multiregional, public)
+    # but we only created the regional bucket
+    bucket_name="$(read_bucket_name "regional")"
+    : "${bucket_name:?failed to read bucket name from gcs lock file}"
+
+    gsutil rm -r "gs://${bucket_name}" || true
+    ;;
+
+  azurebs)
+    : "${azure_storage_account:?}"
+    : "${azure_storage_key:?}"
+    export environment="AzureCloud"
+
+    bash "${script_dir}/../azurebs/teardown.sh"
+    ;;
+
+  alioss)
+    : "${alioss_access_key_id:?}"
+    : "${alioss_access_key_secret:?}"
+    : "${alioss_region:=eu-central-1}"
+
+    # translate to names the alioss teardown script expects
+    export access_key_id="${alioss_access_key_id}"
+    export access_key_secret="${alioss_access_key_secret}"
+    export region="${alioss_region}"
+    export test_name="benchmark"
+
+    bash "${script_dir}/../alioss/teardown.sh"
+    ;;
+
+  *)
+    echo "Unknown provider: ${provider}. Must be one of: s3, gcs, azurebs, alioss"
+    exit 1
+    ;;
+esac
+
+echo "Teardown complete."

.github/scripts/gcs/utils.sh

@@ -69,4 +69,12 @@ function gcloud_login {
     echo "${google_json_key_data:-}" > "${keyfile}"
     gcloud auth activate-service-account --key-file="${keyfile}"
     gcloud config set project "${google_project:-}"
+}
+
+
+function read_bucket_name {
+    # mirrors TMP_DIR in .github/scripts/gcs/Makefile
+    local gcs_tmp_dir="/tmp/storage-cli-gcs-${GITHUB_RUN_ID:-${USER}}"
+    local storage_class="${1:?storage class required}"
+    cat "${gcs_tmp_dir}/${storage_class}.lock"
 }

.github/workflows/benchmark-manual.yml

@@ -0,0 +1,104 @@
+name: Benchmark Manual
+
+on:
+  workflow_dispatch:
+    inputs:
+      profile:
+        description: "Benchmark profile"
+        required: true
+        type: choice
+        default: quick
+        options:
+          - quick
+          - full
+      provider:
+        description: "Storage provider"
+        required: true
+        type: choice
+        default: s3
+        options:
+          - s3
+          - gcs
+          - azurebs
+          - alioss
+
+jobs:
+  benchmarking:
+    name: Run Benchmarking
+    runs-on: [ubuntu-latest]
+    env:
+      BENCHMARK_CONFIG_FILE: /tmp/benchmark-config.json
+    steps:
+      - name: Checkout Into Source Code
+        uses: actions/checkout@v7
+
+      - name: Set up Go    
+        uses: actions/setup-go@v7
+        with:
+          go-version-file: go.mod
+
+      - name: Build benchmarker
+        run: |
+              echo "Building Benchmarker."
+              GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -tags benchmark -o "benchmarker" ./benchmark/
+
+      - name: Build storage-cli
+        run: |
+              echo "Building Storage CLI."
+              GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -ldflags "-X main.version=0.0.0" -o "storage-cli" .
+
+      - name: Set up Aliyun CLI
+        if: inputs.provider == 'alioss'
+        uses: aliyun/setup-aliyun-cli-action@v1
+
+      - name: Setup The Environment
+        run: |
+              export aws_access_key_id="${{ secrets.AWS_ACCESS_KEY_ID }}"
+              export aws_secret_access_key="${{ secrets.AWS_SECRET_ACCESS_KEY }}"
+              echo "${{ secrets.GCP_SERVICE_ACCOUNT_BASE64 }}" | base64 -d > /tmp/gcp-key.json
+              export google_json_key_data="$(cat /tmp/gcp-key.json)"
+              export azure_storage_account="${{ secrets.AZURE_STORAGE_ACCOUNT }}"
+              export azure_storage_key="${{ secrets.AZURE_STORAGE_KEY }}"
+              export alioss_access_key_id="${{ secrets.ALI_ACCESS_KEY_ID }}"
+              export alioss_access_key_secret="${{ secrets.ALI_ACCESS_KEY_SECRET }}"
+              export provider="${{ inputs.provider }}"
+              export benchmark_config_file="${{ env.BENCHMARK_CONFIG_FILE }}"
+              ./.github/scripts/benchmark/setup.sh
+
+      - name: Run Benchmark
+        run: |
+              mkdir -p "${GITHUB_WORKSPACE}/benchmark-results"
+              ./benchmarker \
+                -candidate ./storage-cli \
+                -provider  "${{ inputs.provider }}" \
+                -config    "${{ env.BENCHMARK_CONFIG_FILE }}" \
+                -profile   "${{ inputs.profile }}" \
+                -output    "${GITHUB_WORKSPACE}/benchmark-results"
+
+      - name: Upload Artifacts
+        if: always()
+        uses: actions/upload-artifact@v7
+        with:
+          name: benchmark-results-${{ inputs.provider }}-${{ inputs.profile }}
+          path: ${{ github.workspace }}/benchmark-results/
+          retention-days: 90
+
+      - name: Append Summary
+        if: always()
+        run: |
+              cat "${GITHUB_WORKSPACE}/benchmark-results/summary.md" >> "${GITHUB_STEP_SUMMARY}"
+
+      - name: Teardown The Environment
+        if: always()
+        run: |
+              export aws_access_key_id="${{ secrets.AWS_ACCESS_KEY_ID }}"
+              export aws_secret_access_key="${{ secrets.AWS_SECRET_ACCESS_KEY }}"
+              echo "${{ secrets.GCP_SERVICE_ACCOUNT_BASE64 }}" | base64 -d > /tmp/gcp-key.json
+              export google_json_key_data="$(cat /tmp/gcp-key.json)"
+              export azure_storage_account="${{ secrets.AZURE_STORAGE_ACCOUNT }}"
+              export azure_storage_key="${{ secrets.AZURE_STORAGE_KEY }}"
+              export alioss_access_key_id="${{ secrets.ALI_ACCESS_KEY_ID }}"
+              export alioss_access_key_secret="${{ secrets.ALI_ACCESS_KEY_SECRET }}"
+              export provider="${{ inputs.provider }}"
+              export benchmark_config_file="${{ env.BENCHMARK_CONFIG_FILE }}"
+              ./.github/scripts/benchmark/teardown.sh