diff --git a/.github/scripts/benchmark/setup.sh b/.github/scripts/benchmark/setup.sh new file mode 100755 index 00000000..44dd1685 --- /dev/null +++ b/.github/scripts/benchmark/setup.sh @@ -0,0 +1,139 @@ +#!/usr/bin/env bash +set -euo pipefail + +script_dir="$( cd "$(dirname "${0}")" && pwd )" + + +: "${benchmark_config_file:?}" +: "${provider:?provider is required (s3|gcs|azurebs|alioss)}" +CONFIG_FILE="${benchmark_config_file}" + +case "${provider}" in + + s3) + : "${aws_access_key_id:?}" + : "${aws_secret_access_key:?}" + : "${aws_region:=us-east-1}" + : "${aws_stack_name:=s3cli-public-bucket}" + + # translate to names the s3 setup-aws-infrastructure.sh script expects + export access_key_id="${aws_access_key_id}" + export secret_access_key="${aws_secret_access_key}" + export region_name="${aws_region}" + export stack_name="${aws_stack_name}" + + bash "${script_dir}/../s3/setup-aws-infrastructure.sh" + + # source utils to use get_stack_info functions + # re-export AWS vars since subprocess env does not propagate back + export AWS_ACCESS_KEY_ID="${aws_access_key_id}" + export AWS_SECRET_ACCESS_KEY="${aws_secret_access_key}" + export AWS_DEFAULT_REGION="${aws_region}" + source "${script_dir}/../s3/utils.sh" + stack_info=$(get_stack_info "${aws_stack_name}") + bucket_name=$(get_stack_info_of "${stack_info}" "BucketName") + : "${bucket_name:?failed to read bucket name from stack outputs}" + + jq -n \ + --arg access_key_id "${aws_access_key_id}" \ + --arg secret_access_key "${aws_secret_access_key}" \ + --arg bucket_name "${bucket_name}" \ + --arg region "${aws_region}" \ + '{ + access_key_id: $access_key_id, + secret_access_key: $secret_access_key, + bucket_name: $bucket_name, + region: $region + }' > "${CONFIG_FILE}" + ;; + + gcs) + : "${google_json_key_data:?}" + + # login via gcloud + source "${script_dir}/../gcs/utils.sh" + gcloud_login + + # create the regional bucket + make -C "${script_dir}/../gcs" regional-bucket + + # read bucket name from the lock file + bucket_name="$(read_bucket_name "regional")" + : "${bucket_name:?failed to read bucket name from gcs lock file}" + + jq -n \ + --arg json_key "${google_json_key_data}" \ + --arg bucket_name "${bucket_name}" \ + '{ + provider: "Google", + credentials_source: "static", + json_key: $json_key, + bucket_name: $bucket_name, + uniform_bucket_level_access: true + }' > "${CONFIG_FILE}" + ;; + + azurebs) + : "${azure_storage_account:?}" + : "${azure_storage_key:?}" + export environment="AzureCloud" + + # create the container + bash "${script_dir}/../azurebs/setup.sh" + + # read the container name from the lock file + source "${script_dir}/../azurebs/utils.sh" + container_name="$(read_container_name_from_file "${environment}")" + : "${container_name:?failed to read container name from lock file}" + + jq -n \ + --arg account_name "${azure_storage_account}" \ + --arg account_key "${azure_storage_key}" \ + --arg container_name "${container_name}" \ + '{ + account_name: $account_name, + account_key: $account_key, + container_name: $container_name + }' > "${CONFIG_FILE}" + ;; + + alioss) + : "${alioss_access_key_id:?}" + : "${alioss_access_key_secret:?}" + : "${alioss_region:=eu-central-1}" + + # translate to names the alioss script expects + export access_key_id="${alioss_access_key_id}" + export access_key_secret="${alioss_access_key_secret}" + export region="${alioss_region}" + export test_name="benchmark" + + # create the bucket + bash "${script_dir}/../alioss/setup.sh" + + # read the bucket name from the lock file + source "${script_dir}/../alioss/utils.sh" + bucket_name="$(read_bucket_name_from_file "${test_name}")" + : "${bucket_name:?failed to read bucket name from alioss lock file}" + + jq -n \ + --arg access_key_id "${alioss_access_key_id}" \ + --arg access_key_secret "${alioss_access_key_secret}" \ + --arg bucket_name "${bucket_name}" \ + --arg endpoint "oss-${alioss_region}.aliyuncs.com" \ + '{ + access_key_id: $access_key_id, + access_key_secret: $access_key_secret, + bucket_name: $bucket_name, + endpoint: $endpoint + }' > "${CONFIG_FILE}" + ;; + + *) + echo "Unknown provider: ${provider}. Must be one of: s3, gcs, azurebs, alioss" + exit 1 + ;; +esac + +echo "Setup complete. Config written to ${CONFIG_FILE}" +chmod 600 "${CONFIG_FILE}" diff --git a/.github/scripts/benchmark/teardown.sh b/.github/scripts/benchmark/teardown.sh new file mode 100755 index 00000000..9ccd1eea --- /dev/null +++ b/.github/scripts/benchmark/teardown.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash +set -euo pipefail + +script_dir="$( cd "$(dirname "${0}")" && pwd )" + +: "${benchmark_config_file:?}" +: "${provider:?provider is required (s3|gcs|azurebs|alioss)}" +CONFIG_FILE="${benchmark_config_file}" + +# always remove the config file even if teardown fails +trap 'rm -f "${CONFIG_FILE}"' EXIT + +case "${provider}" in + + s3) + : "${aws_access_key_id:?}" + : "${aws_secret_access_key:?}" + : "${aws_region:=us-east-1}" + : "${aws_stack_name:=s3cli-public-bucket}" + + # translate to names the s3 teardown script expects + export access_key_id="${aws_access_key_id}" + export secret_access_key="${aws_secret_access_key}" + export region_name="${aws_region}" + export stack_name="${aws_stack_name}" + + bash "${script_dir}/../s3/teardown-infrastructure.sh" + ;; + + gcs) + : "${google_json_key_data:?}" + + source "${script_dir}/../gcs/utils.sh" + gcloud_login + + # read bucket name and delete — cannot use make clean-gcs since it + # requires all three lock files (regional, multiregional, public) + # but we only created the regional bucket + bucket_name="$(read_bucket_name "regional")" + : "${bucket_name:?failed to read bucket name from gcs lock file}" + + gsutil rm -r "gs://${bucket_name}" || true + ;; + + azurebs) + : "${azure_storage_account:?}" + : "${azure_storage_key:?}" + export environment="AzureCloud" + + bash "${script_dir}/../azurebs/teardown.sh" + ;; + + alioss) + : "${alioss_access_key_id:?}" + : "${alioss_access_key_secret:?}" + : "${alioss_region:=eu-central-1}" + + # translate to names the alioss teardown script expects + export access_key_id="${alioss_access_key_id}" + export access_key_secret="${alioss_access_key_secret}" + export region="${alioss_region}" + export test_name="benchmark" + + bash "${script_dir}/../alioss/teardown.sh" + ;; + + *) + echo "Unknown provider: ${provider}. Must be one of: s3, gcs, azurebs, alioss" + exit 1 + ;; +esac + +echo "Teardown complete." diff --git a/.github/scripts/gcs/utils.sh b/.github/scripts/gcs/utils.sh index ba4f5a5c..3f505507 100644 --- a/.github/scripts/gcs/utils.sh +++ b/.github/scripts/gcs/utils.sh @@ -69,4 +69,12 @@ function gcloud_login { echo "${google_json_key_data:-}" > "${keyfile}" gcloud auth activate-service-account --key-file="${keyfile}" gcloud config set project "${google_project:-}" +} + + +function read_bucket_name { + # mirrors TMP_DIR in .github/scripts/gcs/Makefile + local gcs_tmp_dir="/tmp/storage-cli-gcs-${GITHUB_RUN_ID:-${USER}}" + local storage_class="${1:?storage class required}" + cat "${gcs_tmp_dir}/${storage_class}.lock" } \ No newline at end of file diff --git a/.github/workflows/benchmark-manual.yml b/.github/workflows/benchmark-manual.yml new file mode 100644 index 00000000..c4043a81 --- /dev/null +++ b/.github/workflows/benchmark-manual.yml @@ -0,0 +1,104 @@ +name: Benchmark Manual + +on: + workflow_dispatch: + inputs: + profile: + description: "Benchmark profile" + required: true + type: choice + default: quick + options: + - quick + - full + provider: + description: "Storage provider" + required: true + type: choice + default: s3 + options: + - s3 + - gcs + - azurebs + - alioss + +jobs: + benchmarking: + name: Run Benchmarking + runs-on: [ubuntu-latest] + env: + BENCHMARK_CONFIG_FILE: /tmp/benchmark-config.json + steps: + - name: Checkout Into Source Code + uses: actions/checkout@v7 + + - name: Set up Go + uses: actions/setup-go@v7 + with: + go-version-file: go.mod + + - name: Build benchmarker + run: | + echo "Building Benchmarker." + GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -tags benchmark -o "benchmarker" ./benchmark/ + + - name: Build storage-cli + run: | + echo "Building Storage CLI." + GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -ldflags "-X main.version=0.0.0" -o "storage-cli" . + + - name: Set up Aliyun CLI + if: inputs.provider == 'alioss' + uses: aliyun/setup-aliyun-cli-action@v1 + + - name: Setup The Environment + run: | + export aws_access_key_id="${{ secrets.AWS_ACCESS_KEY_ID }}" + export aws_secret_access_key="${{ secrets.AWS_SECRET_ACCESS_KEY }}" + echo "${{ secrets.GCP_SERVICE_ACCOUNT_BASE64 }}" | base64 -d > /tmp/gcp-key.json + export google_json_key_data="$(cat /tmp/gcp-key.json)" + export azure_storage_account="${{ secrets.AZURE_STORAGE_ACCOUNT }}" + export azure_storage_key="${{ secrets.AZURE_STORAGE_KEY }}" + export alioss_access_key_id="${{ secrets.ALI_ACCESS_KEY_ID }}" + export alioss_access_key_secret="${{ secrets.ALI_ACCESS_KEY_SECRET }}" + export provider="${{ inputs.provider }}" + export benchmark_config_file="${{ env.BENCHMARK_CONFIG_FILE }}" + ./.github/scripts/benchmark/setup.sh + + - name: Run Benchmark + run: | + mkdir -p "${GITHUB_WORKSPACE}/benchmark-results" + ./benchmarker \ + -candidate ./storage-cli \ + -provider "${{ inputs.provider }}" \ + -config "${{ env.BENCHMARK_CONFIG_FILE }}" \ + -profile "${{ inputs.profile }}" \ + -output "${GITHUB_WORKSPACE}/benchmark-results" + + - name: Upload Artifacts + if: always() + uses: actions/upload-artifact@v7 + with: + name: benchmark-results-${{ inputs.provider }}-${{ inputs.profile }} + path: ${{ github.workspace }}/benchmark-results/ + retention-days: 90 + + - name: Append Summary + if: always() + run: | + cat "${GITHUB_WORKSPACE}/benchmark-results/summary.md" >> "${GITHUB_STEP_SUMMARY}" + + - name: Teardown The Environment + if: always() + run: | + export aws_access_key_id="${{ secrets.AWS_ACCESS_KEY_ID }}" + export aws_secret_access_key="${{ secrets.AWS_SECRET_ACCESS_KEY }}" + echo "${{ secrets.GCP_SERVICE_ACCOUNT_BASE64 }}" | base64 -d > /tmp/gcp-key.json + export google_json_key_data="$(cat /tmp/gcp-key.json)" + export azure_storage_account="${{ secrets.AZURE_STORAGE_ACCOUNT }}" + export azure_storage_key="${{ secrets.AZURE_STORAGE_KEY }}" + export alioss_access_key_id="${{ secrets.ALI_ACCESS_KEY_ID }}" + export alioss_access_key_secret="${{ secrets.ALI_ACCESS_KEY_SECRET }}" + export provider="${{ inputs.provider }}" + export benchmark_config_file="${{ env.BENCHMARK_CONFIG_FILE }}" + ./.github/scripts/benchmark/teardown.sh \ No newline at end of file diff --git a/.github/workflows/release-cron.yml b/.github/workflows/release-cron.yml index 840a7481..b9a0c545 100644 --- a/.github/workflows/release-cron.yml +++ b/.github/workflows/release-cron.yml @@ -116,6 +116,8 @@ jobs: needs: [check, tests, integration-s3, integration-gcs, integration-azurebs, integration-alioss, integration-dav] if: needs.tests.result == 'success' runs-on: ubuntu-latest + outputs: + version_number: ${{ steps.version.outputs.VERSION_NUMBER }} steps: - name: Checkout Into Source Code uses: actions/checkout@v7 @@ -201,3 +203,86 @@ jobs: draft: false makeLatest: true generateReleaseNotes: true + + benchmark: + name: Performance Benchmark - ${{ matrix.provider }} + needs: [build-and-release] + if: needs.build-and-release.result == 'success' + runs-on: ubuntu-latest + env: + BENCHMARK_CONFIG_FILE: /tmp/benchmark-config.json + strategy: + fail-fast: false + matrix: + provider: [s3, gcs, azurebs, alioss] + steps: + - name: Checkout Into Source Code + uses: actions/checkout@v7 + + - name: Set up Go + uses: actions/setup-go@v7 + with: + go-version-file: go.mod + + - name: Set up Aliyun CLI + uses: aliyun/setup-aliyun-cli-action@v1 + + - name: Build benchmarker + run: | + GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -tags benchmark -o "benchmarker" ./benchmark/ + + - name: Build storage-cli + run: | + GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -ldflags "-X main.version=${{ needs.build-and-release.outputs.version_number }}" -o "storage-cli" . + + - name: Setup The Environment + run: | + export aws_access_key_id="${{ secrets.AWS_ACCESS_KEY_ID }}" + export aws_secret_access_key="${{ secrets.AWS_SECRET_ACCESS_KEY }}" + echo "${{ secrets.GCP_SERVICE_ACCOUNT_BASE64 }}" | base64 -d > /tmp/gcp-key.json + export google_json_key_data="$(cat /tmp/gcp-key.json)" + export azure_storage_account="${{ secrets.AZURE_STORAGE_ACCOUNT }}" + export azure_storage_key="${{ secrets.AZURE_STORAGE_KEY }}" + export alioss_access_key_id="${{ secrets.ALI_ACCESS_KEY_ID }}" + export alioss_access_key_secret="${{ secrets.ALI_ACCESS_KEY_SECRET }}" + export provider="${{ matrix.provider }}" + export benchmark_config_file="${{ env.BENCHMARK_CONFIG_FILE }}" + ./.github/scripts/benchmark/setup.sh + + - name: Run Benchmark + run: | + mkdir -p "${GITHUB_WORKSPACE}/benchmark-results" + ./benchmarker \ + -candidate ./storage-cli \ + -provider "${{ matrix.provider }}" \ + -config "${BENCHMARK_CONFIG_FILE}" \ + -profile full \ + -output "${GITHUB_WORKSPACE}/benchmark-results" + + - name: Upload Artifacts + if: always() + uses: actions/upload-artifact@v7 + with: + name: benchmark-results-${{ matrix.provider }}-full + path: ${{ github.workspace }}/benchmark-results/ + retention-days: 90 + + - name: Append Summary + if: always() + run: | + cat "${GITHUB_WORKSPACE}/benchmark-results/summary.md" >> "${GITHUB_STEP_SUMMARY}" + + - name: Teardown The Environment + if: always() + run: | + export aws_access_key_id="${{ secrets.AWS_ACCESS_KEY_ID }}" + export aws_secret_access_key="${{ secrets.AWS_SECRET_ACCESS_KEY }}" + export azure_storage_account="${{ secrets.AZURE_STORAGE_ACCOUNT }}" + export azure_storage_key="${{ secrets.AZURE_STORAGE_KEY }}" + export alioss_access_key_id="${{ secrets.ALI_ACCESS_KEY_ID }}" + export alioss_access_key_secret="${{ secrets.ALI_ACCESS_KEY_SECRET }}" + echo "${{ secrets.GCP_SERVICE_ACCOUNT_BASE64 }}" | base64 -d > /tmp/gcp-key.json + export google_json_key_data="$(cat /tmp/gcp-key.json)" + export provider="${{ matrix.provider }}" + export benchmark_config_file="${{ env.BENCHMARK_CONFIG_FILE }}" + ./.github/scripts/benchmark/teardown.sh diff --git a/benchmark/README.md b/benchmark/README.md new file mode 100644 index 00000000..92dde385 --- /dev/null +++ b/benchmark/README.md @@ -0,0 +1,94 @@ +# Benchmarker +Standalone benchmark harness for storage-cli. Measures wall-clock transfer performance of the CLI binary against real provider buckets. + +## How it works +The harness invokes the real `storage-cli` binary as a subprocess for every `put` and `get`, measures elapsed wall-clock time, and verifies downloaded content against a SHA-256 computed before the timer starts. This exercises flags, config parsing, client construction, SDK behaviour, and exit status at exactly the CLI boundary used by Cloud Controller. + +Failed or timed-out samples are retained in the output and mark the run as failed; they never appear as successful timing samples. + +## Test Profiles +| Profile | Object sizes | Samples | When to use | +|---------|-------------|---------|-------------| +| `quick` | 1 KiB, 1 MiB, 10 MiB, 128 MiB | 3 measured round trips per size | Fast sanity check on relevant changes — covers latency-dominated (1 KiB) through large-object (128 MiB) transfers without a warm-up | +| `full` | 1 KiB, 1 MiB, 10 MiB, 128 MiB, 1 GiB | 5 measured round trips per size | Weekly runs, pre-release validation, and investigations — the extra 1 GiB size exercises transfers well beyond provider multipart/chunk thresholds (Azure 32 MiB, GCS 100 MiB, S3 5 MiB parts) | + +## Usage +```sh +# Build the harness +go build -tags benchmark -o benchmarker ./benchmarks + +# Quick run against AWS S3 +./benchmarker \ + -candidate ./storage-cli-linux-amd64 \ + -provider s3 \ + -config /path/to/config-s3.json \ + -profile quick \ + -output ./results/s3 + +# Full run with baseline comparison against AWS ceph +./benchmarker \ + -candidate ./storage-cli-linux-amd64 \ + -baseline ./storage-cli-1.0.12-linux-amd64 \ + -provider s3 \ + -config /path/to/config-s3.json \ + -backend-label ceph \ + -profile full \ + -timeout 20m \ + -namespace benchmark/my-run \ + -output ./results/ceph +``` + +## Flags +| Flag | Default | Description | +|------|---------|-------------| +| `-candidate` | *(required)* | Path to the CLI binary under test | +| `-baseline` | *(none)* | Path to a previous release binary for comparison | +| `-provider` | *(required)* | CLI storage selector: `s3`, `gcs`, `azurebs`, `alioss` | +| `-config` | *(required)* | Path to provider JSON configuration file | +| `-backend-label` | *(same as -provider)* | Label used in reports to identify the actual storage backend. Defaults to the provider name. Set explicitly when the backend differs, e.g. `-provider s3 -backend-label ceph` | +| `-profile` | `quick` | `quick` or `full` | +| `-timeout` | `10m` | Per-operation timeout applied to each put and get. Must include a unit, e.g. `30s`, `10m`, `1h` | +| `-namespace` | `benchmark` | Top-level prefix for every object key written to the bucket, e.g. `benchmark/1731.../s3/candidate/1KiB/sample-0`. Used to group benchmark objects so bucket lifecycle rules and cleanup stay scoped away from other data in the same bucket. | +| `-run-id` | *(timestamp)* | Unique identifier embedded in all object keys | +| `-output` | `.` | Directory for output files | + +## Output Files +| File | Contents | +|------|----------| +| `results.json` | Raw per-sample timings, statuses, and metadata | +| `summary.md` | Markdown table with median latency and throughput; suitable for GitHub job summary | + +## Configuration Files +Provider configuration is read from the JSON file specified by `-config`. The format is the same as the file CC passes to the CLI (provider-specific). +Example structure (AWS S3 — exact field names match the `s3/config` package): + +```json +{ + "access_key_id": "...", + "secret_access_key": "...", + "bucket_name": "my-benchmark-bucket", + "region": "eu-central-1" +} +``` + +## Infrastructure Requirements +- Use a dedicated benchmark bucket or container, not a production or shared integration bucket. The harness deletes objects it creates, but lifecycle expiration should be configured as a backstop for interrupted runs. +- The IAM/service-account used for benchmarking needs only object-level read/write/delete on the dedicated bucket; no bucket-management permissions are required. +- Object keys are namespaced as `///