From 0b754c6fc3ae17e2f0e96eb61b58eb5e6a4bf60c Mon Sep 17 00:00:00 2001 From: Shradha Katyal Date: Tue, 8 Sep 2026 06:22:50 +0000 Subject: [PATCH 1/5] ci(storage): add GCS read microbenchmark runner and Cloud Build config Add automated Cloud Build CI workflow and test runner script for high-bandwidth GCS DirectPath read microbenchmarks against rapid zonal buckets. - Add benchmarks-cloudbuild.yaml to orchestrate VM lifecycle and private SSH runs. - Add run_benchmark_tests.sh for VM environment setup and pytest execution. - Display formatted performance results table directly in Cloud Build logs. --- .../cloudbuild/benchmarks-cloudbuild.yaml | 163 ++++++++++++++++++ .../cloudbuild/run_benchmark_tests.sh | 155 +++++++++++++++++ 2 files changed, 318 insertions(+) create mode 100644 packages/google-cloud-storage/cloudbuild/benchmarks-cloudbuild.yaml create mode 100755 packages/google-cloud-storage/cloudbuild/run_benchmark_tests.sh diff --git a/packages/google-cloud-storage/cloudbuild/benchmarks-cloudbuild.yaml b/packages/google-cloud-storage/cloudbuild/benchmarks-cloudbuild.yaml new file mode 100644 index 000000000000..e7c007434fe3 --- /dev/null +++ b/packages/google-cloud-storage/cloudbuild/benchmarks-cloudbuild.yaml @@ -0,0 +1,163 @@ +substitutions: + _ZONE: "us-west4-a" + _VM_NAME: "gcs-benchmark-runner-us-west4-a" + _ULIMIT: "65536" + _PROCESSES: "48" + _COROS: "1" + _FILE_SIZE_MIB: "10240" + _CHUNK_SIZE_KIB: "102400" + _ROUNDS: "2" + _ZONAL_BUCKET: "gcs-read-bench-zb-us-west4-a" + _PR_NUMBER: "" + _REPO: "googleapis/google-cloud-python" + +steps: + # Step 0: Generate a temporary SSH key for this build run and register with OS Login + - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" + id: "generate-ssh-key" + entrypoint: "bash" + args: + - "-c" + - | + mkdir -p /workspace/.ssh + ssh-keygen -t rsa -f /workspace/.ssh/google_compute_engine -N '' -C gcb + cat /workspace/.ssh/google_compute_engine.pub > /workspace/gcb_ssh_key.pub + gcloud compute os-login ssh-keys add \ + --key-file=/workspace/.ssh/google_compute_engine.pub \ + --ttl=1h + waitFor: ["-"] + + # Step 1: Package google-cloud-storage directory for direct transfer to VM + - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" + id: "package-code" + entrypoint: "bash" + args: + - "-c" + - | + tar --exclude='.nox' --exclude='venv_*' --exclude='.pytest_cache' --exclude='__pycache__' --exclude='.git' \ + -czf /workspace/google-cloud-storage.tar.gz -C /workspace/packages google-cloud-storage + waitFor: ["-"] + + # Step 2: Start the standing high-bandwidth VM + - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" + id: "start-vm" + entrypoint: "bash" + args: + - "-c" + - | + echo "Starting standing VM ${_VM_NAME} in zone ${_ZONE}..." + gcloud compute instances start "${_VM_NAME}" --zone="${_ZONE}" + waitFor: ["-"] + + # Step 3: Run the benchmark directly on the VM via private internal IP SSH, fetch results, and stop the VM + - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" + id: "run-benchmark-on-vm" + entrypoint: "bash" + args: + - "-c" + - | + set -e + echo "Waiting for VM ${_VM_NAME} to become accessible over internal SSH..." + for i in $(seq 1 20); do + if gcloud compute ssh "${_VM_NAME}" --zone="${_ZONE}" --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine --command="echo VM is ready" 2>/dev/null; then + echo "VM internal SSH connection established successfully." + break + fi + echo "Waiting for VM internal SSH availability... (attempt $$i/20)" + sleep 10 + done + + echo "Copying package archive and runner script to VM over internal IP..." + gcloud compute scp /workspace/google-cloud-storage.tar.gz \ + packages/google-cloud-storage/cloudbuild/run_benchmark_tests.sh \ + "${_VM_NAME}":~ --zone="${_ZONE}" --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine + + echo "Executing benchmark test suite directly on VM via SSH..." + set +e + gcloud compute ssh "${_VM_NAME}" --zone="${_ZONE}" --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine \ + --command="tar -xzf google-cloud-storage.tar.gz && cd google-cloud-storage && ulimit -n ${_ULIMIT}; PROCESSES=${_PROCESSES} COROS=${_COROS} FILE_SIZE_MIB=${_FILE_SIZE_MIB} CHUNK_SIZE_KIB=${_CHUNK_SIZE_KIB} ROUNDS=${_ROUNDS} TARGET_BUCKET=${_ZONAL_BUCKET} bash cloudbuild/run_benchmark_tests.sh" + TEST_EXIT_CODE=$? + set -e + + # Copy JSON report back from VM to Cloud Build workspace + mkdir -p /workspace/report + echo "Fetching benchmark result JSON from VM..." + gcloud compute scp "${_VM_NAME}":~/bench_result.json /workspace/report/bench_result.json \ + --zone="${_ZONE}" --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine 2>/dev/null || true + + # Turn off the standing VM to save quota and cost + echo "Stopping VM ${_VM_NAME}..." + gcloud compute instances stop "${_VM_NAME}" --zone="${_ZONE}" --quiet || true + + exit $$TEST_EXIT_CODE + waitFor: + - "start-vm" + - "generate-ssh-key" + - "package-code" + + # Step 4: Display benchmark performance summary table in Cloud Build logs + - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" + id: "display-benchmark-results" + entrypoint: "python3" + args: + - "-c" + - | + import json, os, sys + + result_path = "/workspace/report/bench_result.json" + if not os.path.exists(result_path): + print("ERROR: Benchmark result file not found at " + result_path, file=sys.stderr) + sys.exit(1) + + with open(result_path) as f: + data = json.load(f) + + benchmarks = data.get("benchmarks", []) + if not benchmarks: + print("No benchmarks found in result file.") + sys.exit(0) + + print("\n" + "=" * 88) + print(" GCS DIRECTPATH READ BENCHMARK PERFORMANCE RESULTS") + print("=" * 88) + header = f"| {'Workload Pattern':<36} | {'Avg Throughput':<17} | {'Network Bandwidth':<22} | {'CPU Usage':<9} |" + print(header) + print("|" + "-" * 38 + "|" + "-" * 19 + "|" + "-" * 24 + "|" + "-" * 11 + "|") + for b in benchmarks: + name = b.get("name", "").replace("test_downloads_multi_proc_multi_coro[", "").replace("]", "") + extra = b.get("extra_info", {}) + avg_mib = extra.get("avg_throughput_mib_s", "N/A") + net_mb = extra.get("net_throughput_mb_s") + if net_mb: + try: + net_str = f"{float(net_mb):,.1f} MB/s ({float(net_mb)*0.008:.1f} Gbps)" + except Exception: + net_str = str(net_mb) + else: + net_str = "N/A" + cpu = extra.get("cpu_max_global", "N/A") + print(f"| {name:<36} | {str(avg_mib) + ' MiB/s':<17} | {net_str:<22} | {str(cpu):<9} |") + print("=" * 88 + "\n") + waitFor: + - "run-benchmark-on-vm" + + # Step 5: Clean up SSH key from OS Login profile + - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" + id: "cleanup-ssh-key" + entrypoint: "bash" + args: + - "-c" + - | + echo "Removing temporary build SSH key from OS Login profile..." + gcloud compute os-login ssh-keys remove \ + --key-file=/workspace/gcb_ssh_key.pub || true + waitFor: + - "display-benchmark-results" + +timeout: "3600s" + +options: + logging: CLOUD_LOGGING_ONLY + dynamicSubstitutions: true + pool: + name: "projects/${PROJECT_ID}/locations/us-west4/workerPools/benchmark-worker-pool" diff --git a/packages/google-cloud-storage/cloudbuild/run_benchmark_tests.sh b/packages/google-cloud-storage/cloudbuild/run_benchmark_tests.sh new file mode 100755 index 000000000000..efb03a1f2c53 --- /dev/null +++ b/packages/google-cloud-storage/cloudbuild/run_benchmark_tests.sh @@ -0,0 +1,155 @@ +#!/bin/bash +# ============================================================================== +# Automated Google Cloud Storage Read Microbenchmark Runner +# Intended for GitHub CI/CD & GCE High-Bandwidth Tier-1 VMs (C4/N2/C3 series) +# Location: packages/google-cloud-storage/cloudbuild/run_benchmark_tests.sh +# ============================================================================== + +set -eo pipefail + +# Configurable defaults +PROCESSES="${PROCESSES:-48}" +COROS="${COROS:-1}" +FILE_SIZE_MIB="${FILE_SIZE_MIB:-10240}" # 10 GiB files by default +CHUNK_SIZE_KIB="${CHUNK_SIZE_KIB:-102400}" # ~100 MiB read chunks by default +ROUNDS="${ROUNDS:-2}" # Run benchmark 2 rounds by default +BUCKET_TYPE="${BUCKET_TYPE:-zonal}" # "zonal" uses BidiReadObject gRPC DirectPath +TARGET_BUCKET="${DEFAULT_RAPID_ZONAL_BUCKET:-gcs-read-bench-zb-us-west4-a}" +OUT_JSON="${OUT_JSON:-${HOME:-/tmp}/bench_result.json}" +UPLOAD_GCS_PREFIX="${UPLOAD_GCS_PREFIX:-}" + +echo "========================================================================" +echo " GCS Read Microbenchmark Runner (gRPC BidiReadObject / REST)" +echo " Processes: ${PROCESSES}" +echo " Coroutines/proc: ${COROS}" +echo " File Size: ${FILE_SIZE_MIB} MiB" +echo " Chunk Size: ${CHUNK_SIZE_KIB} KiB" +echo " Rounds: ${ROUNDS}" +echo " Bucket Type: ${BUCKET_TYPE} (zonal = BidiReadObject gRPC DirectPath)" +echo " Target Bucket: gs://${TARGET_BUCKET}" +echo "========================================================================" + +# Ensure HOME is exported for gRPC / ALTS Application Default Credentials +export HOME="${HOME:-/root}" +export DEFAULT_RAPID_ZONAL_BUCKET="${TARGET_BUCKET}" +export DEFAULT_STANDARD_BUCKET="${TARGET_BUCKET}" +export USE_PRESEEDED_BENCHMARK_OBJECTS="1" + +# Determine repository root +REPO_ROOT="$(git rev-parse --show-toplevel 2>/dev/null || pwd)" +cd "${REPO_ROOT}/packages/google-cloud-storage" 2>/dev/null || cd "$(pwd)" + +echo "--- 1. Setting up Python environment ---" +# Ensure python3-pip and python3-venv are present on the VM +if ! command -v pip3 &>/dev/null || ! python3 -c "import venv" 2>/dev/null; then + echo "Installing python3-pip and python3-venv on VM..." + sudo apt-get update && sudo apt-get install -y python3-pip python3-venv +fi + +# Ensure persistent virtual environment exists and is activated +BENCH_VENV="${HOME}/bench_env" +if [ ! -d "${BENCH_VENV}" ]; then + echo "Creating virtual environment at ${BENCH_VENV}..." + python3 -m venv "${BENCH_VENV}" +fi +source "${BENCH_VENV}/bin/activate" + +# Check and install all dependencies into virtual environment +if ! python3 -c "import pytest, psutil, yaml, google.cloud.storage" 2>/dev/null; then + echo "Installing dependencies into virtual environment..." + pip install --upgrade pip + pip install -e ".[grpc,testing]" + pip install google-cloud-kms +fi + +# Ensure latest source code is linked +pip install --no-deps -e . + +CONFIG_PATH="tests/perf/microbenchmarks/time_based/reads/config.yaml" +if [ ! -f "${CONFIG_PATH}" ]; then + echo "ERROR: Could not find ${CONFIG_PATH}. Please run from google-cloud-storage root." + exit 1 +fi + +echo "--- 2. Updating ${CONFIG_PATH} parameters (rounds=${ROUNDS}) ---" +python3 -c " +import yaml +path = '${CONFIG_PATH}' +with open(path) as f: + d = yaml.safe_load(f) +if isinstance(d, dict): + common = d.get('common') + if isinstance(common, dict): + common['file_sizes_mib'] = [${FILE_SIZE_MIB}] + common['chunk_sizes_kib'] = [${CHUNK_SIZE_KIB}] + common['bucket_types'] = ['${BUCKET_TYPE}'] + common['rounds'] = int('${ROUNDS}') + workloads = d.get('workload') + if isinstance(workloads, list): + for w in workloads: + if isinstance(w, dict): + w['processes'] = [${PROCESSES}] + w['coros'] = [${COROS}] +with open(path, 'w') as f: + yaml.dump(d, f) +" + +# Patch config.py so 1-to-1 process-to-file indexing prevents 404 on multi-coroutine runs +sed -i 's/num_files = num_processes \* num_coros/num_files = num_processes/g' tests/perf/microbenchmarks/time_based/reads/config.py || true +sed -i 's/num_files = num_processes \* num_coros/num_files = num_processes/g' tests/perf/microbenchmarks/reads/config.py || true + +# Patch conftest.py at runtime on VM to use pre-seeded test objects and bypass 480GB re-upload +python3 -c " +path = 'tests/perf/microbenchmarks/conftest.py' +try: + with open(path) as f: + s = f.read() + if '_create_files(' in s: + s = s.replace('files_names = _create_files(\n params.num_files,\n params.bucket_name,\n params.bucket_type,\n params.file_size_bytes,\n )', 'files_names = [f\"fio-go_storage_fio.0.{i}\" for i in range(params.num_files)]') + with open(path, 'w') as f: + f.write(s) +except Exception as e: + print(f'Warning patching conftest.py: {e}') +" + +echo "--- 3. Executing pytest benchmark suite (${ROUNDS} rounds) ---" +rm -f "${OUT_JSON}" 2>/dev/null || true +python3 -m pytest --benchmark-json="${OUT_JSON}" \ + -rA \ + tests/perf/microbenchmarks/time_based/reads/test_reads.py + +if [ -s "${OUT_JSON}" ]; then + python3 -c " +import json +with open('${OUT_JSON}') as f: + d = json.load(f) +benchmarks = d.get('benchmarks', []) +print('\n' + '='*85) +print(' GCS DIRECTPATH READ BENCHMARK PERFORMANCE RESULTS') +print('='*85) +header = f'| {\"Workload Pattern\":<36} | {\"Avg Throughput\":<17} | {\"Network Bandwidth\":<22} | {\"CPU Usage\":<9} |' +print(header) +print('|' + '-'*38 + '|' + '-'*19 + '|' + '-'*24 + '|' + '-'*11 + '|') +for b in benchmarks: + name = b.get('name', '').replace('test_downloads_multi_proc_multi_coro[', '').replace(']', '') + extra = b.get('extra_info', {}) + avg_mib = extra.get('avg_throughput_mib_s', 'N/A') + net_mb = extra.get('net_throughput_mb_s') + if net_mb: + net_str = f'{float(net_mb):,.1f} MB/s ({float(net_mb)*0.008:.1f} Gbps)' + else: + net_str = 'N/A' + cpu = extra.get('cpu_max_global', 'N/A') + row = f'| {name:<36} | {avg_mib + \" MiB/s\":<17} | {net_str:<22} | {str(cpu):<9} |' + print(row) +print('='*85 + '\n') +" + + if [ -n "${UPLOAD_GCS_PREFIX}" ]; then + GCS_DEST="${UPLOAD_GCS_PREFIX}/test_result_$(hostname)_$(date +%s).json" + echo "Uploading JSON report to ${GCS_DEST}..." + gcloud storage cp "${OUT_JSON}" "${GCS_DEST}" + fi +fi + +echo "--- Benchmark Run Complete ---" From e682ba48fe3ead28a9b5ca6c6cea49d71cdf153c Mon Sep 17 00:00:00 2001 From: Shradha Katyal Date: Tue, 8 Sep 2026 13:34:20 +0000 Subject: [PATCH 2/5] fix(cloudbuild): address automated code review feedback on VM cleanup and type validation - Add EXIT trap to ensure standing VM shutdown on failure. - Check SSH connection success and fail if connection times out. - Add target existence validation when patching conftest.py. - Add defensive type validation (isinstance) for parsed JSON results. - Wrap float conversion for network throughput in try-except. --- .../cloudbuild/benchmarks-cloudbuild.yaml | 28 +++++++++++++++---- .../cloudbuild/run_benchmark_tests.sh | 25 ++++++++++++++--- 2 files changed, 44 insertions(+), 9 deletions(-) diff --git a/packages/google-cloud-storage/cloudbuild/benchmarks-cloudbuild.yaml b/packages/google-cloud-storage/cloudbuild/benchmarks-cloudbuild.yaml index e7c007434fe3..365f1e8f6d2c 100644 --- a/packages/google-cloud-storage/cloudbuild/benchmarks-cloudbuild.yaml +++ b/packages/google-cloud-storage/cloudbuild/benchmarks-cloudbuild.yaml @@ -57,16 +57,30 @@ steps: - "-c" - | set -e + cleanup() { + set +e + echo "Stopping VM ${_VM_NAME}..." + gcloud compute instances stop "${_VM_NAME}" --zone="${_ZONE}" --quiet + } + trap cleanup EXIT + echo "Waiting for VM ${_VM_NAME} to become accessible over internal SSH..." + SSH_READY=0 for i in $(seq 1 20); do if gcloud compute ssh "${_VM_NAME}" --zone="${_ZONE}" --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine --command="echo VM is ready" 2>/dev/null; then echo "VM internal SSH connection established successfully." + SSH_READY=1 break fi echo "Waiting for VM internal SSH availability... (attempt $$i/20)" sleep 10 done + if [ $$SSH_READY -ne 1 ]; then + echo "ERROR: VM internal SSH connection could not be established." >&2 + exit 1 + fi + echo "Copying package archive and runner script to VM over internal IP..." gcloud compute scp /workspace/google-cloud-storage.tar.gz \ packages/google-cloud-storage/cloudbuild/run_benchmark_tests.sh \ @@ -85,10 +99,6 @@ steps: gcloud compute scp "${_VM_NAME}":~/bench_result.json /workspace/report/bench_result.json \ --zone="${_ZONE}" --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine 2>/dev/null || true - # Turn off the standing VM to save quota and cost - echo "Stopping VM ${_VM_NAME}..." - gcloud compute instances stop "${_VM_NAME}" --zone="${_ZONE}" --quiet || true - exit $$TEST_EXIT_CODE waitFor: - "start-vm" @@ -112,8 +122,12 @@ steps: with open(result_path) as f: data = json.load(f) + if not isinstance(data, dict): + print("ERROR: Invalid JSON structure in benchmark result file.", file=sys.stderr) + sys.exit(1) + benchmarks = data.get("benchmarks", []) - if not benchmarks: + if not isinstance(benchmarks, list) or not benchmarks: print("No benchmarks found in result file.") sys.exit(0) @@ -124,8 +138,12 @@ steps: print(header) print("|" + "-" * 38 + "|" + "-" * 19 + "|" + "-" * 24 + "|" + "-" * 11 + "|") for b in benchmarks: + if not isinstance(b, dict): + continue name = b.get("name", "").replace("test_downloads_multi_proc_multi_coro[", "").replace("]", "") extra = b.get("extra_info", {}) + if not isinstance(extra, dict): + extra = {} avg_mib = extra.get("avg_throughput_mib_s", "N/A") net_mb = extra.get("net_throughput_mb_s") if net_mb: diff --git a/packages/google-cloud-storage/cloudbuild/run_benchmark_tests.sh b/packages/google-cloud-storage/cloudbuild/run_benchmark_tests.sh index efb03a1f2c53..2c523ce59996 100755 --- a/packages/google-cloud-storage/cloudbuild/run_benchmark_tests.sh +++ b/packages/google-cloud-storage/cloudbuild/run_benchmark_tests.sh @@ -105,7 +105,11 @@ try: with open(path) as f: s = f.read() if '_create_files(' in s: - s = s.replace('files_names = _create_files(\n params.num_files,\n params.bucket_name,\n params.bucket_type,\n params.file_size_bytes,\n )', 'files_names = [f\"fio-go_storage_fio.0.{i}\" for i in range(params.num_files)]') + target = 'files_names = _create_files(\n params.num_files,\n params.bucket_name,\n params.bucket_type,\n params.file_size_bytes,\n)' + replacement = 'files_names = [f\"fio-go_storage_fio.0.{i}\" for i in range(params.num_files)]' + if target not in s: + raise ValueError('Exact _create_files call signature not found in conftest.py') + s = s.replace(target, replacement) with open(path, 'w') as f: f.write(s) except Exception as e: @@ -120,10 +124,16 @@ python3 -m pytest --benchmark-json="${OUT_JSON}" \ if [ -s "${OUT_JSON}" ]; then python3 -c " -import json +import json, sys with open('${OUT_JSON}') as f: d = json.load(f) +if not isinstance(d, dict): + print('ERROR: Invalid JSON structure in benchmark result file.', file=sys.stderr) + sys.exit(1) benchmarks = d.get('benchmarks', []) +if not isinstance(benchmarks, list) or not benchmarks: + print('No benchmarks found in result file.') + sys.exit(0) print('\n' + '='*85) print(' GCS DIRECTPATH READ BENCHMARK PERFORMANCE RESULTS') print('='*85) @@ -131,16 +141,23 @@ header = f'| {\"Workload Pattern\":<36} | {\"Avg Throughput\":<17} | {\"Network print(header) print('|' + '-'*38 + '|' + '-'*19 + '|' + '-'*24 + '|' + '-'*11 + '|') for b in benchmarks: + if not isinstance(b, dict): + continue name = b.get('name', '').replace('test_downloads_multi_proc_multi_coro[', '').replace(']', '') extra = b.get('extra_info', {}) + if not isinstance(extra, dict): + extra = {} avg_mib = extra.get('avg_throughput_mib_s', 'N/A') net_mb = extra.get('net_throughput_mb_s') if net_mb: - net_str = f'{float(net_mb):,.1f} MB/s ({float(net_mb)*0.008:.1f} Gbps)' + try: + net_str = f'{float(net_mb):,.1f} MB/s ({float(net_mb)*0.008:.1f} Gbps)' + except Exception: + net_str = str(net_mb) else: net_str = 'N/A' cpu = extra.get('cpu_max_global', 'N/A') - row = f'| {name:<36} | {avg_mib + \" MiB/s\":<17} | {net_str:<22} | {str(cpu):<9} |' + row = f'| {name:<36} | {str(avg_mib) + \" MiB/s\":<17} | {net_str:<22} | {str(cpu):<9} |' print(row) print('='*85 + '\n') " From 6c0f88c1dee227a3d4db73b00c8f2d9f5cef9f50 Mon Sep 17 00:00:00 2001 From: Shradha Katyal Date: Thu, 10 Sep 2026 16:22:22 +0000 Subject: [PATCH 3/5] fix(cloudbuild): separate zonal and regional buckets for benchmark runner - Add _BUCKET_TYPE and _REGIONAL_BUCKET substitutions in benchmarks-cloudbuild.yaml. - Update run_benchmark_tests.sh to export distinct DEFAULT_RAPID_ZONAL_BUCKET and DEFAULT_STANDARD_BUCKET. - Pass BUCKET_TYPE, ZONAL_BUCKET, and REGIONAL_BUCKET to run_benchmark_tests.sh over SSH. --- .../cloudbuild/benchmarks-cloudbuild.yaml | 4 ++- .../cloudbuild/run_benchmark_tests.sh | 28 ++++++++++++++----- 2 files changed, 24 insertions(+), 8 deletions(-) diff --git a/packages/google-cloud-storage/cloudbuild/benchmarks-cloudbuild.yaml b/packages/google-cloud-storage/cloudbuild/benchmarks-cloudbuild.yaml index 365f1e8f6d2c..f598c605080d 100644 --- a/packages/google-cloud-storage/cloudbuild/benchmarks-cloudbuild.yaml +++ b/packages/google-cloud-storage/cloudbuild/benchmarks-cloudbuild.yaml @@ -7,7 +7,9 @@ substitutions: _FILE_SIZE_MIB: "10240" _CHUNK_SIZE_KIB: "102400" _ROUNDS: "2" + _BUCKET_TYPE: "zonal" _ZONAL_BUCKET: "gcs-read-bench-zb-us-west4-a" + _REGIONAL_BUCKET: "gcs-read-bench-rb-us-west4" _PR_NUMBER: "" _REPO: "googleapis/google-cloud-python" @@ -89,7 +91,7 @@ steps: echo "Executing benchmark test suite directly on VM via SSH..." set +e gcloud compute ssh "${_VM_NAME}" --zone="${_ZONE}" --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine \ - --command="tar -xzf google-cloud-storage.tar.gz && cd google-cloud-storage && ulimit -n ${_ULIMIT}; PROCESSES=${_PROCESSES} COROS=${_COROS} FILE_SIZE_MIB=${_FILE_SIZE_MIB} CHUNK_SIZE_KIB=${_CHUNK_SIZE_KIB} ROUNDS=${_ROUNDS} TARGET_BUCKET=${_ZONAL_BUCKET} bash cloudbuild/run_benchmark_tests.sh" + --command="tar -xzf google-cloud-storage.tar.gz && cd google-cloud-storage && ulimit -n ${_ULIMIT}; PROCESSES=${_PROCESSES} COROS=${_COROS} FILE_SIZE_MIB=${_FILE_SIZE_MIB} CHUNK_SIZE_KIB=${_CHUNK_SIZE_KIB} ROUNDS=${_ROUNDS} BUCKET_TYPE=${_BUCKET_TYPE} ZONAL_BUCKET=${_ZONAL_BUCKET} REGIONAL_BUCKET=${_REGIONAL_BUCKET} bash cloudbuild/run_benchmark_tests.sh" TEST_EXIT_CODE=$? set -e diff --git a/packages/google-cloud-storage/cloudbuild/run_benchmark_tests.sh b/packages/google-cloud-storage/cloudbuild/run_benchmark_tests.sh index 2c523ce59996..ca24cb20cf6e 100755 --- a/packages/google-cloud-storage/cloudbuild/run_benchmark_tests.sh +++ b/packages/google-cloud-storage/cloudbuild/run_benchmark_tests.sh @@ -13,8 +13,16 @@ COROS="${COROS:-1}" FILE_SIZE_MIB="${FILE_SIZE_MIB:-10240}" # 10 GiB files by default CHUNK_SIZE_KIB="${CHUNK_SIZE_KIB:-102400}" # ~100 MiB read chunks by default ROUNDS="${ROUNDS:-2}" # Run benchmark 2 rounds by default -BUCKET_TYPE="${BUCKET_TYPE:-zonal}" # "zonal" uses BidiReadObject gRPC DirectPath -TARGET_BUCKET="${DEFAULT_RAPID_ZONAL_BUCKET:-gcs-read-bench-zb-us-west4-a}" +BUCKET_TYPE="${BUCKET_TYPE:-zonal}" # "zonal" uses BidiReadObject gRPC DirectPath, "regional" uses REST/gRPC standard +ZONAL_BUCKET="${ZONAL_BUCKET:-${DEFAULT_RAPID_ZONAL_BUCKET:-gcs-read-bench-zb-us-west4-a}}" +REGIONAL_BUCKET="${REGIONAL_BUCKET:-${DEFAULT_STANDARD_BUCKET:-gcs-read-bench-rb-us-west4}}" +if [ -n "${TARGET_BUCKET:-}" ]; then + if [ "${BUCKET_TYPE}" = "regional" ]; then + REGIONAL_BUCKET="${TARGET_BUCKET}" + else + ZONAL_BUCKET="${TARGET_BUCKET}" + fi +fi OUT_JSON="${OUT_JSON:-${HOME:-/tmp}/bench_result.json}" UPLOAD_GCS_PREFIX="${UPLOAD_GCS_PREFIX:-}" @@ -25,14 +33,15 @@ echo " Coroutines/proc: ${COROS}" echo " File Size: ${FILE_SIZE_MIB} MiB" echo " Chunk Size: ${CHUNK_SIZE_KIB} KiB" echo " Rounds: ${ROUNDS}" -echo " Bucket Type: ${BUCKET_TYPE} (zonal = BidiReadObject gRPC DirectPath)" -echo " Target Bucket: gs://${TARGET_BUCKET}" +echo " Bucket Type: ${BUCKET_TYPE}" +echo " Zonal Bucket: gs://${ZONAL_BUCKET}" +echo " Regional Bucket: gs://${REGIONAL_BUCKET}" echo "========================================================================" # Ensure HOME is exported for gRPC / ALTS Application Default Credentials export HOME="${HOME:-/root}" -export DEFAULT_RAPID_ZONAL_BUCKET="${TARGET_BUCKET}" -export DEFAULT_STANDARD_BUCKET="${TARGET_BUCKET}" +export DEFAULT_RAPID_ZONAL_BUCKET="${ZONAL_BUCKET}" +export DEFAULT_STANDARD_BUCKET="${REGIONAL_BUCKET}" export USE_PRESEEDED_BENCHMARK_OBJECTS="1" # Determine repository root @@ -78,11 +87,16 @@ path = '${CONFIG_PATH}' with open(path) as f: d = yaml.safe_load(f) if isinstance(d, dict): + defaults = d.get('defaults') + if isinstance(defaults, dict): + defaults['DEFAULT_RAPID_ZONAL_BUCKET'] = '${ZONAL_BUCKET}' + defaults['DEFAULT_STANDARD_BUCKET'] = '${REGIONAL_BUCKET}' common = d.get('common') if isinstance(common, dict): common['file_sizes_mib'] = [${FILE_SIZE_MIB}] common['chunk_sizes_kib'] = [${CHUNK_SIZE_KIB}] - common['bucket_types'] = ['${BUCKET_TYPE}'] + b_types = [b.strip() for b in '${BUCKET_TYPE}'.split(',') if b.strip()] + common['bucket_types'] = b_types if b_types else ['zonal'] common['rounds'] = int('${ROUNDS}') workloads = d.get('workload') if isinstance(workloads, list): From 607cedd5f6c2bd2ffdc430f8bf7107b5662a37dd Mon Sep 17 00:00:00 2001 From: Shradha Katyal Date: Thu, 10 Sep 2026 16:34:32 +0000 Subject: [PATCH 4/5] fix(cloudbuild): address review comments on runner defaults and dependencies - Update default benchmark rounds from 2 to 3. - Rename OUT_JSON to OUTPUT_JSON_PATH. - Echo OUTPUT_JSON_PATH and UPLOAD_GCS_PREFIX in runner banner. - Remove redundant google-cloud-kms and duplicate editable pip install commands. --- .../cloudbuild/benchmarks-cloudbuild.yaml | 2 +- .../cloudbuild/run_benchmark_tests.sh | 36 +++++++++---------- 2 files changed, 18 insertions(+), 20 deletions(-) diff --git a/packages/google-cloud-storage/cloudbuild/benchmarks-cloudbuild.yaml b/packages/google-cloud-storage/cloudbuild/benchmarks-cloudbuild.yaml index f598c605080d..16dd00eb941d 100644 --- a/packages/google-cloud-storage/cloudbuild/benchmarks-cloudbuild.yaml +++ b/packages/google-cloud-storage/cloudbuild/benchmarks-cloudbuild.yaml @@ -6,7 +6,7 @@ substitutions: _COROS: "1" _FILE_SIZE_MIB: "10240" _CHUNK_SIZE_KIB: "102400" - _ROUNDS: "2" + _ROUNDS: "3" _BUCKET_TYPE: "zonal" _ZONAL_BUCKET: "gcs-read-bench-zb-us-west4-a" _REGIONAL_BUCKET: "gcs-read-bench-rb-us-west4" diff --git a/packages/google-cloud-storage/cloudbuild/run_benchmark_tests.sh b/packages/google-cloud-storage/cloudbuild/run_benchmark_tests.sh index ca24cb20cf6e..ff3e96856906 100755 --- a/packages/google-cloud-storage/cloudbuild/run_benchmark_tests.sh +++ b/packages/google-cloud-storage/cloudbuild/run_benchmark_tests.sh @@ -12,7 +12,7 @@ PROCESSES="${PROCESSES:-48}" COROS="${COROS:-1}" FILE_SIZE_MIB="${FILE_SIZE_MIB:-10240}" # 10 GiB files by default CHUNK_SIZE_KIB="${CHUNK_SIZE_KIB:-102400}" # ~100 MiB read chunks by default -ROUNDS="${ROUNDS:-2}" # Run benchmark 2 rounds by default +ROUNDS="${ROUNDS:-3}" # Run benchmark 3 rounds by default BUCKET_TYPE="${BUCKET_TYPE:-zonal}" # "zonal" uses BidiReadObject gRPC DirectPath, "regional" uses REST/gRPC standard ZONAL_BUCKET="${ZONAL_BUCKET:-${DEFAULT_RAPID_ZONAL_BUCKET:-gcs-read-bench-zb-us-west4-a}}" REGIONAL_BUCKET="${REGIONAL_BUCKET:-${DEFAULT_STANDARD_BUCKET:-gcs-read-bench-rb-us-west4}}" @@ -23,19 +23,21 @@ if [ -n "${TARGET_BUCKET:-}" ]; then ZONAL_BUCKET="${TARGET_BUCKET}" fi fi -OUT_JSON="${OUT_JSON:-${HOME:-/tmp}/bench_result.json}" +OUTPUT_JSON_PATH="${OUTPUT_JSON_PATH:-${OUT_JSON:-${HOME:-/tmp}/bench_result.json}}" UPLOAD_GCS_PREFIX="${UPLOAD_GCS_PREFIX:-}" echo "========================================================================" echo " GCS Read Microbenchmark Runner (gRPC BidiReadObject / REST)" -echo " Processes: ${PROCESSES}" -echo " Coroutines/proc: ${COROS}" -echo " File Size: ${FILE_SIZE_MIB} MiB" -echo " Chunk Size: ${CHUNK_SIZE_KIB} KiB" -echo " Rounds: ${ROUNDS}" -echo " Bucket Type: ${BUCKET_TYPE}" -echo " Zonal Bucket: gs://${ZONAL_BUCKET}" -echo " Regional Bucket: gs://${REGIONAL_BUCKET}" +echo " Processes: ${PROCESSES}" +echo " Coroutines/proc: ${COROS}" +echo " File Size: ${FILE_SIZE_MIB} MiB" +echo " Chunk Size: ${CHUNK_SIZE_KIB} KiB" +echo " Rounds: ${ROUNDS}" +echo " Bucket Type: ${BUCKET_TYPE}" +echo " Zonal Bucket: gs://${ZONAL_BUCKET}" +echo " Regional Bucket: gs://${REGIONAL_BUCKET}" +echo " Output JSON Path: ${OUTPUT_JSON_PATH}" +echo " Upload GCS Path: ${UPLOAD_GCS_PREFIX:-None}" echo "========================================================================" # Ensure HOME is exported for gRPC / ALTS Application Default Credentials @@ -68,12 +70,8 @@ if ! python3 -c "import pytest, psutil, yaml, google.cloud.storage" 2>/dev/null; echo "Installing dependencies into virtual environment..." pip install --upgrade pip pip install -e ".[grpc,testing]" - pip install google-cloud-kms fi -# Ensure latest source code is linked -pip install --no-deps -e . - CONFIG_PATH="tests/perf/microbenchmarks/time_based/reads/config.yaml" if [ ! -f "${CONFIG_PATH}" ]; then echo "ERROR: Could not find ${CONFIG_PATH}. Please run from google-cloud-storage root." @@ -131,15 +129,15 @@ except Exception as e: " echo "--- 3. Executing pytest benchmark suite (${ROUNDS} rounds) ---" -rm -f "${OUT_JSON}" 2>/dev/null || true -python3 -m pytest --benchmark-json="${OUT_JSON}" \ +rm -f "${OUTPUT_JSON_PATH}" 2>/dev/null || true +python3 -m pytest --benchmark-json="${OUTPUT_JSON_PATH}" \ -rA \ tests/perf/microbenchmarks/time_based/reads/test_reads.py -if [ -s "${OUT_JSON}" ]; then +if [ -s "${OUTPUT_JSON_PATH}" ]; then python3 -c " import json, sys -with open('${OUT_JSON}') as f: +with open('${OUTPUT_JSON_PATH}') as f: d = json.load(f) if not isinstance(d, dict): print('ERROR: Invalid JSON structure in benchmark result file.', file=sys.stderr) @@ -179,7 +177,7 @@ print('='*85 + '\n') if [ -n "${UPLOAD_GCS_PREFIX}" ]; then GCS_DEST="${UPLOAD_GCS_PREFIX}/test_result_$(hostname)_$(date +%s).json" echo "Uploading JSON report to ${GCS_DEST}..." - gcloud storage cp "${OUT_JSON}" "${GCS_DEST}" + gcloud storage cp "${OUTPUT_JSON_PATH}" "${GCS_DEST}" fi fi From 5c6c050ea5e35462b443da429edb99e85647d48e Mon Sep 17 00:00:00 2001 From: Shradha Katyal Date: Thu, 10 Sep 2026 16:43:30 +0000 Subject: [PATCH 5/5] refactor(cloudbuild): extract benchmark result formatting into standalone script - Add display_benchmark_results.py to format and display test results. - Invoke display_benchmark_results.py from run_benchmark_tests.sh. - Update Cloud Build Step 4 to reuse display_benchmark_results.py. --- .../cloudbuild/benchmarks-cloudbuild.yaml | 51 ++------------ .../cloudbuild/display_benchmark_results.py | 69 +++++++++++++++++++ .../cloudbuild/run_benchmark_tests.sh | 46 ++----------- 3 files changed, 80 insertions(+), 86 deletions(-) create mode 100644 packages/google-cloud-storage/cloudbuild/display_benchmark_results.py diff --git a/packages/google-cloud-storage/cloudbuild/benchmarks-cloudbuild.yaml b/packages/google-cloud-storage/cloudbuild/benchmarks-cloudbuild.yaml index 16dd00eb941d..2085762cbdc5 100644 --- a/packages/google-cloud-storage/cloudbuild/benchmarks-cloudbuild.yaml +++ b/packages/google-cloud-storage/cloudbuild/benchmarks-cloudbuild.yaml @@ -83,9 +83,10 @@ steps: exit 1 fi - echo "Copying package archive and runner script to VM over internal IP..." + echo "Copying package archive and runner scripts to VM over internal IP..." gcloud compute scp /workspace/google-cloud-storage.tar.gz \ packages/google-cloud-storage/cloudbuild/run_benchmark_tests.sh \ + packages/google-cloud-storage/cloudbuild/display_benchmark_results.py \ "${_VM_NAME}":~ --zone="${_ZONE}" --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine echo "Executing benchmark test suite directly on VM via SSH..." @@ -112,52 +113,8 @@ steps: id: "display-benchmark-results" entrypoint: "python3" args: - - "-c" - - | - import json, os, sys - - result_path = "/workspace/report/bench_result.json" - if not os.path.exists(result_path): - print("ERROR: Benchmark result file not found at " + result_path, file=sys.stderr) - sys.exit(1) - - with open(result_path) as f: - data = json.load(f) - - if not isinstance(data, dict): - print("ERROR: Invalid JSON structure in benchmark result file.", file=sys.stderr) - sys.exit(1) - - benchmarks = data.get("benchmarks", []) - if not isinstance(benchmarks, list) or not benchmarks: - print("No benchmarks found in result file.") - sys.exit(0) - - print("\n" + "=" * 88) - print(" GCS DIRECTPATH READ BENCHMARK PERFORMANCE RESULTS") - print("=" * 88) - header = f"| {'Workload Pattern':<36} | {'Avg Throughput':<17} | {'Network Bandwidth':<22} | {'CPU Usage':<9} |" - print(header) - print("|" + "-" * 38 + "|" + "-" * 19 + "|" + "-" * 24 + "|" + "-" * 11 + "|") - for b in benchmarks: - if not isinstance(b, dict): - continue - name = b.get("name", "").replace("test_downloads_multi_proc_multi_coro[", "").replace("]", "") - extra = b.get("extra_info", {}) - if not isinstance(extra, dict): - extra = {} - avg_mib = extra.get("avg_throughput_mib_s", "N/A") - net_mb = extra.get("net_throughput_mb_s") - if net_mb: - try: - net_str = f"{float(net_mb):,.1f} MB/s ({float(net_mb)*0.008:.1f} Gbps)" - except Exception: - net_str = str(net_mb) - else: - net_str = "N/A" - cpu = extra.get("cpu_max_global", "N/A") - print(f"| {name:<36} | {str(avg_mib) + ' MiB/s':<17} | {net_str:<22} | {str(cpu):<9} |") - print("=" * 88 + "\n") + - "packages/google-cloud-storage/cloudbuild/display_benchmark_results.py" + - "/workspace/report/bench_result.json" waitFor: - "run-benchmark-on-vm" diff --git a/packages/google-cloud-storage/cloudbuild/display_benchmark_results.py b/packages/google-cloud-storage/cloudbuild/display_benchmark_results.py new file mode 100644 index 000000000000..031dc1ce2fa9 --- /dev/null +++ b/packages/google-cloud-storage/cloudbuild/display_benchmark_results.py @@ -0,0 +1,69 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Helper script to format and display GCS benchmark performance results.""" + +import json +import os +import sys + + +def display_results(result_path: str) -> None: + """Reads benchmark JSON result and prints a formatted summary table.""" + if not os.path.exists(result_path): + print(f"ERROR: Benchmark result file not found at {result_path}", file=sys.stderr) + sys.exit(1) + + with open(result_path) as f: + data = json.load(f) + + if not isinstance(data, dict): + print("ERROR: Invalid JSON structure in benchmark result file.", file=sys.stderr) + sys.exit(1) + + benchmarks = data.get("benchmarks", []) + if not isinstance(benchmarks, list) or not benchmarks: + print("No benchmarks found in result file.") + sys.exit(0) + + print("\n" + "=" * 88) + print(" GCS DIRECTPATH READ BENCHMARK PERFORMANCE RESULTS") + print("=" * 88) + header = f"| {'Workload Pattern':<36} | {'Avg Throughput':<17} | {'Network Bandwidth':<22} | {'CPU Usage':<9} |" + print(header) + print("|" + "-" * 38 + "|" + "-" * 19 + "|" + "-" * 24 + "|" + "-" * 11 + "|") + for b in benchmarks: + if not isinstance(b, dict): + continue + name = b.get("name", "").replace("test_downloads_multi_proc_multi_coro[", "").replace("]", "") + extra = b.get("extra_info", {}) + if not isinstance(extra, dict): + extra = {} + avg_mib = extra.get("avg_throughput_mib_s", "N/A") + net_mb = extra.get("net_throughput_mb_s") + if net_mb: + try: + net_str = f"{float(net_mb):,.1f} MB/s ({float(net_mb)*0.008:.1f} Gbps)" + except Exception: + net_str = str(net_mb) + else: + net_str = "N/A" + cpu = extra.get("cpu_max_global", "N/A") + print(f"| {name:<36} | {str(avg_mib) + ' MiB/s':<17} | {net_str:<22} | {str(cpu):<9} |") + print("=" * 88 + "\n") + + +if __name__ == "__main__": + path = sys.argv[1] if len(sys.argv) > 1 else "/workspace/report/bench_result.json" + display_results(path) diff --git a/packages/google-cloud-storage/cloudbuild/run_benchmark_tests.sh b/packages/google-cloud-storage/cloudbuild/run_benchmark_tests.sh index ff3e96856906..0d1be176f0cd 100755 --- a/packages/google-cloud-storage/cloudbuild/run_benchmark_tests.sh +++ b/packages/google-cloud-storage/cloudbuild/run_benchmark_tests.sh @@ -46,7 +46,8 @@ export DEFAULT_RAPID_ZONAL_BUCKET="${ZONAL_BUCKET}" export DEFAULT_STANDARD_BUCKET="${REGIONAL_BUCKET}" export USE_PRESEEDED_BENCHMARK_OBJECTS="1" -# Determine repository root +# Determine script directory and repository root +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" REPO_ROOT="$(git rev-parse --show-toplevel 2>/dev/null || pwd)" cd "${REPO_ROOT}/packages/google-cloud-storage" 2>/dev/null || cd "$(pwd)" @@ -135,44 +136,11 @@ python3 -m pytest --benchmark-json="${OUTPUT_JSON_PATH}" \ tests/perf/microbenchmarks/time_based/reads/test_reads.py if [ -s "${OUTPUT_JSON_PATH}" ]; then - python3 -c " -import json, sys -with open('${OUTPUT_JSON_PATH}') as f: - d = json.load(f) -if not isinstance(d, dict): - print('ERROR: Invalid JSON structure in benchmark result file.', file=sys.stderr) - sys.exit(1) -benchmarks = d.get('benchmarks', []) -if not isinstance(benchmarks, list) or not benchmarks: - print('No benchmarks found in result file.') - sys.exit(0) -print('\n' + '='*85) -print(' GCS DIRECTPATH READ BENCHMARK PERFORMANCE RESULTS') -print('='*85) -header = f'| {\"Workload Pattern\":<36} | {\"Avg Throughput\":<17} | {\"Network Bandwidth\":<22} | {\"CPU Usage\":<9} |' -print(header) -print('|' + '-'*38 + '|' + '-'*19 + '|' + '-'*24 + '|' + '-'*11 + '|') -for b in benchmarks: - if not isinstance(b, dict): - continue - name = b.get('name', '').replace('test_downloads_multi_proc_multi_coro[', '').replace(']', '') - extra = b.get('extra_info', {}) - if not isinstance(extra, dict): - extra = {} - avg_mib = extra.get('avg_throughput_mib_s', 'N/A') - net_mb = extra.get('net_throughput_mb_s') - if net_mb: - try: - net_str = f'{float(net_mb):,.1f} MB/s ({float(net_mb)*0.008:.1f} Gbps)' - except Exception: - net_str = str(net_mb) - else: - net_str = 'N/A' - cpu = extra.get('cpu_max_global', 'N/A') - row = f'| {name:<36} | {str(avg_mib) + \" MiB/s\":<17} | {net_str:<22} | {str(cpu):<9} |' - print(row) -print('='*85 + '\n') -" + DISPLAY_SCRIPT="${SCRIPT_DIR}/display_benchmark_results.py" + if [ ! -f "${DISPLAY_SCRIPT}" ]; then + DISPLAY_SCRIPT="cloudbuild/display_benchmark_results.py" + fi + python3 "${DISPLAY_SCRIPT}" "${OUTPUT_JSON_PATH}" if [ -n "${UPLOAD_GCS_PREFIX}" ]; then GCS_DEST="${UPLOAD_GCS_PREFIX}/test_result_$(hostname)_$(date +%s).json"