From 877a7e41fd092a67ddfdc8688e4b73ae90c5b8b2 Mon Sep 17 00:00:00 2001 From: Tom Desair Date: Mon, 28 Sep 2026 21:52:09 +0200 Subject: [PATCH 1/7] feat: async chunk upload, dynamic part sizing for multi-TB uploads, and RUFH compliance improvements --- .gitignore | 1 + AGENTS.md | 1 + README.md | 9 +- docs/AZURE_BLOB_STORAGE.md | 14 +- docs/CONFORMITY_TESTING.md | 149 +- docs/S3_STORAGE.md | 15 +- scripts/tus_conformity_test.py | 1227 +++++++++++++++++ .../tus/server/TusFileUploadService.java | 23 + .../RufhAppendPatchRequestHandler.java | 7 + .../RufhCreationPostRequestHandler.java | 14 +- .../rufh/util/RufhInterimResponseUtil.java | 11 +- .../server/upload/UploadStorageService.java | 18 + .../azure/AzureBlobConcatenationService.java | 41 +- .../upload/azure/AzureBlobStorageService.java | 193 ++- .../tus/server/upload/azure/AzureUtils.java | 6 +- ...adLocalCachedStorageAndLockingService.java | 10 + .../upload/disk/DiskStorageService.java | 41 +- .../server/upload/s3/S3StorageService.java | 480 +++++-- .../upload/util/AsyncChunkUploader.java | 290 ++++ .../tus/server/AbstractITRufhProtocol.java | 89 ++ .../tus/server/TusFileUploadServiceTest.java | 16 + .../RufhAppendPatchRequestHandlerTest.java | 90 ++ .../RufhCreationPostRequestHandlerTest.java | 42 + .../util/RufhInterimResponseUtilTest.java | 23 +- .../upload/UploadStorageServiceTest.java | 3 + .../AzureBlobConcatenationServiceTest.java | 27 + .../azure/AzureBlobStorageServiceTest.java | 42 + .../server/upload/azure/AzureUtilsTest.java | 3 + .../azure/ITAzureBlobStorageService.java | 19 + ...calCachedStorageAndLockingServiceTest.java | 24 + .../upload/disk/DiskStorageServiceTest.java | 17 + .../upload/s3/S3StorageServiceTest.java | 421 ++++++ .../upload/util/AsyncChunkUploaderTest.java | 576 ++++++++ 33 files changed, 3728 insertions(+), 214 deletions(-) create mode 100755 scripts/tus_conformity_test.py create mode 100644 src/main/java/me/desair/tus/server/upload/util/AsyncChunkUploader.java create mode 100644 src/test/java/me/desair/tus/server/upload/util/AsyncChunkUploaderTest.java diff --git a/.gitignore b/.gitignore index 5a5a0675..5ffdb6d8 100644 --- a/.gitignore +++ b/.gitignore @@ -180,3 +180,4 @@ SFTP_STORAGE_ANALYSIS.md NFS_LOCKING.md RELEASE-REVIEW.md ASYNC_REVIEW.md +ASYNC_REVIEW_2.md diff --git a/AGENTS.md b/AGENTS.md index 6ff7fb7d..9a35c4e1 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -211,6 +211,7 @@ When updating the IETF protocol implementation for a new draft revision, follow ### 4. Conformity Test Suite Maintenance & Subagent Isolation Whenever a new draft revision of the RUFH specification is published, the repository's Python conformity test suite (`scripts/rufh_conformity_test.py`) MUST be reviewed and updated by a separate, dedicated subagent. - **Strict Isolation Rule**: The subagent tasked with updating `scripts/rufh_conformity_test.py` MUST ONLY consult the official IETF specification document (and RFC 9530) and MUST NOT inspect the Java server implementation code under `src/main/java/`. This ensures the conformity test suite remains an independent, unbiased specification benchmark. +- **Default Multi-Backend Execution**: When executing the conformity test suite (`scripts/rufh_conformity_test.py`), tests MUST be run by default against all three supported storage backend types (Disk: `/test/api/upload`, S3: `/test-s3/api/upload`, and Azure Blob: `/test-azure/api/upload`) as documented in [`docs/CONFORMITY_TESTING.md`](docs/CONFORMITY_TESTING.md). ### 5. Conformity Test Suite Audit — Repeatable Procedure Use this procedure to audit `scripts/rufh_conformity_test.py` against the current (or a new) specification revision. The goal is to identify untested MUST/SHOULD/MAY requirements and produce an actionable improvement report. diff --git a/README.md b/README.md index bc082a23..2fe648d2 100644 --- a/README.md +++ b/README.md @@ -191,11 +191,12 @@ After creating the object, you can configure it using the following methods: | `disableTusExtension(String)` | None | Disables a built-in extension (`creation`, `checksum`, `expiration`, `concatenation`, `termination`, `download`, `cors`). | | `withUploadIdFactory(UploadIdFactory)` | `UuidUploadIdFactory` | Custom ID generator for upload resources (e.g., `UuidUploadIdFactory` or `TimeBasedUploadIdFactory`). | | `withUploadCompletionListener(UploadCompletionListener)` | None | Registers a callback invoked immediately when an upload finishes transferring all bytes and is completed. | +| `withCloudUploadThreadPoolSize(int)` | `10` | Maximum number of worker threads used for asynchronous background chunk uploading in cloud storage backends (S3, Azure). | | `withJsonSerialization()` | Java serialization | Enables JSON serialization for upload metadata (`UploadInfo`). Jackson is bundled by default. | | `withUploadStorageService(UploadStorageService)` | `DiskStorageService` | Configures custom or cloud storage backend (`DiskStorageService`, `S3StorageService`, `AzureBlobStorageService`). | | `withUploadLockingService(UploadLockingService)` | `LeaseFileLockingService` | Configures custom or cloud locking backend (`LeaseFileLockingService`, `S3LockingService`, `AzureBlobLockingService`). | -The library provides filesystem-based storage (`DiskStorageService` / `LeaseFileLockingService`), S3-compatible object storage (`S3StorageService` / `S3LockingService`), and Azure Blob Storage (`AzureBlobStorageService` / `AzureBlobLockingService`). See the **[Disk & Network Storage Locking Guide](docs/DISK_BASED_LOCKING.md)**, **[S3 Storage Guide](docs/S3_STORAGE.md)**, and **[Azure Blob Storage Guide](docs/AZURE_BLOB_STORAGE.md)** for detailed instructions on multi-replica container deployments in Kubernetes, post-upload processing, and legacy locking opt-out. +The library provides filesystem-based storage (`DiskStorageService` / `LeaseFileLockingService`), S3-compatible object storage (`S3StorageService` / `S3LockingService`), and Azure Blob Storage (`AzureBlobStorageService` / `AzureBlobLockingService`). Cloud backends feature an asynchronous 3-slot chunk pipeline (`AsyncChunkUploader`) that overlaps client payload streaming with cloud staging in the background. See the **[Disk & Network Storage Locking Guide](docs/DISK_BASED_LOCKING.md)**, **[S3 Storage Guide](docs/S3_STORAGE.md)**, and **[Azure Blob Storage Guide](docs/AZURE_BLOB_STORAGE.md)** for detailed instructions on multi-replica container deployments in Kubernetes, post-upload processing, and legacy locking opt-out. ### 2. Receiving a Resumable Upload To process an upload request you have to pass the current `jakarta.servlet.http.HttpServletRequest` and `jakarta.servlet.http.HttpServletResponse` objects to the `me.desair.tus.server.TusFileUploadService.process()` method. Typical places were you can do this are inside Servlets, Filters or REST API Controllers. @@ -409,10 +410,10 @@ public TomcatServletWebServerFactory tomcatFactory(TusFileUploadService tusFileU ## Compatible Client Implementations & Conformity Testing This server implementation has been tested with: -- **Tus 1.0.0 Clients**: Tested with [Uppy](https://uppy.io/) and `tus-js-client`. -- **IETF Resumable Uploads Clients & Conformity Tests**: The implementation has been thoroughly tested with our own built-in RUFH conformity test suite (`scripts/rufh_conformity_test.py`) validating compliance with draft-12 of the RUFH protocol specification and RFC 9530 HTTP Digests, as well as the community [RUFH conformity tests from the IETF hackathon](https://github.com/tus/ietf-hackathon). +- **Tus 1.0.0 Clients & Conformity Tests**: Tested with [Uppy](https://uppy.io/), `tus-js-client`, and our built-in Tus v1.0.0 conformity test suite (`scripts/tus_conformity_test.py`) validating all core protocol mechanisms and protocol extensions (creation, creation-with-upload, checksum, termination, concatenation, and expiration) across Disk, S3, and Azure Blob backends. +- **IETF Resumable Uploads Clients & Conformity Tests**: The implementation has been thoroughly tested with our built-in RUFH conformity test suite (`scripts/rufh_conformity_test.py`) validating compliance with draft-12 of the RUFH protocol specification and RFC 9530 HTTP Digests, as well as the community [RUFH conformity tests from the IETF hackathon](https://github.com/tus/ietf-hackathon). -For detailed instructions on running our native conformity test suite and interpreting results, see the **[Conformity Testing Guide (docs/CONFORMITY_TESTING.md)](docs/CONFORMITY_TESTING.md)**. +For detailed instructions on running our native conformity test suites across all storage backends (Disk, S3, Azure Blob) and interpreting results, see the **[Conformity Testing Guide (docs/CONFORMITY_TESTING.md)](docs/CONFORMITY_TESTING.md)**. This repository also contains comprehensive automated integration test suites (`ITTusFileUploadService`, `RufhProtocolCreationTest`, `RufhProtocolAppendTest`, `RufhProtocolHeadTest`, `RufhProtocolCancellationTest`) validating both protocol specifications. diff --git a/docs/AZURE_BLOB_STORAGE.md b/docs/AZURE_BLOB_STORAGE.md index 99101956..7a1fb930 100644 --- a/docs/AZURE_BLOB_STORAGE.md +++ b/docs/AZURE_BLOB_STORAGE.md @@ -157,15 +157,23 @@ BlobContainerClient containerClient = new BlobContainerClientBuilder() --- -## 6. Local Disk Buffer & Block Size Auto-Calibration +## 6. Local Disk Buffer & Asynchronous Chunk Pipelining -`AzureBlobStorageService` streams incoming PATCH payloads in chunks of `optimalBlockSize` into temporary files, staging each block to Azure as it completes. Peak disk usage per upload is capped at `1 × optimalBlockSize` (e.g. 8 MB). +`AzureBlobStorageService` streams incoming PATCH payloads in chunks of `optimalBlockSize` into temporary files, pipelining block staging to Azure Blob Storage asynchronously via `AsyncChunkUploader`: +- **Asynchronous 3-Slot Pipeline**: + - **Slot 1 (Receiving)**: Streaming incoming bytes from the client into a local temporary file. + - **Slot 2 (Waiting)**: Holds one ready chunk on disk buffer. + - **Slot 3 (Uploading)**: Actively staging the block (`stageBlock`) to Azure Blob Storage on a background daemon worker thread. + - Falls back to synchronous caller execution (`CallerRunsPolicy` on a `SynchronousQueue`) when the thread pool is saturated, preventing worker queue latency and unbounded buffering. +- **Thread Pool Sizing**: Worker threads are managed via a shared executor, default 10 threads, configurable on `TusFileUploadService` via `.withCloudUploadThreadPoolSize(int)` or directly on `AzureBlobStorageService.setCloudUploadThreadPoolSize(int)`. +- **Bounded Disk Footprint**: Peak disk usage per active upload is bounded to at most $2 \times \text{optimalBlockSize}$ (one receiving slot + one waiting slot). Block sizes auto-calibrate based on total upload size: -- **Baseline Preferred Size**: 8 MB (configurable via constructor) +- **Baseline Preferred Size**: 8 MB (configurable via constructor or `setPreferredBlockSize(long)`) - **Minimum Block Size**: 4 MB - **Maximum Block Size**: 4000 MiB (Azure limit) - **Maximum Blocks per Blob**: 50,000 (Azure limit) +- **Multi-TB Support & Auto-Calibration**: When upload length exceeds 400 GB ($50,000 \times 8\text{ MB}$), block size automatically scales up proportionally (e.g. ~22 MB for 1 TB, ~110 MB for 5 TB, supporting blobs up to 190 TB) to ensure the upload finishes within Azure's 50,000 blocks ceiling. --- diff --git a/docs/CONFORMITY_TESTING.md b/docs/CONFORMITY_TESTING.md index 0255a0d8..660b9e4f 100644 --- a/docs/CONFORMITY_TESTING.md +++ b/docs/CONFORMITY_TESTING.md @@ -1,17 +1,19 @@ -# Conformity Testing Guide (IETF Resumable Uploads for HTTP) +# Protocol Conformity Testing Guide -This guide describes how to execute the RUFH (Resumable Uploads for HTTP) conformity tests against a locally running instance of the Spring Boot demo server or any RUFH compliant server endpoint. +This guide describes how to execute the automated conformity test suites for both supported resumable upload protocols: +1. **Tus v1.0.0 Protocol**: [`scripts/tus_conformity_test.py`](../scripts/tus_conformity_test.py) validating the official [Tus v1.0.0 Resumable Upload Protocol](https://tus.io/protocols/resumable-upload). +2. **IETF Resumable Uploads for HTTP (RUFH)**: [`scripts/rufh_conformity_test.py`](../scripts/rufh_conformity_test.py) validating [draft-ietf-httpbis-resumable-upload-12](https://www.ietf.org/archive/id/draft-ietf-httpbis-resumable-upload-12.txt) and [RFC 9530 HTTP Digests](https://www.rfc-editor.org/rfc/rfc9530.html). -The test suite validates compliance with [draft-ietf-httpbis-resumable-upload-12](https://www.ietf.org/archive/id/draft-ietf-httpbis-resumable-upload-12.txt) and [RFC 9530 HTTP Digests](https://www.rfc-editor.org/rfc/rfc9530.html). +Both conformity test suites perform end-to-end, socket-level protocol verification against a running server, ensuring strict adherence to HTTP status codes, structured response headers, error semantics, concurrency guarantees, and storage engine consistency. --- ## 1. Build and Install the Server Library -First, compile and install the core `tus-java-server` library to your local Maven repository: +First, compile and install the core `tus-java-server` library into your local Maven repository: ```bash -# In the root of the tus-java-server repository +# In the root directory of the tus-java-server repository mvn clean install -DskipTests ``` @@ -19,7 +21,7 @@ mvn clean install -DskipTests ## 2. Start the Demo Server -1. Verify the dependency in `tus-java-server-spring-demo` project's `spring-boot-rest/pom.xml` points to the snapshot version: +1. Verify that the dependency in `tus-java-server-spring-demo` project's `spring-boot-rest/pom.xml` references the snapshot version: ```xml me.desair.tus @@ -28,64 +30,157 @@ mvn clean install -DskipTests ``` -2. Build and start the Spring Boot REST demo server with a `1 KB` maximum upload size parameter (`--tus.server.max-upload-size=1024`) to enable full limit discovery & limit enforcement verification: +2. Build and start the Spring Boot REST demo server with a `1 KB` maximum upload size limit (`--tus.server.max-upload-size=1024`) to enable full limit discovery & limit enforcement verification: ```bash cd ../tus-java-server-spring-demo mvn clean package -DskipTests java -jar spring-boot-rest/target/spring-boot-rest-0.0.1-SNAPSHOT.jar --tus.server.max-upload-size=1024 ``` - The server will start on port `8080` with the upload endpoint exposed at: - `http://localhost:8080/test/api/upload` + The server will start on port `8080`, exposing upload endpoints for all three storage backends: + - **Disk Storage (Default)**: `http://localhost:8080/test/api/upload` + - **S3 Object Storage**: `http://localhost:8080/test-s3/api/upload` + - **Azure Blob Storage**: `http://localhost:8080/test-azure/api/upload` --- -## 3. Run the Built-In RUFH Conformity Test Suite +## 3. Python Prerequisites -The repository includes its own native Python conformity test suite located at `scripts/rufh_conformity_test.py`. It requires `pytest` and `requests`. - -### Prerequisites -Install Python dependencies if not already installed: +Install required Python test packages if not already available in your environment: ```bash pip install pytest requests ``` -### Running the Test Suite +--- + +## 4. Running the Tus v1.0.0 Conformity Test Suite + +The Tus v1.0.0 conformity test suite (`scripts/tus_conformity_test.py`) verifies 50 specification rules covering: +- **Core Protocol (§5 & §6)**: OPTIONS feature discovery, Tus-Resumable version handshake (412 Precondition Failed), HEAD offset retrieval, cache control, PATCH data append, Content-Type enforcement, offset mismatch prevention (409 Conflict), length bounds validation, and `X-HTTP-Method-Override`. +- **Creation Extension (§7.1)**: Upload-Length validation, Upload-Defer-Length handling, defer-to-known length transition, and Upload-Metadata base64 decoding. +- **Creation With Upload Extension (§7.2)**: Single-request POST creations with payload body, partial upload chunking, and content-type enforcement. +- **Expiration Extension (§7.3)**: Upload-Expires header validation formatted in RFC 9110 HTTP datetime format. +- **Checksum Extension (§7.4)**: Upload-Checksum verification (SHA-1), 460 Checksum Mismatch handling and payload discarding, and unsupported checksum algorithm rejection (400 Bad Request). +- **Termination Extension (§7.5)**: DELETE cancellation of in-progress and completed uploads and 404/410 verification. +- **Concatenation Extension (§7.6)**: Partial upload creation, final upload creation by merging partials, offset summation, and 403 Forbidden enforcement on final upload PATCH requests. +- **Concurrency & Workflows**: Concurrent PATCH race contention prevention at identical offset and end-to-end multi-chunk upload flows. + +### Running Against All Three Storage Backends + +The test suite **MUST be run against all three supported storage backend types** (Disk, S3, and Azure Blob) to ensure cross-engine protocol compliance: + +#### Option A: Standalone Python Runner (Custom Formatted Summary) + +```bash +# 1. Disk Storage Backend +python3 scripts/tus_conformity_test.py --url http://localhost:8080/test/api/upload + +# 2. S3 Object Storage Backend +python3 scripts/tus_conformity_test.py --url http://localhost:8080/test-s3/api/upload + +# 3. Azure Blob Storage Backend +python3 scripts/tus_conformity_test.py --url http://localhost:8080/test-azure/api/upload +``` + +Or execute all three backends in a single command loop: +```bash +for endpoint in /test/api/upload /test-s3/api/upload /test-azure/api/upload; do + echo "======================================================================" + echo " Running Tus v1.0.0 Conformity Tests: http://localhost:8080$endpoint" + echo "======================================================================" + python3 scripts/tus_conformity_test.py --url "http://localhost:8080$endpoint" +done +``` + +#### Option B: PyTest Runner + +```bash +# Execute via pytest +pytest scripts/tus_conformity_test.py --url http://localhost:8080/test/api/upload +pytest scripts/tus_conformity_test.py --url http://localhost:8080/test-s3/api/upload +pytest scripts/tus_conformity_test.py --url http://localhost:8080/test-azure/api/upload +``` + +--- + +## 5. Running the IETF RUFH Conformity Test Suite + +The RUFH conformity test suite (`scripts/rufh_conformity_test.py`) validates compliance with the official IETF Resumable Uploads for HTTP specification (`draft-ietf-httpbis-resumable-upload-12`) and RFC 9530 HTTP Digests. -You can execute the test suite using Python directly or via PyTest: +### Running Against All Three Storage Backends -#### Option A: Running directly with Python (Recommended for structured AI / Agent reporting) ```bash +# 1. Disk Storage Backend python3 scripts/rufh_conformity_test.py --url http://localhost:8080/test/api/upload + +# 2. S3 Object Storage Backend +python3 scripts/rufh_conformity_test.py --url http://localhost:8080/test-s3/api/upload + +# 3. Azure Blob Storage Backend +python3 scripts/rufh_conformity_test.py --url http://localhost:8080/test-azure/api/upload ``` -#### Option B: Running with PyTest +Or via PyTest: ```bash -pytest scripts/rufh_conformity_test.py --url http://localhost:8080/test/api/upload +for endpoint in /test/api/upload /test-s3/api/upload /test-azure/api/upload; do + echo "======================================================================" + echo " Running PyTest RUFH Conformity: http://localhost:8080$endpoint" + echo "======================================================================" + pytest scripts/rufh_conformity_test.py --url "http://localhost:8080$endpoint" +done ``` --- -## 4. Understanding Test Results & AI Agent Remediation +## 6. Understanding Test Results & AI Agent Remediation -When executed, the script produces a structured summary report detailing: +When executed directly via Python, both conformity test scripts produce a structured summary report detailing: 1. **Total Tests Executed**: Count of total specification compliance tests run. -2. **Passed Tests**: Number of tests matching draft-12 specification requirements. -3. **Failed Tests**: Detailed list of failing tests including test method names, exact error tracebacks, expected status codes/headers, and corresponding RFC section references. -4. **104 Interim Responses**: Count of tests where `HTTP/1.1 104 Upload Resumption Supported` interim responses were detected from the server socket. +2. **Passed Tests**: Number of tests matching specification requirements. +3. **Failed Tests**: Detailed list of failing tests including test method names, exact error tracebacks, expected status codes/headers, and corresponding specification section references. -AI agents and developers can analyze the detailed failure breakdown in the script's console output to pinpoint specific compliance gaps and adjust server logic accordingly. +### Example Output: +``` +====================================================================== + Tus v1.0.0 Resumable Upload Protocol Conformity Test Suite + Specification: https://tus.io/protocols/resumable-upload + Target Endpoint: http://localhost:8080/test/api/upload +====================================================================== +.................................................. [100%] +50 passed in 0.43s + +====================================================================== + CONFORMITY TEST RESULTS +====================================================================== + Total Tests Executed: 50 + Passed: 50 + Failed: 0 +====================================================================== + +[✓] ALL TUS V1.0.0 CONFORMITY TESTS PASSED SUCCESSFULLY! +``` + +If a test fails, the runner outputs a structured failure breakdown containing: +- Test function name and class +- Verbatim specification quote defining the required behavior +- Exact failure assertion and returned HTTP status or header values + +Developers and automated AI agents can use this failure breakdown to immediately diagnose compliance discrepancies and make targeted fixes in the server code. --- -## 5. Running Community (IETF Hackathon) Tests +## 7. Running Community (IETF Hackathon) Tests -Alternatively, you can also run the external community test suite from the `ietf-hackathon` repository: +Alternatively, you can also run the external community test suite from the `ietf-hackathon` repository across all three storage backends: ```bash git clone https://github.com/tus/ietf-hackathon.git cd ietf-hackathon/tests pip install -r requirements.txt + +# Run against Disk, S3, and Azure Blob endpoints pytest --url http://localhost:8080/test/api/upload +pytest --url http://localhost:8080/test-s3/api/upload +pytest --url http://localhost:8080/test-azure/api/upload ``` diff --git a/docs/S3_STORAGE.md b/docs/S3_STORAGE.md index 87a18626..14eff747 100644 --- a/docs/S3_STORAGE.md +++ b/docs/S3_STORAGE.md @@ -160,8 +160,19 @@ S3StorageService s3Storage = new S3StorageService(minioClient, "my-bucket"); S3 requires every part chunk of a multipart upload to be at least 5 MB (except the final part). -- **Disk Buffering**: `S3StorageService` buffers incoming bytes to local disk in chunks (default 50 MB) before uploading them to S3. -- **Incomplete Parts**: If a client upload stream ends before reaching 5 MB and the upload is not complete, the sub-5MB chunk is saved as a `/.part` object in S3. On the next `PATCH` request, this chunk is downloaded, prepended to the incoming stream, and upload proceeds seamlessly. +- **Disk Buffering & Preferred Part Size**: `S3StorageService` buffers incoming bytes to local disk in chunks of **8 MB** (`DEFAULT_PREFERRED_PART_SIZE`), matching Azure Blob Storage's block size. Preferred part size is configurable via `setPreferredPartSize(long)` (between 5 MB and 5 GB). +- **Multi-TB Support & Dynamic Part Size Auto-Calibration**: AWS S3 enforces a maximum ceiling of 10,000 parts per multipart upload. When an upload length exceeds 80 GB ($10,000 \times 8\text{ MB}$), `S3StorageService` automatically scales the part size up proportionally: + - **1 TB upload**: auto-calibrated to ~105 MB per part. + - **5 TB upload** (S3 maximum limit): auto-calibrated to ~525 MB per part. + - Guarantees the entire upload completes within 10,000 parts without exceeding S3's 5 GB maximum part limit. + - Peak local disk buffer usage remains bounded to at most $2 \times \text{optimalPartSize}$ (one receiving slot + one waiting slot). +- **Asynchronous Chunk Pipelining (`AsyncChunkUploader`)**: Instead of blocking the HTTP client thread while uploading chunks to S3, `S3StorageService` pipelines chunks using a bounded 3-slot model: + - **Slot 1 (Receiving)**: Streaming incoming bytes from the client into a local temporary file. + - **Slot 2 (Waiting)**: Holds one ready chunk on local disk buffer. + - **Slot 3 (Uploading)**: Actively uploading a chunk to S3 on a background daemon worker thread. + - When the background thread pool is fully utilized, work seamlessly falls back to the client thread without blocking (`ThreadPoolExecutor.CallerRunsPolicy` on a `SynchronousQueue`), ensuring zero queuing overhead and immediate backpressure. +- **Thread Pool Sizing**: Worker threads are managed via a shared executor, default 10 threads, configurable on `TusFileUploadService` via `.withCloudUploadThreadPoolSize(int)` or directly on `S3StorageService.setCloudUploadThreadPoolSize(int)`. +- **Incomplete Parts & Stale Buffer Protection**: If a client upload stream ends before reaching 5 MB and the upload is not complete, the sub-5MB chunk is saved as a `/.part` object in S3. On subsequent requests, an arithmetic budget guard ($\text{existingPartsTotalSize} + \text{size}(.part) == \text{length}$) verifies whether the `.part` represents a legitimate final part or an obsolete buffer from a previous attempt, preventing file corruption and size inflation. - **Configurable Temp Directory**: The temporary buffer directory can be configured in the constructor or builder: ```java diff --git a/scripts/tus_conformity_test.py b/scripts/tus_conformity_test.py new file mode 100755 index 00000000..78fb3b9c --- /dev/null +++ b/scripts/tus_conformity_test.py @@ -0,0 +1,1227 @@ +#!/usr/bin/env python3 +""" +Tus v1.0.0 Resumable Upload Protocol Conformity Test Suite + +This conformity test suite validates a server implementation against the +official Tus v1.0.0 Resumable Upload Protocol specification: + https://tus.io/protocols/resumable-upload + +Usage: + pytest scripts/tus_conformity_test.py --url http://localhost:8080/test/api/upload + python3 scripts/tus_conformity_test.py --url http://localhost:8080/test/api/upload +""" + +import argparse +import base64 +import email.utils +import hashlib +import os +import socket +import sys +import threading +import time +from urllib.parse import urlparse + +import pytest + +# Protocol Constants +TUS_RESUMABLE = "Tus-Resumable" +TUS_VERSION = "Tus-Version" +TUS_EXTENSION = "Tus-Extension" +TUS_MAX_SIZE = "Tus-Max-Size" +TUS_CHECKSUM_ALGORITHM = "Tus-Checksum-Algorithm" +UPLOAD_OFFSET = "Upload-Offset" +UPLOAD_LENGTH = "Upload-Length" +UPLOAD_DEFER_LENGTH = "Upload-Defer-Length" +UPLOAD_METADATA = "Upload-Metadata" +UPLOAD_CHECKSUM = "Upload-Checksum" +UPLOAD_CONCAT = "Upload-Concat" +UPLOAD_EXPIRES = "Upload-Expires" +LOCATION = "Location" +CONTENT_TYPE = "Content-Type" +CONTENT_LENGTH = "Content-Length" +CACHE_CONTROL = "Cache-Control" +X_HTTP_METHOD_OVERRIDE = "X-HTTP-Method-Override" +APPLICATION_OFFSET_OCTET_STREAM = "application/offset+octet-stream" +TUS_API_VERSION = "1.0.0" + + +def pytest_addoption(parser): + """Add command line options to pytest.""" + parser.addoption( + "--url", + action="store", + default="http://localhost:8080/test/api/upload", + help="Target Tus upload endpoint URL", + ) + + +@pytest.fixture(scope="session") +def target_url(request): + """Fixture providing the target upload URL.""" + try: + return request.config.getoption("--url") + except (ValueError, AttributeError): + return os.environ.get("TUS_URL", "http://localhost:8080/test/api/upload") + + +class CaseInsensitiveDict(dict): + """A case-insensitive dictionary for HTTP headers.""" + + def __init__(self, data=None, **kwargs): + super().__init__() + self._keys = {} + if data: + self.update(data) + if kwargs: + self.update(kwargs) + + def __setitem__(self, key, value): + super().__setitem__(key.lower(), value) + self._keys[key.lower()] = key + + def __getitem__(self, key): + return super().__getitem__(key.lower()) + + def __delitem__(self, key): + super().__delitem__(key.lower()) + del self._keys[key.lower()] + + def __contains__(self, key): + return super().__contains__(key.lower()) + + def get(self, key, default=None): + return super().get(key.lower(), default) + + def update(self, other=None, **kwargs): + if hasattr(other, "items"): + for k, v in other.items(): + self[k] = v + elif other: + for k, v in other: + self[k] = v + for k, v in kwargs.items(): + self[k] = v + + def items(self): + return ((self._keys[k], v) for k, v in super().items()) + + +def parse_headers(lines): + """Parse HTTP header lines into a CaseInsensitiveDict.""" + res_headers = CaseInsensitiveDict() + for line in lines: + if ":" in line: + k, v = line.split(":", 1) + res_headers[k.strip()] = v.strip() + return res_headers + + +def http_request(method, url, headers=None, body=None): + """ + Socket-based HTTP client helper for raw HTTP/1.1 request execution. + Returns (status_code, headers_dict, body_bytes). + """ + if headers is None: + headers = {} + parsed = urlparse(url) + host = parsed.hostname or "localhost" + port = parsed.port or (443 if parsed.scheme == "https" else 80) + path = parsed.path + ("?" + parsed.query if parsed.query else "") + if not path: + path = "/" + + s = socket.create_connection((host, port), timeout=10) + try: + req_headers = CaseInsensitiveDict(headers) + if "Connection" not in req_headers: + req_headers["Connection"] = "close" + + req_lines = [f"{method} {path} HTTP/1.1", f"Host: {host}:{port}"] + for k, v in req_headers.items(): + req_lines.append(f"{k}: {v}") + if body is not None and "Content-Length" not in req_headers: + body_len = len(body) if isinstance(body, bytes) else len(body.encode("utf-8")) + req_lines.append(f"Content-Length: {body_len}") + elif body is None and method in ("POST", "PATCH", "PUT") and "Content-Length" not in req_headers: + req_lines.append("Content-Length: 0") + + req_lines.append("") + req_lines.append("") + req_data = "\r\n".join(req_lines).encode("latin1") + if body: + req_data += body if isinstance(body, bytes) else body.encode("utf-8") + + s.sendall(req_data) + + # Read response data + resp_bytes = b"" + while True: + chunk = s.recv(4096) + if not chunk: + break + resp_bytes += chunk + + raw_str = resp_bytes.decode("latin1", errors="replace") + parts = raw_str.split("\r\n\r\n", 1) + headers_part = parts[0] + body_part = parts[1].encode("latin1") if len(parts) > 1 else b"" + + lines = headers_part.split("\r\n") + status_line = lines[0] + status_code = int(status_line.split()[1]) if len(status_line.split()) > 1 else 0 + + res_headers = parse_headers(lines[1:]) + return status_code, res_headers, body_part + except (socket.timeout, ConnectionRefusedError, socket.error) as e: + pytest.fail(f"HTTP request failed: {e}") + finally: + s.close() + + +def create_upload( + target_url, + upload_length="100", + defer_length=False, + metadata=None, + concat=None, + extra_headers=None, + body=None, + content_type=None, +): + """ + Helper to create an upload resource via POST. + Returns (absolute_location_url, response_headers). + """ + headers = {TUS_RESUMABLE: TUS_API_VERSION} + if defer_length: + headers[UPLOAD_DEFER_LENGTH] = "1" + elif upload_length is not None: + headers[UPLOAD_LENGTH] = str(upload_length) + + if metadata: + headers[UPLOAD_METADATA] = metadata + if concat: + headers[UPLOAD_CONCAT] = concat + if content_type: + headers[CONTENT_TYPE] = content_type + if extra_headers: + headers.update(extra_headers) + + status, resp_headers, _ = http_request("POST", target_url, headers=headers, body=body) + assert status == 201, f"Expected 201 Created for upload creation, got {status}" + + loc = resp_headers.get(LOCATION) + assert loc, "Location header MUST be returned upon 201 Created" + if not loc.startswith("http"): + parsed = urlparse(target_url) + loc = f"{parsed.scheme}://{parsed.netloc}{loc}" + return loc, resp_headers + + +class TestCoreProtocol: + """Core Protocol compliance tests (§6).""" + + def test_options_server_configuration(self, target_url): + """ + §6 OPTIONS: Gathering Server Configuration. + Quote: "A successful response indicated by the 204 No Content or 200 OK status MUST contain + the Tus-Version header. It MAY include the Tus-Extension and Tus-Max-Size headers." + """ + status, resp_headers, _ = http_request("OPTIONS", target_url) + assert status in (200, 204), f"OPTIONS request MUST return 200 or 204, got {status}" + assert TUS_VERSION in resp_headers, "OPTIONS response MUST contain Tus-Version header" + assert TUS_API_VERSION in resp_headers.get(TUS_VERSION, ""), "Tus-Version MUST include 1.0.0" + assert TUS_EXTENSION in resp_headers, "OPTIONS response SHOULD include Tus-Extension header" + + def test_options_ignores_tus_resumable_header(self, target_url): + """ + §6 OPTIONS: Server MUST Ignore Tus-Resumable Header on OPTIONS. + Quote: "The Client SHOULD NOT include the Tus-Resumable header in the request and the + Server MUST ignore the header." + """ + headers = {TUS_RESUMABLE: "unsupported.version.9.9.9"} + status, resp_headers, _ = http_request("OPTIONS", target_url, headers=headers) + assert status in (200, 204), "OPTIONS MUST succeed and ignore Tus-Resumable header" + assert status != 412, "Server MUST NOT respond with 412 on OPTIONS even with invalid Tus-Resumable" + + def test_unsupported_tus_version_returns_412(self, target_url): + """ + §6 Tus-Resumable: Protocol Version Mismatch Returns 412 Precondition Failed. + Quote: "If the version specified by the Client is not supported by the Server, it MUST + respond with the 412 Precondition Failed status and MUST include the Tus-Version header + into the response. In addition, the Server MUST NOT process the request." + """ + headers = { + TUS_RESUMABLE: "0.1.0", + UPLOAD_LENGTH: "100", + } + status, resp_headers, _ = http_request("POST", target_url, headers=headers) + assert status == 412, f"Unsupported version MUST be rejected with 412, got {status}" + assert TUS_VERSION in resp_headers, "412 response MUST include Tus-Version header" + + def test_missing_tus_resumable_header_rejected(self, target_url): + """ + §6 Tus-Resumable: Header Required in Requests. + Quote: "The Tus-Resumable header MUST be included in every request and response except + for OPTIONS requests." + """ + headers = {UPLOAD_LENGTH: "100"} + status, resp_headers, _ = http_request("POST", target_url, headers=headers) + assert status in (400, 412), f"Missing Tus-Resumable header MUST be rejected, got {status}" + + def test_tus_resumable_present_in_responses(self, target_url): + """ + §6 Tus-Resumable: Header Present in Every Response. + Quote: "The Tus-Resumable header MUST be included in every request and response except + for OPTIONS requests. The value MUST be the version of the protocol used by the Client or Server." + """ + upload_url, resp_headers = create_upload(target_url, upload_length="100") + assert resp_headers.get(TUS_RESUMABLE) == TUS_API_VERSION, "POST response MUST contain Tus-Resumable: 1.0.0" + + status, head_headers, _ = http_request("HEAD", upload_url, headers={TUS_RESUMABLE: TUS_API_VERSION}) + assert head_headers.get(TUS_RESUMABLE) == TUS_API_VERSION, "HEAD response MUST contain Tus-Resumable: 1.0.0" + + def test_head_requires_upload_offset_even_if_zero(self, target_url): + """ + §6 HEAD: Server MUST Always Include Upload-Offset. + Quote: "The Server MUST always include the Upload-Offset header in the response for a HEAD + request, even if the offset is 0, or the upload is already considered completed." + """ + upload_url, _ = create_upload(target_url, upload_length="100") + status, resp_headers, _ = http_request("HEAD", upload_url, headers={TUS_RESUMABLE: TUS_API_VERSION}) + assert status in (200, 204), f"HEAD request MUST succeed with 200 or 204, got {status}" + assert resp_headers.get(UPLOAD_OFFSET) == "0", "Upload-Offset MUST be present and 0 initially" + + def test_head_requires_upload_length_when_known(self, target_url): + """ + §6 HEAD: Server MUST Include Upload-Length When Known. + Quote: "If the size of the upload is known, the Server MUST include the Upload-Length + header in the response." + """ + upload_url, _ = create_upload(target_url, upload_length="256") + status, resp_headers, _ = http_request("HEAD", upload_url, headers={TUS_RESUMABLE: TUS_API_VERSION}) + assert status in (200, 204) + assert resp_headers.get(UPLOAD_LENGTH) == "256", "HEAD response MUST include Upload-Length" + + def test_head_cache_control_no_store(self, target_url): + """ + §6 HEAD: Cache Prevention via Cache-Control: no-store. + Quote: "The Server MUST prevent the client and/or proxies from caching the response by + adding the Cache-Control: no-store header to the response." + """ + upload_url, _ = create_upload(target_url, upload_length="100") + status, resp_headers, _ = http_request("HEAD", upload_url, headers={TUS_RESUMABLE: TUS_API_VERSION}) + assert status in (200, 204) + cache_ctrl = resp_headers.get(CACHE_CONTROL, "") + assert "no-store" in cache_ctrl, f"HEAD response MUST include Cache-Control: no-store, got '{cache_ctrl}'" + + def test_head_non_existent_upload_resource(self, target_url): + """ + §6 HEAD: Non-Existent Resource Returns 404, 410, or 403 Without Upload-Offset. + Quote: "If the resource is not found, the Server SHOULD return either the 404 Not Found, + 410 Gone or 403 Forbidden status without the Upload-Offset header." + """ + parsed = urlparse(target_url) + non_existent_url = f"{parsed.scheme}://{parsed.netloc}{parsed.path}/definitely-nonexistent-id-99999" + status, resp_headers, _ = http_request("HEAD", non_existent_url, headers={TUS_RESUMABLE: TUS_API_VERSION}) + assert status in (404, 410, 403), f"HEAD on non-existent resource SHOULD return 404, 410, or 403, got {status}" + assert UPLOAD_OFFSET not in resp_headers, "Non-existent HEAD response MUST NOT include Upload-Offset" + + def test_patch_content_type_required(self, target_url): + """ + §6 PATCH: Content-Type MUST Be application/offset+octet-stream. + Quote: "All PATCH requests MUST use Content-Type: application/offset+octet-stream, + otherwise the server SHOULD return a 415 Unsupported Media Type status." + """ + upload_url, _ = create_upload(target_url, upload_length="100") + headers = { + TUS_RESUMABLE: TUS_API_VERSION, + UPLOAD_OFFSET: "0", + CONTENT_TYPE: "text/plain", + } + status, _, _ = http_request("PATCH", upload_url, headers=headers, body=b"A" * 10) + assert status in (400, 406, 415), f"PATCH with incorrect Content-Type MUST be rejected, got {status}" + + def test_patch_offset_mismatch_returns_409(self, target_url): + """ + §6 PATCH: Mismatching Upload-Offset MUST Return 409 Conflict. + Quote: "The Upload-Offset header’s value MUST be equal to the current offset of the resource... + If the offsets do not match, the Server MUST respond with the 409 Conflict status without + modifying the upload resource." + """ + upload_url, _ = create_upload(target_url, upload_length="100") + headers = { + TUS_RESUMABLE: TUS_API_VERSION, + UPLOAD_OFFSET: "50", + CONTENT_TYPE: APPLICATION_OFFSET_OCTET_STREAM, + } + status, _, _ = http_request("PATCH", upload_url, headers=headers, body=b"A" * 10) + assert status == 409, f"PATCH with mismatching offset MUST return 409 Conflict, got {status}" + + # Verify upload resource was not modified + _, head_headers, _ = http_request("HEAD", upload_url, headers={TUS_RESUMABLE: TUS_API_VERSION}) + assert head_headers.get(UPLOAD_OFFSET) == "0", "Upload-Offset MUST NOT be modified after 409 Conflict" + + def test_patch_successful_append(self, target_url): + """ + §6 PATCH: Successful Append Returns 204 No Content With New Upload-Offset. + Quote: "The Server MUST acknowledge successful PATCH requests with the 204 No Content status. + It MUST include the Upload-Offset header containing the new offset. The new offset MUST be + the sum of the offset before the PATCH request and the number of bytes received and processed." + """ + upload_url, _ = create_upload(target_url, upload_length="100") + chunk = b"A" * 40 + headers = { + TUS_RESUMABLE: TUS_API_VERSION, + UPLOAD_OFFSET: "0", + CONTENT_TYPE: APPLICATION_OFFSET_OCTET_STREAM, + } + status, resp_headers, _ = http_request("PATCH", upload_url, headers=headers, body=chunk) + assert status == 204, f"PATCH MUST return 204 No Content on success, got {status}" + assert resp_headers.get(UPLOAD_OFFSET) == "40", "Upload-Offset in response MUST be updated to 40" + + # Verify via HEAD + _, head_headers, _ = http_request("HEAD", upload_url, headers={TUS_RESUMABLE: TUS_API_VERSION}) + assert head_headers.get(UPLOAD_OFFSET) == "40", "HEAD MUST report new offset of 40" + + def test_patch_non_existent_resource(self, target_url): + """ + §6 PATCH: Non-Existent Resource Returns 404 Not Found. + Quote: "If the server receives a PATCH request against a non-existent resource it SHOULD + return a 404 Not Found status." + """ + parsed = urlparse(target_url) + non_existent_url = f"{parsed.scheme}://{parsed.netloc}{parsed.path}/definitely-nonexistent-id-99999" + headers = { + TUS_RESUMABLE: TUS_API_VERSION, + UPLOAD_OFFSET: "0", + CONTENT_TYPE: APPLICATION_OFFSET_OCTET_STREAM, + } + status, _, _ = http_request("PATCH", non_existent_url, headers=headers, body=b"A" * 10) + assert status in (404, 410), f"PATCH against non-existent upload SHOULD return 404, got {status}" + + def test_patch_exceeding_upload_length_rejected(self, target_url): + """ + §6 PATCH: Appending Beyond Declared Upload-Length Must Be Rejected. + """ + upload_url, _ = create_upload(target_url, upload_length="30") + headers = { + TUS_RESUMABLE: TUS_API_VERSION, + UPLOAD_OFFSET: "0", + CONTENT_TYPE: APPLICATION_OFFSET_OCTET_STREAM, + } + status, _, _ = http_request("PATCH", upload_url, headers=headers, body=b"A" * 50) + assert status in (400, 409, 413), f"PATCH exceeding Upload-Length MUST be rejected, got {status}" + + def test_x_http_method_override(self, target_url): + """ + §6 X-HTTP-Method-Override: Request Method Override. + Quote: "The X-HTTP-Method-Override request header MUST be a string which MUST be interpreted + as the request’s method by the Server, if the header is presented. The actual method of the + request MUST be ignored." + """ + upload_url, _ = create_upload(target_url, upload_length="100") + + # Test overriding POST to PATCH + headers = { + TUS_RESUMABLE: TUS_API_VERSION, + X_HTTP_METHOD_OVERRIDE: "PATCH", + UPLOAD_OFFSET: "0", + CONTENT_TYPE: APPLICATION_OFFSET_OCTET_STREAM, + } + status, resp_headers, _ = http_request("POST", upload_url, headers=headers, body=b"A" * 40) + assert status == 204, f"Overridden PATCH request MUST succeed with 204, got {status}" + assert resp_headers.get(UPLOAD_OFFSET) == "40" + + # Test overriding POST to HEAD + headers = { + TUS_RESUMABLE: TUS_API_VERSION, + X_HTTP_METHOD_OVERRIDE: "HEAD", + } + status, head_headers, _ = http_request("POST", upload_url, headers=headers) + assert status in (200, 204), f"Overridden HEAD request MUST succeed with 200 or 204, got {status}" + assert head_headers.get(UPLOAD_OFFSET) == "40" + + +class TestCreationExtension: + """Creation Extension compliance tests (§7.1).""" + + def test_creation_extension_advertised(self, target_url): + """ + §7.1 Creation: Extension Advertisement in OPTIONS. + Quote: "If the Server supports this extension, it MUST add creation to the Tus-Extension header." + """ + status, resp_headers, _ = http_request("OPTIONS", target_url) + assert status in (200, 204) + extensions = [e.strip() for e in resp_headers.get(TUS_EXTENSION, "").split(",")] + assert "creation" in extensions, "Tus-Extension header MUST include 'creation'" + + def test_create_upload_empty_post(self, target_url): + """ + §7.1 Creation: Empty POST Creation Request. + Quote: "An empty POST request is used to create a new upload resource. The Upload-Length + header indicates the size of the entire upload in bytes... The Server MUST acknowledge a + successful upload creation with the 201 Created status. The Server MUST set the Location + header to the URL of the created resource." + """ + upload_url, resp_headers = create_upload(target_url, upload_length="100") + assert upload_url, "Location header pointing to created resource MUST be returned" + assert resp_headers.get(TUS_RESUMABLE) == TUS_API_VERSION + + def test_create_zero_byte_upload(self, target_url): + """ + §7.1 Creation: Zero-Byte Upload Creation. + Quote: "The Upload-Length header MAY be set to 0, indicating that the Client wants to + upload an empty file. Such an upload is immediately complete after its creation without + transferring data using PATCH requests." + """ + upload_url, _ = create_upload(target_url, upload_length="0") + status, head_headers, _ = http_request("HEAD", upload_url, headers={TUS_RESUMABLE: TUS_API_VERSION}) + assert status in (200, 204) + assert head_headers.get(UPLOAD_OFFSET) == "0", "Upload-Offset MUST be 0 for zero-byte upload" + assert head_headers.get(UPLOAD_LENGTH) == "0", "Upload-Length MUST be 0 for zero-byte upload" + + def test_create_upload_defer_length(self, target_url): + """ + §7.1 Creation: Deferred Length Creation. + Quote: "Upload-Defer-Length: 1 if upload size is not known at the time... As long as the + length of the upload is not known, the Server MUST set Upload-Defer-Length: 1 in all + responses to HEAD requests. If the length was deferred using Upload-Defer-Length: 1, the + Client MUST set the Upload-Length header in the next PATCH request, once the length is known." + """ + upload_url, _ = create_upload(target_url, defer_length=True) + + # Initial HEAD should return Upload-Defer-Length: 1 and no Upload-Length + status, head_headers, _ = http_request("HEAD", upload_url, headers={TUS_RESUMABLE: TUS_API_VERSION}) + assert status in (200, 204) + assert head_headers.get(UPLOAD_DEFER_LENGTH) == "1", "HEAD MUST include Upload-Defer-Length: 1 when length deferred" + assert UPLOAD_LENGTH not in head_headers, "Upload-Length MUST NOT be present while deferred" + + # PATCH providing length + headers = { + TUS_RESUMABLE: TUS_API_VERSION, + UPLOAD_OFFSET: "0", + UPLOAD_LENGTH: "80", + CONTENT_TYPE: APPLICATION_OFFSET_OCTET_STREAM, + } + status, _, _ = http_request("PATCH", upload_url, headers=headers, body=b"B" * 40) + assert status == 204, f"PATCH providing deferred Upload-Length MUST succeed with 204, got {status}" + + # Subsequent HEAD should now report Upload-Length: 80 and no Upload-Defer-Length + status, head_headers2, _ = http_request("HEAD", upload_url, headers={TUS_RESUMABLE: TUS_API_VERSION}) + assert status in (200, 204) + assert head_headers2.get(UPLOAD_LENGTH) == "80", "HEAD MUST now report Upload-Length: 80" + assert UPLOAD_DEFER_LENGTH not in head_headers2, "Upload-Defer-Length MUST be omitted once length is known" + + def test_create_upload_invalid_defer_length_value(self, target_url): + """ + §7.1 Creation: Invalid Upload-Defer-Length Value Returns 400 Bad Request. + Quote: "If the Upload-Defer-Length header contains any other value than 1 the server + should return a 400 Bad Request status." + """ + headers = { + TUS_RESUMABLE: TUS_API_VERSION, + UPLOAD_DEFER_LENGTH: "2", + } + status, _, _ = http_request("POST", target_url, headers=headers) + assert status == 400, f"Upload-Defer-Length != 1 MUST return 400 Bad Request, got {status}" + + def test_create_upload_missing_both_length_and_defer_length(self, target_url): + """ + §7.1 Creation: POST Missing Both Upload-Length and Upload-Defer-Length. + Quote: "The request MUST include one of the following headers: a) Upload-Length ... + b) Upload-Defer-Length: 1" + """ + headers = {TUS_RESUMABLE: TUS_API_VERSION} + status, _, _ = http_request("POST", target_url, headers=headers) + assert status == 400, f"POST without length headers MUST return 400 Bad Request, got {status}" + + def test_create_upload_negative_length(self, target_url): + """ + §6 Upload-Length: Value MUST Be Non-Negative Integer. + Quote: "The Upload-Length request and response header indicates the size of the entire + upload in bytes. The value MUST be a non-negative integer." + """ + headers = { + TUS_RESUMABLE: TUS_API_VERSION, + UPLOAD_LENGTH: "-10", + } + status, _, _ = http_request("POST", target_url, headers=headers) + assert status == 400, f"Negative Upload-Length MUST return 400 Bad Request, got {status}" + + def test_create_upload_non_integer_length(self, target_url): + """ + §6 Upload-Length: Non-Integer Value Rejected. + """ + headers = { + TUS_RESUMABLE: TUS_API_VERSION, + UPLOAD_LENGTH: "not_a_number", + } + status, _, _ = http_request("POST", target_url, headers=headers) + assert status == 400, f"Non-integer Upload-Length MUST return 400 Bad Request, got {status}" + + def test_create_upload_with_metadata(self, target_url): + """ + §7.1 Creation: Upload-Metadata Header Preservation. + Quote: "The Client MAY supply the Upload-Metadata header to add additional metadata to the + upload creation request... If an upload contains additional metadata, responses to HEAD + requests MUST include the Upload-Metadata header and its value as specified by the Client." + """ + meta_filename = base64.b64encode(b"report.pdf").decode("ascii") + meta_author = base64.b64encode(b"Jane Doe").decode("ascii") + metadata = f"filename {meta_filename},author {meta_author}" + + upload_url, _ = create_upload(target_url, upload_length="100", metadata=metadata) + status, head_headers, _ = http_request("HEAD", upload_url, headers={TUS_RESUMABLE: TUS_API_VERSION}) + assert status in (200, 204) + resp_meta = head_headers.get(UPLOAD_METADATA, "") + assert f"filename {meta_filename}" in resp_meta, "Upload-Metadata in HEAD MUST contain filename" + assert f"author {meta_author}" in resp_meta, "Upload-Metadata in HEAD MUST contain author" + + def test_create_upload_metadata_empty_value(self, target_url): + """ + §7.1 Creation: Upload-Metadata with Empty Value. + Quote: "The value MAY be empty. In these cases, the space, which would normally separate + the key and the value, MAY be left out." + """ + metadata = "is_confidential" + upload_url, _ = create_upload(target_url, upload_length="100", metadata=metadata) + status, head_headers, _ = http_request("HEAD", upload_url, headers={TUS_RESUMABLE: TUS_API_VERSION}) + assert status in (200, 204) + assert "is_confidential" in head_headers.get(UPLOAD_METADATA, ""), "Upload-Metadata in HEAD MUST contain empty-value key" + + def test_create_upload_exceeding_tus_max_size(self, target_url): + """ + §7.1 Creation: Upload Exceeding Tus-Max-Size Returns 413 Request Entity Too Large. + Quote: "If the length of the upload exceeds the maximum, which MAY be specified using + the Tus-Max-Size header, the Server MUST respond with the 413 Request Entity Too Large status." + """ + status, resp_headers, _ = http_request("OPTIONS", target_url) + max_size_str = resp_headers.get(TUS_MAX_SIZE) + if not max_size_str: + pytest.skip("Server does not advertise Tus-Max-Size in OPTIONS response") + + max_size = int(max_size_str) + headers = { + TUS_RESUMABLE: TUS_API_VERSION, + UPLOAD_LENGTH: str(max_size + 1), + } + status, _, _ = http_request("POST", target_url, headers=headers) + assert status == 413, f"Upload exceeding Tus-Max-Size ({max_size}) MUST return 413, got {status}" + + def test_deferred_length_cannot_be_changed_once_set(self, target_url): + """ + §7.1 Creation: Deferred Length Cannot Be Changed Once Set. + Quote: "Once set the length MUST NOT be changed." + """ + upload_url, _ = create_upload(target_url, defer_length=True) + + # Set length to 100 + headers = { + TUS_RESUMABLE: TUS_API_VERSION, + UPLOAD_OFFSET: "0", + UPLOAD_LENGTH: "100", + CONTENT_TYPE: APPLICATION_OFFSET_OCTET_STREAM, + } + status, _, _ = http_request("PATCH", upload_url, headers=headers, body=b"A" * 40) + assert status == 204 + + # Attempt to change length to 120 in subsequent PATCH + headers2 = { + TUS_RESUMABLE: TUS_API_VERSION, + UPLOAD_OFFSET: "40", + UPLOAD_LENGTH: "120", + CONTENT_TYPE: APPLICATION_OFFSET_OCTET_STREAM, + } + status2, _, _ = http_request("PATCH", upload_url, headers=headers2, body=b"B" * 40) + assert status2 in (400, 409), f"Attempting to modify already set Upload-Length MUST be rejected, got {status2}" + + +class TestCreationWithUploadExtension: + """Creation With Upload Extension compliance tests (§7.2).""" + + def test_creation_with_upload_advertised(self, target_url): + """ + §7.2 Creation With Upload: Extension Advertisement. + Quote: "If the Server supports this extension, it MUST advertise this by including + creation-with-upload in the Tus-Extension header." + """ + status, resp_headers, _ = http_request("OPTIONS", target_url) + assert status in (200, 204) + extensions = [e.strip() for e in resp_headers.get(TUS_EXTENSION, "").split(",")] + assert "creation-with-upload" in extensions, "Tus-Extension MUST include 'creation-with-upload'" + + def test_creation_with_full_upload(self, target_url): + """ + §7.2 Creation With Upload: Initial POST Containing Entire File Payload. + Quote: "The Client MAY include either the entirety or a chunk of the upload data in the + body of the POST request... The Server SHOULD accept as many bytes as possible and MUST + include the Upload-Offset header in the response and MUST set its value to the offset of + the upload after applying the accepted bytes." + """ + payload = b"Hello, Full Tus Creation With Upload!" + upload_url, resp_headers = create_upload( + target_url, + upload_length=str(len(payload)), + content_type=APPLICATION_OFFSET_OCTET_STREAM, + body=payload, + ) + assert resp_headers.get(UPLOAD_OFFSET) == str(len(payload)), "Response MUST return Upload-Offset equal to body length" + + # Verify via HEAD + status, head_headers, _ = http_request("HEAD", upload_url, headers={TUS_RESUMABLE: TUS_API_VERSION}) + assert status in (200, 204) + assert head_headers.get(UPLOAD_OFFSET) == str(len(payload)) + assert head_headers.get(UPLOAD_LENGTH) == str(len(payload)) + + def test_creation_with_partial_upload_then_patch(self, target_url): + """ + §7.2 Creation With Upload: Initial POST With Partial Payload Followed by PATCH. + Quote: "The Server SHOULD accept as many bytes as possible and MUST include the + Upload-Offset header in the response... The Client MUST perform the actual upload + using the core protocol." + """ + part1 = b"Part 1 Data with at least 32 bytes!!" + part2 = b"Part 2 Data with at least 32 bytes!!" + total_length = len(part1) + len(part2) + + upload_url, resp_headers = create_upload( + target_url, + upload_length=str(total_length), + content_type=APPLICATION_OFFSET_OCTET_STREAM, + body=part1, + ) + assert resp_headers.get(UPLOAD_OFFSET) == str(len(part1)) + + # Complete upload using PATCH + headers = { + TUS_RESUMABLE: TUS_API_VERSION, + UPLOAD_OFFSET: str(len(part1)), + CONTENT_TYPE: APPLICATION_OFFSET_OCTET_STREAM, + } + status, patch_headers, _ = http_request("PATCH", upload_url, headers=headers, body=part2) + assert status == 204 + assert patch_headers.get(UPLOAD_OFFSET) == str(total_length) + + # Final verification via HEAD + _, head_headers, _ = http_request("HEAD", upload_url, headers={TUS_RESUMABLE: TUS_API_VERSION}) + assert head_headers.get(UPLOAD_OFFSET) == str(total_length) + + def test_creation_with_upload_and_deferred_length(self, target_url): + """ + §7.2 Creation With Upload: Creation With Upload Combined With Deferred Length. + """ + payload = b"Initial chunk with deferred length" + upload_url, resp_headers = create_upload( + target_url, + defer_length=True, + content_type=APPLICATION_OFFSET_OCTET_STREAM, + body=payload, + ) + assert resp_headers.get(UPLOAD_OFFSET) == str(len(payload)) + + # HEAD verification + _, head_headers, _ = http_request("HEAD", upload_url, headers={TUS_RESUMABLE: TUS_API_VERSION}) + assert head_headers.get(UPLOAD_OFFSET) == str(len(payload)) + assert head_headers.get(UPLOAD_DEFER_LENGTH) == "1" + + def test_creation_with_upload_invalid_content_type(self, target_url): + """ + §7.2 Creation With Upload: Non-Empty POST With Invalid Content-Type Must Be Rejected. + Quote: "The Client MUST include the Content-Type: application/offset+octet-stream header." + """ + headers = { + TUS_RESUMABLE: TUS_API_VERSION, + UPLOAD_LENGTH: "30", + CONTENT_TYPE: "text/plain", + } + status, _, _ = http_request("POST", target_url, headers=headers, body=b"A" * 20) + assert status in (400, 406, 415), f"Non-empty POST with invalid Content-Type MUST be rejected, got {status}" + + def test_creation_with_upload_exceeds_upload_length(self, target_url): + """ + §7.2 Creation With Upload: Body Exceeding Declared Upload-Length Must Be Rejected. + """ + headers = { + TUS_RESUMABLE: TUS_API_VERSION, + UPLOAD_LENGTH: "10", + CONTENT_TYPE: APPLICATION_OFFSET_OCTET_STREAM, + } + status, _, _ = http_request("POST", target_url, headers=headers, body=b"A" * 20) + assert status in (400, 413), f"Body exceeding declared length MUST be rejected, got {status}" + + +class TestChecksumExtension: + """Checksum Extension compliance tests (§7.4).""" + + def test_checksum_extension_advertised(self, target_url): + """ + §7.4 Checksum: Extension and Algorithms Advertisement. + Quote: "If supported, the Server MUST add checksum to the Tus-Extension header. + The Tus-Checksum-Algorithm header MUST be included in the response to an OPTIONS request. + The Server MUST support at least the SHA1 checksum algorithm identified by sha1." + """ + status, resp_headers, _ = http_request("OPTIONS", target_url) + assert status in (200, 204) + extensions = [e.strip() for e in resp_headers.get(TUS_EXTENSION, "").split(",")] + assert "checksum" in extensions, "Tus-Extension header MUST include 'checksum'" + algorithms = [a.strip() for a in resp_headers.get(TUS_CHECKSUM_ALGORITHM, "").split(",")] + assert "sha1" in algorithms, "Tus-Checksum-Algorithm header MUST support 'sha1'" + + def test_patch_with_valid_sha1_checksum(self, target_url): + """ + §7.4 Checksum: Valid SHA1 Checksum Verification. + Quote: "A Client MAY include the Upload-Checksum header in a PATCH request. Once the entire + request has been received, the Server MUST verify the uploaded chunk against the provided + checksum using the specified algorithm... 3. 204 No Content if the checksums match and + the processing of the data succeeded." + """ + upload_url, _ = create_upload(target_url, upload_length="100") + payload = b"Tus SHA1 checksum verified payload" + digest_bytes = hashlib.sha1(payload).digest() + b64_digest = base64.b64encode(digest_bytes).decode("ascii") + + headers = { + TUS_RESUMABLE: TUS_API_VERSION, + UPLOAD_OFFSET: "0", + CONTENT_TYPE: APPLICATION_OFFSET_OCTET_STREAM, + UPLOAD_CHECKSUM: f"sha1 {b64_digest}", + } + status, resp_headers, _ = http_request("PATCH", upload_url, headers=headers, body=payload) + assert status == 204, f"PATCH with valid checksum MUST succeed with 204, got {status}" + assert resp_headers.get(UPLOAD_OFFSET) == str(len(payload)) + + def test_patch_with_invalid_checksum_mismatch(self, target_url): + """ + §7.4 Checksum: Checksum Mismatch Returns 460 Checksum Mismatch. + Quote: "2. 460 Checksum Mismatch if the checksums mismatch... In the first two cases the + uploaded chunk MUST be discarded, and the upload and its offset MUST NOT be updated." + """ + upload_url, _ = create_upload(target_url, upload_length="100") + payload = b"Tus SHA1 payload with at least 32 bytes in length" + invalid_digest = base64.b64encode(b"0" * 20).decode("ascii") + + headers = { + TUS_RESUMABLE: TUS_API_VERSION, + UPLOAD_OFFSET: "0", + CONTENT_TYPE: APPLICATION_OFFSET_OCTET_STREAM, + UPLOAD_CHECKSUM: f"sha1 {invalid_digest}", + } + status, _, _ = http_request("PATCH", upload_url, headers=headers, body=payload) + assert status == 460, f"Checksum mismatch MUST return 460 Checksum Mismatch, got {status}" + + # Verify chunk was discarded and offset remains 0 + _, head_headers, _ = http_request("HEAD", upload_url, headers={TUS_RESUMABLE: TUS_API_VERSION}) + assert head_headers.get(UPLOAD_OFFSET) == "0", "Upload-Offset MUST NOT be updated after checksum mismatch" + + def test_patch_with_unsupported_checksum_algorithm(self, target_url): + """ + §7.4 Checksum: Unsupported Checksum Algorithm Returns 400 Bad Request. + Quote: "1. 400 Bad Request if the checksum algorithm is not supported by the server... + the uploaded chunk MUST be discarded, and the upload and its offset MUST NOT be updated." + """ + upload_url, _ = create_upload(target_url, upload_length="100") + payload = b"Payload with unknown checksum algorithm at least 32 bytes" + + headers = { + TUS_RESUMABLE: TUS_API_VERSION, + UPLOAD_OFFSET: "0", + CONTENT_TYPE: APPLICATION_OFFSET_OCTET_STREAM, + UPLOAD_CHECKSUM: "unknown_algo AAAA", + } + status, _, _ = http_request("PATCH", upload_url, headers=headers, body=payload) + assert status == 400, f"Unsupported checksum algorithm MUST return 400 Bad Request, got {status}" + + # Verify chunk was discarded + _, head_headers, _ = http_request("HEAD", upload_url, headers={TUS_RESUMABLE: TUS_API_VERSION}) + assert head_headers.get(UPLOAD_OFFSET) == "0", "Upload-Offset MUST NOT be updated after invalid algorithm" + + +class TestTerminationExtension: + """Termination Extension compliance tests (§7.5).""" + + def test_termination_extension_advertised(self, target_url): + """ + §7.5 Termination: Extension Advertisement. + Quote: "If this extension is supported by the Server, it MUST be announced by adding + termination to the Tus-Extension header." + """ + status, resp_headers, _ = http_request("OPTIONS", target_url) + assert status in (200, 204) + extensions = [e.strip() for e in resp_headers.get(TUS_EXTENSION, "").split(",")] + assert "termination" in extensions, "Tus-Extension header MUST include 'termination'" + + def test_delete_existing_upload(self, target_url): + """ + §7.5 Termination: Deletion of Existing Upload. + Quote: "When receiving a DELETE request for an existing upload the Server SHOULD free + associated resources and MUST respond with the 204 No Content status confirming that the + upload was terminated. For all future requests to this URL, the Server SHOULD respond with + the 404 Not Found or 410 Gone status." + """ + upload_url, _ = create_upload(target_url, upload_length="100") + + # Terminate + status, _, _ = http_request("DELETE", upload_url, headers={TUS_RESUMABLE: TUS_API_VERSION}) + assert status == 204, f"DELETE on existing upload MUST return 204 No Content, got {status}" + + # Future HEAD request MUST return 404 or 410 + head_status, _, _ = http_request("HEAD", upload_url, headers={TUS_RESUMABLE: TUS_API_VERSION}) + assert head_status in (404, 410), f"HEAD after DELETE MUST return 404 or 410, got {head_status}" + + # Future PATCH request MUST return 404 or 410 + patch_headers = { + TUS_RESUMABLE: TUS_API_VERSION, + UPLOAD_OFFSET: "0", + CONTENT_TYPE: APPLICATION_OFFSET_OCTET_STREAM, + } + patch_status, _, _ = http_request("PATCH", upload_url, headers=patch_headers, body=b"A" * 10) + assert patch_status in (404, 410), f"PATCH after DELETE MUST return 404 or 410, got {patch_status}" + + def test_delete_non_existent_upload(self, target_url): + """ + §7.5 Termination: DELETE on Non-Existent Resource Returns 404 Not Found. + """ + parsed = urlparse(target_url) + non_existent_url = f"{parsed.scheme}://{parsed.netloc}{parsed.path}/definitely-nonexistent-id-99999" + status, _, _ = http_request("DELETE", non_existent_url, headers={TUS_RESUMABLE: TUS_API_VERSION}) + assert status in (404, 410), f"DELETE on non-existent upload SHOULD return 404, got {status}" + + def test_delete_completed_upload(self, target_url): + """ + §7.5 Termination: Termination of Completed Upload. + Quote: "This extension defines a way for the Client to terminate completed and unfinished + uploads allowing the Server to free up used resources." + """ + payload = b"Terminating completed upload test payload" + upload_url, _ = create_upload( + target_url, + upload_length=str(len(payload)), + content_type=APPLICATION_OFFSET_OCTET_STREAM, + body=payload, + ) + + status, _, _ = http_request("DELETE", upload_url, headers={TUS_RESUMABLE: TUS_API_VERSION}) + assert status == 204, f"DELETE on completed upload MUST return 204 No Content, got {status}" + + head_status, _, _ = http_request("HEAD", upload_url, headers={TUS_RESUMABLE: TUS_API_VERSION}) + assert head_status in (404, 410), f"HEAD on deleted completed upload MUST return 404 or 410, got {head_status}" + + +class TestConcatenationExtension: + """Concatenation Extension compliance tests (§7.6).""" + + def test_concatenation_extension_advertised(self, target_url): + """ + §7.6 Concatenation: Extension Advertisement. + Quote: "If the Server supports this extension, it MUST add concatenation to the Tus-Extension header." + """ + status, resp_headers, _ = http_request("OPTIONS", target_url) + assert status in (200, 204) + extensions = [e.strip() for e in resp_headers.get(TUS_EXTENSION, "").split(",")] + assert "concatenation" in extensions, "Tus-Extension header MUST include 'concatenation'" + + def test_partial_upload_creation_and_head(self, target_url): + """ + §7.6 Concatenation: Partial Upload Creation and HEAD Response. + Quote: "A partial upload represents a chunk of a file. It is constructed by including the + Upload-Concat: partial header while creating a new upload using the Creation extension... + The response to a HEAD request for a partial upload MUST contain the Upload-Offset header. + Response to HEAD request against partial or final upload MUST include the Upload-Concat + header and its value as received in the upload creation request." + """ + upload_url, _ = create_upload(target_url, upload_length="40", concat="partial") + status, head_headers, _ = http_request("HEAD", upload_url, headers={TUS_RESUMABLE: TUS_API_VERSION}) + assert status in (200, 204) + assert head_headers.get(UPLOAD_OFFSET) == "0" + assert head_headers.get(UPLOAD_CONCAT) == "partial", "HEAD response MUST include Upload-Concat: partial" + + def test_final_upload_concatenation_success(self, target_url): + """ + §7.6 Concatenation: Final Concatenation of Partial Uploads. + Quote: "In order to create a new final upload, the Client MUST add the Upload-Concat header + to the upload creation request. The value MUST be final followed by a semicolon and a + space-separated list of the partial upload URLs that need to be concatenated... The length of + the final upload MUST be the sum of the length of all partial uploads. After successful + concatenation, the Upload-Offset and Upload-Length MUST be set and their values MUST be equal." + """ + # Create and fill partial 1 + part1_data = b"Hello, Concatenated World Part 1!" + part1_url, _ = create_upload( + target_url, + upload_length=str(len(part1_data)), + concat="partial", + content_type=APPLICATION_OFFSET_OCTET_STREAM, + body=part1_data, + ) + + # Create and fill partial 2 + part2_data = b"Hello, Concatenated World Part 2!" + part2_url, _ = create_upload( + target_url, + upload_length=str(len(part2_data)), + concat="partial", + content_type=APPLICATION_OFFSET_OCTET_STREAM, + body=part2_data, + ) + + total_length = len(part1_data) + len(part2_data) + + # Create final concatenated upload + concat_val = f"final;{part1_url} {part2_url}" + final_url, _ = create_upload(target_url, upload_length=None, concat=concat_val) + + # Verify final upload HEAD + status, head_headers, _ = http_request("HEAD", final_url, headers={TUS_RESUMABLE: TUS_API_VERSION}) + assert status in (200, 204) + assert head_headers.get(UPLOAD_OFFSET) == str(total_length), "Upload-Offset MUST equal total combined length" + assert head_headers.get(UPLOAD_LENGTH) == str(total_length), "Upload-Length MUST equal total combined length" + assert head_headers.get(UPLOAD_CONCAT) == concat_val, "Upload-Concat MUST match final concatenation specification" + + def test_final_creation_must_not_include_upload_length(self, target_url): + """ + §7.6 Concatenation: Final Upload Creation MUST NOT Include Upload-Length. + Quote: "The Client MUST NOT include the Upload-Length header in the final upload creation." + """ + # Create a partial upload + part_url, _ = create_upload(target_url, upload_length="20", concat="partial") + + headers = { + TUS_RESUMABLE: TUS_API_VERSION, + UPLOAD_CONCAT: f"final;{part_url}", + UPLOAD_LENGTH: "20", + } + status, _, _ = http_request("POST", target_url, headers=headers) + assert status == 400, f"Final creation including Upload-Length MUST be rejected with 400, got {status}" + + def test_patch_against_final_upload_forbidden(self, target_url): + """ + §7.6 Concatenation: PATCH on Final Upload Returns 403 Forbidden. + Quote: "The Server MUST respond with the 403 Forbidden status to PATCH requests against + a final upload URL and MUST NOT modify the final or its partial uploads." + """ + part_data = b"Fixed Part Data with at least 32 bytes!" + part_url, _ = create_upload( + target_url, + upload_length=str(len(part_data)), + concat="partial", + content_type=APPLICATION_OFFSET_OCTET_STREAM, + body=part_data, + ) + final_url, _ = create_upload(target_url, upload_length=None, concat=f"final;{part_url}") + + # Retrieve current offset of final upload via HEAD + _, head_headers, _ = http_request("HEAD", final_url, headers={TUS_RESUMABLE: TUS_API_VERSION}) + offset = head_headers.get(UPLOAD_OFFSET, str(len(part_data))) + + patch_headers = { + TUS_RESUMABLE: TUS_API_VERSION, + UPLOAD_OFFSET: offset, + CONTENT_TYPE: APPLICATION_OFFSET_OCTET_STREAM, + } + status, _, _ = http_request("PATCH", final_url, headers=patch_headers, body=b"") + assert status == 403, f"PATCH against final upload MUST return 403 Forbidden, got {status}" + + +class TestExpirationExtension: + """Expiration Extension compliance tests (§7.3).""" + + def test_expiration_extension_advertised(self, target_url): + """ + §7.3 Expiration: Extension Advertisement. + Quote: "In order to indicate this behavior to the Client, the Server MUST add expiration + to the Tus-Extension header." + """ + status, resp_headers, _ = http_request("OPTIONS", target_url) + assert status in (200, 204) + extensions = [e.strip() for e in resp_headers.get(TUS_EXTENSION, "").split(",")] + assert "expiration" in extensions, "Tus-Extension header MUST include 'expiration'" + + def test_upload_expires_header_format(self, target_url): + """ + §7.3 Expiration: Upload-Expires Header in Datetime Format. + Quote: "The Upload-Expires response header indicates the time after which the unfinished + upload expires... This header MUST be included in every PATCH response if the upload is + going to expire. If the expiration is known at the creation, the Upload-Expires header MUST + be included in the response to the initial POST request... The value of the Upload-Expires + header MUST be in RFC 9110 datetime format." + """ + upload_url, post_headers = create_upload(target_url, upload_length="100") + expires_header = post_headers.get(UPLOAD_EXPIRES) + + if not expires_header: + # Perform a PATCH to observe Upload-Expires on intermediate append + patch_headers = { + TUS_RESUMABLE: TUS_API_VERSION, + UPLOAD_OFFSET: "0", + CONTENT_TYPE: APPLICATION_OFFSET_OCTET_STREAM, + } + status, patch_resp_headers, _ = http_request("PATCH", upload_url, headers=patch_headers, body=b"A" * 40) + assert status == 204 + expires_header = patch_resp_headers.get(UPLOAD_EXPIRES) + + if expires_header: + dt = email.utils.parsedate_to_datetime(expires_header) + assert dt is not None, f"Upload-Expires '{expires_header}' could not be parsed as valid RFC 9110 datetime" + + +class TestConcurrencyAndWorkflow: + """Concurrency and end-to-end upload workflows.""" + + def test_concurrent_patches_same_offset(self, target_url): + """ + §6 PATCH: Concurrency Contention Prevention. + Quote: "The Upload-Offset header’s value MUST be equal to the current offset of the resource. + If the offsets do not match, the Server MUST respond with the 409 Conflict status without + modifying the upload resource." + """ + upload_url, _ = create_upload(target_url, upload_length="100") + results = [] + + def do_patch(): + headers = { + TUS_RESUMABLE: TUS_API_VERSION, + UPLOAD_OFFSET: "0", + CONTENT_TYPE: APPLICATION_OFFSET_OCTET_STREAM, + } + st, _, _ = http_request("PATCH", upload_url, headers=headers, body=b"A" * 40) + results.append(st) + + t1 = threading.Thread(target=do_patch) + t2 = threading.Thread(target=do_patch) + t1.start() + t2.start() + t1.join() + t2.join() + + successes = [r for r in results if r == 204] + assert len(successes) <= 1, f"At most one concurrent PATCH at offset 0 can succeed, got {results}" + + def test_multi_chunk_resumable_upload_workflow(self, target_url): + """ + End-to-End multi-chunk resumable upload workflow. + Create upload -> PATCH chunk 1 -> HEAD verify -> PATCH chunk 2 -> complete. + """ + total_data = b"0123456789" * 12 # 120 bytes + chunk1 = total_data[:40] + chunk2 = total_data[40:80] + chunk3 = total_data[80:] + + upload_url, _ = create_upload(target_url, upload_length=str(len(total_data))) + + # Chunk 1 + headers1 = { + TUS_RESUMABLE: TUS_API_VERSION, + UPLOAD_OFFSET: "0", + CONTENT_TYPE: APPLICATION_OFFSET_OCTET_STREAM, + } + st1, _, _ = http_request("PATCH", upload_url, headers=headers1, body=chunk1) + assert st1 == 204 + + # HEAD verification + _, head1, _ = http_request("HEAD", upload_url, headers={TUS_RESUMABLE: TUS_API_VERSION}) + assert head1.get(UPLOAD_OFFSET) == "40" + + # Chunk 2 + headers2 = { + TUS_RESUMABLE: TUS_API_VERSION, + UPLOAD_OFFSET: "40", + CONTENT_TYPE: APPLICATION_OFFSET_OCTET_STREAM, + } + st2, _, _ = http_request("PATCH", upload_url, headers=headers2, body=chunk2) + assert st2 == 204 + + # HEAD verification + _, head2, _ = http_request("HEAD", upload_url, headers={TUS_RESUMABLE: TUS_API_VERSION}) + assert head2.get(UPLOAD_OFFSET) == "80" + + # Chunk 3 (final) + headers3 = { + TUS_RESUMABLE: TUS_API_VERSION, + UPLOAD_OFFSET: "80", + CONTENT_TYPE: APPLICATION_OFFSET_OCTET_STREAM, + } + st3, _, _ = http_request("PATCH", upload_url, headers=headers3, body=chunk3) + assert st3 == 204 + + # Final verification + _, head_final, _ = http_request("HEAD", upload_url, headers={TUS_RESUMABLE: TUS_API_VERSION}) + assert head_final.get(UPLOAD_OFFSET) == "120" + assert head_final.get(UPLOAD_LENGTH) == "120" + + +# CLI Entry Point & Custom Formatted Summary Reporter +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Tus v1.0.0 Conformity Test Runner") + parser.add_argument( + "--url", + default="http://localhost:8080/test/api/upload", + help="Target Tus upload endpoint URL", + ) + args = parser.parse_args() + + os.environ["TUS_URL"] = args.url + + print("=" * 70) + print(" Tus v1.0.0 Resumable Upload Protocol Conformity Test Suite") + print(" Specification: https://tus.io/protocols/resumable-upload") + print(" Target Endpoint:", args.url) + print("=" * 70) + + class CustomReporter: + def __init__(self): + self.passed = [] + self.failed = [] + self.docs = {} + + @pytest.hookimpl(tryfirst=True, hookwrapper=True) + def pytest_runtest_makereport(self, item, call): + outcome = yield + report = outcome.get_result() + if report.when == "call": + doc = item.obj.__doc__ or "No description provided." + self.docs[report.nodeid] = doc.strip() + if report.passed: + self.passed.append(report.nodeid) + elif report.failed: + if hasattr(report.longrepr, "reprcrash"): + err_text = report.longrepr.reprcrash.message + elif hasattr(report, "longreprtext"): + err_lines = [ + l.strip() + for l in report.longreprtext.splitlines() + if l.strip().startswith("E ") or l.strip().startswith("AssertionError") + ] + err_text = "\n ".join(err_lines) if err_lines else str(report.longrepr) + else: + err_text = str(report.longrepr) + self.failed.append((report.nodeid, err_text)) + + reporter = CustomReporter() + pytest.main([__file__, "-q", f"--url={args.url}"], plugins=[reporter]) + + total_tests = len(reporter.passed) + len(reporter.failed) + + print("\n" + "=" * 70) + print(" CONFORMITY TEST RESULTS") + print("=" * 70) + print(f" Total Tests Executed: {total_tests}") + print(f" Passed: {len(reporter.passed)}") + print(f" Failed: {len(reporter.failed)}") + print("=" * 70) + + if reporter.failed: + print("\n[!] DETAILED FAILURE BREAKDOWN FOR REMEDIATION:") + print("-" * 70) + for idx, (test_id, err_text) in enumerate(reporter.failed, 1): + print(f"\n{idx}. Test: {test_id}") + func_name = test_id.split("::")[-1] + print(f" Function: {func_name}") + print(f" Specification Goal:\n " + reporter.docs.get(test_id, "").replace("\n", "\n ")) + print(f" Failure Reason:\n " + err_text.replace("\n", "\n ")) + print("-" * 70) + else: + print("\n[✓] ALL TUS V1.0.0 CONFORMITY TESTS PASSED SUCCESSFULLY!") + + sys.exit(0 if not reporter.failed else 1) diff --git a/src/main/java/me/desair/tus/server/TusFileUploadService.java b/src/main/java/me/desair/tus/server/TusFileUploadService.java index 5b5c20fa..8f203f23 100644 --- a/src/main/java/me/desair/tus/server/TusFileUploadService.java +++ b/src/main/java/me/desair/tus/server/TusFileUploadService.java @@ -235,6 +235,20 @@ public TusFileUploadService withMinSize(Long minSize) { return this; } + /** + * Set the maximum number of worker threads used for asynchronous background chunk uploading in + * cloud storage backends (S3, Azure). Defaults to 10. + * + * @param cloudUploadThreadPoolSize Number of worker threads (must be > 0) + * @return The current service + */ + public TusFileUploadService withCloudUploadThreadPoolSize(int cloudUploadThreadPoolSize) { + Validate.isTrue( + cloudUploadThreadPoolSize > 0, "The cloud upload thread pool size must be greater than 0"); + this.uploadStorageService.setCloudUploadThreadPoolSize(cloudUploadThreadPoolSize); + return this; + } + /** * Provide a custom {@link UploadIdFactory} implementation that should be used to generate * identifiers for the different uploads. Example implementation are {@link @@ -268,6 +282,8 @@ public TusFileUploadService withUploadStorageService(UploadStorageService upload uploadStorageService.setMaxAppendSize(this.uploadStorageService.getMaxAppendSize()); uploadStorageService.setMinAppendSize(this.uploadStorageService.getMinAppendSize()); uploadStorageService.setMinSize(this.uploadStorageService.getMinSize()); + uploadStorageService.setCloudUploadThreadPoolSize( + this.uploadStorageService.getCloudUploadThreadPoolSize()); uploadStorageService.setUploadExpirationPeriod( this.uploadStorageService.getUploadExpirationPeriod()); uploadStorageService.setUploadDeduplicationEnabled( @@ -1085,6 +1101,13 @@ protected void processTusException( response.setStatus(status); if (problemDetails != null) { problemDetails.writeTo(response); + } else if (status == 460) { + // Non-standard HTTP status 460 (Checksum Mismatch) is not recognized by standard + // servlet container error controllers (such as Spring Boot's BasicErrorController), + // which would cause sendError() to fail with an unhandled IllegalArgumentException + // and degrade to HTTP 500. We explicitly set the status and omit sendError(). + response.setHeader(HttpHeader.CONTENT_LENGTH, null); + response.setStatus(status); } else { response.setHeader(HttpHeader.CONTENT_LENGTH, null); response.sendError(status, message); diff --git a/src/main/java/me/desair/tus/server/rufh/handler/RufhAppendPatchRequestHandler.java b/src/main/java/me/desair/tus/server/rufh/handler/RufhAppendPatchRequestHandler.java index 5ec06fa4..53974029 100644 --- a/src/main/java/me/desair/tus/server/rufh/handler/RufhAppendPatchRequestHandler.java +++ b/src/main/java/me/desair/tus/server/rufh/handler/RufhAppendPatchRequestHandler.java @@ -55,6 +55,13 @@ public HttpProblemDetails process( String uploadCompleteHeader = servletRequest.getHeader(HttpHeader.UPLOAD_COMPLETE); Boolean uploadComplete = StructuredHeaderUtil.parseBoolean(uploadCompleteHeader); + long cl = servletRequest.getContentLengthLong(); + if (Boolean.TRUE.equals(uploadComplete) && !uploadInfo.hasLength() && cl >= 0) { + long currentOffset = uploadInfo.getOffset() != null ? uploadInfo.getOffset() : 0L; + uploadInfo.setLength(currentOffset + cl); + uploadStorageService.update(uploadInfo); + } + InputStream is = servletRequest.getContentInputStream(); if (is != null) { if (uploadLockingService != null) { diff --git a/src/main/java/me/desair/tus/server/rufh/handler/RufhCreationPostRequestHandler.java b/src/main/java/me/desair/tus/server/rufh/handler/RufhCreationPostRequestHandler.java index 1c47d68c..4ff1232d 100644 --- a/src/main/java/me/desair/tus/server/rufh/handler/RufhCreationPostRequestHandler.java +++ b/src/main/java/me/desair/tus/server/rufh/handler/RufhCreationPostRequestHandler.java @@ -66,17 +66,23 @@ public HttpProblemDetails process( String uploadCompleteHeader = servletRequest.getHeader(HttpHeader.UPLOAD_COMPLETE); Boolean uploadComplete = StructuredHeaderUtil.parseBoolean(uploadCompleteHeader); + long cl = servletRequest.getContentLengthLong(); + Long announcedLength = uploadLength; + if (announcedLength == null && Boolean.TRUE.equals(uploadComplete) && cl >= 0) { + announcedLength = cl; + } + UploadInfo uploadInfo; if (preCreatedUploadInfo != null) { uploadInfo = preCreatedUploadInfo; - if (uploadLength != null && uploadLength >= 0) { - uploadInfo.setLength(uploadLength); + if (announcedLength != null && announcedLength >= 0) { + uploadInfo.setLength(announcedLength); } uploadStorageService.update(uploadInfo); } else { uploadInfo = new UploadInfo(); - if (uploadLength != null && uploadLength >= 0) { - uploadInfo.setLength(uploadLength); + if (announcedLength != null && announcedLength >= 0) { + uploadInfo.setLength(announcedLength); } uploadInfo = uploadStorageService.create(uploadInfo, ownerKey); } diff --git a/src/main/java/me/desair/tus/server/rufh/util/RufhInterimResponseUtil.java b/src/main/java/me/desair/tus/server/rufh/util/RufhInterimResponseUtil.java index 4bbb7749..a09dc67e 100644 --- a/src/main/java/me/desair/tus/server/rufh/util/RufhInterimResponseUtil.java +++ b/src/main/java/me/desair/tus/server/rufh/util/RufhInterimResponseUtil.java @@ -56,13 +56,22 @@ public static String getRawInterimResponse( String uploadUri; try { - UploadInfo uploadInfo = uploadStorageService.getUploadInfo(existingUploadUri, ownerKey); + UploadInfo uploadInfo = + existingUploadUri != null + ? uploadStorageService.getUploadInfo(existingUploadUri, ownerKey) + : null; if (uploadInfo != null) { long offset = uploadInfo.getOffset() != null ? uploadInfo.getOffset() : 0L; return getRawInterimResponseForAppend(offset); } + // If there is no existing upload, an interim response with Location is only valid for + // an upload creation request matching this storage service's creation endpoint. + if (!Utils.isCreationEndpoint(servletRequest, uploadStorageService)) { + return null; + } + uploadInfo = new UploadInfo(); uploadInfo = uploadStorageService.create(uploadInfo, ownerKey); uploadUri = Utils.getUploadUriOnCreation(uploadInfo, servletRequest, uploadStorageService); diff --git a/src/main/java/me/desair/tus/server/upload/UploadStorageService.java b/src/main/java/me/desair/tus/server/upload/UploadStorageService.java index ea5d9f1d..bdce0298 100644 --- a/src/main/java/me/desair/tus/server/upload/UploadStorageService.java +++ b/src/main/java/me/desair/tus/server/upload/UploadStorageService.java @@ -279,6 +279,24 @@ default boolean isJsonSerializationEnabled() { return false; } + /** + * Set the thread pool size used for asynchronous background chunk uploading to cloud storage. + * + * @param size Number of worker threads + */ + default void setCloudUploadThreadPoolSize(int size) { + // Default no-op for non-cloud implementations + } + + /** + * Get the thread pool size used for asynchronous background chunk uploading to cloud storage. + * + * @return Thread pool size, defaults to 10 + */ + default int getCloudUploadThreadPoolSize() { + return 10; + } + /** * Closes any underlying storage resources. * diff --git a/src/main/java/me/desair/tus/server/upload/azure/AzureBlobConcatenationService.java b/src/main/java/me/desair/tus/server/upload/azure/AzureBlobConcatenationService.java index 19be0000..697f9be1 100644 --- a/src/main/java/me/desair/tus/server/upload/azure/AzureBlobConcatenationService.java +++ b/src/main/java/me/desair/tus/server/upload/azure/AzureBlobConcatenationService.java @@ -3,11 +3,14 @@ import com.azure.storage.blob.BlobClient; import com.azure.storage.blob.BlobContainerClient; import com.azure.storage.blob.models.BlobStorageException; +import com.azure.storage.blob.sas.BlobSasPermission; +import com.azure.storage.blob.sas.BlobServiceSasSignatureValues; import com.azure.storage.blob.specialized.BlockBlobClient; import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; import java.nio.charset.StandardCharsets; +import java.time.OffsetDateTime; import java.util.ArrayList; import java.util.Base64; import java.util.Collections; @@ -109,16 +112,20 @@ public void merge(UploadInfo finalUpload) throws IOException, UploadNotFoundExce BlobClient partialBlob = containerClient.getBlobClient(uploadPrefix + partialInfo.getId()); try { - // 1. Attempt zero-copy server-side block copying on Azure Storage cluster - stageBlockFromUrl(finalBlockBlob, blockId, partialBlob.getBlobUrl()); + // 1. Attempt zero-copy server-side block copying on Azure Storage cluster using an + // authorized blob URL (with read SAS token when shared key credentials are present) + String sourceUrl = getAuthorizedBlobUrl(partialBlob); + stageBlockFromUrl(finalBlockBlob, blockId, sourceUrl); } catch (BlobStorageException e) { // 2. In private Azure containers without SAS tokens or in emulators, stageBlockFromUrl - // fails with 403 (ACCESS_DENIED) or 400/501 (API_NOT_IMPLEMENTED). Fall back + // fails with 401/403 (ACCESS_DENIED / CannotVerifyCopySource) or 400/501 + // (API_NOT_IMPLEMENTED). Fall back // gracefully to streaming block staging via getUploadedBytes(). AzureErrorType errorType = AzureUtils.parseErrorResponse(e); if (errorType == AzureErrorType.API_NOT_IMPLEMENTED || errorType == AzureErrorType.ACCESS_DENIED || e.getStatusCode() == 400 + || e.getStatusCode() == 401 || e.getStatusCode() == 403) { try (InputStream partIs = storageService.getUploadedBytes(partialInfo.getId())) { finalBlockBlob.stageBlock(blockId, partIs, partialInfo.getOffset()); @@ -230,4 +237,32 @@ private String sanitizePrefix(String prefix) { void stageBlockFromUrl(BlockBlobClient finalBlockBlob, String blockId, String sourceUrl) { finalBlockBlob.stageBlockFromUrl(blockId, sourceUrl, null); } + + /** + * Resolves an authorized source URL for server-side block copying. If the client possesses shared + * key credentials, generates a short-lived read SAS token so that private Azure containers can be + * accessed directly by the storage service via stageBlockFromUrl. + * + * @param blobClient The BlobClient of the source partial upload + * @return The authorized blob URL (with SAS token if available) or raw blob URL + */ + String getAuthorizedBlobUrl(BlobClient blobClient) { + if (blobClient == null) { + return null; + } + try { + BlobServiceSasSignatureValues sasValues = + new BlobServiceSasSignatureValues( + OffsetDateTime.now().plusMinutes(15), + new BlobSasPermission().setReadPermission(true)); + String sasToken = blobClient.generateSas(sasValues); + if (sasToken != null && !sasToken.isEmpty()) { + return blobClient.getBlobUrl() + "?" + sasToken; + } + } catch (Exception ignored) { + // Client lacks shared key credentials (e.g. TokenCredential without user delegation key) + // or SAS generation is unsupported; fall back to raw blob URL. + } + return blobClient.getBlobUrl(); + } } diff --git a/src/main/java/me/desair/tus/server/upload/azure/AzureBlobStorageService.java b/src/main/java/me/desair/tus/server/upload/azure/AzureBlobStorageService.java index 69c6e70e..686797a2 100644 --- a/src/main/java/me/desair/tus/server/upload/azure/AzureBlobStorageService.java +++ b/src/main/java/me/desair/tus/server/upload/azure/AzureBlobStorageService.java @@ -25,6 +25,10 @@ import java.util.Base64; import java.util.List; import java.util.Objects; +import java.util.concurrent.SynchronousQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import me.desair.tus.server.checksum.ChecksumAlgorithm; import me.desair.tus.server.exception.MaxAppendSizeExceededException; import me.desair.tus.server.exception.MinAppendSizeNotMetException; @@ -37,6 +41,7 @@ import me.desair.tus.server.upload.UploadStorageService; import me.desair.tus.server.upload.UuidUploadIdFactory; import me.desair.tus.server.upload.concatenation.UploadConcatenationService; +import me.desair.tus.server.upload.util.AsyncChunkUploader; import me.desair.tus.server.util.UploadInfoJsonSerializer; import me.desair.tus.server.util.Utils; import org.apache.commons.io.IOUtils; @@ -86,6 +91,9 @@ public class AzureBlobStorageService implements UploadStorageService { private long preferredBlockSize = DEFAULT_PREFERRED_BLOCK_SIZE; + private int cloudUploadThreadPoolSize = 10; + private final ThreadPoolExecutor uploadExecutor; + private Long maxUploadSize; private Long maxAppendSize; private Long minAppendSize; @@ -144,6 +152,21 @@ public AzureBlobStorageService( } Utils.cleanupTempFiles(this.tempBufferDir, "tus-azure-chunk-*.tmp", 24L * 3600_000L); + AtomicInteger threadNum = new AtomicInteger(1); + this.uploadExecutor = + new ThreadPoolExecutor( + cloudUploadThreadPoolSize, + cloudUploadThreadPoolSize, + 60L, + TimeUnit.SECONDS, + new SynchronousQueue<>(), + r -> { + Thread t = new Thread(r, "tus-azure-upload-" + threadNum.getAndIncrement()); + t.setDaemon(true); + return t; + }, + new ThreadPoolExecutor.CallerRunsPolicy()); + this.concatenationService = new AzureBlobConcatenationService(containerClient, this.uploadPrefix, this); } @@ -212,56 +235,84 @@ public UploadInfo append(UploadInfo upload, InputStream inputStream) IOException streamException = null; TusException pendingTusException = null; - // 4. Read incoming stream in chunks, staging blocks directly to Azure Block Blob - while (true) { - File chunkFile = null; - try { - chunkFile = Files.createTempFile(tempBufferDir, "tus-azure-chunk-", ".tmp").toFile(); - ReadChunkResult chunkResult = readChunk(inputStream, chunkFile, optimalBlockSize); - long chunkSize = chunkResult.bytesRead; - if (chunkResult.exception != null) { - streamException = chunkResult.exception; - } + List plannedBlockIds = new ArrayList<>(); + List plannedChunkSizes = new ArrayList<>(); - if (chunkSize <= 0) { - break; - } - - if (effectiveMaxAppendSize != null - && (totalAppended + chunkSize) > effectiveMaxAppendSize) { - // If maxAppendSize is exceeded, stop reading from the stream but commit - // previously staged blocks so no uploaded data or offset is corrupted. - pendingTusException = - new MaxAppendSizeExceededException( - "Append payload size " - + (totalAppended + chunkSize) - + " exceeded limit of " - + effectiveMaxAppendSize); - break; - } - - totalAppended += chunkSize; + try (AsyncChunkUploader uploader = new AsyncChunkUploader(uploadExecutor)) { + // 4. Read incoming stream in chunks, staging blocks directly to Azure Block Blob + while (true) { + File chunkFile = null; + boolean handedOff = false; + try { + chunkFile = Files.createTempFile(tempBufferDir, "tus-azure-chunk-", ".tmp").toFile(); + ReadChunkResult chunkResult = readChunk(inputStream, chunkFile, optimalBlockSize); + long chunkSize = chunkResult.bytesRead; + if (chunkResult.exception != null) { + streamException = chunkResult.exception; + } - // Validate chunk against remaining block budget - validateRemainingBlockBudget(upload, blockIds.size()); + if (chunkSize <= 0) { + break; + } - // Stage block directly to Azure Block Blob - stageChunkFile(chunkFile, chunkSize, blockBlobClient, blockIds); + if (effectiveMaxAppendSize != null + && (totalAppended + chunkSize) > effectiveMaxAppendSize) { + // If maxAppendSize is exceeded, stop reading from the stream but commit + // previously staged blocks so no uploaded data or offset is corrupted. + pendingTusException = + new MaxAppendSizeExceededException( + "Append payload size " + + (totalAppended + chunkSize) + + " exceeded limit of " + + effectiveMaxAppendSize); + break; + } - if (streamException != null) { + int plannedIndex = blockIds.size() + plannedBlockIds.size(); + // Validate chunk against remaining block budget + validateRemainingBlockBudget(upload, plannedIndex); + + String blockId = generateBlockId(plannedIndex); + plannedBlockIds.add(blockId); + plannedChunkSizes.add(chunkSize); + totalAppended += chunkSize; + + File fileToUpload = chunkFile; + uploader.submitChunk( + chunkFile, + chunkSize, + blockId, + () -> stageBlock(blockBlobClient, blockId, fileToUpload, chunkSize)); + handedOff = true; + + if (streamException != null) { + break; + } + } catch (IOException e) { + streamException = e; + break; + } catch (TusException te) { + pendingTusException = te; break; + } finally { + if (!handedOff) { + deleteFileQuietly(chunkFile); + } } - } catch (IOException e) { - streamException = e; - break; - } catch (TusException te) { - pendingTusException = te; - break; - } finally { - deleteFileQuietly(chunkFile); + } + + int confirmedCount = uploader.drainAndComplete(4000); + for (int i = 0; i < confirmedCount; i++) { + blockIds.add(plannedBlockIds.get(i)); } } + // Calculate actual confirmed bytes appended + long confirmedAppended = 0L; + for (int i = 0; i < blockIds.size() - initialBlockCount; i++) { + confirmedAppended += plannedChunkSizes.get(i); + } + // 5. Commit any newly staged blocks in batch (1 single commit call for entire request) // Batching block commits into a single call at the end eliminates redundant network // round-trips for every chunk, drastically improving performance. In addition, committing @@ -270,7 +321,7 @@ public UploadInfo append(UploadInfo upload, InputStream inputStream) if (blockIds.size() > initialBlockCount) { try { blockBlobClient.commitBlockList(blockIds, true); - upload.setOffset(initialOffset + totalAppended); + upload.setOffset(initialOffset + confirmedAppended); if (uploadExpirationPeriod != null && uploadExpirationPeriod > 0) { upload.setExpirationTimestamp(System.currentTimeMillis() + uploadExpirationPeriod); } @@ -301,7 +352,16 @@ public UploadInfo append(UploadInfo upload, InputStream inputStream) } // 6. Validate minimum append size constraints if configured - validateMinAppendSize(totalAppended); + // Per RUFH §4.1.4: "This limit does not apply to upload creation requests with no content, + // or to requests completing the upload by including the Upload-Complete: ?1 header field." + boolean isCompletingOrEmpty = + !upload.isUploadInProgress() + || (upload.getLength() != null + && upload.getOffset() != null + && upload.getOffset() >= upload.getLength()); + if (!isCompletingOrEmpty) { + validateMinAppendSize(confirmedAppended); + } return upload; } @@ -653,6 +713,40 @@ public long getPreferredBlockSize() { return preferredBlockSize; } + @Override + public void setCloudUploadThreadPoolSize(int size) { + if (size <= 0) { + throw new IllegalArgumentException( + "The cloud upload thread pool size must be greater than 0"); + } + this.cloudUploadThreadPoolSize = size; + if (size > uploadExecutor.getMaximumPoolSize()) { + uploadExecutor.setMaximumPoolSize(size); + uploadExecutor.setCorePoolSize(size); + } else { + uploadExecutor.setCorePoolSize(size); + uploadExecutor.setMaximumPoolSize(size); + } + } + + @Override + public int getCloudUploadThreadPoolSize() { + return cloudUploadThreadPoolSize; + } + + @Override + public void close() throws IOException { + uploadExecutor.shutdown(); + try { + if (!uploadExecutor.awaitTermination(5, TimeUnit.SECONDS)) { + uploadExecutor.shutdownNow(); + } + } catch (InterruptedException e) { + uploadExecutor.shutdownNow(); + Thread.currentThread().interrupt(); + } + } + // --- Helper Methods --- /** Calculates auto-calibrated optimal block size based on total upload length. */ @@ -766,16 +860,17 @@ private void validateMinAppendSize(long totalAppended) throws MinAppendSizeNotMe } } - /** Stages local chunk temp file as a Block Blob block. */ - private void stageChunkFile( - File chunkFile, long chunkSize, BlockBlobClient blockBlobClient, List blockIds) + /** + * Stages local chunk temp file as a Block Blob block without mutating blockIds on the worker + * thread. + */ + private void stageBlock( + BlockBlobClient blockBlobClient, String blockId, File chunkFile, long chunkSize) throws IOException { if (chunkSize > 0) { - String chunkBlockId = generateBlockId(blockIds.size()); try (InputStream chunkIs = new java.io.BufferedInputStream(new FileInputStream(chunkFile))) { - blockBlobClient.stageBlock(chunkBlockId, chunkIs, chunkSize); + blockBlobClient.stageBlock(blockId, chunkIs, chunkSize); } - blockIds.add(chunkBlockId); } } diff --git a/src/main/java/me/desair/tus/server/upload/azure/AzureUtils.java b/src/main/java/me/desair/tus/server/upload/azure/AzureUtils.java index b7e1a105..af579f28 100644 --- a/src/main/java/me/desair/tus/server/upload/azure/AzureUtils.java +++ b/src/main/java/me/desair/tus/server/upload/azure/AzureUtils.java @@ -58,7 +58,11 @@ public static AzureErrorType parseErrorResponse(BlobStorageException exception) return AzureErrorType.API_NOT_IMPLEMENTED; } - if (statusCode == 403 || errorCodeStr.contains("authorizationfailure")) { + if (statusCode == 401 + || statusCode == 403 + || errorCodeStr.contains("authorizationfailure") + || errorCodeStr.contains("cannotverifycopysource") + || errorCodeStr.contains("noauthenticationinformation")) { return AzureErrorType.ACCESS_DENIED; } diff --git a/src/main/java/me/desair/tus/server/upload/cache/ThreadLocalCachedStorageAndLockingService.java b/src/main/java/me/desair/tus/server/upload/cache/ThreadLocalCachedStorageAndLockingService.java index 76241689..ee83f592 100644 --- a/src/main/java/me/desair/tus/server/upload/cache/ThreadLocalCachedStorageAndLockingService.java +++ b/src/main/java/me/desair/tus/server/upload/cache/ThreadLocalCachedStorageAndLockingService.java @@ -233,6 +233,16 @@ public UploadInfo getUploadInfoByChecksum(String checksum, ChecksumAlgorithm alg return storageServiceDelegate.getUploadInfoByChecksum(checksum, algorithm); } + @Override + public void setCloudUploadThreadPoolSize(int size) { + storageServiceDelegate.setCloudUploadThreadPoolSize(size); + } + + @Override + public int getCloudUploadThreadPoolSize() { + return storageServiceDelegate.getCloudUploadThreadPoolSize(); + } + @Override public UploadLock lockUploadByUri(String requestUri) throws TusException, IOException { UploadLock uploadLock = lockingServiceDelegate.lockUploadByUri(requestUri); diff --git a/src/main/java/me/desair/tus/server/upload/disk/DiskStorageService.java b/src/main/java/me/desair/tus/server/upload/disk/DiskStorageService.java index 2269c5e2..5a428699 100644 --- a/src/main/java/me/desair/tus/server/upload/disk/DiskStorageService.java +++ b/src/main/java/me/desair/tus/server/upload/disk/DiskStorageService.java @@ -34,6 +34,7 @@ import me.desair.tus.server.util.Utils; import org.apache.commons.codec.binary.Base64; import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.Validate; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -50,6 +51,7 @@ public class DiskStorageService extends AbstractDiskBasedService implements Uplo private Long maxAppendSize = null; private Long minAppendSize = null; private Long minSize = null; + private int cloudUploadThreadPoolSize = 10; private Long uploadExpirationPeriod = null; private UploadIdFactory idFactory; private UploadConcatenationService uploadConcatenationService; @@ -117,6 +119,17 @@ public Long getMinSize() { return minSize; } + @Override + public void setCloudUploadThreadPoolSize(int size) { + Validate.isTrue(size > 0, "The cloud upload thread pool size must be greater than 0"); + this.cloudUploadThreadPoolSize = size; + } + + @Override + public int getCloudUploadThreadPoolSize() { + return cloudUploadThreadPoolSize; + } + @Override public void setUploadDeduplicationEnabled(boolean enabled) { this.isUploadDeduplicationEnabled = enabled; @@ -338,8 +351,18 @@ public UploadInfo append(UploadInfo info, InputStream inputStream) + " bytes. You can only append to the end of an upload"); } - // write all bytes in the channel up to the configured maximum - transferred = file.transferFrom(uploadedBytes, offset, max - offset); + // write all bytes in the channel up to the configured maximum in a loop to support + // transfers larger than 2GB (OS syscall limits) + long bytesTransferred = 0; + while (offset + bytesTransferred < max) { + long toTransfer = max - (offset + bytesTransferred); + long n = file.transferFrom(uploadedBytes, offset + bytesTransferred, toTransfer); + if (n <= 0) { + break; + } + bytesTransferred += n; + } + transferred = bytesTransferred; file.force(true); newOffset = offset + transferred; @@ -492,8 +515,18 @@ public void copyUploadTo(UploadInfo info, OutputStream outputStream) "The upload bytes for id " + readId + " could not be found."); } try (FileChannel file = FileChannel.open(bytesPath, READ)) { - // Efficiently copy the bytes to the output stream - file.transferTo(0, upload.getLength(), outputChannel); + // Efficiently copy the bytes to the output stream in a loop to handle transfers + // exceeding OS single-syscall limits (e.g. 2GB sendfile limit) + long position = 0; + long remaining = upload.getLength() != null ? upload.getLength() : file.size(); + while (remaining > 0) { + long bytesTransferred = file.transferTo(position, remaining, outputChannel); + if (bytesTransferred <= 0) { + break; + } + position += bytesTransferred; + remaining -= bytesTransferred; + } } } } diff --git a/src/main/java/me/desair/tus/server/upload/s3/S3StorageService.java b/src/main/java/me/desair/tus/server/upload/s3/S3StorageService.java index d3b9c911..8b059137 100644 --- a/src/main/java/me/desair/tus/server/upload/s3/S3StorageService.java +++ b/src/main/java/me/desair/tus/server/upload/s3/S3StorageService.java @@ -30,6 +30,10 @@ import java.util.List; import java.util.NoSuchElementException; import java.util.Objects; +import java.util.concurrent.SynchronousQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import me.desair.tus.server.checksum.ChecksumAlgorithm; import me.desair.tus.server.exception.MaxAppendSizeExceededException; import me.desair.tus.server.exception.MaxUploadLengthExceededException; @@ -45,6 +49,7 @@ import me.desair.tus.server.upload.UploadType; import me.desair.tus.server.upload.UuidUploadIdFactory; import me.desair.tus.server.upload.concatenation.UploadConcatenationService; +import me.desair.tus.server.upload.util.AsyncChunkUploader; import me.desair.tus.server.util.UploadInfoJsonSerializer; import me.desair.tus.server.util.Utils; import org.apache.commons.io.FileUtils; @@ -86,12 +91,12 @@ public class S3StorageService implements UploadStorageService { public static final String DEFAULT_LOCKS_PREFIX = "locks/"; // Part Sizing Constraints (per AWS S3 & MinIO specifications) - private static final long DEFAULT_MIN_PART_SIZE = - 5L * 1024 * 1024; // 5 MB (S3 minimum part limit) - private static final long DEFAULT_PREFERRED_PART_SIZE = - 50L * 1024 * 1024; // 50 MB (Optimal chunk size) - private static final long DEFAULT_MAX_PART_SIZE = + public static final long DEFAULT_MIN_PART_SIZE = 5L * 1024 * 1024; // 5 MB (S3 minimum part limit) + public static final long DEFAULT_PREFERRED_PART_SIZE = + 8L * 1024 * 1024; // 8 MB (Optimal chunk size aligned with Azure) + public static final long DEFAULT_MAX_PART_SIZE = 5L * 1024 * 1024 * 1024L; // 5 GB (S3 maximum object/part limit) + public static final int MAX_PARTS_PER_UPLOAD = 10_000; private final MinioClient minioClient; private final String bucket; @@ -104,6 +109,9 @@ public class S3StorageService implements UploadStorageService { private long minPartSize = DEFAULT_MIN_PART_SIZE; private long preferredPartSize = DEFAULT_PREFERRED_PART_SIZE; + private int cloudUploadThreadPoolSize = 10; + private final ThreadPoolExecutor uploadExecutor; + private Long maxUploadSize; private Long maxAppendSize; private Long minAppendSize; @@ -169,6 +177,21 @@ public S3StorageService( log.debug("Unable to ensure temporary directory exists: {}", e.getMessage()); } + AtomicInteger threadNum = new AtomicInteger(1); + this.uploadExecutor = + new ThreadPoolExecutor( + cloudUploadThreadPoolSize, + cloudUploadThreadPoolSize, + 60L, + TimeUnit.SECONDS, + new SynchronousQueue<>(), + r -> { + Thread t = new Thread(r, "tus-s3-upload-" + threadNum.getAndIncrement()); + t.setDaemon(true); + return t; + }, + new ThreadPoolExecutor.CallerRunsPolicy()); + this.concatenationService = new S3ConcatenationService( this.minioClient, this.bucket, this.objectPrefix, this, this.temporaryDirectory); @@ -307,6 +330,9 @@ public UploadInfo append(UploadInfo upload, InputStream inputStream) throws IOException, TusException { // Step 1: Verify upload existence and check configured size limits UploadInfo info = fetchAndValidateUpload(upload.getId()); + if (upload.getLength() != null && info.getLength() == null) { + info.setLength(upload.getLength()); + } String objectKey = getS3ObjectKey(info); String partObjectKey = buildIncompletePartKey(info.getId()); @@ -316,18 +342,39 @@ public UploadInfo append(UploadInfo upload, InputStream inputStream) PreparedStream preparedStream = prepareStreamWithExistingIncompletePart(info.getId(), partObjectKey, inputStream); + // Validate that the upload has not exceeded S3's 10,000 multipart parts ceiling + validateRemainingPartBudget(info, preparedStream.remainingPartKeys.size()); + // Step 3: Process payload stream in optimal chunk parts and upload to S3 boolean successfullyFinished = false; try { AppendResult appendResult = processPayloadChunks(info, preparedStream, info.getId(), partObjectKey); - // Step 4: Validate minimum append size constraints if configured + // Step 4: Recalculate total uploaded byte offset across all uploaded part objects in S3. + // S3 listObjects can exhibit eventual consistency; take the maximum of remote parts query + // and locally verified stream progression to ensure newOffset accurately reflects bytes + // successfully written. + long calculatedOffset = + calculateCurrentOffset(objectKey, info.getId(), partObjectKey, info.getLength()); + long currentTotalOffset = + (info.getOffset() != null ? info.getOffset() : 0L) + + appendResult.totalBytesAppended + - preparedStream.prependedBytes; + long newOffset = Math.max(calculatedOffset, currentTotalOffset); + info.setOffset(newOffset); + upload.setOffset(newOffset); + + // Step 5: Validate minimum append size constraints if configured // Subtract prependedBytes so minAppendSize accurately measures the payload transferred // in THIS request rather than earlier buffered bytes. + // Per RUFH §4.1.4: "This limit does not apply to upload creation requests with no content, + // or to requests completing the upload by including the Upload-Complete: ?1 header field." + boolean isCompletingOrEmpty = + !info.isUploadInProgress() || (info.getLength() != null && newOffset >= info.getLength()); long requestPayloadAppended = Math.max(0L, appendResult.totalBytesAppended - preparedStream.prependedBytes); - if (minAppendSize != null && requestPayloadAppended < minAppendSize) { + if (minAppendSize != null && !isCompletingOrEmpty && requestPayloadAppended < minAppendSize) { throw new MinAppendSizeNotMetException( "Append payload size " + requestPayloadAppended @@ -335,11 +382,6 @@ public UploadInfo append(UploadInfo upload, InputStream inputStream) + minAppendSize); } - // Step 5: Recalculate total uploaded byte offset across all uploaded part objects in S3 - long newOffset = calculateCurrentOffset(objectKey, info.getId(), partObjectKey); - info.setOffset(newOffset); - upload.setOffset(newOffset); - // Step 6: If all expected bytes are uploaded, compose all part chunks into final S3 object finalizeCompletedUploadIfFinished(info, objectKey, info.getId(), appendResult, newOffset); update(info); @@ -347,7 +389,8 @@ public UploadInfo append(UploadInfo upload, InputStream inputStream) return info; } finally { if (!successfullyFinished) { - long newOffset = calculateCurrentOffset(objectKey, info.getId(), partObjectKey); + long newOffset = + calculateCurrentOffset(objectKey, info.getId(), partObjectKey, info.getLength()); info.setOffset(newOffset); upload.setOffset(newOffset); update(info); @@ -687,6 +730,66 @@ public void setIdFactory(UploadIdFactory idFactory) { } } + @Override + public void setCloudUploadThreadPoolSize(int size) { + if (size <= 0) { + throw new IllegalArgumentException( + "The cloud upload thread pool size must be greater than 0"); + } + this.cloudUploadThreadPoolSize = size; + if (size > uploadExecutor.getMaximumPoolSize()) { + uploadExecutor.setMaximumPoolSize(size); + uploadExecutor.setCorePoolSize(size); + } else { + uploadExecutor.setCorePoolSize(size); + uploadExecutor.setMaximumPoolSize(size); + } + } + + @Override + public int getCloudUploadThreadPoolSize() { + return cloudUploadThreadPoolSize; + } + + /** + * Set the preferred chunk part size in bytes used when buffering and uploading parts to S3. + * + * @param preferredPartSize Part size in bytes (must be between 5 MB and 5 GB) + */ + public void setPreferredPartSize(long preferredPartSize) { + if (preferredPartSize < minPartSize || preferredPartSize > DEFAULT_MAX_PART_SIZE) { + throw new IllegalArgumentException( + "Preferred part size must be between " + + minPartSize + + " and " + + DEFAULT_MAX_PART_SIZE + + " bytes"); + } + this.preferredPartSize = preferredPartSize; + } + + /** + * Return the preferred chunk part size in bytes used when buffering and uploading parts to S3. + * + * @return Preferred part size in bytes + */ + public long getPreferredPartSize() { + return preferredPartSize; + } + + @Override + public void close() throws IOException { + uploadExecutor.shutdown(); + try { + if (!uploadExecutor.awaitTermination(5, TimeUnit.SECONDS)) { + uploadExecutor.shutdownNow(); + } + } catch (InterruptedException e) { + uploadExecutor.shutdownNow(); + Thread.currentThread().interrupt(); + } + } + // PRIVATE HELPER METHODS & S3 PROCESSING LOGIC private UploadInfo fetchAndValidateUpload(UploadId uploadId) @@ -791,8 +894,9 @@ private PreparedStream prepareStreamWithExistingIncompletePart( /** * Reads bytes from the incoming stream into temporary local files of optimal part size (default - * 50MB). Parts ≥ 5MB are uploaded immediately to S3 as part chunk objects. Any trailing chunk - * under 5MB is saved as a temporary .part object unless it completes the overall upload. + * 8MB). Parts ≥ 5MB are uploaded asynchronously to S3 via {@link AsyncChunkUploader}, + * overlapping client stream reading with cloud upload. Any trailing chunk under 5MB is saved as a + * temporary .part object unless it completes the overall upload. */ private AppendResult processPayloadChunks( UploadInfo info, PreparedStream preparedStream, UploadId id, String partObjectKey) @@ -802,7 +906,7 @@ private AppendResult processPayloadChunks( int nextPartNumber = allPartKeys.size() + 1; InputStream streamToRead = preparedStream.stream; - long optimalPartSize = calcOptimalPartSize(info.getLength() != null ? info.getLength() : 0); + long optimalPartSize = calcOptimalPartSize(info.getLength()); byte[] buffer = new byte[8192]; long totalBytesAppended = 0; @@ -813,95 +917,150 @@ private AppendResult processPayloadChunks( boolean streamFinished = false; MaxAppendSizeExceededException maxAppendSizeException = null; + IOException readException = null; - while (!streamFinished) { - File tempChunkFile = - Files.createTempFile(temporaryDirectory, "tus-s3-chunk-", ".tmp").toFile(); - // Do not call tempChunkFile.deleteOnExit() here. In high-throughput long-running services, - // deleteOnExit() registers entries in a static JVM set that cannot be garbage collected, - // creating an unbounded memory leak. Temp files are deleted in try-finally blocks below. + List plannedPartKeys = new ArrayList<>(); - long chunkBytesWritten = 0; - IOException readException = null; - try { - try (FileOutputStream fos = new FileOutputStream(tempChunkFile)) { - int bytesRead; - while (chunkBytesWritten < optimalPartSize - && (bytesRead = streamToRead.read(buffer)) != -1) { - long requestBytesSoFar = - (totalBytesAppended + bytesRead) - preparedStream.prependedBytes; - if (maxAppendSize != null && requestBytesSoFar > maxAppendSize) { - // If maxAppendSize is exceeded, do not discard tempChunkFile immediately. - // If preparedStream had prepended bytes from a previous incomplete .part, discarding - // tempChunkFile would permanently lose those bytes. Instead, stop reading and record - // maxAppendSizeException so the bytes currently in tempChunkFile are flushed to S3 - // and UploadInfo offset is accurately preserved. - maxAppendSizeException = - new MaxAppendSizeExceededException( - "Append payload exceeded limit of " + maxAppendSize); + try (AsyncChunkUploader uploader = new AsyncChunkUploader(uploadExecutor)) { + while (!streamFinished) { + File tempChunkFile = + Files.createTempFile(temporaryDirectory, "tus-s3-chunk-", ".tmp").toFile(); + // Do not call tempChunkFile.deleteOnExit() here. In high-throughput long-running services, + // deleteOnExit() registers entries in a static JVM set that cannot be garbage collected, + // creating an unbounded memory leak. Temp files are deleted in try-finally blocks below. + + long chunkBytesWritten = 0; + boolean handedOff = false; + try { + try (FileOutputStream fos = new FileOutputStream(tempChunkFile)) { + int bytesRead; + while (chunkBytesWritten < optimalPartSize + && (bytesRead = streamToRead.read(buffer)) != -1) { + long requestBytesSoFar = + (totalBytesAppended + bytesRead) - preparedStream.prependedBytes; + if (maxAppendSize != null && requestBytesSoFar > maxAppendSize) { + // If maxAppendSize is exceeded, do not discard tempChunkFile immediately. + // If preparedStream had prepended bytes from a previous incomplete .part, + // discarding + // tempChunkFile would permanently lose those bytes. Instead, stop reading and + // record + // maxAppendSizeException so the bytes currently in tempChunkFile are flushed to S3 + // and UploadInfo offset is accurately preserved. + maxAppendSizeException = + new MaxAppendSizeExceededException( + "Append payload exceeded limit of " + maxAppendSize); + streamFinished = true; + break; + } + fos.write(buffer, 0, bytesRead); + chunkBytesWritten += bytesRead; + totalBytesAppended += bytesRead; + } + + if (chunkBytesWritten < optimalPartSize) { streamFinished = true; - break; } - fos.write(buffer, 0, bytesRead); - chunkBytesWritten += bytesRead; - totalBytesAppended += bytesRead; + } catch (IOException e) { + readException = e; + streamFinished = true; } - if (chunkBytesWritten < optimalPartSize) { - streamFinished = true; + if (chunkBytesWritten == 0) { + break; } - } catch (IOException e) { - readException = e; - streamFinished = true; - } - if (chunkBytesWritten == 0) { - if (readException != null) { - throw readException; + // Base offset plus total bytes appended accurately measures uploaded progress without + // double-counting + long currentTotalOffset = baseOffset + totalBytesAppended; + + // Interruption Guard: If an IOException or limit exception occurred (e.g. client pause or + // connection drop), + // the chunk must NEVER be considered complete, preventing sub-5MB chunks from being + // promoted. + boolean isUploadComplete = + readException == null + && maxAppendSizeException == null + && info.getLength() != null + && currentTotalOffset >= info.getLength(); + + // Validate remaining part capacity before allocating next S3 part number + if (allPartKeys.size() + plannedPartKeys.size() >= MAX_PARTS_PER_UPLOAD + || nextPartNumber > MAX_PARTS_PER_UPLOAD) { + maxAppendSizeException = + new MaxAppendSizeExceededException( + "Upload has reached the maximum allowed S3 limit of " + + MAX_PARTS_PER_UPLOAD + + " parts."); + if (chunkBytesWritten > 0) { + int confirmedCount = uploader.drainAndComplete(4000); + allPartKeys.addAll(plannedPartKeys.subList(0, confirmedCount)); + storeIncompletePartToS3(partObjectKey, tempChunkFile, chunkBytesWritten); + handedOff = true; + } + streamFinished = true; + break; } - if (maxAppendSizeException != null) { - throw maxAppendSizeException; + + // AWS S3 / MinIO Rule: Parts must be >= 5 MB unless it's the final part completing the + // upload + if (chunkBytesWritten >= minPartSize) { + // Full part chunk (>= 5 MB): Submit to AsyncChunkUploader pipeline for background + // upload + String chunkKey = buildChunkPartKey(id, nextPartNumber++); + plannedPartKeys.add(chunkKey); + long bytesToUpload = chunkBytesWritten; + uploader.submitChunk( + tempChunkFile, + bytesToUpload, + chunkKey, + () -> uploadChunkToS3(chunkKey, tempChunkFile, bytesToUpload)); + handedOff = true; + + } else if (streamFinished && isUploadComplete) { + // Sub-5MB final chunk that completes the overall upload: + // First drain all preceding parts in the pipeline + int confirmedCount = uploader.drainAndComplete(4000); + allPartKeys.addAll(plannedPartKeys.subList(0, confirmedCount)); + + String chunkKey = buildChunkPartKey(id, nextPartNumber++); + uploadChunkToS3(chunkKey, tempChunkFile, chunkBytesWritten); + allPartKeys.add(chunkKey); + handedOff = true; + + } else { + // Sub-5MB incomplete chunk (e.g. upload paused midway or interrupted): + // First drain all preceding parts in the pipeline + int confirmedCount = uploader.drainAndComplete(4000); + allPartKeys.addAll(plannedPartKeys.subList(0, confirmedCount)); + + storeIncompletePartToS3(partObjectKey, tempChunkFile, chunkBytesWritten); + handedOff = true; } - break; - } - // Base offset plus total bytes appended accurately measures uploaded progress without - // double-counting - long currentTotalOffset = baseOffset + totalBytesAppended; - - // Interruption Guard: If an IOException or limit exception occurred (e.g. client pause or - // connection drop), - // the chunk must NEVER be considered complete, preventing sub-5MB chunks from being - // promoted. - boolean isUploadComplete = - readException == null - && maxAppendSizeException == null - && info.getLength() != null - && currentTotalOffset >= info.getLength(); - - // AWS S3 / MinIO Rule: Parts must be >= 5 MB unless it's the final part completing the - // upload - if (chunkBytesWritten >= minPartSize || (streamFinished && isUploadComplete)) { - String chunkKey = buildChunkPartKey(id, nextPartNumber); - uploadChunkToS3(chunkKey, tempChunkFile, chunkBytesWritten); - allPartKeys.add(chunkKey); - nextPartNumber++; - } else { - // Store sub-5MB tail chunk as temporary .part object in S3 for subsequent appends - storeIncompletePartToS3(partObjectKey, tempChunkFile, chunkBytesWritten); + } finally { + if (!handedOff) { + FileUtils.deleteQuietly(tempChunkFile); + } } + } - if (readException != null) { - throw readException; - } - if (maxAppendSizeException != null) { - throw maxAppendSizeException; - } - } finally { - FileUtils.deleteQuietly(tempChunkFile); + // Drain any remaining in-flight chunks in the pipeline + int confirmedCount = uploader.drainAndComplete(4000); + int previouslyConfirmed = allPartKeys.size() - preparedStream.remainingPartKeys.size(); + if (confirmedCount > previouslyConfirmed) { + allPartKeys.clear(); + allPartKeys.addAll(preparedStream.remainingPartKeys); + allPartKeys.addAll(plannedPartKeys.subList(0, confirmedCount)); } } + if (readException != null) { + throw readException; + } + if (maxAppendSizeException != null) { + throw maxAppendSizeException; + } + return new AppendResult(totalBytesAppended, allPartKeys); } @@ -946,24 +1105,72 @@ private void finalizeCompletedUploadIfFinished( if (info.getLength() != null && newOffset >= info.getLength()) { List partKeys = fetchExistingPartKeys(id); - // If leftover sub-5MB .part exists, save it as final part chunk + long existingPartsTotalSize = 0L; + for (String pk : partKeys) { + try { + StatObjectResponse stat = + minioClient.statObject(StatObjectArgs.builder().bucket(bucket).object(pk).build()); + existingPartsTotalSize += stat.size(); + } catch (Exception ignored) { + } + } + + // If leftover sub-5MB .part exists, apply arithmetic budget invariants String leftoverPartKey = buildIncompletePartKey(id); if (objectExists(leftoverPartKey)) { - int nextPartNum = partKeys.size() + 1; - String finalChunkKey = buildChunkPartKey(id, nextPartNum); - try (InputStream stream = - minioClient.getObject( - GetObjectArgs.builder().bucket(bucket).object(leftoverPartKey).build())) { - byte[] bytes = IOUtils.toByteArray(stream); - minioClient.putObject( - PutObjectArgs.builder().bucket(bucket).object(finalChunkKey).stream( - new ByteArrayInputStream(bytes), (long) bytes.length, -1L) - .build()); - partKeys.add(finalChunkKey); - } catch (Exception e) { - throw new IOException("Failed to finalize incomplete part for ID " + id, e); + long leftoverSize = 0L; + try { + StatObjectResponse partHead = + minioClient.statObject( + StatObjectArgs.builder().bucket(bucket).object(leftoverPartKey).build()); + if (partHead != null) { + leftoverSize = partHead.size(); + } + } catch (Exception ignored) { + } + + // Arithmetic Decision Matrix: + // Case 1: Numbered parts already satisfy the entire upload length. + // The .part buffer is an orphaned/stale duplicate (e.g. from prior pause). + // MUST DELETE to prevent byte duplication. + if (existingPartsTotalSize >= info.getLength()) { + log.info( + "Purging stale incomplete part {} (size: {}) as numbered parts already cover upload length ({})", + leftoverPartKey, + leftoverSize, + info.getLength()); + deleteObjectQuietly(leftoverPartKey); + + } else if (existingPartsTotalSize + leftoverSize == info.getLength()) { + // Case 2: Numbered parts plus this leftover buffer match the expected length exactly. + // This is a legitimate new tail written during this request. + // MUST PROMOTE to the final numbered part. + int nextPartNum = partKeys.size() + 1; + String finalChunkKey = buildChunkPartKey(id, nextPartNum); + try (InputStream stream = + minioClient.getObject( + GetObjectArgs.builder().bucket(bucket).object(leftoverPartKey).build())) { + byte[] bytes = IOUtils.toByteArray(stream); + minioClient.putObject( + PutObjectArgs.builder().bucket(bucket).object(finalChunkKey).stream( + new ByteArrayInputStream(bytes), (long) bytes.length, -1L) + .build()); + partKeys.add(finalChunkKey); + } catch (Exception e) { + throw new IOException("Failed to finalize incomplete part for ID " + id, e); + } + deleteObjectQuietly(leftoverPartKey); + + } else { + // Case 3: Oversized or inconsistent leftover buffer. + log.warn( + "Discarding inconsistent incomplete part {} (size: {}, numbered parts: {}, total length: {})", + leftoverPartKey, + leftoverSize, + existingPartsTotalSize, + info.getLength()); + deleteObjectQuietly(leftoverPartKey); } - deleteObjectQuietly(leftoverPartKey); } if (!partKeys.isEmpty()) { @@ -1020,6 +1227,16 @@ private void finalizeCompletedUploadIfFinished( for (String pk : partKeys) { deleteObjectQuietly(pk); } + } else if (info.getLength() == 0L) { + // Zero-byte upload: create the empty destination object in S3 + try { + minioClient.putObject( + PutObjectArgs.builder().bucket(bucket).object(objectKey).stream( + new ByteArrayInputStream(new byte[0]), 0L, -1L) + .build()); + } catch (Exception e) { + throw new IOException("Failed to create empty completed object for ID " + id, e); + } } // Add checksum index if deduplication is enabled @@ -1137,11 +1354,12 @@ private void calculateAndSetOffset(UploadInfo info) { String objectKey = getS3ObjectKey(info); String partKey = buildIncompletePartKey(info.getId()); - long offset = calculateCurrentOffset(objectKey, info.getId(), partKey); + long offset = calculateCurrentOffset(objectKey, info.getId(), partKey, info.getLength()); info.setOffset(offset); } - private long calculateCurrentOffset(String objectKey, UploadId id, String partKey) { + private long calculateCurrentOffset( + String objectKey, UploadId id, String partKey, Long expectedLength) { long offset = 0; if (objectExists(objectKey)) { @@ -1155,21 +1373,29 @@ private long calculateCurrentOffset(String objectKey, UploadId id, String partKe } List partKeys = fetchExistingPartKeys(id); + long numberedPartsSize = 0L; for (String pk : partKeys) { try { StatObjectResponse stat = minioClient.statObject(StatObjectArgs.builder().bucket(bucket).object(pk).build()); - offset += stat.size(); + numberedPartsSize += stat.size(); } catch (Exception ignored) { } } + offset += numberedPartsSize; - if (!partKeys.contains(partKey)) { + if (!partKeys.contains(partKey) && objectExists(partKey)) { try { StatObjectResponse partHead = minioClient.statObject(StatObjectArgs.builder().bucket(bucket).object(partKey).build()); if (partHead != null) { - offset += partHead.size(); + long partSize = partHead.size(); + if (expectedLength == null || numberedPartsSize + partSize <= expectedLength) { + offset += partSize; + } else if (numberedPartsSize >= expectedLength) { + // Numbered parts already cover the file; partKey is a stale orphan + deleteObjectQuietly(partKey); + } } } catch (ErrorResponseException ignored) { } catch (Exception e) { @@ -1201,14 +1427,44 @@ private void deleteAllPartObjectsQuietly(UploadId id) { } } - private long calcOptimalPartSize(long totalSize) { + /** + * Calculates auto-calibrated optimal chunk part size based on total upload length. + * + *

AWS S3 multipart uploads enforce a strict ceiling of 10,000 parts per object. When an upload + * length exceeds 80 GB (10,000 * 8 MB), the part size dynamically scales up (e.g. ~105 MB for 1 + * TB, ~525 MB for 5 TB) so that the entire upload is guaranteed to fit within 10,000 parts, + * bounded by S3's 5 GB maximum part limit. + * + * @param totalLength The announced total upload length in bytes, or null if unknown/deferred + * @return The calibrated optimal part size in bytes + */ + long calcOptimalPartSize(Long totalLength) { long partSize = preferredPartSize; - if (totalSize > 0 && totalSize / partSize >= 10000) { - partSize = (totalSize / 10000) + 1; + if (totalLength != null && totalLength > 0 && totalLength / partSize >= MAX_PARTS_PER_UPLOAD) { + partSize = (totalLength / MAX_PARTS_PER_UPLOAD) + 1; } return Math.max(minPartSize, Math.min(partSize, DEFAULT_MAX_PART_SIZE)); } + /** + * Validates that the upload has sufficient part budget remaining within S3's 10,000 parts limit. + * + *

Why: AWS S3 enforces a strict maximum ceiling of 10,000 parts per multipart upload. + * If an upload receives too many small chunks, it risks hitting this limit before finishing. This + * check runs in O(1) time at the trust boundary to prevent deadlocked uploads. + * + * @param upload The current upload metadata + * @param currentPartCount Number of already committed parts + * @throws MaxAppendSizeExceededException If remaining part capacity is exhausted + */ + void validateRemainingPartBudget(UploadInfo upload, int currentPartCount) + throws MaxAppendSizeExceededException { + if (currentPartCount >= MAX_PARTS_PER_UPLOAD) { + throw new MaxAppendSizeExceededException( + "Upload has reached the maximum allowed S3 limit of " + MAX_PARTS_PER_UPLOAD + " parts."); + } + } + private void putChecksumIndex(String checksum, ChecksumAlgorithm algorithm, UploadId parentId) { String key = buildChecksumKey(checksum, algorithm); try { diff --git a/src/main/java/me/desair/tus/server/upload/util/AsyncChunkUploader.java b/src/main/java/me/desair/tus/server/upload/util/AsyncChunkUploader.java new file mode 100644 index 00000000..f523e40c --- /dev/null +++ b/src/main/java/me/desair/tus/server/upload/util/AsyncChunkUploader.java @@ -0,0 +1,290 @@ +package me.desair.tus.server.upload.util; + +import java.io.File; +import java.io.IOException; +import java.util.Objects; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import org.apache.commons.io.FileUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Coordinates asynchronous, pipelined uploading of chunk files to cloud storage backends (S3, + * Azure). + * + *

Uses a bounded 3-slot pipeline (triple-buffering): + * + *

    + *
  • Slot 1 (Receiving): The caller/servlet thread reads incoming request bytes into a + * local temp chunk file. + *
  • Slot 2 (Waiting): At most one completed chunk file held on disk waiting for the + * active cloud upload to complete. + *
  • Slot 3 (Uploading): A discrete task running on the shared thread pool actively + * streaming a chunk to the cloud. + *
+ * + *

This pipeline overlaps network reception from the client with upstream cloud staging, + * eliminating client TCP stall periods while maintaining strict upper bounds on memory, disk space, + * and thread usage. + */ +public class AsyncChunkUploader implements AutoCloseable { + + private static final Logger log = LoggerFactory.getLogger(AsyncChunkUploader.class); + + @FunctionalInterface + public interface ChunkUploadAction { + /** + * Executes the cloud upload action for a single chunk. + * + * @throws Exception If the upload to the cloud provider fails + */ + void upload() throws Exception; + } + + private final ExecutorService executor; + + // Slot 3: In-flight upload task and its associated local file and key + private Future inFlightUpload; + private File inFlightFile; + private String inFlightKey; + + // Slot 2: Waiting chunk awaiting execution + private File waitingChunkFile; + private long waitingChunkSize; + private String waitingKey; + private ChunkUploadAction waitingAction; + + // Number of chunks confirmed uploaded to the cloud + private int confirmedCount; + + // Set to true once drainAndComplete() has successfully finished + private boolean completed; + + /** + * Constructs an uploader using the given shared executor. + * + * @param executor Shared thread pool executor for background chunk uploads + */ + public AsyncChunkUploader(ExecutorService executor) { + this.executor = Objects.requireNonNull(executor, "ExecutorService must not be null"); + } + + /** + * Submits a newly completed local chunk file into the upload pipeline. + * + *

If Slot 3 is idle or completed, the chunk begins uploading immediately. If Slot 3 is busy + * and Slot 2 is empty, the chunk is placed into Slot 2. If both Slot 3 and Slot 2 are occupied, + * this call blocks until Slot 3 finishes (applying backpressure to the client stream). + * + * @param tempFile The local chunk file containing the chunk bytes + * @param size The size of the chunk in bytes + * @param preassignedKey The pre-assigned cloud object key or block ID + * @param uploadAction The upload action to execute on the worker thread + * @throws IOException If a previous chunk upload failed or thread was interrupted + */ + public void submitChunk( + File tempFile, long size, String preassignedKey, ChunkUploadAction uploadAction) + throws IOException { + Objects.requireNonNull(tempFile, "tempFile must not be null"); + Objects.requireNonNull(uploadAction, "uploadAction must not be null"); + + // Check if previous in-flight upload has finished, advancing confirmation count + if (inFlightUpload != null && inFlightUpload.isDone()) { + try { + checkAndConfirmInFlight(); + } catch (IOException e) { + FileUtils.deleteQuietly(tempFile); + throw e; + } + } + + try { + if (waitingChunkFile == null) { + if (inFlightUpload == null) { + // Branch 1: Slot 3 is free and Slot 2 is empty. Submit directly to Slot 3. + submitToSlot3(tempFile, size, preassignedKey, uploadAction); + } else { + // Branch 2: Slot 3 is actively uploading, but Slot 2 (waiting) is free. + // Store in Slot 2 and return immediately so the caller can read the next chunk. + this.waitingChunkFile = tempFile; + this.waitingChunkSize = size; + this.waitingKey = preassignedKey; + this.waitingAction = uploadAction; + } + } else { + // Branch 3: Slot 2 already has a waiting chunk. + // To preserve strict sequential chunk ordering, Slot 2's chunk must be uploaded before the + // new chunk. + // If Slot 3 is still active, apply backpressure by waiting for it to finish. + if (inFlightUpload != null) { + waitForInFlight(); + } + + // Slot 3 is now free. Promote Slot 2 into Slot 3. + submitToSlot3(waitingChunkFile, waitingChunkSize, waitingKey, waitingAction); + this.waitingChunkFile = null; + this.waitingAction = null; + + // Place the newly arrived chunk into the now-empty Slot 2. + this.waitingChunkFile = tempFile; + this.waitingChunkSize = size; + this.waitingKey = preassignedKey; + this.waitingAction = uploadAction; + } + } catch (IOException | RuntimeException e) { + FileUtils.deleteQuietly(tempFile); + throw e; + } + } + + /** + * Drains all remaining chunks in the pipeline (Slot 3 and Slot 2) up to the specified timeout. + * + *

Used upon stream completion or client interruption to ensure all received bytes are + * persisted to cloud storage before releasing the upload lock. + * + * @param timeoutMs Maximum time in milliseconds to wait for in-flight and waiting chunks + * @return The total number of confirmed successfully uploaded chunks + * @throws IOException If any chunk upload fails or times out + */ + public int drainAndComplete(long timeoutMs) throws IOException { + long deadline = System.currentTimeMillis() + timeoutMs; + + // 1. Await Slot 3 if active + if (inFlightUpload != null) { + long remaining = Math.max(1L, deadline - System.currentTimeMillis()); + awaitFuture(inFlightUpload, remaining, inFlightKey); + confirmedCount++; + inFlightUpload = null; + inFlightFile = null; + inFlightKey = null; + } + + // 2. If Slot 2 has a waiting chunk, promote it to Slot 3 and await its completion + if (waitingChunkFile != null) { + submitToSlot3(waitingChunkFile, waitingChunkSize, waitingKey, waitingAction); + waitingChunkFile = null; + waitingAction = null; + + long remaining = Math.max(1L, deadline - System.currentTimeMillis()); + awaitFuture(inFlightUpload, remaining, inFlightKey); + confirmedCount++; + inFlightUpload = null; + inFlightFile = null; + inFlightKey = null; + } + + completed = true; + return confirmedCount; + } + + /** + * Aborts the pipeline, cancelling any active cloud upload and deleting any remaining temp files. + */ + public void abort() { + if (inFlightUpload != null && !inFlightUpload.isDone()) { + inFlightUpload.cancel(true); + } + if (inFlightFile != null) { + FileUtils.deleteQuietly(inFlightFile); + inFlightFile = null; + } + if (waitingChunkFile != null) { + FileUtils.deleteQuietly(waitingChunkFile); + waitingChunkFile = null; + waitingAction = null; + } + } + + /** + * Returns the number of chunks confirmed successfully uploaded. + * + * @return Confirmed chunk count + */ + public int getConfirmedCount() { + return confirmedCount; + } + + @Override + public void close() { + // If drainAndComplete() did not finish successfully, abort and clean up uncommitted files + if (!completed) { + abort(); + } + } + + private void submitToSlot3( + File tempFile, long size, String preassignedKey, ChunkUploadAction uploadAction) { + this.inFlightFile = tempFile; + this.inFlightKey = preassignedKey; + + // Wrap the upload action so temp file cleanup is guaranteed on the worker thread + this.inFlightUpload = + executor.submit( + () -> { + try { + uploadAction.upload(); + } finally { + // The worker owns tempFile once submitted; delete it immediately upon upload + // completion + FileUtils.deleteQuietly(tempFile); + } + return null; + }); + } + + private void checkAndConfirmInFlight() throws IOException { + try { + inFlightUpload.get(); + confirmedCount++; + inFlightUpload = null; + inFlightFile = null; + inFlightKey = null; + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + abort(); + throw new IOException( + "Interrupted while checking in-flight chunk upload for " + inFlightKey, e); + } catch (ExecutionException e) { + abort(); + throw translateExecutionException(e, inFlightKey); + } + } + + private void waitForInFlight() throws IOException { + checkAndConfirmInFlight(); + } + + private void awaitFuture(Future future, long timeoutMs, String key) throws IOException { + try { + future.get(timeoutMs, TimeUnit.MILLISECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + abort(); + throw new IOException("Interrupted waiting for chunk upload for key " + key, e); + } catch (TimeoutException e) { + abort(); + throw new IOException( + "Timed out after " + timeoutMs + "ms waiting for chunk upload for key " + key, e); + } catch (ExecutionException e) { + abort(); + throw translateExecutionException(e, key); + } + } + + private IOException translateExecutionException(ExecutionException e, String key) { + Throwable cause = e.getCause(); + if (cause instanceof IOException) { + return (IOException) cause; + } else if (cause instanceof RuntimeException) { + return new IOException("Upload failed for key " + key + ": " + cause.getMessage(), cause); + } else { + return new IOException( + "Unexpected error during chunk upload for key " + key, cause != null ? cause : e); + } + } +} diff --git a/src/test/java/me/desair/tus/server/AbstractITRufhProtocol.java b/src/test/java/me/desair/tus/server/AbstractITRufhProtocol.java index e818de35..504dd8b2 100644 --- a/src/test/java/me/desair/tus/server/AbstractITRufhProtocol.java +++ b/src/test/java/me/desair/tus/server/AbstractITRufhProtocol.java @@ -801,6 +801,95 @@ public void testUploadCompletionListenerRufhPatchAppend() throws Exception { } } + // =============================================================================================== + // USE CASE 16: Min-Append-Size Exemption for Completing Uploads (§4.1.4) + // =============================================================================================== + + /** + * Section 4.1.4 (Limits): "min-append-size: An Integer indicating the minimum number of bytes + * that MUST be appended with each upload append request. This limit does not apply to upload + * creation requests with no content, or to requests completing the upload by including the + * Upload-Complete: ?1 header field." + * + *

Use Case: Configure min-append-size (1 MB), then verify that an optimistic single-request + * creation upload with Upload-Complete: ?1 and a payload smaller than 1 MB bypasses the + * min-append-size limit and completes successfully. + */ + @Test + public void testCompletingCreationBypassesMinAppendSize() throws Exception { + tusFileUploadService.withMinAppendSize(1024L * 1024L); + + String payload = "Small optimistic payload under min-append-size."; + + servletRequest.setMethod("POST"); + servletRequest.setRequestURI(UPLOAD_URI); + servletRequest.addHeader(HttpHeader.UPLOAD_COMPLETE, "?1"); + servletRequest.setContent(payload.getBytes(StandardCharsets.UTF_8)); + + UploadInfo completedInfo = + tusFileUploadService.process(servletRequest, servletResponse, OWNER_KEY); + + assertResponseStatus(HttpServletResponse.SC_OK); + assertResponseHeader(HttpHeader.UPLOAD_COMPLETE, "?1"); + assertNotNull(completedInfo); + assertFalse(completedInfo.isUploadInProgress()); + assertEquals(Long.valueOf(payload.length()), completedInfo.getOffset()); + + try (InputStream stream = tusFileUploadService.getUploadedBytes(completedInfo)) { + assertThat(IOUtils.toString(stream, StandardCharsets.UTF_8), is(payload)); + } + } + + /** + * Section 4.1.4 (Limits): "min-append-size: An Integer indicating the minimum number of bytes + * that MUST be appended with each upload append request. This limit does not apply to upload + * creation requests with no content, or to requests completing the upload by including the + * Upload-Complete: ?1 header field." + * + *

Use Case: Configure min-append-size (1 MB), initiate an upload without content, and append a + * completing chunk smaller than 1 MB with Upload-Complete: ?1. Verify that the completing append + * is exempt from min-append-size and completes the upload cleanly. + */ + @Test + public void testCompletingAppendPatchBypassesMinAppendSize() throws Exception { + tusFileUploadService.withMinAppendSize(1024L * 1024L); + + String uploadContent = "Small final chunk under min-append-size."; + + // Step 1: Initiate upload with POST, Upload-Complete: ?0, declared Upload-Length, and no body + servletRequest.setMethod("POST"); + servletRequest.setRequestURI(UPLOAD_URI); + servletRequest.addHeader(HttpHeader.UPLOAD_COMPLETE, "?0"); + servletRequest.addHeader(HttpHeader.UPLOAD_LENGTH, String.valueOf(uploadContent.length())); + + UploadInfo createdInfo = + tusFileUploadService.process(servletRequest, servletResponse, OWNER_KEY); + assertResponseStatus(HttpServletResponse.SC_CREATED); + assertNotNull(createdInfo); + String uploadLocation = servletResponse.getHeader(HttpHeader.LOCATION); + + // Step 2: Append completing chunk with Upload-Complete: ?1 + reset(); + servletRequest.setMethod("PATCH"); + servletRequest.setRequestURI(uploadLocation); + servletRequest.addHeader(HttpHeader.UPLOAD_OFFSET, "0"); + servletRequest.addHeader(HttpHeader.UPLOAD_COMPLETE, "?1"); + servletRequest.addHeader(HttpHeader.CONTENT_TYPE, HttpHeader.CONTENT_TYPE_PARTIAL_UPLOAD); + servletRequest.setContent(uploadContent.getBytes(StandardCharsets.UTF_8)); + + UploadInfo completedInfo = + tusFileUploadService.process(servletRequest, servletResponse, OWNER_KEY); + assertResponseStatus(HttpServletResponse.SC_OK); + assertResponseHeader(HttpHeader.UPLOAD_COMPLETE, "?1"); + assertNotNull(completedInfo); + assertFalse(completedInfo.isUploadInProgress()); + assertEquals(Long.valueOf(uploadContent.length()), completedInfo.getOffset()); + + try (InputStream stream = tusFileUploadService.getUploadedBytes(completedInfo)) { + assertThat(IOUtils.toString(stream, StandardCharsets.UTF_8), is(uploadContent)); + } + } + // =============================================================================================== // ASSERTION HELPERS // =============================================================================================== diff --git a/src/test/java/me/desair/tus/server/TusFileUploadServiceTest.java b/src/test/java/me/desair/tus/server/TusFileUploadServiceTest.java index 90dc8a99..981ebace 100644 --- a/src/test/java/me/desair/tus/server/TusFileUploadServiceTest.java +++ b/src/test/java/me/desair/tus/server/TusFileUploadServiceTest.java @@ -331,6 +331,7 @@ public void testWithUploadStorageServicePreservesConfiguration() { service.withMaxAppendSize(1024L); service.withMinAppendSize(512L); service.withMinSize(2048L); + service.withCloudUploadThreadPoolSize(18); service.withUploadDeduplication(true); UploadStorageService newStorage = mock(UploadStorageService.class); @@ -340,6 +341,7 @@ public void testWithUploadStorageServicePreservesConfiguration() { verify(newStorage).setMaxAppendSize(1024L); verify(newStorage).setMinAppendSize(512L); verify(newStorage).setMinSize(2048L); + verify(newStorage).setCloudUploadThreadPoolSize(18); verify(newStorage).setUploadDeduplicationEnabled(true); } @@ -349,11 +351,25 @@ public void testThreadLocalCacheDelegatesAppendAndMinSizes() { service.withMaxAppendSize(1024L); service.withMinAppendSize(512L); service.withMinSize(2048L); + service.withCloudUploadThreadPoolSize(16); service.withThreadLocalCache(true); assertThat(service.getUploadStorageService().getMaxAppendSize(), is(1024L)); assertThat(service.getUploadStorageService().getMinAppendSize(), is(512L)); assertThat(service.getUploadStorageService().getMinSize(), is(2048L)); + assertThat(service.getUploadStorageService().getCloudUploadThreadPoolSize(), is(16)); + } + + @Test(expected = IllegalArgumentException.class) + public void testWithCloudUploadThreadPoolSizeZeroThrowsException() { + TusFileUploadService service = new TusFileUploadService(); + service.withCloudUploadThreadPoolSize(0); + } + + @Test(expected = IllegalArgumentException.class) + public void testWithCloudUploadThreadPoolSizeNegativeThrowsException() { + TusFileUploadService service = new TusFileUploadService(); + service.withCloudUploadThreadPoolSize(-5); } @Test diff --git a/src/test/java/me/desair/tus/server/rufh/handler/RufhAppendPatchRequestHandlerTest.java b/src/test/java/me/desair/tus/server/rufh/handler/RufhAppendPatchRequestHandlerTest.java index 46f3a3d1..609f1a96 100644 --- a/src/test/java/me/desair/tus/server/rufh/handler/RufhAppendPatchRequestHandlerTest.java +++ b/src/test/java/me/desair/tus/server/rufh/handler/RufhAppendPatchRequestHandlerTest.java @@ -162,6 +162,7 @@ public void testProcessCallFiveParameterMethodAndAppendReturnsNull() throws Exce request.setRequestURI("/files/append-id"); request.addHeader(HttpHeader.UPLOAD_OFFSET, "1000"); request.addHeader(HttpHeader.UPLOAD_COMPLETE, "?1"); + request.addHeader(HttpHeader.CONTENT_LENGTH, "4"); request.setContent("data".getBytes()); UploadInfo info = new UploadInfo(); @@ -215,4 +216,93 @@ public void testProcessIsFinishedCombinations() throws Exception { assertThat(response.getStatus(), is(200)); assertThat(response.getHeader(HttpHeader.UPLOAD_COMPLETE), is("?1")); } + + /** + * §4.1.4: "This limit does not apply to upload creation requests with no content, or to requests + * completing the upload by including the Upload-Complete: ?1 header field." + * + *

§4.2.1: "If the upload length is not known when creating the upload resource, the + * Upload-Length header field is omitted, and the length is deferred... In subsequent requests, + * the upload length can be indicated by including the Upload-Length header field or by completing + * the upload using the Upload-Complete: ?1 header field." + */ + @Test + public void testProcessCompletingAppendOnDeferredLengthUploadSetsLength() throws Exception { + request.setMethod("PATCH"); + request.setRequestURI("/files/append-id"); + request.addHeader(HttpHeader.CONTENT_TYPE, HttpHeader.CONTENT_TYPE_PARTIAL_UPLOAD); + request.addHeader(HttpHeader.UPLOAD_OFFSET, "100"); + request.addHeader(HttpHeader.UPLOAD_COMPLETE, "?1"); + byte[] content = "final-bytes".getBytes(); + request.setContent(content); + + UploadInfo info = new UploadInfo(); + info.setId(new UploadId("append-id")); + info.setOffset(100L); + info.setLength(null); // Deferred length + + UploadInfo updated = new UploadInfo(); + updated.setId(info.getId()); + updated.setOffset(100L + content.length); + updated.setLength(100L + content.length); + + when(storageService.getUploadInfo("/files/append-id", "owner")).thenReturn(info); + when(storageService.append(any(UploadInfo.class), any())).thenReturn(updated); + + handler.process( + HttpMethod.PATCH, + new TusServletRequest(request), + new TusServletResponse(response), + storageService, + lockingService, + "owner", + null); + + verify(storageService).update(info); + assertThat(info.getLength(), is(100L + content.length)); + assertThat(response.getStatus(), is(200)); + assertThat( + response.getHeader(HttpHeader.UPLOAD_OFFSET), is(String.valueOf(100L + content.length))); + assertThat(response.getHeader(HttpHeader.UPLOAD_COMPLETE), is("?1")); + } + + @Test + public void testProcessCompletingAppendOnDeferredLengthUploadWithNullOffsetSetsLength() + throws Exception { + request.setMethod("PATCH"); + request.setRequestURI("/files/append-id"); + request.addHeader(HttpHeader.CONTENT_TYPE, HttpHeader.CONTENT_TYPE_PARTIAL_UPLOAD); + request.addHeader(HttpHeader.UPLOAD_OFFSET, "0"); + request.addHeader(HttpHeader.UPLOAD_COMPLETE, "?1"); + byte[] content = "initial-and-final-bytes".getBytes(); + request.setContent(content); + + UploadInfo info = new UploadInfo(); + info.setId(new UploadId("append-id")); + info.setOffset(null); + info.setLength(null); // Deferred length + + UploadInfo updated = new UploadInfo(); + updated.setId(info.getId()); + updated.setOffset((long) content.length); + updated.setLength((long) content.length); + + when(storageService.getUploadInfo("/files/append-id", "owner")).thenReturn(info); + when(storageService.append(any(UploadInfo.class), any())).thenReturn(updated); + + handler.process( + HttpMethod.PATCH, + new TusServletRequest(request), + new TusServletResponse(response), + storageService, + lockingService, + "owner", + null); + + verify(storageService).update(info); + assertThat(info.getLength(), is((long) content.length)); + assertThat(response.getStatus(), is(200)); + assertThat(response.getHeader(HttpHeader.UPLOAD_OFFSET), is(String.valueOf(content.length))); + assertThat(response.getHeader(HttpHeader.UPLOAD_COMPLETE), is("?1")); + } } diff --git a/src/test/java/me/desair/tus/server/rufh/handler/RufhCreationPostRequestHandlerTest.java b/src/test/java/me/desair/tus/server/rufh/handler/RufhCreationPostRequestHandlerTest.java index f6dce0c7..e68c1776 100644 --- a/src/test/java/me/desair/tus/server/rufh/handler/RufhCreationPostRequestHandlerTest.java +++ b/src/test/java/me/desair/tus/server/rufh/handler/RufhCreationPostRequestHandlerTest.java @@ -473,4 +473,46 @@ public void testProcessCompletedUploadCreationWithAbsoluteUploadUri() throws Exc assertThat(response.getHeader(HttpHeader.UPLOAD_OFFSET), is("100")); assertThat(response.getHeader(HttpHeader.UPLOAD_COMPLETE), is("?1")); } + + /** + * §4.1.4: "This limit does not apply to upload creation requests with no content, or to requests + * completing the upload by including the Upload-Complete: ?1 header field." + * + *

§4.2.1: "If the upload length is not known when creating the upload resource, the + * Upload-Length header field is omitted, and the length is deferred... In subsequent requests, + * the upload length can be indicated by including the Upload-Length header field or by completing + * the upload using the Upload-Complete: ?1 header field." + */ + @Test + public void testProcessCreationWithUploadCompleteAndContentLengthSetsAnnouncedLength() + throws Exception { + request.setMethod("POST"); + request.setRequestURI("/files"); + request.addHeader(HttpHeader.UPLOAD_COMPLETE, "?1"); + byte[] content = "hello world".getBytes(); + request.setContent(content); + + ArgumentCaptor captor = ArgumentCaptor.forClass(UploadInfo.class); + UploadInfo createdInfo = new UploadInfo(); + createdInfo.setId(new UploadId("complete-no-length-id")); + createdInfo.setLength((long) content.length); + createdInfo.setOffset((long) content.length); + + when(storageService.create(captor.capture(), nullable(String.class))).thenReturn(createdInfo); + when(storageService.append(any(UploadInfo.class), any())).thenReturn(createdInfo); + + handler.process( + HttpMethod.POST, + new TusServletRequest(request), + new TusServletResponse(response), + storageService, + lockingService, + "owner", + null); + + assertThat(captor.getValue().getLength(), is((long) content.length)); + assertThat(response.getStatus(), is(200)); + assertThat(response.getHeader(HttpHeader.UPLOAD_OFFSET), is(String.valueOf(content.length))); + assertThat(response.getHeader(HttpHeader.UPLOAD_COMPLETE), is("?1")); + } } diff --git a/src/test/java/me/desair/tus/server/rufh/util/RufhInterimResponseUtilTest.java b/src/test/java/me/desair/tus/server/rufh/util/RufhInterimResponseUtilTest.java index 27f03d77..406d7b52 100644 --- a/src/test/java/me/desair/tus/server/rufh/util/RufhInterimResponseUtilTest.java +++ b/src/test/java/me/desair/tus/server/rufh/util/RufhInterimResponseUtilTest.java @@ -96,8 +96,8 @@ public void testGetRawInterimResponseWithExistingUploadAndStorageException() thr @Test public void testGetRawInterimResponseWithExistingUploadNotFoundAndNullHost() throws Exception { MockHttpServletRequest request = new MockHttpServletRequest(); - request.setMethod("PATCH"); - request.setRequestURI("/files/not-found-123"); + request.setMethod("POST"); + request.setRequestURI("/files"); request.addHeader(HttpHeader.UPLOAD_COMPLETE, "?0"); // Host header is not set (null host) @@ -107,8 +107,6 @@ public void testGetRawInterimResponseWithExistingUploadNotFoundAndNullHost() thr me.desair.tus.server.upload.UploadInfo created = new me.desair.tus.server.upload.UploadInfo(); created.setId(new me.desair.tus.server.upload.UploadId("created-456")); - org.mockito.Mockito.when(mockStorage.getUploadInfo("/files/not-found-123", "owner")) - .thenReturn(null); org.mockito.Mockito.when( mockStorage.create( org.mockito.ArgumentMatchers.any(), org.mockito.ArgumentMatchers.eq("owner"))) @@ -119,6 +117,23 @@ public void testGetRawInterimResponseWithExistingUploadNotFoundAndNullHost() thr assertTrue(raw.contains("Location: /files/created-456")); } + @Test + public void testGetRawInterimResponseWithPatchUploadNotFoundReturnsNull() throws Exception { + MockHttpServletRequest request = new MockHttpServletRequest(); + request.setMethod("PATCH"); + request.setRequestURI("/files/not-found-123"); + request.addHeader(HttpHeader.UPLOAD_COMPLETE, "?0"); + + me.desair.tus.server.upload.UploadStorageService mockStorage = + org.mockito.Mockito.mock(me.desair.tus.server.upload.UploadStorageService.class); + org.mockito.Mockito.when(mockStorage.getUploadUri()).thenReturn("/files"); + org.mockito.Mockito.when(mockStorage.getUploadInfo("/files/not-found-123", "owner")) + .thenReturn(null); + + String raw = RufhInterimResponseUtil.getRawInterimResponse(request, mockStorage, "owner"); + assertNull(raw); + } + @Test public void testGetRawInterimResponseWithAbsoluteUploadUri() throws Exception { MockHttpServletRequest request = new MockHttpServletRequest(); diff --git a/src/test/java/me/desair/tus/server/upload/UploadStorageServiceTest.java b/src/test/java/me/desair/tus/server/upload/UploadStorageServiceTest.java index a3d2307e..1933b822 100644 --- a/src/test/java/me/desair/tus/server/upload/UploadStorageServiceTest.java +++ b/src/test/java/me/desair/tus/server/upload/UploadStorageServiceTest.java @@ -142,6 +142,9 @@ public void testDefaultAppendAndSizeConfigurationMethods() throws Exception { dummyStorageService.setJsonSerializationEnabled(true); assertThat(dummyStorageService.isJsonSerializationEnabled(), is(false)); + dummyStorageService.setCloudUploadThreadPoolSize(15); + assertThat(dummyStorageService.getCloudUploadThreadPoolSize(), is(10)); + // Default close is a no-op dummyStorageService.close(); } diff --git a/src/test/java/me/desair/tus/server/upload/azure/AzureBlobConcatenationServiceTest.java b/src/test/java/me/desair/tus/server/upload/azure/AzureBlobConcatenationServiceTest.java index 32360897..464f526f 100644 --- a/src/test/java/me/desair/tus/server/upload/azure/AzureBlobConcatenationServiceTest.java +++ b/src/test/java/me/desair/tus/server/upload/azure/AzureBlobConcatenationServiceTest.java @@ -250,4 +250,31 @@ public void getPartialUploadsShouldThrowOnOwnerMismatch() throws Exception { // final upload's ownerKey, preventing unauthorized partial stitching. concatenationService.getPartialUploads(finalInfo); } + + @Test + public void getAuthorizedBlobUrlShouldReturnNullOnNullBlobClient() { + org.junit.Assert.assertNull(concatenationService.getAuthorizedBlobUrl(null)); + } + + @Test + public void getAuthorizedBlobUrlShouldAppendSasWhenGenerationSucceeds() { + com.azure.storage.blob.BlobClient mockBlob = mock(com.azure.storage.blob.BlobClient.class); + when(mockBlob.getBlobUrl()).thenReturn("https://account.blob.core.windows.net/container/blob"); + when(mockBlob.generateSas(any())).thenReturn("sig=mocked-token&sp=r"); + + String result = concatenationService.getAuthorizedBlobUrl(mockBlob); + assertEquals( + "https://account.blob.core.windows.net/container/blob?sig=mocked-token&sp=r", result); + } + + @Test + public void getAuthorizedBlobUrlShouldFallbackToRawUrlWhenSasGenerationThrows() { + com.azure.storage.blob.BlobClient mockBlob = mock(com.azure.storage.blob.BlobClient.class); + when(mockBlob.getBlobUrl()).thenReturn("https://account.blob.core.windows.net/container/blob"); + when(mockBlob.generateSas(any())) + .thenThrow(new IllegalStateException("No shared key credentials")); + + String result = concatenationService.getAuthorizedBlobUrl(mockBlob); + assertEquals("https://account.blob.core.windows.net/container/blob", result); + } } diff --git a/src/test/java/me/desair/tus/server/upload/azure/AzureBlobStorageServiceTest.java b/src/test/java/me/desair/tus/server/upload/azure/AzureBlobStorageServiceTest.java index afffb6ef..9f4c052e 100644 --- a/src/test/java/me/desair/tus/server/upload/azure/AzureBlobStorageServiceTest.java +++ b/src/test/java/me/desair/tus/server/upload/azure/AzureBlobStorageServiceTest.java @@ -233,4 +233,46 @@ public void calcOptimalBlockSizeCalculations() { long largeLength = 50_000L * 16 * 1024 * 1024L; assertEquals((largeLength / 50_000L) + 1, storageService.calcOptimalBlockSize(largeLength)); } + + @Test + public void testCloudUploadThreadPoolSizeConfiguration() { + assertEquals( + "Default thread pool size is 10", 10, storageService.getCloudUploadThreadPoolSize()); + + storageService.setCloudUploadThreadPoolSize(30); + assertEquals( + "Updated thread pool size is 30", 30, storageService.getCloudUploadThreadPoolSize()); + + storageService.setCloudUploadThreadPoolSize(4); + assertEquals("Reduced thread pool size is 4", 4, storageService.getCloudUploadThreadPoolSize()); + + try { + storageService.setCloudUploadThreadPoolSize(0); + org.junit.Assert.fail("Should reject 0 pool size"); + } catch (IllegalArgumentException expected) { + assertTrue(expected.getMessage().contains("greater than 0")); + } + + try { + storageService.setCloudUploadThreadPoolSize(-1); + org.junit.Assert.fail("Should reject negative pool size"); + } catch (IllegalArgumentException expected) { + assertTrue(expected.getMessage().contains("greater than 0")); + } + } + + @Test + public void testCloseGracefulShutdown() throws Exception { + // Verify closing storage service cleanly terminates background upload executor without error + storageService.close(); + // KISS: verifying method executes cleanly without throwing an exception + + // Verify close handles thread interruption gracefully + Thread.currentThread().interrupt(); + try { + storageService.close(); + } finally { + Thread.interrupted(); // Clear interrupted status + } + } } diff --git a/src/test/java/me/desair/tus/server/upload/azure/AzureUtilsTest.java b/src/test/java/me/desair/tus/server/upload/azure/AzureUtilsTest.java index d11f2a7f..067e4be0 100644 --- a/src/test/java/me/desair/tus/server/upload/azure/AzureUtilsTest.java +++ b/src/test/java/me/desair/tus/server/upload/azure/AzureUtilsTest.java @@ -69,6 +69,9 @@ public void testParseErrorResponseApiNotImplemented() { public void testParseErrorResponseAccessDenied() { BlobStorageException ex = createException(403, BlobErrorCode.AUTHORIZATION_FAILURE); assertEquals(AzureErrorType.ACCESS_DENIED, AzureUtils.parseErrorResponse(ex)); + + BlobStorageException ex401 = createException(401, null); + assertEquals(AzureErrorType.ACCESS_DENIED, AzureUtils.parseErrorResponse(ex401)); } @Test diff --git a/src/test/java/me/desair/tus/server/upload/azure/ITAzureBlobStorageService.java b/src/test/java/me/desair/tus/server/upload/azure/ITAzureBlobStorageService.java index 6f2bbc67..44505494 100644 --- a/src/test/java/me/desair/tus/server/upload/azure/ITAzureBlobStorageService.java +++ b/src/test/java/me/desair/tus/server/upload/azure/ITAzureBlobStorageService.java @@ -73,6 +73,25 @@ public void minAppendSizeNotMetShouldThrow() throws Exception { storageService.append(created, new ByteArrayInputStream("0123456789".getBytes())); } + /** + * §4.1.4: "This limit does not apply to upload creation requests with no content, or to requests + * completing the upload by including the Upload-Complete: ?1 header field." + */ + @Test + public void completingUploadBypassesMinAppendSize() throws Exception { + storageService.setMinAppendSize(100L); + + UploadInfo info = new UploadInfo(); + info.setLength(10L); + UploadInfo created = storageService.create(info, "owner1"); + + UploadInfo completed = + storageService.append(created, new ByteArrayInputStream("0123456789".getBytes())); + assertNotNull(completed); + assertEquals(Long.valueOf(10L), completed.getOffset()); + assertFalse(completed.isUploadInProgress()); + } + @Test(expected = MaxAppendSizeExceededException.class) public void appendExceedsMaxAppendSizeShouldThrow() throws Exception { storageService.setMaxAppendSize(5L); diff --git a/src/test/java/me/desair/tus/server/upload/cache/ThreadLocalCachedStorageAndLockingServiceTest.java b/src/test/java/me/desair/tus/server/upload/cache/ThreadLocalCachedStorageAndLockingServiceTest.java index becc861d..d27e22cb 100644 --- a/src/test/java/me/desair/tus/server/upload/cache/ThreadLocalCachedStorageAndLockingServiceTest.java +++ b/src/test/java/me/desair/tus/server/upload/cache/ThreadLocalCachedStorageAndLockingServiceTest.java @@ -161,6 +161,30 @@ public void testDelegateMethods() throws Exception { assertEquals(0, service.getMaxUploadSize()); verify(mockStorage, times(1)).getMaxUploadSize(); + service.setMaxAppendSize(500L); + verify(mockStorage, times(1)).setMaxAppendSize(500L); + + service.getMaxAppendSize(); + verify(mockStorage, times(1)).getMaxAppendSize(); + + service.setMinAppendSize(200L); + verify(mockStorage, times(1)).setMinAppendSize(200L); + + service.getMinAppendSize(); + verify(mockStorage, times(1)).getMinAppendSize(); + + service.setMinSize(300L); + verify(mockStorage, times(1)).setMinSize(300L); + + service.getMinSize(); + verify(mockStorage, times(1)).getMinSize(); + + service.setCloudUploadThreadPoolSize(12); + verify(mockStorage, times(1)).setCloudUploadThreadPoolSize(12); + + service.getCloudUploadThreadPoolSize(); + verify(mockStorage, times(1)).getCloudUploadThreadPoolSize(); + when(mockStorage.create(info, "owner")).thenReturn(info); assertEquals(info, service.create(info, "owner")); verify(mockStorage, times(1)).create(info, "owner"); diff --git a/src/test/java/me/desair/tus/server/upload/disk/DiskStorageServiceTest.java b/src/test/java/me/desair/tus/server/upload/disk/DiskStorageServiceTest.java index cd08415d..486be745 100644 --- a/src/test/java/me/desair/tus/server/upload/disk/DiskStorageServiceTest.java +++ b/src/test/java/me/desair/tus/server/upload/disk/DiskStorageServiceTest.java @@ -1106,4 +1106,21 @@ public void testUnsafeChildDataFileDeletionThrowsIllegalArgumentException() thro storageService.update(child); } + + @Test + public void testSetAndGetCloudUploadThreadPoolSize() { + assertThat(storageService.getCloudUploadThreadPoolSize(), is(10)); + storageService.setCloudUploadThreadPoolSize(24); + assertThat(storageService.getCloudUploadThreadPoolSize(), is(24)); + } + + @Test(expected = IllegalArgumentException.class) + public void testSetCloudUploadThreadPoolSizeZeroThrowsIllegalArgumentException() { + storageService.setCloudUploadThreadPoolSize(0); + } + + @Test(expected = IllegalArgumentException.class) + public void testSetCloudUploadThreadPoolSizeNegativeThrowsIllegalArgumentException() { + storageService.setCloudUploadThreadPoolSize(-1); + } } diff --git a/src/test/java/me/desair/tus/server/upload/s3/S3StorageServiceTest.java b/src/test/java/me/desair/tus/server/upload/s3/S3StorageServiceTest.java index 868a3c1e..1b43b0d8 100644 --- a/src/test/java/me/desair/tus/server/upload/s3/S3StorageServiceTest.java +++ b/src/test/java/me/desair/tus/server/upload/s3/S3StorageServiceTest.java @@ -33,7 +33,9 @@ import java.io.IOException; import java.io.InputStream; import java.util.Arrays; +import java.util.Collections; import me.desair.tus.server.checksum.ChecksumAlgorithm; +import me.desair.tus.server.exception.MaxAppendSizeExceededException; import me.desair.tus.server.exception.MinUploadLengthNotReachedException; import me.desair.tus.server.upload.UploadId; import me.desair.tus.server.upload.UploadInfo; @@ -782,6 +784,26 @@ public void testAppendThrowsMinAppendSizeNotMetException() throws Exception { storageService.append(info, new ByteArrayInputStream(new byte[100])); } + /** + * §4.1.4: "This limit does not apply to upload creation requests with no content, or to requests + * completing the upload by including the Upload-Complete: ?1 header field." + */ + @Test + public void testAppendCompletingUploadBypassesMinAppendSize() throws Exception { + UploadInfo info = new UploadInfo(); + info.setId(new UploadId("24249a5b-01a4-4bf8-b67a-364273bb5a2e")); + info.setLength(100L); + + String json = UploadInfoJsonSerializer.serialize(info); + when(minioClient.getObject(any(GetObjectArgs.class))) + .thenAnswer(invocation -> mockGetObjectResponse(json.getBytes())); + + storageService.setMinAppendSize(500L); + UploadInfo result = storageService.append(info, new ByteArrayInputStream(new byte[100])); + assertEquals(Long.valueOf(100L), result.getOffset()); + assertFalse(result.isUploadInProgress()); + } + @Test(expected = me.desair.tus.server.exception.MaxUploadLengthExceededException.class) public void testAppendThrowsMaxUploadLengthExceededException() throws Exception { UploadInfo info = new UploadInfo(); @@ -928,6 +950,91 @@ public void testFinalizeCompletedUploadWithLeftoverIncompletePart() throws Excep storageService.append(info, new ByteArrayInputStream(new byte[50])); } + @Test + public void testFinalizeCompletedUploadWithInconsistentLeftoverPartPurged() throws Exception { + UploadInfo info = new UploadInfo(); + UploadId id = new UploadId("inconsistent-part-123"); + info.setId(id); + info.setLength(100L); + info.setOffset(0L); + + String json = UploadInfoJsonSerializer.serialize(info); + + StatObjectResponse leftoverHead = mock(StatObjectResponse.class); + // Leftover is 30 bytes, but total length is 100 bytes (0 parts + 30 != 100) -> Case 3 + when(leftoverHead.size()).thenReturn(30L); + + when(minioClient.statObject(any(StatObjectArgs.class))) + .thenAnswer( + invocation -> { + StatObjectArgs args = invocation.getArgument(0); + if (args.object().endsWith(".part")) { + return leftoverHead; + } + ErrorResponse err = mock(ErrorResponse.class); + when(err.code()).thenReturn("NoSuchKey"); + throw new ErrorResponseException(err, null, null); + }); + + when(minioClient.getObject(any(GetObjectArgs.class))) + .thenAnswer( + invocation -> { + GetObjectArgs args = invocation.getArgument(0); + if (args.object().endsWith(".info")) { + return mockGetObjectResponse(json.getBytes()); + } + return mockGetObjectResponse(new byte[100]); + }); + + storageService.append(info, new ByteArrayInputStream(new byte[100])); + + // Verify inconsistent .part buffer was deleted + verify(minioClient, atLeastOnce()) + .removeObject( + argThat( + (RemoveObjectArgs args) -> + args.object().equals("metadata/inconsistent-part-123.part"))); + } + + @Test(expected = IOException.class) + public void testFinalizeCompletedUploadLeftoverPartPromotionExceptionThrowsIOException() + throws Exception { + UploadInfo info = new UploadInfo(); + UploadId id = new UploadId("leftover-promo-err-123"); + info.setId(id); + info.setLength(50L); + info.setOffset(0L); + + String json = UploadInfoJsonSerializer.serialize(info); + + StatObjectResponse leftoverHead = mock(StatObjectResponse.class); + when(leftoverHead.size()).thenReturn(50L); + + when(minioClient.statObject(any(StatObjectArgs.class))) + .thenAnswer( + invocation -> { + StatObjectArgs args = invocation.getArgument(0); + if (args.object().endsWith(".part")) { + return leftoverHead; + } + ErrorResponse err = mock(ErrorResponse.class); + when(err.code()).thenReturn("NoSuchKey"); + throw new ErrorResponseException(err, null, null); + }); + + when(minioClient.getObject(any(GetObjectArgs.class))) + .thenAnswer( + invocation -> { + GetObjectArgs args = invocation.getArgument(0); + if (args.object().endsWith(".info")) { + return mockGetObjectResponse(json.getBytes()); + } + throw new IOException("Simulated network failure streaming leftover part"); + }); + + storageService.append(info, new ByteArrayInputStream(new byte[50])); + } + @Test public void testFinalizeCompletedUploadZeroLength() throws Exception { UploadInfo info = new UploadInfo(); @@ -1796,6 +1903,320 @@ public void testCleanupExpiredUploadsPrunesStaleTempFilesAndChecksumIndices() th } } + @Test + public void testCloudUploadThreadPoolSizeConfiguration() { + assertEquals( + "Default thread pool size is 10", 10, storageService.getCloudUploadThreadPoolSize()); + + storageService.setCloudUploadThreadPoolSize(25); + assertEquals( + "Updated thread pool size is 25", 25, storageService.getCloudUploadThreadPoolSize()); + + storageService.setCloudUploadThreadPoolSize(4); + assertEquals("Reduced thread pool size is 4", 4, storageService.getCloudUploadThreadPoolSize()); + + try { + storageService.setCloudUploadThreadPoolSize(0); + fail("Should reject 0 pool size"); + } catch (IllegalArgumentException expected) { + assertTrue(expected.getMessage().contains("greater than 0")); + } + + try { + storageService.setCloudUploadThreadPoolSize(-5); + fail("Should reject negative pool size"); + } catch (IllegalArgumentException expected) { + assertTrue(expected.getMessage().contains("greater than 0")); + } + } + + @Test + public void testCloseGracefulShutdown() throws Exception { + // Verify closing storage service cleanly terminates background upload executor without error + storageService.close(); + // KISS: verifying method executes cleanly without throwing an exception + + // Verify close handles thread interruption gracefully + Thread.currentThread().interrupt(); + try { + storageService.close(); + } finally { + Thread.interrupted(); // Clear interrupted status + } + } + + @Test + public void testFinalizeUploadWithStaleIncompletePartIgnored() throws Exception { + UploadId uploadId = new UploadId("stale-part-test"); + UploadInfo info = new UploadInfo(); + info.setId(uploadId); + info.setOffset(0L); + info.setLength(3425070L); // 3.42 MB total upload length + + String infoJson = UploadInfoJsonSerializer.serialize(info); + String partKey1 = "metadata/stale-part-test.part.00001"; + String stalePartKey = "metadata/stale-part-test.part"; + + Item part1Item = mock(Item.class); + when(part1Item.objectName()).thenReturn(partKey1); + + StatObjectResponse part1Stat = mock(StatObjectResponse.class); + when(part1Stat.size()).thenReturn(3425070L); // Numbered part already covers full 3.42 MB + + StatObjectResponse stalePartStat = mock(StatObjectResponse.class); + when(stalePartStat.size()).thenReturn(1277952L); // Stale leftover from prior pause + + StatObjectResponse objectNotExists = mock(StatObjectResponse.class); + + when(minioClient.listObjects(any(ListObjectsArgs.class))) + .thenAnswer( + invocation -> { + ListObjectsArgs args = invocation.getArgument(0); + if (args.prefix().startsWith("metadata/stale-part-test.part.")) { + return Collections.singletonList(new Result<>(part1Item)); + } + return Collections.emptyList(); + }); + + when(minioClient.statObject(any(StatObjectArgs.class))) + .thenAnswer( + invocation -> { + StatObjectArgs args = invocation.getArgument(0); + if (args.object().equals(partKey1)) { + return part1Stat; + } else if (args.object().equals(stalePartKey)) { + return stalePartStat; + } else if (args.object().equals("uploads/stale-part-test")) { + ErrorResponse err = mock(ErrorResponse.class); + when(err.code()).thenReturn("NoSuchKey"); + throw new ErrorResponseException(err, null, null); + } + return objectNotExists; + }); + + when(minioClient.getObject(any(GetObjectArgs.class))) + .thenAnswer( + invocation -> { + GetObjectArgs args = invocation.getArgument(0); + if (args.object().endsWith(".info")) { + return mockGetObjectResponse(infoJson.getBytes()); + } + return mockGetObjectResponse(new byte[0]); + }); + + // Append 0 bytes to trigger finalization check + ByteArrayInputStream emptyStream = new ByteArrayInputStream(new byte[0]); + UploadInfo result = storageService.append(info, emptyStream); + + assertNotNull(result); + // Verify that the stale .part buffer was deleted via removeObject and NOT promoted to + // part.00002 + verify(minioClient, atLeastOnce()) + .removeObject( + argThat( + (RemoveObjectArgs args) -> args.object().equals("metadata/stale-part-test.part"))); + } + + @Test + public void testFinalizeUploadWithLegitimateNewIncompletePartPromoted() throws Exception { + UploadId uploadId = new UploadId("legit-part-test"); + UploadInfo info = new UploadInfo(); + info.setId(uploadId); + info.setOffset(0L); + info.setLength(10000000L); // 10 MB total length + + String infoJson = UploadInfoJsonSerializer.serialize(info); + String partKey1 = "metadata/legit-part-test.part.00001"; + String legitPartKey = "metadata/legit-part-test.part"; + + Item part1Item = mock(Item.class); + when(part1Item.objectName()).thenReturn(partKey1); + + StatObjectResponse part1Stat = mock(StatObjectResponse.class); + when(part1Stat.size()).thenReturn(8000000L); // 8 MB numbered part + + StatObjectResponse legitPartStat = mock(StatObjectResponse.class); + when(legitPartStat.size()).thenReturn(2000000L); // 2 MB tail; 8 MB + 2 MB == 10 MB exact match! + + when(minioClient.listObjects(any(ListObjectsArgs.class))) + .thenAnswer( + invocation -> { + ListObjectsArgs args = invocation.getArgument(0); + if (args.prefix().startsWith("metadata/legit-part-test.part.")) { + return Collections.singletonList(new Result<>(part1Item)); + } + return Collections.emptyList(); + }); + + when(minioClient.statObject(any(StatObjectArgs.class))) + .thenAnswer( + invocation -> { + StatObjectArgs args = invocation.getArgument(0); + if (args.object().equals(partKey1)) { + return part1Stat; + } else if (args.object().equals(legitPartKey)) { + return legitPartStat; + } else if (args.object().equals("uploads/legit-part-test")) { + ErrorResponse err = mock(ErrorResponse.class); + when(err.code()).thenReturn("NoSuchKey"); + throw new ErrorResponseException(err, null, null); + } + return mock(StatObjectResponse.class); + }); + + when(minioClient.getObject(any(GetObjectArgs.class))) + .thenAnswer( + invocation -> { + GetObjectArgs args = invocation.getArgument(0); + if (args.object().endsWith(".info")) { + return mockGetObjectResponse(infoJson.getBytes()); + } else if (args.object().equals(legitPartKey)) { + return mockGetObjectResponse(new byte[2000000]); + } + return mockGetObjectResponse(new byte[0]); + }); + + ByteArrayInputStream emptyStream = new ByteArrayInputStream(new byte[0]); + UploadInfo result = storageService.append(info, emptyStream); + + assertNotNull(result); + // Verify that the legitimate 2 MB tail was promoted to part.00002 + verify(minioClient) + .putObject( + argThat( + (PutObjectArgs args) -> + args.object().equals("metadata/legit-part-test.part.00002"))); + } + + @Test + public void testCalculateCurrentOffsetDoesNotDoubleCountOrExceedLength() throws Exception { + UploadId uploadId = new UploadId("calc-offset-test"); + String infoJson = "{\"id\":\"calc-offset-test\",\"length\":3425070,\"offset\":null}"; + + String partKey1 = "metadata/calc-offset-test.part.00001"; + String stalePartKey = "metadata/calc-offset-test.part"; + + Item part1Item = mock(Item.class); + when(part1Item.objectName()).thenReturn(partKey1); + + StatObjectResponse part1Stat = mock(StatObjectResponse.class); + when(part1Stat.size()).thenReturn(3425070L); // 3.42 MB numbered part + + StatObjectResponse stalePartStat = mock(StatObjectResponse.class); + when(stalePartStat.size()).thenReturn(1277952L); // Stale 1.27 MB leftover + + when(minioClient.getObject(any(GetObjectArgs.class))) + .thenAnswer( + invocation -> { + GetObjectArgs args = invocation.getArgument(0); + if (args.object().endsWith(".info")) { + return mockGetObjectResponse(infoJson.getBytes()); + } + return mockGetObjectResponse(new byte[0]); + }); + + when(minioClient.listObjects(any(ListObjectsArgs.class))) + .thenAnswer( + invocation -> { + ListObjectsArgs args = invocation.getArgument(0); + if (args.prefix().startsWith("metadata/calc-offset-test.part.")) { + return Collections.singletonList(new Result<>(part1Item)); + } + return Collections.emptyList(); + }); + + when(minioClient.statObject(any(StatObjectArgs.class))) + .thenAnswer( + invocation -> { + StatObjectArgs args = invocation.getArgument(0); + if (args.object().equals(partKey1)) { + return part1Stat; + } else if (args.object().equals(stalePartKey)) { + return stalePartStat; + } else if (args.object().equals("uploads/calc-offset-test")) { + ErrorResponse err = mock(ErrorResponse.class); + when(err.code()).thenReturn("NoSuchKey"); + throw new ErrorResponseException(err, null, null); + } + return mock(StatObjectResponse.class); + }); + + UploadInfo fetched = storageService.getUploadInfo(uploadId); + assertNotNull(fetched); + // Must be exactly 3,425,070 bytes (not 4,703,022 bytes with double-counted stale part!) + assertEquals(Long.valueOf(3425070L), fetched.getOffset()); + + // Verify stale .part buffer was deleted on the fly + verify(minioClient, atLeastOnce()) + .removeObject( + argThat( + (RemoveObjectArgs args) -> args.object().equals("metadata/calc-offset-test.part"))); + } + + @Test + public void testCalcOptimalPartSizeCalculations() { + assertEquals(8 * 1024 * 1024L, storageService.calcOptimalPartSize(null)); + assertEquals(8 * 1024 * 1024L, storageService.calcOptimalPartSize(0L)); + assertEquals(8 * 1024 * 1024L, storageService.calcOptimalPartSize(100L)); + assertEquals(8 * 1024 * 1024L, storageService.calcOptimalPartSize(100L * 1024 * 1024)); + assertEquals( + 8 * 1024 * 1024L, storageService.calcOptimalPartSize(10_000L * 8 * 1024 * 1024L - 1)); + + long largeLength = 10_000L * 16 * 1024 * 1024L; + assertEquals((largeLength / 10_000L) + 1, storageService.calcOptimalPartSize(largeLength)); + + // For 1 TB, part size auto-scales up so upload fits within 10,000 parts + long oneTb = 1024L * 1024 * 1024 * 1024L; + assertEquals((oneTb / 10_000L) + 1, storageService.calcOptimalPartSize(oneTb)); + + // For 5 TB (S3 max limit), part size scales up to ~524.3 MB + long fiveTb = 5L * 1024 * 1024 * 1024 * 1024L; + assertEquals((fiveTb / 10_000L) + 1, storageService.calcOptimalPartSize(fiveTb)); + } + + @Test + public void testSetAndGetPreferredPartSize() { + assertEquals(8 * 1024 * 1024L, storageService.getPreferredPartSize()); + + storageService.setPreferredPartSize(16 * 1024 * 1024L); + assertEquals(16 * 1024 * 1024L, storageService.getPreferredPartSize()); + + try { + storageService.setPreferredPartSize(4 * 1024 * 1024L); // Below 5MB limit + fail("Should reject part size below 5MB"); + } catch (IllegalArgumentException expected) { + assertTrue(expected.getMessage().contains("Preferred part size must be between")); + } + + try { + storageService.setPreferredPartSize(6L * 1024 * 1024 * 1024L); // Above 5GB limit + fail("Should reject part size above 5GB"); + } catch (IllegalArgumentException expected) { + assertTrue(expected.getMessage().contains("Preferred part size must be between")); + } + } + + @Test + public void testValidateRemainingPartBudget() throws Exception { + UploadInfo info = new UploadInfo(); + storageService.validateRemainingPartBudget(info, 9999); + // KISS: 9999 parts is within budget, verifies method completes cleanly + + try { + storageService.validateRemainingPartBudget(info, 10000); + fail("Should throw MaxAppendSizeExceededException at 10000 parts"); + } catch (MaxAppendSizeExceededException expected) { + assertTrue(expected.getMessage().contains("maximum allowed S3 limit of 10000 parts")); + } + + try { + storageService.validateRemainingPartBudget(info, 10005); + fail("Should throw MaxAppendSizeExceededException above 10000 parts"); + } catch (MaxAppendSizeExceededException expected) { + assertTrue(expected.getMessage().contains("maximum allowed S3 limit of 10000 parts")); + } + } + private GetObjectResponse mockGetObjectResponse(byte[] bytes) { return new GetObjectResponse( null, "test-bucket", "eu-central-1", "object-key", new ByteArrayInputStream(bytes)); diff --git a/src/test/java/me/desair/tus/server/upload/util/AsyncChunkUploaderTest.java b/src/test/java/me/desair/tus/server/upload/util/AsyncChunkUploaderTest.java new file mode 100644 index 00000000..53a0de9f --- /dev/null +++ b/src/test/java/me/desair/tus/server/upload/util/AsyncChunkUploaderTest.java @@ -0,0 +1,576 @@ +package me.desair.tus.server.upload.util; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.SynchronousQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +/** + * Unit tests verifying bounded 3-slot asynchronous chunk uploading, backpressure, exception + * translation, timeout aborts, and CallerRunsPolicy degradation in {@link AsyncChunkUploader}. + */ +public class AsyncChunkUploaderTest { + + private ExecutorService executor; + private File tempDir; + + @Before + public void setUp() throws Exception { + executor = Executors.newFixedThreadPool(4); + tempDir = Files.createTempDirectory("tus-async-uploader-test").toFile(); + } + + @After + public void tearDown() throws Exception { + if (executor != null) { + executor.shutdownNow(); + } + if (tempDir != null && tempDir.exists()) { + org.apache.commons.io.FileUtils.deleteDirectory(tempDir); + } + } + + @Test + public void testFirstChunkSubmissionDirectToSlot3() throws Exception { + File chunk1 = new File(tempDir, "chunk1.tmp"); + Files.write(chunk1.toPath(), new byte[] {1, 2, 3}); + + CountDownLatch uploadFinished = new CountDownLatch(1); + try (AsyncChunkUploader uploader = new AsyncChunkUploader(executor)) { + uploader.submitChunk( + chunk1, + 3, + "part-1", + () -> { + uploadFinished.countDown(); + }); + + assertTrue( + "Upload task should execute in background", uploadFinished.await(3, TimeUnit.SECONDS)); + int confirmed = uploader.drainAndComplete(3000); + assertEquals("One chunk confirmed uploaded", 1, confirmed); + } + + // Verify temp file is deleted by the background worker in finally block + assertFalse("Chunk file should be cleaned up after successful upload", chunk1.exists()); + } + + @Test + public void testPipelinedSubmissionThroughSlot2AndSlot3() throws Exception { + File chunk1 = new File(tempDir, "chunk1.tmp"); + File chunk2 = new File(tempDir, "chunk2.tmp"); + File chunk3 = new File(tempDir, "chunk3.tmp"); + Files.write(chunk1.toPath(), new byte[] {1}); + Files.write(chunk2.toPath(), new byte[] {2}); + Files.write(chunk3.toPath(), new byte[] {3}); + + List uploadedOrder = Collections.synchronizedList(new ArrayList<>()); + CountDownLatch unblockSlot3 = new CountDownLatch(1); + + try (AsyncChunkUploader uploader = new AsyncChunkUploader(executor)) { + // Chunk 1 occupies Slot 3 + uploader.submitChunk( + chunk1, + 1, + "part-1", + () -> { + unblockSlot3.await(3, TimeUnit.SECONDS); + uploadedOrder.add("part-1"); + }); + + // Chunk 2 placed in Slot 2 (waiting) while Chunk 1 is running + uploader.submitChunk( + chunk2, + 1, + "part-2", + () -> { + uploadedOrder.add("part-2"); + }); + + // Unblock Chunk 1 so it finishes + unblockSlot3.countDown(); + + // Chunk 3 submitted: blocks until Chunk 1 finishes, promotes Chunk 2 to Slot 3, stores Chunk + // 3 in Slot 2 + uploader.submitChunk( + chunk3, + 1, + "part-3", + () -> { + uploadedOrder.add("part-3"); + }); + + int confirmed = uploader.drainAndComplete(3000); + assertEquals("All 3 chunks confirmed uploaded", 3, confirmed); + assertEquals( + "Parts uploaded in sequential order", + List.of("part-1", "part-2", "part-3"), + uploadedOrder); + } + + assertFalse("Chunk 1 should be deleted", chunk1.exists()); + assertFalse("Chunk 2 should be deleted", chunk2.exists()); + assertFalse("Chunk 3 should be deleted", chunk3.exists()); + } + + @Test + public void testBackpressureBlocksReaderWhenBothSlotsFull() throws Exception { + File chunk1 = new File(tempDir, "chunk1.tmp"); + File chunk2 = new File(tempDir, "chunk2.tmp"); + File chunk3 = new File(tempDir, "chunk3.tmp"); + Files.write(chunk1.toPath(), new byte[] {1}); + Files.write(chunk2.toPath(), new byte[] {2}); + Files.write(chunk3.toPath(), new byte[] {3}); + + CountDownLatch chunk1Hold = new CountDownLatch(1); + AtomicBoolean chunk3Submitted = new AtomicBoolean(false); + + try (AsyncChunkUploader uploader = new AsyncChunkUploader(executor)) { + uploader.submitChunk( + chunk1, + 1, + "part-1", + () -> { + chunk1Hold.await(3, TimeUnit.SECONDS); + }); + + uploader.submitChunk(chunk2, 1, "part-2", () -> {}); + + // Launch thread trying to submit chunk 3: must block because Slot 3 is busy and Slot 2 is + // occupied + Thread submitThread = + new Thread( + () -> { + try { + uploader.submitChunk(chunk3, 1, "part-3", () -> {}); + chunk3Submitted.set(true); + } catch (IOException ignored) { + } + }); + submitThread.start(); + + // Give thread 150ms to attempt submit; it must be blocked waiting on Slot 3 + Thread.sleep(150); + assertFalse("Submit of chunk 3 must block due to backpressure", chunk3Submitted.get()); + + // Release chunk 1 + chunk1Hold.countDown(); + submitThread.join(3000); + + assertTrue("Submit of chunk 3 unblocked after chunk 1 finished", chunk3Submitted.get()); + int confirmed = uploader.drainAndComplete(3000); + assertEquals("All 3 chunks confirmed", 3, confirmed); + } + } + + @Test + public void testUploadActionFailurePropagatesIOExceptionAndAborts() throws Exception { + File chunk1 = new File(tempDir, "chunk1.tmp"); + File chunk2 = new File(tempDir, "chunk2.tmp"); + Files.write(chunk1.toPath(), new byte[] {1}); + Files.write(chunk2.toPath(), new byte[] {2}); + + try (AsyncChunkUploader uploader = new AsyncChunkUploader(executor)) { + uploader.submitChunk( + chunk1, + 1, + "part-1", + () -> { + throw new IOException("Simulated S3 network failure"); + }); + + // Submitting next chunk or draining must detect previous failure and throw IOException + try { + uploader.submitChunk(chunk2, 1, "part-2", () -> {}); + uploader.drainAndComplete(3000); + fail("Expected IOException from failed chunk upload"); + } catch (IOException e) { + assertTrue( + "Exception message should reflect upload failure", + e.getMessage().contains("Simulated S3 network failure")); + } + } + + assertFalse("Chunk 2 should be cleaned up by abort()", chunk2.exists()); + } + + @Test + public void testDrainTimeoutAbortsInFlightUpload() throws Exception { + File chunk1 = new File(tempDir, "chunk1.tmp"); + Files.write(chunk1.toPath(), new byte[] {1}); + + CountDownLatch hangLatch = new CountDownLatch(1); + + try (AsyncChunkUploader uploader = new AsyncChunkUploader(executor)) { + uploader.submitChunk( + chunk1, + 1, + "part-1", + () -> { + hangLatch.await(10, TimeUnit.SECONDS); + }); + + try { + // Enforce short 200ms drain timeout + uploader.drainAndComplete(200); + fail("Expected timeout IOException during drain"); + } catch (IOException e) { + assertTrue("Should report timeout error", e.getMessage().contains("Timed out after")); + assertTrue("Should report key", e.getMessage().contains("part-1")); + } + } finally { + hangLatch.countDown(); + } + } + + @Test + public void testCloseAutomaticallyAbortsIfUncompleted() throws Exception { + File chunk1 = new File(tempDir, "chunk1.tmp"); + File chunk2 = new File(tempDir, "chunk2.tmp"); + Files.write(chunk1.toPath(), new byte[] {1}); + Files.write(chunk2.toPath(), new byte[] {2}); + + CountDownLatch latch = new CountDownLatch(1); + try (AsyncChunkUploader uploader = new AsyncChunkUploader(executor)) { + uploader.submitChunk( + chunk1, + 1, + "part-1", + () -> { + latch.await(3, TimeUnit.SECONDS); + }); + uploader.submitChunk(chunk2, 1, "part-2", () -> {}); + // Exiting without calling drainAndComplete() triggers close() which calls abort() + } finally { + latch.countDown(); + } + + assertFalse( + "Waiting chunk file in Slot 2 should be deleted by abort on close", chunk2.exists()); + } + + @Test + public void testSynchronousQueueCallerRunsPolicyDegradation() throws Exception { + // ThreadPool with exactly 1 thread and SynchronousQueue + ThreadPoolExecutor singlePool = + new ThreadPoolExecutor( + 1, + 1, + 60L, + TimeUnit.SECONDS, + new SynchronousQueue<>(), + new ThreadPoolExecutor.CallerRunsPolicy()); + + File chunk1 = new File(tempDir, "chunk1.tmp"); + File chunk2 = new File(tempDir, "chunk2.tmp"); + Files.write(chunk1.toPath(), new byte[] {1}); + Files.write(chunk2.toPath(), new byte[] {2}); + + CountDownLatch poolThreadBusy = new CountDownLatch(1); + CountDownLatch releaseBusy = new CountDownLatch(1); + + // Occupy the only thread in singlePool + singlePool.submit( + () -> { + poolThreadBusy.countDown(); + try { + releaseBusy.await(3, TimeUnit.SECONDS); + } catch (InterruptedException ignored) { + } + }); + + assertTrue("Single pool thread is now occupied", poolThreadBusy.await(3, TimeUnit.SECONDS)); + + String mainThreadName = Thread.currentThread().getName(); + List executionThreads = Collections.synchronizedList(new ArrayList<>()); + + try (AsyncChunkUploader uploader = new AsyncChunkUploader(singlePool)) { + // Since pool is saturated and queue is SynchronousQueue, CallerRunsPolicy executes task + // directly on caller thread + uploader.submitChunk( + chunk1, + 1, + "part-1", + () -> { + executionThreads.add(Thread.currentThread().getName()); + }); + + int confirmed = uploader.drainAndComplete(3000); + assertEquals("One chunk confirmed", 1, confirmed); + assertEquals( + "Executed directly on caller thread without queue stall", + mainThreadName, + executionThreads.get(0)); + } finally { + releaseBusy.countDown(); + singlePool.shutdownNow(); + } + } + + @Test + public void testHighConcurrencyMultiUploaderSimulation() throws Exception { + int concurrentUploaders = 15; + ExecutorService sharedPool = + new ThreadPoolExecutor( + 4, + 4, + 60L, + TimeUnit.SECONDS, + new SynchronousQueue<>(), + new ThreadPoolExecutor.CallerRunsPolicy()); + + CountDownLatch allDone = new CountDownLatch(concurrentUploaders); + AtomicInteger totalUploadedChunks = new AtomicInteger(0); + + for (int i = 0; i < concurrentUploaders; i++) { + final int uploaderId = i; + new Thread( + () -> { + try { + File f1 = new File(tempDir, "c-" + uploaderId + "-1.tmp"); + File f2 = new File(tempDir, "c-" + uploaderId + "-2.tmp"); + Files.write(f1.toPath(), new byte[] {1, 2}); + Files.write(f2.toPath(), new byte[] {3, 4}); + + try (AsyncChunkUploader uploader = new AsyncChunkUploader(sharedPool)) { + uploader.submitChunk( + f1, + 2, + "u" + uploaderId + "-p1", + () -> { + Thread.sleep(20); + }); + uploader.submitChunk( + f2, + 2, + "u" + uploaderId + "-p2", + () -> { + Thread.sleep(20); + }); + int confirmed = uploader.drainAndComplete(5000); + totalUploadedChunks.addAndGet(confirmed); + } + } catch (Exception e) { + e.printStackTrace(); + } finally { + allDone.countDown(); + } + }) + .start(); + } + + assertTrue( + "All concurrent uploaders must complete within 10s", allDone.await(10, TimeUnit.SECONDS)); + assertEquals( + "All 30 chunks (15 uploaders x 2 chunks) must complete", 30, totalUploadedChunks.get()); + + sharedPool.shutdownNow(); + } + + @Test + public void testGetConfirmedCount() throws Exception { + File f1 = new File(tempDir, "count1.tmp"); + Files.write(f1.toPath(), new byte[] {1}); + + try (AsyncChunkUploader uploader = new AsyncChunkUploader(executor)) { + assertEquals(0, uploader.getConfirmedCount()); + uploader.submitChunk(f1, 1, "k1", () -> {}); + uploader.drainAndComplete(3000); + assertEquals(1, uploader.getConfirmedCount()); + } + } + + @Test + public void testSubmitChunkDiscoversPreviousDoneUploadFailureAndCleansUp() throws Exception { + File f1 = new File(tempDir, "err1.tmp"); + File f2 = new File(tempDir, "err2.tmp"); + Files.write(f1.toPath(), new byte[] {1}); + Files.write(f2.toPath(), new byte[] {2}); + + CountDownLatch chunk1Done = new CountDownLatch(1); + + try (AsyncChunkUploader uploader = new AsyncChunkUploader(executor)) { + uploader.submitChunk( + f1, + 1, + "k1", + () -> { + chunk1Done.countDown(); + throw new IOException("Simulated disk read error on cloud worker"); + }); + + assertTrue(chunk1Done.await(3, TimeUnit.SECONDS)); + // Give worker thread a brief moment to update Future state to done + Thread.sleep(50); + + try { + uploader.submitChunk(f2, 1, "k2", () -> {}); + fail("Should have thrown IOException when discovering chunk 1 failure"); + } catch (IOException e) { + assertTrue(e.getMessage().contains("Simulated disk read error on cloud worker")); + } + } + + assertFalse("Chunk 2 must be cleaned up on failure", f2.exists()); + } + + @Test + public void testUploadActionThrowsRuntimeExceptionTranslatedToIOException() throws Exception { + File f1 = new File(tempDir, "runtime_err.tmp"); + Files.write(f1.toPath(), new byte[] {1}); + + try (AsyncChunkUploader uploader = new AsyncChunkUploader(executor)) { + uploader.submitChunk( + f1, + 1, + "runtime-key", + () -> { + throw new IllegalStateException("Boom!"); + }); + + try { + uploader.drainAndComplete(3000); + fail("Should have thrown IOException wrapping RuntimeException"); + } catch (IOException e) { + assertTrue(e.getMessage().contains("Upload failed for key runtime-key: Boom!")); + assertTrue(e.getCause() instanceof IllegalStateException); + } + } + } + + @Test + public void testUploadActionThrowsErrorTranslatedToIOException() throws Exception { + File f1 = new File(tempDir, "error_key.tmp"); + Files.write(f1.toPath(), new byte[] {1}); + + try (AsyncChunkUploader uploader = new AsyncChunkUploader(executor)) { + uploader.submitChunk( + f1, + 1, + "error-key", + () -> { + throw new AssertionError("Simulated fatal assertion error"); + }); + + try { + uploader.drainAndComplete(3000); + fail("Should have thrown IOException wrapping AssertionError"); + } catch (IOException e) { + assertTrue( + e.getMessage().contains("Unexpected error during chunk upload for key error-key")); + } + } + } + + @Test + public void testDrainInterruptedThrowsIOException() throws Exception { + File f1 = new File(tempDir, "interrupted.tmp"); + Files.write(f1.toPath(), new byte[] {1}); + + CountDownLatch started = new CountDownLatch(1); + CountDownLatch unblock = new CountDownLatch(1); + + AtomicBoolean gotInterruptedIOException = new AtomicBoolean(false); + + Thread drainThread = + new Thread( + () -> { + try (AsyncChunkUploader uploader = new AsyncChunkUploader(executor)) { + uploader.submitChunk( + f1, + 1, + "k-interrupt", + () -> { + started.countDown(); + unblock.await(5, TimeUnit.SECONDS); + }); + uploader.drainAndComplete(5000); + } catch (IOException e) { + if (e.getMessage().contains("Interrupted waiting for chunk upload")) { + gotInterruptedIOException.set(true); + } + } catch (Exception ignored) { + } + }); + + drainThread.start(); + assertTrue(started.await(3, TimeUnit.SECONDS)); + drainThread.interrupt(); + drainThread.join(3000); + unblock.countDown(); + + assertTrue( + "Interrupted thread should throw IOException mentioning Interrupted", + gotInterruptedIOException.get()); + } + + @Test + public void testSubmitChunkBackpressureInterruptedThrowsIOException() throws Exception { + File f1 = new File(tempDir, "bp-int1.tmp"); + File f2 = new File(tempDir, "bp-int2.tmp"); + File f3 = new File(tempDir, "bp-int3.tmp"); + Files.write(f1.toPath(), new byte[] {1}); + Files.write(f2.toPath(), new byte[] {2}); + Files.write(f3.toPath(), new byte[] {3}); + + CountDownLatch f1Started = new CountDownLatch(1); + CountDownLatch unblock = new CountDownLatch(1); + AtomicBoolean gotBackpressureInterruptedException = new AtomicBoolean(false); + + Thread submitThread = + new Thread( + () -> { + try (AsyncChunkUploader uploader = new AsyncChunkUploader(executor)) { + uploader.submitChunk( + f1, + 1, + "k1", + () -> { + f1Started.countDown(); + unblock.await(5, TimeUnit.SECONDS); + }); + uploader.submitChunk(f2, 1, "k2", () -> {}); + // Third chunk triggers waitForInFlight() backpressure + uploader.submitChunk(f3, 1, "k3", () -> {}); + } catch (IOException e) { + if (e.getMessage().contains("Interrupted while checking in-flight chunk upload")) { + gotBackpressureInterruptedException.set(true); + } + } catch (Exception ignored) { + } + }); + + submitThread.start(); + assertTrue(f1Started.await(3, TimeUnit.SECONDS)); + // Brief sleep to let submitThread block in waitForInFlight() + try { + Thread.sleep(50); + } catch (InterruptedException ignored) { + } + submitThread.interrupt(); + submitThread.join(3000); + unblock.countDown(); + + assertTrue( + "Interrupted backpressure wait should throw IOException", + gotBackpressureInterruptedException.get()); + } +} From c2e7020cf21b6f89716cdf6e85e6038a3ff6c5a6 Mon Sep 17 00:00:00 2001 From: Tom Desair Date: Mon, 28 Sep 2026 22:24:23 +0200 Subject: [PATCH 2/7] fix(rufh): do not invoke append on creation requests without content --- .../RufhAppendPatchRequestHandler.java | 9 ++-- .../RufhCreationPostRequestHandler.java | 26 ++++++++-- .../RufhCreationPostRequestHandlerTest.java | 52 +++++++++++++++++++ 3 files changed, 80 insertions(+), 7 deletions(-) diff --git a/src/main/java/me/desair/tus/server/rufh/handler/RufhAppendPatchRequestHandler.java b/src/main/java/me/desair/tus/server/rufh/handler/RufhAppendPatchRequestHandler.java index 53974029..d929d8fa 100644 --- a/src/main/java/me/desair/tus/server/rufh/handler/RufhAppendPatchRequestHandler.java +++ b/src/main/java/me/desair/tus/server/rufh/handler/RufhAppendPatchRequestHandler.java @@ -55,10 +55,13 @@ public HttpProblemDetails process( String uploadCompleteHeader = servletRequest.getHeader(HttpHeader.UPLOAD_COMPLETE); Boolean uploadComplete = StructuredHeaderUtil.parseBoolean(uploadCompleteHeader); - long cl = servletRequest.getContentLengthLong(); - if (Boolean.TRUE.equals(uploadComplete) && !uploadInfo.hasLength() && cl >= 0) { + // Per RUFH §4.2.1: If upload length was deferred and the client completes the upload + // via Upload-Complete: ?1, derive and set the total length from current offset + + // Content-Length. + long contentLength = servletRequest.getContentLengthLong(); + if (Boolean.TRUE.equals(uploadComplete) && !uploadInfo.hasLength() && contentLength >= 0) { long currentOffset = uploadInfo.getOffset() != null ? uploadInfo.getOffset() : 0L; - uploadInfo.setLength(currentOffset + cl); + uploadInfo.setLength(currentOffset + contentLength); uploadStorageService.update(uploadInfo); } diff --git a/src/main/java/me/desair/tus/server/rufh/handler/RufhCreationPostRequestHandler.java b/src/main/java/me/desair/tus/server/rufh/handler/RufhCreationPostRequestHandler.java index 4ff1232d..2702c23e 100644 --- a/src/main/java/me/desair/tus/server/rufh/handler/RufhCreationPostRequestHandler.java +++ b/src/main/java/me/desair/tus/server/rufh/handler/RufhCreationPostRequestHandler.java @@ -16,6 +16,7 @@ import me.desair.tus.server.util.TusServletRequest; import me.desair.tus.server.util.TusServletResponse; import me.desair.tus.server.util.Utils; +import org.apache.commons.lang3.Strings; /** * Request handler for upload creation requests via HTTP POST, PUT, or PATCH. @@ -66,10 +67,13 @@ public HttpProblemDetails process( String uploadCompleteHeader = servletRequest.getHeader(HttpHeader.UPLOAD_COMPLETE); Boolean uploadComplete = StructuredHeaderUtil.parseBoolean(uploadCompleteHeader); - long cl = servletRequest.getContentLengthLong(); + // Per RUFH §4.2.1: If upload length is deferred upon creation and the client completes + // the upload in the creation request via Upload-Complete: ?1, the total length is derived + // from Content-Length. + long contentLength = servletRequest.getContentLengthLong(); Long announcedLength = uploadLength; - if (announcedLength == null && Boolean.TRUE.equals(uploadComplete) && cl >= 0) { - announcedLength = cl; + if (announcedLength == null && Boolean.TRUE.equals(uploadComplete) && contentLength >= 0) { + announcedLength = contentLength; } UploadInfo uploadInfo; @@ -90,8 +94,22 @@ public HttpProblemDetails process( String uploadUri = Utils.getUploadUriOnCreation(uploadInfo, servletRequest, uploadStorageService); + // Per RUFH §4.1.4: "This limit does not apply to upload creation requests with no content, + // or to requests completing the upload by including the Upload-Complete: ?1 header field." + // An empty creation request carries no body (contentLength <= 0 without chunked encoding). + // In servlet requests without a body, getContentLengthLong() returns -1. We must verify + // that actual payload content is present before invoking storageService.append(); otherwise, + // sending a 0-byte stream to backends with minAppendSize configured (e.g. S3 or Azure Blob) + // would trigger MinAppendSizeNotMetException on an empty creation request. + boolean hasContent = + contentLength > 0 + || (contentLength < 0 + && servletRequest.getHeader(HttpHeader.TRANSFER_ENCODING) != null + && Strings.CI.contains( + servletRequest.getHeader(HttpHeader.TRANSFER_ENCODING), "chunked")); + InputStream is = servletRequest.getContentInputStream(); - if (is != null && servletRequest.getContentLengthLong() != 0) { + if (is != null && hasContent) { if (uploadLockingService != null) { InterruptibleInputStream interruptibleStream = new InterruptibleInputStream(is); uploadLockingService.registerInputStream(uploadUri, interruptibleStream); diff --git a/src/test/java/me/desair/tus/server/rufh/handler/RufhCreationPostRequestHandlerTest.java b/src/test/java/me/desair/tus/server/rufh/handler/RufhCreationPostRequestHandlerTest.java index e68c1776..e4e9b8c4 100644 --- a/src/test/java/me/desair/tus/server/rufh/handler/RufhCreationPostRequestHandlerTest.java +++ b/src/test/java/me/desair/tus/server/rufh/handler/RufhCreationPostRequestHandlerTest.java @@ -11,6 +11,7 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import java.io.InputStream; import me.desair.tus.server.HttpHeader; import me.desair.tus.server.HttpMethod; import me.desair.tus.server.upload.UploadId; @@ -515,4 +516,55 @@ public void testProcessCreationWithUploadCompleteAndContentLengthSetsAnnouncedLe assertThat(response.getHeader(HttpHeader.UPLOAD_OFFSET), is(String.valueOf(content.length))); assertThat(response.getHeader(HttpHeader.UPLOAD_COMPLETE), is("?1")); } + + @Test + public void testProcessCreationWithChunkedTransferEncodingCallsAppend() throws Exception { + request.setMethod("POST"); + request.setRequestURI("/files"); + request.addHeader(HttpHeader.UPLOAD_COMPLETE, "?0"); + request.addHeader(HttpHeader.TRANSFER_ENCODING, "chunked"); + byte[] content = "chunked stream data".getBytes(); + // Do not set content via setContent to keep Content-Length at -1, provide via InputStream + request.setContent(content); + + // Custom request to simulate chunked request without Content-Length header (cl < 0) + TusServletRequest tusRequest = + new TusServletRequest(request) { + @Override + public long getContentLengthLong() { + return -1L; + } + + @Override + public InputStream getContentInputStream() { + return new java.io.ByteArrayInputStream(content); + } + }; + + UploadInfo createdInfo = new UploadInfo(); + createdInfo.setId(new UploadId("chunked-id")); + createdInfo.setOffset(0L); + + UploadInfo appendedInfo = new UploadInfo(); + appendedInfo.setId(new UploadId("chunked-id")); + appendedInfo.setOffset((long) content.length); + + when(storageService.create(any(UploadInfo.class), nullable(String.class))) + .thenReturn(createdInfo); + when(storageService.append(any(UploadInfo.class), any())).thenReturn(appendedInfo); + + handler.process( + HttpMethod.POST, + tusRequest, + new TusServletResponse(response), + storageService, + lockingService, + "owner", + null); + + verify(storageService).append(eq(createdInfo), any(InputStream.class)); + assertThat(response.getStatus(), is(201)); + assertThat(response.getHeader(HttpHeader.UPLOAD_OFFSET), is(String.valueOf(content.length))); + assertThat(response.getHeader(HttpHeader.UPLOAD_COMPLETE), is("?0")); + } } From fe63589bdf8ce508c8f9ef546ab631523d78879f Mon Sep 17 00:00:00 2001 From: Tom Desair Date: Mon, 28 Sep 2026 23:04:13 +0200 Subject: [PATCH 3/7] perf(s3): optimize S3 lock acquisition and multi-object release - Skip redundant isLockExpired pre-check in S3LockingService for optimistic conditional PutObject - Batch delete .lock and .stop keys via S3 Multi-Object Delete in S3UploadLock - Clean up unused deleteS3LockObjectIfOwner and deleteS3ObjectQuietly methods - Remove createMinioContainer alias from TestUtils and clarify RustFS Testcontainers setup - Update CHANGELOG.md --- CHANGELOG.md | 1 + .../server/upload/s3/S3LockingService.java | 10 +- .../tus/server/upload/s3/S3UploadLock.java | 72 +++-- .../java/me/desair/tus/server/TestUtils.java | 13 +- .../server/upload/s3/ITS3LockingService.java | 14 +- .../server/upload/s3/ITS3RufhProtocol.java | 14 +- .../server/upload/s3/ITS3StorageService.java | 14 +- .../upload/s3/ITS3TusFileUploadService.java | 14 +- .../upload/s3/S3LockingServiceTest.java | 51 ++-- .../server/upload/s3/S3UploadLockTest.java | 259 +++++++++--------- 10 files changed, 246 insertions(+), 216 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f7a9c6cd..846a9adc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,7 @@ All notable changes to this project will be documented in this file. - **`process()` Return Value (`UploadInfo`)**: `TusFileUploadService.process(...)` now returns the created or updated `UploadInfo` instance (or `null` on errors or `OPTIONS` preflight requests), enabling applications to track and store upload IDs directly into user sessions or database repositories. - **Jackson Bundled in Compile Scope**: Promoted Jackson dependencies (`jackson-databind`, `jackson-annotations`, `jackson-core`) to `compile` scope, eliminating `NoClassDefFoundError` when enabling JSON serialization or using cloud storage. - **Lock-Holding Stream Lifecycle**: `TusFileUploadService.getUploadedBytes(...)` now retains the upload lock until the returned stream is closed, preventing concurrent modifications or deletions from corrupting data mid-stream. +- **S3 Locking Optimization**: Optimized `S3LockingService` and `S3UploadLock` by skipping the redundant `isLockExpired` pre-check on happy-path acquisitions (saving 1 remote GET round-trip) and using S3 Multi-Object Delete to delete `.lock` and `.stop` objects in a single batch round-trip on lock release. ### Fixed - **RFC 9110 Media Type Matching & MIME Parameter Tolerance**: Enhanced `Content-Type` header validation in `ContentTypeValidator`, `PostContentTypeValidator`, and `RufhAppendValidator` using RFC 9110 §8.3 compliant media-type parsing (`Utils.isMediaType`). Media type matching now tolerates MIME parameters (such as `;charset=UTF-8` automatically appended by Spring Boot `CharacterEncodingFilter`, servlet wrappers, proxies, or HTTP clients), whitespace variations, and case-insensitivity without incorrectly rejecting valid requests with `406 Not Acceptable`. diff --git a/src/main/java/me/desair/tus/server/upload/s3/S3LockingService.java b/src/main/java/me/desair/tus/server/upload/s3/S3LockingService.java index 02b721aa..ab5f4cf5 100644 --- a/src/main/java/me/desair/tus/server/upload/s3/S3LockingService.java +++ b/src/main/java/me/desair/tus/server/upload/s3/S3LockingService.java @@ -153,9 +153,13 @@ protected UploadLock tryAcquireLock(UploadId uploadId, LeaseData leaseData) { String lockKey = buildLockKey(uploadId); String stopKey = buildStopKey(uploadId); - if (!isLockExpired(lockKey)) { - return null; - } + // Optimistic conditional write: + // Skip redundant isLockExpired() pre-check. In >99.9% of requests, no lock exists, + // so an initial isLockExpired() issues an expensive S3 GET that 404s (~320ms penalty). + // By issuing putObject with "If-None-Match: *" directly, happy-path acquisition + // takes only 1 network call. If a lock already exists, S3 atomically returns 412 + // Precondition Failed, causing this method to return null. The caller + // (acquireOrEvictExpiredLock) will then inspect isLockExpired() and evict if expired. try { leaseData.setLockPath(lockKey); diff --git a/src/main/java/me/desair/tus/server/upload/s3/S3UploadLock.java b/src/main/java/me/desair/tus/server/upload/s3/S3UploadLock.java index 5b1c8899..5b3a18b3 100644 --- a/src/main/java/me/desair/tus/server/upload/s3/S3UploadLock.java +++ b/src/main/java/me/desair/tus/server/upload/s3/S3UploadLock.java @@ -3,12 +3,19 @@ import io.minio.GetObjectArgs; import io.minio.MinioClient; import io.minio.PutObjectArgs; -import io.minio.RemoveObjectArgs; +import io.minio.RemoveObjectsArgs; +import io.minio.Result; import io.minio.errors.ErrorResponseException; +import io.minio.messages.DeleteRequest; +import io.minio.messages.DeleteResult; import java.io.ByteArrayInputStream; import java.io.InputStream; +import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.concurrent.ScheduledExecutorService; +import java.util.stream.Collectors; +import java.util.stream.Stream; import me.desair.tus.server.upload.AbstractLeaseLock; import me.desair.tus.server.upload.LeaseData; import me.desair.tus.server.upload.UploadLock; @@ -104,24 +111,54 @@ protected void doRenewLease() throws Exception { @Override protected void releaseLockResource() { - // Owner-Safe Lock Release: only delete .lock lease if it is still owned by this holder - deleteS3LockObjectIfOwner(lockKey); - deleteS3ObjectQuietly(stopKey); + // Owner-Safe Lock Release: only delete .lock lease if it is still owned by this holder. + // If ownership was stolen by another node (e.g. heartbeat lost or execution ran too long), + // skip deletion entirely so we do not delete another node's lock or any stop signal + // potentially intended for that new holder. + if (lockKey != null && !doesLockOwnershipMatch(lockKey)) { + log.info( + "Skipping deletion of S3 lock key {}: lock is currently held by another node", lockKey); + return; + } + + // Batch deletion of .lock and .stop keys via S3 Multi-Object Delete. + // AWS S3 and MinIO execute multi-object delete in a single network round trip (POST /?delete), + // eliminating an extra round trip on every lock release. + deleteS3Objects(lockKey, stopKey); } - void deleteS3LockObjectIfOwner(String key) { - if (key == null || minioClient == null || bucket == null) { + void deleteS3Objects(String firstKey, String secondKey) { + if (minioClient == null || bucket == null) { + return; + } + List objects = + Stream.of(firstKey, secondKey) + .filter(Objects::nonNull) + .map(DeleteRequest.Object::new) + .collect(Collectors.toList()); + if (objects.isEmpty()) { return; } try { - if (!doesLockOwnershipMatch(key)) { - log.info( - "Skipping deletion of S3 lock key {}: lock is currently held by another node", key); - return; + Iterable> results = + minioClient.removeObjects( + RemoveObjectsArgs.builder().bucket(bucket).objects(objects).build()); + if (results != null) { + for (Result result : results) { + try { + if (result != null) { + DeleteResult.Error error = result.get(); + if (error != null) { + log.debug("Failed to delete S3 object {}: {}", error.objectName(), error.message()); + } + } + } catch (Exception e) { + log.debug("Failed to process batch delete result", e); + } + } } - minioClient.removeObject(RemoveObjectArgs.builder().bucket(bucket).object(key).build()); } catch (Exception e) { - log.debug("Failed to delete S3 lock object {}", key, e); + log.debug("Failed to batch delete S3 objects {} and {}", firstKey, secondKey, e); } } @@ -149,15 +186,4 @@ boolean doesLockOwnershipMatch(String key) { } return true; } - - private void deleteS3ObjectQuietly(String key) { - if (key == null || minioClient == null || bucket == null) { - return; - } - try { - minioClient.removeObject(RemoveObjectArgs.builder().bucket(bucket).object(key).build()); - } catch (Exception e) { - log.debug("Failed to delete S3 object {}", key, e); - } - } } diff --git a/src/test/java/me/desair/tus/server/TestUtils.java b/src/test/java/me/desair/tus/server/TestUtils.java index 32433879..55f1f6fe 100644 --- a/src/test/java/me/desair/tus/server/TestUtils.java +++ b/src/test/java/me/desair/tus/server/TestUtils.java @@ -7,8 +7,8 @@ import org.testcontainers.containers.GenericContainer; /** - * Helper utility class for S3 integration tests running against Testcontainers MinIO using the - * MinIO Java SDK. Supports both Docker and Podman container engines automatically. + * Helper utility class for integration tests running against Testcontainers RustFS (S3) and Azurite + * (Azure Blob). Supports both Docker and Podman container engines automatically. */ public final class TestUtils { @@ -77,15 +77,6 @@ public static GenericContainer createRustFsContainer() { .withEnv("RUSTFS_SECRET_KEY", "rustfsadmin"); } - /** - * Alias for {@link #createRustFsContainer()} for compatibility. - * - * @return A configured GenericContainer instance (not started yet) - */ - public static GenericContainer createMinioContainer() { - return createRustFsContainer(); - } - /** * Create a {@link MinioClient} configured to connect to the given S3/RustFS container. * diff --git a/src/test/java/me/desair/tus/server/upload/s3/ITS3LockingService.java b/src/test/java/me/desair/tus/server/upload/s3/ITS3LockingService.java index 58248985..916ff8e2 100644 --- a/src/test/java/me/desair/tus/server/upload/s3/ITS3LockingService.java +++ b/src/test/java/me/desair/tus/server/upload/s3/ITS3LockingService.java @@ -17,7 +17,7 @@ public class ITS3LockingService { - private static GenericContainer minio; + private static GenericContainer rustfsContainer; private static MinioClient minioClient; private static final String BUCKET = "test-locking-service-bucket"; @@ -26,20 +26,20 @@ public class ITS3LockingService { @BeforeClass public static void setUpClass() { org.junit.Assume.assumeTrue( - "Container runtime is not available; skipping Testcontainers MinIO test", + "Container runtime is not available; skipping Testcontainers S3 (RustFS) test", TestUtils.isContainerRuntimeAvailable()); - minio = TestUtils.createMinioContainer(); - minio.start(); + rustfsContainer = TestUtils.createRustFsContainer(); + rustfsContainer.start(); - minioClient = TestUtils.createMinioClient(minio); + minioClient = TestUtils.createMinioClient(rustfsContainer); TestUtils.createBucket(minioClient, BUCKET); } @AfterClass public static void tearDownClass() { - if (minio != null) { - minio.stop(); + if (rustfsContainer != null) { + rustfsContainer.stop(); } } diff --git a/src/test/java/me/desair/tus/server/upload/s3/ITS3RufhProtocol.java b/src/test/java/me/desair/tus/server/upload/s3/ITS3RufhProtocol.java index f281926c..60b21270 100644 --- a/src/test/java/me/desair/tus/server/upload/s3/ITS3RufhProtocol.java +++ b/src/test/java/me/desair/tus/server/upload/s3/ITS3RufhProtocol.java @@ -14,27 +14,27 @@ */ public class ITS3RufhProtocol extends AbstractITRufhProtocol { - private static org.testcontainers.containers.GenericContainer minio; + private static org.testcontainers.containers.GenericContainer rustfsContainer; private static MinioClient minioClient; private static final String BUCKET = "test-rufh-s3-bucket"; @BeforeClass public static void setUpClass() { org.junit.Assume.assumeTrue( - "Container runtime is not available; skipping Testcontainers MinIO test", + "Container runtime is not available; skipping Testcontainers S3 (RustFS) test", TestUtils.isContainerRuntimeAvailable()); - minio = TestUtils.createMinioContainer(); - minio.start(); + rustfsContainer = TestUtils.createRustFsContainer(); + rustfsContainer.start(); - minioClient = TestUtils.createMinioClient(minio); + minioClient = TestUtils.createMinioClient(rustfsContainer); TestUtils.createBucket(minioClient, BUCKET); } @AfterClass public static void tearDownClass() { - if (minio != null) { - minio.stop(); + if (rustfsContainer != null) { + rustfsContainer.stop(); } } diff --git a/src/test/java/me/desair/tus/server/upload/s3/ITS3StorageService.java b/src/test/java/me/desair/tus/server/upload/s3/ITS3StorageService.java index 544147a0..c2f10768 100644 --- a/src/test/java/me/desair/tus/server/upload/s3/ITS3StorageService.java +++ b/src/test/java/me/desair/tus/server/upload/s3/ITS3StorageService.java @@ -21,7 +21,7 @@ public class ITS3StorageService { - private static GenericContainer minio; + private static GenericContainer rustfsContainer; private static MinioClient minioClient; private static final String BUCKET = "test-storage-service-bucket"; @@ -30,20 +30,20 @@ public class ITS3StorageService { @BeforeClass public static void setUpClass() { org.junit.Assume.assumeTrue( - "Container runtime is not available; skipping Testcontainers MinIO test", + "Container runtime is not available; skipping Testcontainers S3 (RustFS) test", TestUtils.isContainerRuntimeAvailable()); - minio = TestUtils.createMinioContainer(); - minio.start(); + rustfsContainer = TestUtils.createRustFsContainer(); + rustfsContainer.start(); - minioClient = TestUtils.createMinioClient(minio); + minioClient = TestUtils.createMinioClient(rustfsContainer); TestUtils.createBucket(minioClient, BUCKET); } @AfterClass public static void tearDownClass() { - if (minio != null) { - minio.stop(); + if (rustfsContainer != null) { + rustfsContainer.stop(); } } diff --git a/src/test/java/me/desair/tus/server/upload/s3/ITS3TusFileUploadService.java b/src/test/java/me/desair/tus/server/upload/s3/ITS3TusFileUploadService.java index af3c0436..c3a0a008 100644 --- a/src/test/java/me/desair/tus/server/upload/s3/ITS3TusFileUploadService.java +++ b/src/test/java/me/desair/tus/server/upload/s3/ITS3TusFileUploadService.java @@ -16,27 +16,27 @@ */ public class ITS3TusFileUploadService extends AbstractITTusFileUploadService { - private static GenericContainer minio; + private static GenericContainer rustfsContainer; private static MinioClient minioClient; private static final String BUCKET = "test-service-s3-bucket"; @BeforeClass public static void setUpClass() { org.junit.Assume.assumeTrue( - "Container runtime is not available; skipping Testcontainers MinIO test", + "Container runtime is not available; skipping Testcontainers S3 (RustFS) test", TestUtils.isContainerRuntimeAvailable()); - minio = TestUtils.createMinioContainer(); - minio.start(); + rustfsContainer = TestUtils.createRustFsContainer(); + rustfsContainer.start(); - minioClient = TestUtils.createMinioClient(minio); + minioClient = TestUtils.createMinioClient(rustfsContainer); TestUtils.createBucket(minioClient, BUCKET); } @AfterClass public static void tearDownClass() { - if (minio != null) { - minio.stop(); + if (rustfsContainer != null) { + rustfsContainer.stop(); } } diff --git a/src/test/java/me/desair/tus/server/upload/s3/S3LockingServiceTest.java b/src/test/java/me/desair/tus/server/upload/s3/S3LockingServiceTest.java index 368d2684..41a9e90d 100644 --- a/src/test/java/me/desair/tus/server/upload/s3/S3LockingServiceTest.java +++ b/src/test/java/me/desair/tus/server/upload/s3/S3LockingServiceTest.java @@ -90,6 +90,26 @@ public void setUp() throws Exception { }) .when(minioClient) .removeObject(Mockito.any(io.minio.RemoveObjectArgs.class)); + + Mockito.doAnswer( + invocation -> { + io.minio.RemoveObjectsArgs args = invocation.getArgument(0); + if (args != null && args.objects() != null) { + for (io.minio.messages.DeleteRequest.Object obj : args.objects()) { + try { + java.lang.reflect.Field field = + io.minio.messages.DeleteRequest.Object.class.getDeclaredField("name"); + field.setAccessible(true); + String name = (String) field.get(obj); + s3StorageMap.remove(name); + } catch (Exception ignored) { + } + } + } + return java.util.Collections.emptyList(); + }) + .when(minioClient) + .removeObjects(Mockito.any(io.minio.RemoveObjectsArgs.class)); } @Test @@ -558,30 +578,17 @@ public void testLockUploadByUriWithContentionOnPostPutVerificationThrows() throw "locks/24249a5b-01a4-4bf8-b67a-364273bb5a2e.stop"); String rivalJson = me.desair.tus.server.util.LeaseDataJsonSerializer.serialize(rivalLock); - // Initial check sees expired/missing, but post-put verification sees rival holder - java.util.concurrent.atomic.AtomicInteger getCallCount = - new java.util.concurrent.atomic.AtomicInteger(0); + // PutObject succeeds, but post-put read-after-write verification sees rival holder Mockito.when(minioClient.getObject(Mockito.any(GetObjectArgs.class))) - .thenAnswer( - inv -> { - if (getCallCount.incrementAndGet() == 1) { - // First call: check if expired (missing -> not locked) - ErrorResponse errorResponse = Mockito.mock(ErrorResponse.class); - Mockito.when(errorResponse.code()).thenReturn("NoSuchKey"); - throw new io.minio.errors.ErrorResponseException(errorResponse, null, null); - } - // Second call: read-after-write verification sees rivalLock - return new GetObjectResponse( - null, - "test-bucket", - "eu-central-1", - "locks/24249a5b-01a4-4bf8-b67a-364273bb5a2e.lock", - new ByteArrayInputStream(rivalJson.getBytes(StandardCharsets.UTF_8))); - }); + .thenReturn( + new GetObjectResponse( + null, + "test-bucket", + "eu-central-1", + "locks/24249a5b-01a4-4bf8-b67a-364273bb5a2e.lock", + new ByteArrayInputStream(rivalJson.getBytes(StandardCharsets.UTF_8)))); - UploadLock lock = - lockingService.lockUploadByUri("/files/upload/24249a5b-01a4-4bf8-b67a-364273bb5a2e"); - assertNull(lock); + lockingService.lockUploadByUri("/files/upload/24249a5b-01a4-4bf8-b67a-364273bb5a2e"); } @Test diff --git a/src/test/java/me/desair/tus/server/upload/s3/S3UploadLockTest.java b/src/test/java/me/desair/tus/server/upload/s3/S3UploadLockTest.java index 8b7e66b4..4ea4d024 100644 --- a/src/test/java/me/desair/tus/server/upload/s3/S3UploadLockTest.java +++ b/src/test/java/me/desair/tus/server/upload/s3/S3UploadLockTest.java @@ -9,8 +9,13 @@ import io.minio.MinioClient; import io.minio.PutObjectArgs; import io.minio.RemoveObjectArgs; +import io.minio.RemoveObjectsArgs; +import io.minio.Result; +import io.minio.messages.DeleteResult; import java.io.ByteArrayInputStream; import java.io.InputStream; +import java.util.Arrays; +import java.util.List; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import me.desair.tus.server.upload.LeaseData; @@ -87,7 +92,7 @@ public void testRenewLeaseExceptionHandling() throws Exception { public void testLockDeleteQuietlyWithNullKeysAndExceptionHandling() throws Exception { Mockito.doThrow(new RuntimeException("Remove failed")) .when(minioClient) - .removeObject(any(RemoveObjectArgs.class)); + .removeObjects(any(RemoveObjectsArgs.class)); LeaseData leaseData = createLeaseData("holder-123", "/files/upload-1"); S3UploadLock lockWithNullKeys = @@ -109,8 +114,28 @@ public void testLockDeleteQuietlyWithNullKeysAndExceptionHandling() throws Excep } @Test - public void testDeleteS3LockObjectIfOwnerSkipsWhenHolderMismatch() throws Exception { - // Simulate remote lock owned by another holder + public void testCloseHeartbeatExecutorShutdownException() throws Exception { + java.util.concurrent.ScheduledExecutorService mockExecutor = + mock(java.util.concurrent.ScheduledExecutorService.class); + Mockito.doThrow(new RuntimeException("Shutdown error")).when(mockExecutor).shutdownNow(); + + LeaseData myLeaseData = createLeaseData("holder-123", "/files/upload-1"); + S3UploadLock lock = + new S3UploadLock( + myLeaseData, + minioClient, + "test-bucket", + "tus-locks/upload-1.lock", + "tus-locks/upload-1.stop", + inputStreamMap, + mockExecutor); + + lock.close(); + assertEquals("holder-123", lock.getHolderId()); + } + + @Test + public void testRenewLeaseSkipsWhenHolderMismatch() throws Exception { LeaseData otherLock = new LeaseData( "other-holder", @@ -134,6 +159,10 @@ public void testDeleteS3LockObjectIfOwnerSkipsWhenHolderMismatch() throws Except Mockito.when(minioClient.getObject(any(io.minio.GetObjectArgs.class))).thenReturn(response); LeaseData myLeaseData = createLeaseData("my-holder", "/files/upload-1"); + InterruptibleInputStream stream = + new InterruptibleInputStream(new ByteArrayInputStream("data".getBytes())); + inputStreamMap.put("/files/upload-1", stream); + S3UploadLock lock = new S3UploadLock( myLeaseData, @@ -143,18 +172,19 @@ public void testDeleteS3LockObjectIfOwnerSkipsWhenHolderMismatch() throws Except "tus-locks/upload-1.stop", inputStreamMap); - lock.deleteS3LockObjectIfOwner("tus-locks/upload-1.lock"); + lock.renewLease(); + + // On lease ownership loss, renewLease must abort the active input stream immediately + // to prevent writing un-locked bytes to S3. + assertTrue(stream.isInterrupted()); - // Must NOT remove object because it belongs to other-holder - Mockito.verify(minioClient, Mockito.never()) - .removeObject( - Mockito.argThat( - args -> args != null && "tus-locks/upload-1.lock".equals(args.object()))); + // Must NOT call putObject because lock is now held by other-holder + Mockito.verify(minioClient, Mockito.never()).putObject(any(PutObjectArgs.class)); + lock.close(); } @Test - public void testDeleteS3LockObjectIfOwnerDeletesWhenHolderMatches() throws Exception { - // Simulate remote lock owned by this holder + public void testRenewLeaseSucceedsWhenHolderMatches() throws Exception { LeaseData myLock = new LeaseData( "my-holder", @@ -187,25 +217,15 @@ public void testDeleteS3LockObjectIfOwnerDeletesWhenHolderMatches() throws Excep "tus-locks/upload-1.stop", inputStreamMap); - lock.deleteS3LockObjectIfOwner("tus-locks/upload-1.lock"); + lock.renewLease(); - // Must remove object because it matches my-holder - Mockito.verify(minioClient) - .removeObject( - Mockito.argThat( - args -> args != null && "tus-locks/upload-1.lock".equals(args.object()))); + // Must call putObject because holder matches + Mockito.verify(minioClient).putObject(any(PutObjectArgs.class)); + lock.close(); } @Test - public void testDeleteS3LockObjectIfOwnerHandlesNoSuchKey() throws Exception { - io.minio.messages.ErrorResponse errorResponse = - Mockito.mock(io.minio.messages.ErrorResponse.class); - Mockito.when(errorResponse.code()).thenReturn("NoSuchKey"); - io.minio.errors.ErrorResponseException ex = - new io.minio.errors.ErrorResponseException(errorResponse, null, null); - - Mockito.when(minioClient.getObject(any(io.minio.GetObjectArgs.class))).thenThrow(ex); - + public void testDoesLockOwnershipMatchNullChecksAndException() throws Exception { LeaseData myLeaseData = createLeaseData("my-holder", "/files/upload-1"); S3UploadLock lock = new S3UploadLock( @@ -216,56 +236,36 @@ public void testDeleteS3LockObjectIfOwnerHandlesNoSuchKey() throws Exception { "tus-locks/upload-1.stop", inputStreamMap); - lock.deleteS3LockObjectIfOwner("tus-locks/upload-1.lock"); - assertEquals("my-holder", lock.getHolderId()); - } - - @Test - public void testCloseHeartbeatExecutorShutdownException() throws Exception { - java.util.concurrent.ScheduledExecutorService mockExecutor = - mock(java.util.concurrent.ScheduledExecutorService.class); - Mockito.doThrow(new RuntimeException("Shutdown error")).when(mockExecutor).shutdownNow(); + // Null key returns false + org.junit.Assert.assertFalse(lock.doesLockOwnershipMatch(null)); - LeaseData myLeaseData = createLeaseData("holder-123", "/files/upload-1"); - S3UploadLock lock = + // Null minioClient returns false + S3UploadLock nullClientLock = new S3UploadLock( - myLeaseData, - minioClient, - "test-bucket", - "tus-locks/upload-1.lock", - "tus-locks/upload-1.stop", - inputStreamMap, - mockExecutor); - - lock.close(); - assertEquals("holder-123", lock.getHolderId()); - } + myLeaseData, null, "test-bucket", "tus-locks/upload-1.lock", null, inputStreamMap); + org.junit.Assert.assertFalse(nullClientLock.doesLockOwnershipMatch("key")); - @Test - public void testDeleteS3LockObjectIfOwnerNullChecksAndExceptionHandling() throws Exception { - LeaseData myLeaseData = createLeaseData("holder-123", "/files/upload-1"); - S3UploadLock lock = + // Null bucket returns false + S3UploadLock nullBucketLock = new S3UploadLock( - myLeaseData, - minioClient, - "test-bucket", - "tus-locks/upload-1.lock", - "tus-locks/upload-1.stop", - inputStreamMap); + myLeaseData, minioClient, null, "tus-locks/upload-1.lock", null, inputStreamMap); + org.junit.Assert.assertFalse(nullBucketLock.doesLockOwnershipMatch("key")); - // Null key check - lock.deleteS3LockObjectIfOwner(null); + // General exception (non-ErrorResponseException) returns true to allow proceed + Mockito.when(minioClient.getObject(any(io.minio.GetObjectArgs.class))) + .thenThrow(new RuntimeException("Transient S3 error")); + org.junit.Assert.assertTrue(lock.doesLockOwnershipMatch("tus-locks/upload-1.lock")); - // Exception during removeObject - Mockito.doThrow(new RuntimeException("Remove failed")) - .when(minioClient) - .removeObject(any(RemoveObjectArgs.class)); - lock.deleteS3LockObjectIfOwner("tus-locks/upload-1.lock"); - assertEquals("holder-123", lock.getHolderId()); + // Renew lease with null lockKey or null minioClient + nullClientLock.doRenewLease(); + S3UploadLock nullKeyLock = + new S3UploadLock(myLeaseData, minioClient, "test-bucket", null, null, inputStreamMap); + nullKeyLock.doRenewLease(); } @Test - public void testRenewLeaseSkipsWhenHolderMismatch() throws Exception { + public void testReleaseLockResourceWhenOwnershipMismatch() throws Exception { + // Simulate remote lock owned by another holder LeaseData otherLock = new LeaseData( "other-holder", @@ -283,16 +283,11 @@ public void testRenewLeaseSkipsWhenHolderMismatch() throws Exception { "test-bucket", "eu-central-1", "tus-locks/upload-1.lock", - new java.io.ByteArrayInputStream( - json.getBytes(java.nio.charset.StandardCharsets.UTF_8))); + new ByteArrayInputStream(json.getBytes(java.nio.charset.StandardCharsets.UTF_8))); Mockito.when(minioClient.getObject(any(io.minio.GetObjectArgs.class))).thenReturn(response); LeaseData myLeaseData = createLeaseData("my-holder", "/files/upload-1"); - InterruptibleInputStream stream = - new InterruptibleInputStream(new ByteArrayInputStream("data".getBytes())); - inputStreamMap.put("/files/upload-1", stream); - S3UploadLock lock = new S3UploadLock( myLeaseData, @@ -302,41 +297,16 @@ public void testRenewLeaseSkipsWhenHolderMismatch() throws Exception { "tus-locks/upload-1.stop", inputStreamMap); - lock.renewLease(); - - // On lease ownership loss, renewLease must abort the active input stream immediately - // to prevent writing un-locked bytes to S3. - assertTrue(stream.isInterrupted()); - - // Must NOT call putObject because lock is now held by other-holder - Mockito.verify(minioClient, Mockito.never()).putObject(any(PutObjectArgs.class)); lock.close(); + + // Must NOT call removeObjects or removeObject because lock was stolen by another node + // and any stop signal might have been set by a third thread for the new holder + Mockito.verify(minioClient, Mockito.never()).removeObjects(any(RemoveObjectsArgs.class)); + Mockito.verify(minioClient, Mockito.never()).removeObject(any(RemoveObjectArgs.class)); } @Test - public void testRenewLeaseSucceedsWhenHolderMatches() throws Exception { - LeaseData myLock = - new LeaseData( - "my-holder", - "/files/upload-1", - 60000L, - System.currentTimeMillis() + 60000L, - System.currentTimeMillis(), - "tus-locks/upload-1.lock", - "tus-locks/upload-1.stop"); - String json = LeaseDataJsonSerializer.serialize(myLock); - - io.minio.GetObjectResponse response = - new io.minio.GetObjectResponse( - null, - "test-bucket", - "eu-central-1", - "tus-locks/upload-1.lock", - new java.io.ByteArrayInputStream( - json.getBytes(java.nio.charset.StandardCharsets.UTF_8))); - - Mockito.when(minioClient.getObject(any(io.minio.GetObjectArgs.class))).thenReturn(response); - + public void testDeleteS3ObjectsNullChecksAndEmpty() { LeaseData myLeaseData = createLeaseData("my-holder", "/files/upload-1"); S3UploadLock lock = new S3UploadLock( @@ -347,15 +317,29 @@ public void testRenewLeaseSucceedsWhenHolderMatches() throws Exception { "tus-locks/upload-1.stop", inputStreamMap); - lock.renewLease(); + // Null client + S3UploadLock nullClientLock = + new S3UploadLock( + myLeaseData, null, "test-bucket", "tus-locks/upload-1.lock", null, inputStreamMap); + nullClientLock.deleteS3Objects("key1", "key2"); - // Must call putObject because holder matches - Mockito.verify(minioClient).putObject(any(PutObjectArgs.class)); - lock.close(); + // Null bucket + S3UploadLock nullBucketLock = + new S3UploadLock( + myLeaseData, minioClient, null, "tus-locks/upload-1.lock", null, inputStreamMap); + nullBucketLock.deleteS3Objects("key1", "key2"); + + // Both keys null -> empty objects + lock.deleteS3Objects(null, null); + + Mockito.verify(minioClient, Mockito.never()).removeObjects(any(RemoveObjectsArgs.class)); + assertEquals("test-bucket", lock.getBucket()); + assertEquals("tus-locks/upload-1.lock", lock.getLockKey()); + assertEquals("tus-locks/upload-1.stop", lock.getStopKey()); } @Test - public void testDoesLockOwnershipMatchNullChecksAndException() throws Exception { + public void testDeleteS3ObjectsWithErrorsAndExceptions() throws Exception { LeaseData myLeaseData = createLeaseData("my-holder", "/files/upload-1"); S3UploadLock lock = new S3UploadLock( @@ -366,30 +350,47 @@ public void testDoesLockOwnershipMatchNullChecksAndException() throws Exception "tus-locks/upload-1.stop", inputStreamMap); - // Null key returns false - org.junit.Assert.assertFalse(lock.doesLockOwnershipMatch(null)); + DeleteResult.Error deleteError = mock(DeleteResult.Error.class); + Mockito.when(deleteError.objectName()).thenReturn("tus-locks/upload-1.lock"); + Mockito.when(deleteError.message()).thenReturn("Access Denied"); - // Null minioClient returns false - S3UploadLock nullClientLock = - new S3UploadLock( - myLeaseData, null, "test-bucket", "tus-locks/upload-1.lock", null, inputStreamMap); - org.junit.Assert.assertFalse(nullClientLock.doesLockOwnershipMatch("key")); + @SuppressWarnings("unchecked") + Result errorResult = mock(Result.class); + Mockito.when(errorResult.get()).thenReturn(deleteError); - // Null bucket returns false - S3UploadLock nullBucketLock = + @SuppressWarnings("unchecked") + Result throwingResult = mock(Result.class); + Mockito.when(throwingResult.get()).thenThrow(new RuntimeException("Result parsing error")); + + List> resultList = Arrays.asList(errorResult, throwingResult, null); + + Mockito.when(minioClient.removeObjects(any(RemoveObjectsArgs.class))).thenReturn(resultList); + + // Call deleteS3Objects with only firstKey, then with both keys + lock.deleteS3Objects("tus-locks/upload-1.lock", null); + lock.deleteS3Objects(null, "tus-locks/upload-1.stop"); + + Mockito.verify(minioClient, Mockito.times(2)).removeObjects(any(RemoveObjectsArgs.class)); + assertEquals("my-holder", lock.getHolderId()); + } + + @Test + public void testDeleteS3ObjectsWhenMinioClientThrows() { + LeaseData myLeaseData = createLeaseData("my-holder", "/files/upload-1"); + S3UploadLock lock = new S3UploadLock( - myLeaseData, minioClient, null, "tus-locks/upload-1.lock", null, inputStreamMap); - org.junit.Assert.assertFalse(nullBucketLock.doesLockOwnershipMatch("key")); + myLeaseData, + minioClient, + "test-bucket", + "tus-locks/upload-1.lock", + "tus-locks/upload-1.stop", + inputStreamMap); - // General exception (non-ErrorResponseException) returns true to allow proceed - Mockito.when(minioClient.getObject(any(io.minio.GetObjectArgs.class))) - .thenThrow(new RuntimeException("Transient S3 error")); - org.junit.Assert.assertTrue(lock.doesLockOwnershipMatch("tus-locks/upload-1.lock")); + Mockito.when(minioClient.removeObjects(any(RemoveObjectsArgs.class))) + .thenThrow(new RuntimeException("S3 connection error")); - // Renew lease with null lockKey or null minioClient - nullClientLock.doRenewLease(); - S3UploadLock nullKeyLock = - new S3UploadLock(myLeaseData, minioClient, "test-bucket", null, null, inputStreamMap); - nullKeyLock.doRenewLease(); + // Should catch exception quietly without propagating + lock.deleteS3Objects("tus-locks/upload-1.lock", "tus-locks/upload-1.stop"); + assertEquals("my-holder", lock.getHolderId()); } } From 89da876c418a53559801b1b09f0f9957f0b62edd Mon Sep 17 00:00:00 2001 From: Tom Desair Date: Tue, 29 Sep 2026 09:20:29 +0200 Subject: [PATCH 4/7] feat(s3): configurable locking jitter, HTTP 501 non-CAS fallback, and streamlined constructors - Added configurable jitter bounds (withJitter) in AbstractLeaseLockingService and S3LockingService defaulting to 20-60 ms, with zero-jitter support (0, 0) for pure AWS S3 / Cloudflare R2 deployments. - Implemented automatic non-CAS fallback in S3LockingService when If-None-Match: * returns HTTP 501 / NotImplemented (supporting Backblaze B2, Ceph RGW) along with withS3ConditionalWritesSupported(boolean) override. - Streamlined S3LockingService constructors to minimal required set (basic 2-arg and full 6-arg). - Clarified locking documentation and inline comments regarding lock arbitration on non-CAS backends. --- CHANGELOG.md | 2 +- docs/LOCKING.md | 2 +- docs/S3_STORAGE.md | 39 ++++- .../upload/AbstractLeaseLockingService.java | 65 +++++++- .../server/upload/s3/S3LockingService.java | 143 +++++++++++++----- .../server/upload/s3/S3StorageService.java | 3 +- .../desair/tus/server/upload/s3/S3Utils.java | 4 +- .../AbstractLeaseLockingServiceTest.java | 61 +++++++- .../upload/s3/S3LockingServiceTest.java | 136 ++++++++++++++++- .../tus/server/upload/s3/S3UtilsTest.java | 18 +++ 10 files changed, 413 insertions(+), 60 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 846a9adc..6058f827 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,7 +25,7 @@ All notable changes to this project will be documented in this file. - **`process()` Return Value (`UploadInfo`)**: `TusFileUploadService.process(...)` now returns the created or updated `UploadInfo` instance (or `null` on errors or `OPTIONS` preflight requests), enabling applications to track and store upload IDs directly into user sessions or database repositories. - **Jackson Bundled in Compile Scope**: Promoted Jackson dependencies (`jackson-databind`, `jackson-annotations`, `jackson-core`) to `compile` scope, eliminating `NoClassDefFoundError` when enabling JSON serialization or using cloud storage. - **Lock-Holding Stream Lifecycle**: `TusFileUploadService.getUploadedBytes(...)` now retains the upload lock until the returned stream is closed, preventing concurrent modifications or deletions from corrupting data mid-stream. -- **S3 Locking Optimization**: Optimized `S3LockingService` and `S3UploadLock` by skipping the redundant `isLockExpired` pre-check on happy-path acquisitions (saving 1 remote GET round-trip) and using S3 Multi-Object Delete to delete `.lock` and `.stop` objects in a single batch round-trip on lock release. +- **S3 Locking Optimization & Non-CAS Fallback**: Optimized `S3LockingService` and `S3UploadLock` by skipping the redundant `isLockExpired` pre-check on happy-path acquisitions (saving 1 remote GET round-trip) and using S3 Multi-Object Delete to delete `.lock` and `.stop` objects in a single batch round-trip on lock release. Made read-after-write verification jitter bounds configurable via `AbstractLeaseLockingService.setJitter(minMs, maxMs)` and `S3LockingService.withJitter(minMs, maxMs)`. Added automatic non-CAS fallback when `If-None-Match: *` returns `HTTP 501 Not Implemented` (enabling seamless out-of-the-box support for Backblaze B2 and Ceph RGW) along with `withS3ConditionalWritesSupported(boolean)` override. Streamlined `S3LockingService` constructors to minimal required set and documented zero-jitter throughput optimization for pure AWS S3 / Cloudflare R2 deployments in `docs/S3_STORAGE.md`. ### Fixed - **RFC 9110 Media Type Matching & MIME Parameter Tolerance**: Enhanced `Content-Type` header validation in `ContentTypeValidator`, `PostContentTypeValidator`, and `RufhAppendValidator` using RFC 9110 §8.3 compliant media-type parsing (`Utils.isMediaType`). Media type matching now tolerates MIME parameters (such as `;charset=UTF-8` automatically appended by Spring Boot `CharacterEncodingFilter`, servlet wrappers, proxies, or HTTP clients), whitespace variations, and case-insensitivity without incorrectly rejecting valid requests with `406 Not Acceptable`. diff --git a/docs/LOCKING.md b/docs/LOCKING.md index 0b946612..6416525e 100644 --- a/docs/LOCKING.md +++ b/docs/LOCKING.md @@ -86,7 +86,7 @@ public interface UploadLock extends Closeable { |---|---|---|---| | **Disk & Network Filesystems (Default)** | `LeaseFileLockingService` | Atomic sibling mutex directory (`.mutex/`), in-place expired lock takeover, TTL-based JSON lease files with heartbeat renewal, ownership fencing, and `.stop` signal files. Fully safe on NFSv3/v4, AWS EFS, SMB/CIFS, Kubernetes containers, and local disks. | [`docs/DISK_BASED_LOCKING.md`](file:///Users/tom/projects/tus-java-server/docs/DISK_BASED_LOCKING.md) | | **Local File System (Legacy Opt-Out)** | `DiskLockingService` | OS kernel-level exclusive POSIX `FileLock` (`fcntl`) with JVM shutdown hooks and `.stop` signal files. Best for single-node deployments on local disk. | [`docs/DISK_BASED_LOCKING.md`](file:///Users/tom/projects/tus-java-server/docs/DISK_BASED_LOCKING.md) | -| **Amazon S3 / S3-Compatible** | `S3LockingService` | S3 object-backed TTL lease objects (`.lock`), conditional writes (`If-None-Match: *`), jittered read-after-write verification, heartbeat renewal, and cross-pod `.stop` signal object polling watchdog. | [`docs/S3_STORAGE.md`](file:///Users/tom/projects/tus-java-server/docs/S3_STORAGE.md) | +| **Amazon S3 / S3-Compatible** | `S3LockingService` | S3 object-backed TTL lease objects (`.lock`), atomic conditional writes (`If-None-Match: *`), non-CAS lock arbitration with configurable jitter backoff (`withJitter`) for backends lacking CAS (B2, Ceph, Wasabi), heartbeat renewal, and cross-pod `.stop` signal object polling watchdog. | [`docs/S3_STORAGE.md`](file:///Users/tom/projects/tus-java-server/docs/S3_STORAGE.md) | | **Azure Blob Storage** | `AzureBlobLockingService` | Native Azure Blob Storage exclusive 30-second leases (`BlobLeaseClient`), background daemon renewal, and `.stop` signal blob polling watchdog. | [`docs/AZURE_BLOB_STORAGE.md`](file:///Users/tom/projects/tus-java-server/docs/AZURE_BLOB_STORAGE.md) | --- diff --git a/docs/S3_STORAGE.md b/docs/S3_STORAGE.md index 14eff747..ac44c643 100644 --- a/docs/S3_STORAGE.md +++ b/docs/S3_STORAGE.md @@ -211,7 +211,7 @@ Understanding the architectural distinction between disk/file locking and S3 dis - Removing renewal with a short TTL would cause locks to expire mid-upload during long transfers, leading to race conditions and data corruption. - Removing renewal with an infinite/static lock would mean a single pod crash (`kill -9`, node OOM) leaves behind an orphaned `.lock` object in S3, permanently deadlocking that upload ID. -### TOCTOU (Time-of-Check to Time-of-Use) Mitigation in S3 +### Lock Arbitration & TOCTOU Mitigation in S3 In stateless distributed object storage, acquiring a lock via standard `GetObject` followed by `PutObject` is vulnerable to a classic **Time-of-Check to Time-of-Use (TOCTOU)** race condition: 1. **Time of Check (TOC)**: Two contender nodes (Node A and Node B) concurrently check if a `.lock` object exists or is expired. Both observe it as available. @@ -222,15 +222,46 @@ To guarantee waterproof single-winner lock exclusivity across cloud providers an 1. **Conditional Writes (`If-None-Match: *`)**: - `PutObject` requests include the `If-None-Match: *` header. - - On Amazon S3 and compliant object storage engines, S3 atomically rejects the second write with `HTTP 412 Precondition Failed` (`PreconditionFailed`), immediately preventing concurrent overwrites. -2. **Jittered Read-After-Write Verification**: - - For S3-compatible emulators or endpoints that do not strictly enforce conditional writes on `PutObject`, the acquiring node sleeps for a brief randomized jitter (20–60ms) to allow in-flight writes to settle, then re-reads `GetObject`. + - On Amazon S3 and compliant object storage engines, S3 atomically rejects the second write with `HTTP 412 Precondition Failed` (`PreconditionFailed`), immediately preventing concurrent overwrites in a single network round-trip. +2. **Lock Arbitration on Backends Without Atomic Conditional Writes (Jittered Read-After-Write Verification)**: + - Several major S3-compatible engines do not support atomic conditional writes: + | Backend | Conditional Write (`If-None-Match: *`) Behavior | `S3LockingService` Handling | + | :--- | :--- | :--- | + | **AWS S3, Cloudflare R2, LocalStack, RustFS** | Returns `HTTP 412 Precondition Failed` | Optimistic CAS (1 network call) | + | **Backblaze B2, Ceph RGW** | Returns `HTTP 501 Not Implemented` | Auto-downgrades to non-CAS arbitration | + | **Wasabi** | Silently ignored (last-write-wins overwrite) | Layer 2 jitter arbitration or `withS3ConditionalWritesSupported(false)` | + - On backends without conditional write support, contender nodes sleep for a brief randomized jitter (default: 20–60ms) to allow competing writes to settle, then re-read `GetObject`. - If the remote `holderId` does not match its own, the node detects that it was overtaken, rejects the acquisition, and leaves the winner's lock intact. 3. **Safe Expired Lock Eviction**: - When evicting an expired lock, the lock object's expiration is verified again immediately before deletion to prevent evicting a fresh lock created by a winning peer. 4. **Owner-Safe Lock Release**: - In `S3UploadLock.close()`, the node verifies that the remote lock is still owned by its own `holderId` before deleting it. If its lease expired while the process was paused and another node took over ownership, the previous node will never delete the new owner's active lock. +### Jitter Configuration & Zero-Jitter Performance Tuning + +The jitter window can be customized via `.withJitter(minMs, maxMs)` on `S3LockingService`: + +```java +// Option A: Zero-Jitter for AWS S3 & Cloudflare R2 (Maximum Throughput) +// On backends with atomic conditional writes, jitter is not needed. +// Passing (0L, 0L) completely bypasses Thread.sleep for fastest lock acquisition: +S3LockingService s3LockingService = + new S3LockingService(minioClient, bucketName) + .withJitter(0L, 0L); + +// Option B: High-Latency or Distributed Backends (Ceph, multi-datacenter MinIO) +// High-latency backends or geo-replicated clusters may need a wider window to settle writes: +S3LockingService s3LockingService = + new S3LockingService(minioClient, bucketName) + .withJitter(50L, 200L); + +// Option C: Explicit Non-CAS Mode (e.g. Wasabi) +// Pre-checks existing locks before writing unconditionally: +S3LockingService s3LockingService = + new S3LockingService(minioClient, bucketName) + .withS3ConditionalWritesSupported(false); +``` + ### Clock Synchronization & NTP Requirement Distributed TTL lease evaluation relies on wall-clock timestamps (`expiresAt`). While `S3LockingService` incorporates a 2-second safety buffer (`CLOCK_SKEW_SAFETY_MARGIN_MS`) to absorb minor time deviations between pods, all cluster nodes and containers running `tus-java-server` MUST synchronize their system clocks using NTP (Network Time Protocol) or Amazon Time Sync Service (`chrony`). Avoid clock skew exceeding $\pm 1$ second between cluster nodes. diff --git a/src/main/java/me/desair/tus/server/upload/AbstractLeaseLockingService.java b/src/main/java/me/desair/tus/server/upload/AbstractLeaseLockingService.java index a1f87707..93c8023e 100644 --- a/src/main/java/me/desair/tus/server/upload/AbstractLeaseLockingService.java +++ b/src/main/java/me/desair/tus/server/upload/AbstractLeaseLockingService.java @@ -37,9 +37,14 @@ public abstract class AbstractLeaseLockingService extends AbstractCloseableResou */ public static final long CLOCK_SKEW_SAFETY_MARGIN_MS = 2000L; + public static final long DEFAULT_JITTER_MIN_MS = 20L; + public static final long DEFAULT_JITTER_MAX_MS = 60L; + protected final long leaseDurationMs; protected final long pollIntervalMs; protected UploadIdFactory idFactory; + protected long jitterMinMs = DEFAULT_JITTER_MIN_MS; + protected long jitterMaxMs = DEFAULT_JITTER_MAX_MS; protected final Map activeInputStreams = new ConcurrentHashMap<>(); protected final ScheduledExecutorService watchdogExecutor; @@ -251,17 +256,71 @@ public void checkStopSignals() { } /** - * Applies randomized jitter backoff to settle concurrent in-flight writes. + * Applies randomized jitter backoff to settle concurrent in-flight writes and arbitrate lock + * acquisition races on storage backends lacking atomic conditional writes. + * + *

If {@code maxMs <= 0}, this method returns immediately without sleeping, allowing callers to + * bypass jitter on backends with guaranteed atomic conditional writes (e.g. AWS S3). * * @param minMs Minimum jitter duration in milliseconds * @param maxMs Maximum jitter duration in milliseconds */ protected void applyJitter(long minMs, long maxMs) { + if (maxMs <= 0) { + return; + } try { - long jitterMs = minMs + (long) (Math.random() * (maxMs - minMs)); - Thread.sleep(jitterMs); + long jitterMs = (minMs >= maxMs) ? minMs : minMs + (long) (Math.random() * (maxMs - minMs)); + if (jitterMs > 0) { + Thread.sleep(jitterMs); + } } catch (InterruptedException e) { Thread.currentThread().interrupt(); } } + + /** + * Applies randomized jitter backoff using the configured {@link #getJitterMinMs()} and {@link + * #getJitterMaxMs()} bounds. + */ + protected void applyJitter() { + applyJitter(jitterMinMs, jitterMaxMs); + } + + /** + * Returns the configured minimum jitter duration in milliseconds. + * + * @return Minimum jitter duration in milliseconds + */ + public long getJitterMinMs() { + return jitterMinMs; + } + + /** + * Returns the configured maximum jitter duration in milliseconds. + * + * @return Maximum jitter duration in milliseconds + */ + public long getJitterMaxMs() { + return jitterMaxMs; + } + + /** + * Configures the jitter bounds used during lock acquisition read-after-write verification. + * + * @param minMs Minimum jitter duration in milliseconds (must be >= 0) + * @param maxMs Maximum jitter duration in milliseconds (must be >= minMs) + * @throws IllegalArgumentException If minMs is negative or maxMs is less than minMs + */ + public void setJitter(long minMs, long maxMs) { + if (minMs < 0) { + throw new IllegalArgumentException("jitterMinMs cannot be negative: " + minMs); + } + if (maxMs < minMs) { + throw new IllegalArgumentException( + "jitterMaxMs (" + maxMs + ") cannot be less than jitterMinMs (" + minMs + ")"); + } + this.jitterMinMs = minMs; + this.jitterMaxMs = maxMs; + } } diff --git a/src/main/java/me/desair/tus/server/upload/s3/S3LockingService.java b/src/main/java/me/desair/tus/server/upload/s3/S3LockingService.java index ab5f4cf5..f37c1278 100644 --- a/src/main/java/me/desair/tus/server/upload/s3/S3LockingService.java +++ b/src/main/java/me/desair/tus/server/upload/s3/S3LockingService.java @@ -61,6 +61,7 @@ public class S3LockingService extends AbstractLeaseLockingService { private final MinioClient minioClient; private final String bucket; private final String locksPrefix; + private volatile boolean s3ConditionalWritesSupported = true; /** * Basic constructor using default lock prefix ("locks/"), 30s lease duration, and 2s polling @@ -75,30 +76,7 @@ public S3LockingService(MinioClient minioClient, String bucket) { bucket, DEFAULT_LOCKS_PREFIX, DEFAULT_LEASE_DURATION_MS, - DEFAULT_POLL_INTERVAL_MS); - } - - /** - * Full constructor allowing custom configuration for all locking parameters. - * - * @param minioClient Pre-configured MinIO Client - * @param bucket Target S3 bucket name - * @param locksPrefix Object key prefix for locks and stop signals - * @param leaseDurationMs Lock lease duration in milliseconds - * @param pollIntervalMs Watchdog poll interval for lock contention interrupt signals - */ - public S3LockingService( - MinioClient minioClient, - String bucket, - String locksPrefix, - long leaseDurationMs, - long pollIntervalMs) { - this( - minioClient, - bucket, - locksPrefix, - leaseDurationMs, - pollIntervalMs, + DEFAULT_POLL_INTERVAL_MS, new UuidUploadIdFactory()); } @@ -125,6 +103,51 @@ public S3LockingService( this.locksPrefix = sanitizePrefix(locksPrefix); } + /** + * Configures custom jitter bounds used during lock acquisition read-after-write verification. + * + *

On S3-compatible backends lacking atomic conditional writes (e.g., Wasabi, SeaweedFS, Ceph, + * Backblaze B2, older MinIO), randomized jitter backoff resolves last-write-wins races. For + * high-latency or cross-region backends, configure higher bounds (e.g. 50–200 ms). For pure AWS + * S3 or Cloudflare R2 deployments with strong conditional write enforcement, jitter can be + * disabled by passing {@code 0, 0} to maximize throughput. + * + * @param minMs Minimum jitter duration in milliseconds (must be >= 0) + * @param maxMs Maximum jitter duration in milliseconds (must be >= minMs) + * @return This service instance for fluent chaining + */ + public S3LockingService withJitter(long minMs, long maxMs) { + setJitter(minMs, maxMs); + return this; + } + + /** + * Configures whether the underlying S3 endpoint supports atomic conditional writes via {@code + * If-None-Match: *}. + * + *

Defaults to {@code true} (optimistic). If the endpoint returns HTTP 501 Not Implemented, + * this is automatically downgraded to {@code false}. For endpoints known to silently ignore + * {@code If-None-Match: *} (such as Wasabi), setting this to {@code false} ensures the service + * pre-checks existing lock status before writing. + * + * @param supported Whether conditional writes are supported + * @return This service instance for fluent chaining + */ + public S3LockingService withS3ConditionalWritesSupported(boolean supported) { + this.s3ConditionalWritesSupported = supported; + return this; + } + + /** + * Returns whether the underlying S3 endpoint currently supports atomic conditional writes. + * + * @return true if conditional writes are supported or assumed supported; false if downgraded or + * disabled + */ + public boolean isS3ConditionalWritesSupported() { + return s3ConditionalWritesSupported; + } + @Override public void cleanupStaleLocks() throws IOException { try { @@ -167,21 +190,54 @@ protected UploadLock tryAcquireLock(UploadId uploadId, LeaseData leaseData) { byte[] lockContentBytes = LeaseDataJsonSerializer.serializeToBytes(leaseData); - // Layer 1: Conditional PutObject with "If-None-Match: *" - // AWS S3 and compliant servers reject this with 412 Precondition Failed if the object already - // exists - minioClient.putObject( - PutObjectArgs.builder() - .bucket(bucket) - .object(lockKey) - .extraHeaders(Collections.singletonMap("If-None-Match", "*")) - .stream( - new ByteArrayInputStream(lockContentBytes), (long) lockContentBytes.length, -1L) - .build()); + if (s3ConditionalWritesSupported) { + // Layer 1: Optimistic Conditional PutObject with "If-None-Match: *" + // AWS S3 and compliant servers reject this with 412 Precondition Failed if the object + // already + // exists + try { + minioClient.putObject( + PutObjectArgs.builder() + .bucket(bucket) + .object(lockKey) + .extraHeaders(Collections.singletonMap("If-None-Match", "*")) + .stream( + new ByteArrayInputStream(lockContentBytes), + (long) lockContentBytes.length, + -1L) + .build()); + } catch (ErrorResponseException e) { + S3ErrorType errorType = S3Utils.parseErrorResponse(e); + if (errorType == S3ErrorType.API_NOT_IMPLEMENTED) { + // Backend (e.g. Backblaze B2, Ceph RGW) does not support conditional writes. + // Downgrade to non-CAS arbitration mode and proceed with safe pre-check + unconditional + // write. + log.info( + "S3 endpoint does not support conditional writes (If-None-Match: *). " + + "Downgrading to non-CAS lock arbitration for key {}", + lockKey); + s3ConditionalWritesSupported = false; + if (!isLockExpired(lockKey)) { + return null; + } + writeUnconditionalLockObject(lockKey, lockContentBytes); + } else { + throw e; + } + } + } else { + // Non-CAS mode (e.g., Wasabi, Ceph RGW, Backblaze B2): + // Verify no active unexpired lock exists before performing unconditional write + if (!isLockExpired(lockKey)) { + return null; + } + writeUnconditionalLockObject(lockKey, lockContentBytes); + } // Layer 2: Jittered Read-After-Write Verification - // For emulators or S3 backends where If-None-Match is not strictly enforced, - // pause for a small randomized jitter (20-60ms) and verify our holderId is still the owner + // For non-CAS backends or backends where If-None-Match is not strictly enforced, + // pause for a randomized jitter duration to allow competing writes to settle, + // then verify our holderId is still the owner applyJitter(); if (!verifyLockOwnership(lockKey, leaseData.getHolderId())) { return null; @@ -202,6 +258,14 @@ protected UploadLock tryAcquireLock(UploadId uploadId, LeaseData leaseData) { } } + private void writeUnconditionalLockObject(String lockKey, byte[] lockContentBytes) + throws Exception { + minioClient.putObject( + PutObjectArgs.builder().bucket(bucket).object(lockKey).stream( + new ByteArrayInputStream(lockContentBytes), (long) lockContentBytes.length, -1L) + .build()); + } + @Override protected boolean isLockExpired(UploadId uploadId) { if (uploadId == null) { @@ -311,8 +375,9 @@ private void deleteObjectQuietly(String key) { } } - void applyJitter() { - applyJitter(20L, 60L); + @Override + protected void applyJitter() { + super.applyJitter(); } private String sanitizePrefix(String prefix) { diff --git a/src/main/java/me/desair/tus/server/upload/s3/S3StorageService.java b/src/main/java/me/desair/tus/server/upload/s3/S3StorageService.java index 8b059137..6e9c2502 100644 --- a/src/main/java/me/desair/tus/server/upload/s3/S3StorageService.java +++ b/src/main/java/me/desair/tus/server/upload/s3/S3StorageService.java @@ -352,7 +352,8 @@ public UploadInfo append(UploadInfo upload, InputStream inputStream) processPayloadChunks(info, preparedStream, info.getId(), partObjectKey); // Step 4: Recalculate total uploaded byte offset across all uploaded part objects in S3. - // S3 listObjects can exhibit eventual consistency; take the maximum of remote parts query + // S3 listObjects in distributed clusters (e.g. multi-site Ceph, cluster rebalancing, or + // async replication) can exhibit listing delays; take the maximum of remote parts query // and locally verified stream progression to ensure newOffset accurately reflects bytes // successfully written. long calculatedOffset = diff --git a/src/main/java/me/desair/tus/server/upload/s3/S3Utils.java b/src/main/java/me/desair/tus/server/upload/s3/S3Utils.java index 618f41c4..2d0c2eaa 100644 --- a/src/main/java/me/desair/tus/server/upload/s3/S3Utils.java +++ b/src/main/java/me/desair/tus/server/upload/s3/S3Utils.java @@ -43,7 +43,9 @@ public static S3ErrorType parseErrorResponse(ErrorResponseException exception) { return S3ErrorType.ACCESS_DENIED; } - if ("APINotImplemented".equalsIgnoreCase(code) || "NotImplemented".equalsIgnoreCase(code)) { + if ("APINotImplemented".equalsIgnoreCase(code) + || "NotImplemented".equalsIgnoreCase(code) + || (exception.response() != null && exception.response().code() == 501)) { return S3ErrorType.API_NOT_IMPLEMENTED; } diff --git a/src/test/java/me/desair/tus/server/upload/AbstractLeaseLockingServiceTest.java b/src/test/java/me/desair/tus/server/upload/AbstractLeaseLockingServiceTest.java index d547ef9c..09aa108e 100644 --- a/src/test/java/me/desair/tus/server/upload/AbstractLeaseLockingServiceTest.java +++ b/src/test/java/me/desair/tus/server/upload/AbstractLeaseLockingServiceTest.java @@ -4,6 +4,7 @@ import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import static org.mockito.Mockito.mock; @@ -77,6 +78,10 @@ public void testJitter(long min, long max) { applyJitter(min, max); } + public void testJitterDefault() { + applyJitter(); + } + public void testCheckStopSignals() { checkStopSignals(); } @@ -228,8 +233,60 @@ public void testCheckStopSignalsIteratesActiveStreams() { } @Test - public void testApplyJitter() { - service.testJitter(1L, 5L); + public void testDefaultJitterBounds() { + assertEquals(20L, service.getJitterMinMs()); + assertEquals(60L, service.getJitterMaxMs()); + } + + @Test + public void testSetJitterValidBounds() { + service.setJitter(10L, 50L); + assertEquals(10L, service.getJitterMinMs()); + assertEquals(50L, service.getJitterMaxMs()); + } + + @Test(expected = IllegalArgumentException.class) + public void testSetJitterNegativeMinThrows() { + service.setJitter(-1L, 50L); + } + + @Test(expected = IllegalArgumentException.class) + public void testSetJitterMaxLessThanMinThrows() { + service.setJitter(50L, 10L); + } + + @Test + public void testApplyJitterWithZeroOrNegativeMaxMsReturnsImmediately() { + long start = System.currentTimeMillis(); + service.testJitter(0L, 0L); + service.testJitter(0L, -5L); + long elapsed = System.currentTimeMillis() - start; + // KISS: zero/negative maxMs returns immediately without sleep + assertTrue(elapsed < 100L); + } + + @Test + public void testApplyJitterWhenMinGreaterThanOrEqualMax() { + long start = System.currentTimeMillis(); + service.testJitter(2L, 2L); + service.testJitter(3L, 2L); + long elapsed = System.currentTimeMillis() - start; + assertTrue(elapsed >= 2L); + } + + @Test + public void testApplyJitterDefault() { + service.setJitter(1L, 3L); + service.testJitterDefault(); + assertEquals(1L, service.getJitterMinMs()); + assertEquals(3L, service.getJitterMaxMs()); + } + + @Test + public void testApplyJitterPreservesInterrupt() { + Thread.currentThread().interrupt(); + service.testJitter(10L, 20L); + assertTrue(Thread.interrupted()); } @Test diff --git a/src/test/java/me/desair/tus/server/upload/s3/S3LockingServiceTest.java b/src/test/java/me/desair/tus/server/upload/s3/S3LockingServiceTest.java index 41a9e90d..832220d6 100644 --- a/src/test/java/me/desair/tus/server/upload/s3/S3LockingServiceTest.java +++ b/src/test/java/me/desair/tus/server/upload/s3/S3LockingServiceTest.java @@ -4,7 +4,9 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertSame; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; import io.minio.GetObjectArgs; import io.minio.GetObjectResponse; @@ -28,6 +30,7 @@ import me.desair.tus.server.upload.LeaseData; import me.desair.tus.server.upload.UploadId; import me.desair.tus.server.upload.UploadLock; +import me.desair.tus.server.upload.UuidUploadIdFactory; import me.desair.tus.server.util.InterruptibleInputStream; import org.junit.Before; import org.junit.Test; @@ -333,11 +336,12 @@ public void testInterruptStreamStandardStreamCloseException() throws Exception { @Test public void testSanitizePrefixNullOrEmpty() throws Exception { S3LockingService serviceWithEmptyPrefix = - new S3LockingService(minioClient, "test-bucket", "", 30000L, 0L); + new S3LockingService(minioClient, "test-bucket", "", 30000L, 0L, new UuidUploadIdFactory()); assertNotNull(serviceWithEmptyPrefix); S3LockingService serviceWithNullPrefix = - new S3LockingService(minioClient, "test-bucket", null, 30000L, 0L); + new S3LockingService( + minioClient, "test-bucket", null, 30000L, 0L, new UuidUploadIdFactory()); assertNotNull(serviceWithNullPrefix); } @@ -376,12 +380,12 @@ public void testCheckStopSignalForEntryHappyPath() throws Exception { io.minio.StatObjectResponse mockStat = Mockito.mock(io.minio.StatObjectResponse.class); Mockito.when(mockClient.statObject(Mockito.any(StatObjectArgs.class))).thenReturn(mockStat); - S3LockingService service = - new S3LockingService(mockClient, "test-bucket", "locks", 30000L, 50L); me.desair.tus.server.upload.TimeBasedUploadIdFactory idFactory = new me.desair.tus.server.upload.TimeBasedUploadIdFactory(); idFactory.setUploadUri("/files/upload"); - service.setIdFactory(idFactory); + + S3LockingService service = + new S3LockingService(mockClient, "test-bucket", "locks", 30000L, 50L, idFactory); ByteArrayInputStream bais = new ByteArrayInputStream("test".getBytes()); InterruptibleInputStream stream = new InterruptibleInputStream(bais); @@ -411,12 +415,12 @@ public void testCheckStopSignalForEntryNoSuchKey() throws Exception { Mockito.when(mockClient.statObject(Mockito.any(StatObjectArgs.class))) .thenThrow(noSuchKeyException); - S3LockingService service = - new S3LockingService(mockClient, "test-bucket", "locks", 30000L, 50L); me.desair.tus.server.upload.TimeBasedUploadIdFactory idFactory = new me.desair.tus.server.upload.TimeBasedUploadIdFactory(); idFactory.setUploadUri("/files/upload"); - service.setIdFactory(idFactory); + + S3LockingService service = + new S3LockingService(mockClient, "test-bucket", "locks", 30000L, 50L, idFactory); ByteArrayInputStream bais = new ByteArrayInputStream("test".getBytes()); InterruptibleInputStream stream = new InterruptibleInputStream(bais); @@ -862,4 +866,120 @@ public void testEvictExpiredLockHandlesRemoveObjectException() throws Exception boolean evicted = lockingService.evictExpiredLock(uploadId); assertFalse(evicted); } + + @Test + public void testWithJitterConfiguresBoundsAndReturnsSelf() { + S3LockingService returned = lockingService.withJitter(15L, 75L); + assertSame(lockingService, returned); + assertEquals(15L, lockingService.getJitterMinMs()); + assertEquals(75L, lockingService.getJitterMaxMs()); + } + + @Test + public void testLockAcquisitionWithJitterDisabled() throws Exception { + lockingService.withJitter(0L, 0L); + assertEquals(0L, lockingService.getJitterMinMs()); + assertEquals(0L, lockingService.getJitterMaxMs()); + + UploadLock lock = + lockingService.lockUploadByUri("/files/upload/24249a5b-01a4-4bf8-b67a-364273bb5a2e"); + assertNotNull(lock); + lock.close(); + } + + @Test + public void testWithS3ConditionalWritesSupported() { + assertTrue(lockingService.isS3ConditionalWritesSupported()); + S3LockingService returned = lockingService.withS3ConditionalWritesSupported(false); + assertSame(lockingService, returned); + assertFalse(lockingService.isS3ConditionalWritesSupported()); + } + + @Test + public void testConditionalWriteNotImplementedDowngradesToNonCasModeAndSucceeds() + throws Exception { + ErrorResponse errorResponse = Mockito.mock(ErrorResponse.class); + Mockito.when(errorResponse.code()).thenReturn("NotImplemented"); + ErrorResponseException notImplementedEx = + new ErrorResponseException(errorResponse, null, "NotImplemented"); + + // The first PutObject with If-None-Match: * throws NotImplemented (e.g. Backblaze B2, Ceph RGW) + Mockito.doAnswer( + invocation -> { + PutObjectArgs args = invocation.getArgument(0); + if (args.extraHeaders() != null && args.extraHeaders().containsKey("If-None-Match")) { + throw notImplementedEx; + } + // Unconditional fallback write stores bytes in our in-memory map + java.io.ByteArrayOutputStream baos = new java.io.ByteArrayOutputStream(); + byte[] buf = new byte[1024]; + int read; + java.io.InputStream is = args.stream(); + while ((read = is.read(buf)) != -1) { + baos.write(buf, 0, read); + } + s3StorageMap.put(args.object(), baos.toByteArray()); + return null; + }) + .when(minioClient) + .putObject(Mockito.any(PutObjectArgs.class)); + + assertTrue(lockingService.isS3ConditionalWritesSupported()); + + UploadLock lock = + lockingService.lockUploadByUri("/files/upload/24249a5b-01a4-4bf8-b67a-364273bb5a2e"); + assertNotNull(lock); + // Verified that service automatically downgraded to non-CAS mode + assertFalse(lockingService.isS3ConditionalWritesSupported()); + lock.close(); + } + + @Test(expected = UploadAlreadyLockedException.class) + public void testConditionalWriteNotImplementedFailsWhenActiveLockExists() throws Exception { + ErrorResponse errorResponse = Mockito.mock(ErrorResponse.class); + Mockito.when(errorResponse.code()).thenReturn("NotImplemented"); + ErrorResponseException notImplementedEx = + new ErrorResponseException(errorResponse, null, "NotImplemented"); + + // PutObject throws NotImplemented when attempting conditional write + Mockito.doThrow(notImplementedEx).when(minioClient).putObject(Mockito.any(PutObjectArgs.class)); + + // Existing lock in S3 is actively held by another contender + LeaseData rivalLock = + new LeaseData( + "rival-holder", + "/files/upload/24249a5b-01a4-4bf8-b67a-364273bb5a2e", + 30000L, + System.currentTimeMillis() + 30000L, + System.currentTimeMillis(), + "locks/24249a5b-01a4-4bf8-b67a-364273bb5a2e.lock", + "locks/24249a5b-01a4-4bf8-b67a-364273bb5a2e.stop"); + s3StorageMap.put( + "locks/24249a5b-01a4-4bf8-b67a-364273bb5a2e.lock", + me.desair.tus.server.util.LeaseDataJsonSerializer.serializeToBytes(rivalLock)); + + lockingService.lockUploadByUri("/files/upload/24249a5b-01a4-4bf8-b67a-364273bb5a2e"); + } + + @Test + public void testExplicitNonCasModeChecksLockExpirationBeforeUnconditionalWrite() + throws Exception { + lockingService.withS3ConditionalWritesSupported(false); + assertFalse(lockingService.isS3ConditionalWritesSupported()); + + // First lock acquisition succeeds + UploadLock lock1 = + lockingService.lockUploadByUri("/files/upload/24249a5b-01a4-4bf8-b67a-364273bb5a2e"); + assertNotNull(lock1); + + // Second lock acquisition while first is active fails because isLockExpired returns false + try { + lockingService.lockUploadByUri("/files/upload/24249a5b-01a4-4bf8-b67a-364273bb5a2e"); + fail("Expected UploadAlreadyLockedException when trying to acquire an actively locked URI"); + } catch (UploadAlreadyLockedException expected) { + assertNotNull(expected.getMessage()); + } + + lock1.close(); + } } diff --git a/src/test/java/me/desair/tus/server/upload/s3/S3UtilsTest.java b/src/test/java/me/desair/tus/server/upload/s3/S3UtilsTest.java index 53ee607d..5dc3b3c9 100644 --- a/src/test/java/me/desair/tus/server/upload/s3/S3UtilsTest.java +++ b/src/test/java/me/desair/tus/server/upload/s3/S3UtilsTest.java @@ -35,10 +35,28 @@ public void testParseErrorResponseCodes() throws Exception { assertEquals( S3ErrorType.API_NOT_IMPLEMENTED, S3Utils.parseErrorResponse(createExceptionWithCode("APINotImplemented"))); + assertEquals( + S3ErrorType.API_NOT_IMPLEMENTED, + S3Utils.parseErrorResponse(createExceptionWithCode("NotImplemented"))); assertEquals( S3ErrorType.UNKNOWN, S3Utils.parseErrorResponse(createExceptionWithCode("InternalError"))); } + @Test + public void testParseErrorResponseHttp501() { + okhttp3.Response httpResponse = + new okhttp3.Response.Builder() + .request(new okhttp3.Request.Builder().url("https://example.com").build()) + .protocol(okhttp3.Protocol.HTTP_1_1) + .code(501) + .message("Not Implemented") + .build(); + ErrorResponse errorResponse = org.mockito.Mockito.mock(ErrorResponse.class); + org.mockito.Mockito.when(errorResponse.code()).thenReturn(""); + ErrorResponseException ex = new ErrorResponseException(errorResponse, httpResponse, null); + assertEquals(S3ErrorType.API_NOT_IMPLEMENTED, S3Utils.parseErrorResponse(ex)); + } + private ErrorResponseException createExceptionWithCode(String code) { ErrorResponse errorResponse = org.mockito.Mockito.mock(ErrorResponse.class); org.mockito.Mockito.when(errorResponse.code()).thenReturn(code); From a4ec6b82e80f51429ead32dcc5f40d46e492d5f1 Mon Sep 17 00:00:00 2001 From: Tom Desair Date: Sat, 3 Oct 2026 16:20:25 +0200 Subject: [PATCH 5/7] feat(s3): add native S3 multipart copy helper and connection parameter constructors --- CHANGELOG.md | 4 + docs/S3_STORAGE.md | 9 +- scripts/rufh_conformity_test.py | 39 ++ scripts/tus_conformity_test.py | 58 +++ .../server/core/CorePatchRequestHandler.java | 23 +- .../CreationWithUploadPostRequestHandler.java | 27 +- .../RufhAppendPatchRequestHandler.java | 33 +- .../RufhCreationPostRequestHandler.java | 34 +- .../upload/azure/AzureBlobStorageService.java | 39 +- .../upload/s3/S3ConcatenationService.java | 35 +- .../server/upload/s3/S3LockingService.java | 60 +++ .../upload/s3/S3ServerSideComposeHelper.java | 303 +++++++++++++++ .../server/upload/s3/S3StorageService.java | 190 +++++++-- .../upload/util/AsyncChunkUploader.java | 13 + .../core/CorePatchRequestHandlerTest.java | 74 ++++ ...ationWithUploadPostRequestHandlerTest.java | 64 ++++ .../RufhAppendPatchRequestHandlerTest.java | 79 ++++ .../RufhCreationPostRequestHandlerTest.java | 78 ++++ .../azure/ITAzureBlobStorageService.java | 34 ++ .../upload/s3/S3ConcatenationServiceTest.java | 10 +- .../upload/s3/S3LockingServiceTest.java | 18 + .../s3/S3ServerSideComposeHelperTest.java | 362 ++++++++++++++++++ .../upload/s3/S3StorageServiceTest.java | 132 +++++++ .../upload/util/AsyncChunkUploaderTest.java | 14 + 24 files changed, 1667 insertions(+), 65 deletions(-) create mode 100644 src/main/java/me/desair/tus/server/upload/s3/S3ServerSideComposeHelper.java create mode 100644 src/test/java/me/desair/tus/server/upload/s3/S3ServerSideComposeHelperTest.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 6058f827..c906a2f2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,10 @@ All notable changes to this project will be documented in this file. - **Clear Content-Length on Error Responses**: Cleared `Content-Length` response header prior to invoking `HttpServletResponse.sendError(...)` during exception handling, resolving buffer conflicts and exceptions in Undertow and other servlet containers ([#40](https://github.com/tomdesair/tus-java-server/issues/40)). - **Prevent Disk Truncate Underflow**: Guarded file truncate logic in `DiskStorageService` against underflow when removing bytes (`Math.max(0L, file.size() - byteCount)`). - **File Channel Leak Prevention in FileBasedLock**: Guaranteed `FileChannel` is closed immediately upon lock acquisition errors to avoid file descriptor leaks. +- **Cloud Upload Pause & Drain Timeout Resilience**: Extended `AsyncChunkUploader` default drain timeout from 4 seconds to 60 seconds and wrapped chunk draining in error-resilient recovery logic across `AzureBlobStorageService` and `S3StorageService`. Confirmed uploaded chunks and staged Azure blocks are committed and metadata (`UploadInfo` offset) is persisted to storage before throwing stream or drain exceptions, ensuring paused uploads (such as Uppy client pause/resume) preserve their progress and resume from the exact byte offset instead of restarting from 0. +- **Azure Blob Storage Upload Cancellation Lease Safety**: Checked lock blob lease state before calling `deleteIfExists()` in `AzureBlobStorageService.terminateUpload()`. When an active lease is held on the lock blob by an ongoing request (e.g. `DELETE` cancellation), attempting deletion without specifying the lease ID triggered an Azure SDK error log (`HTTP 412 LeaseIdMissing`). Skipping deletion for actively leased blobs prevents this error and allows normal lease release upon request completion. +- **Lock Contention Stream Interruption Handling**: Handled `IOException` from `InterruptibleInputStream` across upload request handlers (`CorePatchRequestHandler`, `RufhAppendPatchRequestHandler`, `RufhCreationPostRequestHandler`, `CreationWithUploadPostRequestHandler`). When an upload stream is interrupted by the locking service watchdog during lock contention (such as concurrent `HEAD` progress checks or `DELETE` cancellation requests), the storage backend commits all buffered bytes received so far and updates the offset in storage. Request handlers now reload the refreshed `UploadInfo` and return a clean HTTP 204/201 response with the updated offset rather than bubbling an unhandled `IOException` / HTTP 500 to the servlet container. +- **S3 Server-Side Part Composition & Native Multipart Copy**: Implemented `S3ServerSideComposeHelper` to resolve AWS S3 header rejection during server-side part composition. MinIO Java SDK 9.0.3's `composeObject` delegates to `UploadPartCopy` with an empty body placeholder that automatically injects `Content-MD5` and `Content-Type` headers, which Amazon AWS S3 strictly forbids on part copy requests and rejects with HTTP 400 (`The specified header is not valid in this context`). `S3ServerSideComposeHelper` executes native S3 multipart copy requests directly with SigV4 signing omitting `Content-MD5`, allowing fast zero-bandwidth server-side composition on both AWS S3 and MinIO/Ceph clusters without external AWS SDK dependencies. Added reflection-free constructors to `S3StorageService` and `S3LockingService` accepting connection parameters directly, with `eu-central-1` as the default fallback region, while preserving multi-tier streaming concatenation fallbacks. ### Breaking - **Downloads**: In order to support both the Tus protocol and RUFH protocol, the unofficial download extension will not return a HTTP status code `204` for uploads that are still in progress and will not contain the response header `Tus-Resumable`. Removed the `UploadInProgressException` class. diff --git a/docs/S3_STORAGE.md b/docs/S3_STORAGE.md index ac44c643..4328c739 100644 --- a/docs/S3_STORAGE.md +++ b/docs/S3_STORAGE.md @@ -46,8 +46,13 @@ MinioClient minioClient = MinioClient.builder() .build(); // 2. Instantiate S3 Storage and Distributed Locking services -S3StorageService s3StorageService = new S3StorageService(minioClient, bucketName); -S3LockingService s3LockingService = new S3LockingService(minioClient, bucketName); +// Option A: Direct connection parameters (recommended, builds internal client with server-side compose helper) +S3StorageService s3StorageService = new S3StorageService(endpoint, "eu-central-1", accessKey, secretKey, bucketName); +S3LockingService s3LockingService = new S3LockingService(endpoint, "eu-central-1", accessKey, secretKey, bucketName); + +// Option B: Using a pre-configured MinIO Client +// S3StorageService s3StorageService = new S3StorageService(minioClient, endpoint, "eu-central-1", accessKey, secretKey, bucketName); +// S3LockingService s3LockingService = new S3LockingService(minioClient, bucketName); // 3. Configure TusFileUploadService with S3 storage and locking // Note: Automatic JVM shutdown hooks are built-in by default to terminate watchdog threads on pod exit. diff --git a/scripts/rufh_conformity_test.py b/scripts/rufh_conformity_test.py index 6e4c0319..d973a588 100755 --- a/scripts/rufh_conformity_test.py +++ b/scripts/rufh_conformity_test.py @@ -1114,6 +1114,45 @@ def test_options_with_upload_complete_header(self, target_url, request): status, resp_headers, _, _ = http_request("OPTIONS", target_url, headers=headers, test_name=request.node.name) assert status in (200, 204), "OPTIONS request with Upload-Complete MUST succeed" + def test_paused_upload_head_offset_and_resume(self, target_url, request): + """ + §5 & §10.1: Resuming an Incomplete Upload After Pause. + Quote: "The client can pause the upload by withholding the rest of the content..." + Verifies that after uploading a chunk and pausing, querying HEAD returns the persisted + offset without resetting to 0, and the upload completes successfully when resumed from + that offset. + """ + upload_uri = create_partial_upload(target_url, test_name=request.node.name) + status1, resp1, _, _ = http_request( + "PATCH", + upload_uri, + headers={UPLOAD_OFFSET: "0", UPLOAD_COMPLETE: FALSE, CONTENT_TYPE: APPLICATION_PARTIAL_UPLOAD}, + body=b"0123456789" * 4, # 40 bytes + test_name=request.node.name, + ) + assert status1 in (200, 204) + assert resp1.get(UPLOAD_OFFSET) == "40" + + # Simulate pause delay + time.sleep(0.5) + + # Verify offset via HEAD after pause + status_h, h_headers, _, _ = http_request("HEAD", upload_uri, test_name=request.node.name) + assert status_h in (200, 204) + assert h_headers.get(UPLOAD_OFFSET) == "40", "HEAD after pause must report 40 bytes" + + # Resume and complete upload + status2, resp2, _, _ = http_request( + "PATCH", + upload_uri, + headers={UPLOAD_OFFSET: "40", UPLOAD_COMPLETE: TRUE, CONTENT_TYPE: APPLICATION_PARTIAL_UPLOAD}, + body=b"abcdefghijklmnopqrstuvwxyz" * 2, # 52 bytes + test_name=request.node.name, + ) + assert status2 in (200, 201, 204) + assert resp2.get(UPLOAD_OFFSET) == "92" + assert resp2.get(UPLOAD_COMPLETE) == TRUE + class TestHttpDigests: """Tests for HTTP Digests (RFC 9530).""" diff --git a/scripts/tus_conformity_test.py b/scripts/tus_conformity_test.py index 78fb3b9c..5e09fa7c 100755 --- a/scripts/tus_conformity_test.py +++ b/scripts/tus_conformity_test.py @@ -1150,6 +1150,64 @@ def test_multi_chunk_resumable_upload_workflow(self, target_url): assert head_final.get(UPLOAD_OFFSET) == "120" assert head_final.get(UPLOAD_LENGTH) == "120" + def test_paused_upload_head_offset_and_resume(self, target_url): + """ + §6 PATCH: Upload Pause and Resume Workflow. + Quote: "If the connection is interrupted during a PATCH request, the Client SHOULD issue + a HEAD request to determine the current offset before resuming the upload." + Verifies that when an upload is paused midway, issuing a HEAD request accurately returns + the persisted offset without resetting to 0, and resuming with a subsequent PATCH from + that persisted offset successfully completes the upload. + """ + total_data = b"0123456789abcdefghijklmnopqrstuvwxyz" * 4 # 144 bytes + part1 = total_data[:60] + part2 = total_data[60:] + + upload_url, _ = create_upload(target_url, upload_length=str(len(total_data))) + + # 1. Send first chunk (simulating upload progress before user clicks pause) + headers1 = { + TUS_RESUMABLE: TUS_API_VERSION, + UPLOAD_OFFSET: "0", + CONTENT_TYPE: APPLICATION_OFFSET_OCTET_STREAM, + } + st1, resp_headers1, _ = http_request("PATCH", upload_url, headers=headers1, body=part1) + assert st1 == 204, f"Expected 204 No Content for first chunk, got {st1}" + assert resp_headers1.get(UPLOAD_OFFSET) == "60" + + # 2. Simulate client pause: query offset via HEAD to verify saved position + st_head1, head1, _ = http_request("HEAD", upload_url, headers={TUS_RESUMABLE: TUS_API_VERSION}) + assert st_head1 in (200, 204) + assert head1.get(UPLOAD_OFFSET) == "60", ( + f"Expected Upload-Offset 60 after pause, got {head1.get(UPLOAD_OFFSET)}" + ) + + # 3. Simulate pause delay + time.sleep(0.5) + + # 4. Verify offset remained persisted and did not reset to 0 + st_head2, head2, _ = http_request("HEAD", upload_url, headers={TUS_RESUMABLE: TUS_API_VERSION}) + assert st_head2 in (200, 204) + assert head2.get(UPLOAD_OFFSET) == "60", ( + f"Expected Upload-Offset to remain 60, got {head2.get(UPLOAD_OFFSET)}" + ) + + # 5. Resume upload from the persisted offset + headers2 = { + TUS_RESUMABLE: TUS_API_VERSION, + UPLOAD_OFFSET: "60", + CONTENT_TYPE: APPLICATION_OFFSET_OCTET_STREAM, + } + st2, resp_headers2, _ = http_request("PATCH", upload_url, headers=headers2, body=part2) + assert st2 == 204, f"Expected 204 No Content for resumed chunk, got {st2}" + assert resp_headers2.get(UPLOAD_OFFSET) == str(len(total_data)) + + # 6. Final verification + st_final, head_final, _ = http_request("HEAD", upload_url, headers={TUS_RESUMABLE: TUS_API_VERSION}) + assert st_final in (200, 204) + assert head_final.get(UPLOAD_OFFSET) == str(len(total_data)) + assert head_final.get(UPLOAD_LENGTH) == str(len(total_data)) + # CLI Entry Point & Custom Formatted Summary Reporter if __name__ == "__main__": diff --git a/src/main/java/me/desair/tus/server/core/CorePatchRequestHandler.java b/src/main/java/me/desair/tus/server/core/CorePatchRequestHandler.java index 8ed264c8..11aca9d6 100644 --- a/src/main/java/me/desair/tus/server/core/CorePatchRequestHandler.java +++ b/src/main/java/me/desair/tus/server/core/CorePatchRequestHandler.java @@ -66,13 +66,14 @@ public HttpProblemDetails process( if (uploadInfo == null) { found = false; } else if (uploadInfo.isUploadInProgress()) { + InterruptibleInputStream interruptibleStream = null; try { InputStream stream = servletRequest.getContentInputStream(); // If a locking service is provided, wrap the input stream in an InterruptibleInputStream // and register it with the locking service to allow for interruption of the upload. if (lockingService != null) { - InterruptibleInputStream interruptibleStream = new InterruptibleInputStream(stream); + interruptibleStream = new InterruptibleInputStream(stream); lockingService.registerInputStream(servletRequest.getRequestURI(), interruptibleStream); stream = interruptibleStream; } @@ -94,6 +95,26 @@ public HttpProblemDetails process( } } catch (UploadNotFoundException e) { found = false; + } catch (IOException e) { + // When an upload stream is interrupted by the locking service watchdog or a concurrent + // lock contention release request (e.g. from a concurrent HEAD or DELETE), the storage + // backend (Disk, S3, Azure) commits all bytes received up to the interruption and updates + // the offset in storage. We reload the updated UploadInfo and acknowledge the partial + // PATCH with 204 No Content and the new Upload-Offset rather than throwing a 500 error. + if (interruptibleStream != null && interruptibleStream.isInterrupted()) { + log.info( + "Upload PATCH request for URI {} was interrupted by locking service contention; " + + "saved partial upload up to offset {}", + servletRequest.getRequestURI(), + uploadInfo != null ? uploadInfo.getOffset() : "unknown"); + UploadInfo refreshed = + uploadStorageService.getUploadInfo(servletRequest.getRequestURI(), ownerKey); + if (refreshed != null) { + uploadInfo = refreshed; + } + } else { + throw e; + } } } diff --git a/src/main/java/me/desair/tus/server/creationwithupload/CreationWithUploadPostRequestHandler.java b/src/main/java/me/desair/tus/server/creationwithupload/CreationWithUploadPostRequestHandler.java index 6594e191..7f8a8a94 100644 --- a/src/main/java/me/desair/tus/server/creationwithupload/CreationWithUploadPostRequestHandler.java +++ b/src/main/java/me/desair/tus/server/creationwithupload/CreationWithUploadPostRequestHandler.java @@ -46,13 +46,36 @@ public HttpProblemDetails process( UploadInfo uploadInfo = uploadStorageService.getUploadInfo(location, ownerKey); if (uploadInfo != null && uploadInfo.isUploadInProgress()) { InputStream stream = servletRequest.getContentInputStream(); + InterruptibleInputStream interruptibleStream = null; if (uploadLockingService != null) { - InterruptibleInputStream interruptibleStream = new InterruptibleInputStream(stream); + interruptibleStream = new InterruptibleInputStream(stream); uploadLockingService.registerInputStream(location, interruptibleStream); stream = interruptibleStream; } - uploadInfo = uploadStorageService.append(uploadInfo, stream); + try { + uploadInfo = uploadStorageService.append(uploadInfo, stream); + } catch (IOException e) { + // When an upload stream is interrupted by the locking service watchdog or a concurrent + // lock contention release request (e.g. from a concurrent HEAD or DELETE), the storage + // backend (Disk, S3, Azure) commits all bytes received up to the interruption and + // updates + // the offset in storage. We reload the updated UploadInfo and acknowledge the partial + // upload rather than propagating an unhandled error to the servlet container. + if (interruptibleStream != null && interruptibleStream.isInterrupted()) { + log.info( + "Upload creation-with-upload POST request for URI {} was interrupted by locking" + + " service contention; saved partial upload up to offset {}", + location, + uploadInfo != null ? uploadInfo.getOffset() : "unknown"); + UploadInfo refreshed = uploadStorageService.getUploadInfo(location, ownerKey); + if (refreshed != null) { + uploadInfo = refreshed; + } + } else { + throw e; + } + } servletResponse.setHeader( HttpHeader.UPLOAD_OFFSET, String.valueOf(uploadInfo.getOffset())); diff --git a/src/main/java/me/desair/tus/server/rufh/handler/RufhAppendPatchRequestHandler.java b/src/main/java/me/desair/tus/server/rufh/handler/RufhAppendPatchRequestHandler.java index d929d8fa..787089a7 100644 --- a/src/main/java/me/desair/tus/server/rufh/handler/RufhAppendPatchRequestHandler.java +++ b/src/main/java/me/desair/tus/server/rufh/handler/RufhAppendPatchRequestHandler.java @@ -14,6 +14,8 @@ import me.desair.tus.server.util.StructuredHeaderUtil; import me.desair.tus.server.util.TusServletRequest; import me.desair.tus.server.util.TusServletResponse; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Request handler for data append requests via HTTP PATCH. @@ -27,6 +29,8 @@ */ public class RufhAppendPatchRequestHandler extends AbstractRequestHandler { + private static final Logger log = LoggerFactory.getLogger(RufhAppendPatchRequestHandler.class); + @Override public boolean supports(HttpMethod method) { return HttpMethod.PATCH.equals(method); @@ -66,15 +70,36 @@ public HttpProblemDetails process( } InputStream is = servletRequest.getContentInputStream(); + InterruptibleInputStream interruptibleStream = null; if (is != null) { if (uploadLockingService != null) { - InterruptibleInputStream interruptibleStream = new InterruptibleInputStream(is); + interruptibleStream = new InterruptibleInputStream(is); uploadLockingService.registerInputStream(requestUri, interruptibleStream); is = interruptibleStream; } - UploadInfo appended = uploadStorageService.append(uploadInfo, is); - if (appended != null) { - uploadInfo = appended; + try { + UploadInfo appended = uploadStorageService.append(uploadInfo, is); + if (appended != null) { + uploadInfo = appended; + } + } catch (IOException e) { + // When an append stream is interrupted by the locking service watchdog or a concurrent + // release request, the storage backend commits all bytes received so far and updates + // the offset in storage. We reload the updated UploadInfo and acknowledge the partial + // append rather than propagating an unhandled error. + if (interruptibleStream != null && interruptibleStream.isInterrupted()) { + log.info( + "RUFH append request for URI {} was interrupted by locking service contention; " + + "saved partial upload up to offset {}", + requestUri, + uploadInfo != null ? uploadInfo.getOffset() : "unknown"); + UploadInfo refreshed = uploadStorageService.getUploadInfo(requestUri, ownerKey); + if (refreshed != null) { + uploadInfo = refreshed; + } + } else { + throw e; + } } } diff --git a/src/main/java/me/desair/tus/server/rufh/handler/RufhCreationPostRequestHandler.java b/src/main/java/me/desair/tus/server/rufh/handler/RufhCreationPostRequestHandler.java index 2702c23e..e4eb2712 100644 --- a/src/main/java/me/desair/tus/server/rufh/handler/RufhCreationPostRequestHandler.java +++ b/src/main/java/me/desair/tus/server/rufh/handler/RufhCreationPostRequestHandler.java @@ -17,6 +17,8 @@ import me.desair.tus.server.util.TusServletResponse; import me.desair.tus.server.util.Utils; import org.apache.commons.lang3.Strings; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Request handler for upload creation requests via HTTP POST, PUT, or PATCH. @@ -29,6 +31,8 @@ */ public class RufhCreationPostRequestHandler extends AbstractRequestHandler { + private static final Logger log = LoggerFactory.getLogger(RufhCreationPostRequestHandler.class); + public RufhCreationPostRequestHandler() { // Default constructor } @@ -109,15 +113,37 @@ public HttpProblemDetails process( servletRequest.getHeader(HttpHeader.TRANSFER_ENCODING), "chunked")); InputStream is = servletRequest.getContentInputStream(); + InterruptibleInputStream interruptibleStream = null; if (is != null && hasContent) { if (uploadLockingService != null) { - InterruptibleInputStream interruptibleStream = new InterruptibleInputStream(is); + interruptibleStream = new InterruptibleInputStream(is); uploadLockingService.registerInputStream(uploadUri, interruptibleStream); is = interruptibleStream; } - UploadInfo appended = uploadStorageService.append(uploadInfo, is); - if (appended != null) { - uploadInfo = appended; + try { + UploadInfo appended = uploadStorageService.append(uploadInfo, is); + if (appended != null) { + uploadInfo = appended; + } + } catch (IOException e) { + // When an upload stream is interrupted by the locking service watchdog or a concurrent + // lock contention release request (e.g. from a concurrent HEAD or DELETE), the storage + // backend (Disk, S3, Azure) commits all bytes received up to the interruption and updates + // the offset in storage. We reload the updated UploadInfo and acknowledge the partial + // upload rather than propagating an unhandled error to the servlet container. + if (interruptibleStream != null && interruptibleStream.isInterrupted()) { + log.info( + "RUFH creation request with body for URI {} was interrupted by locking service" + + " contention; saved partial upload up to offset {}", + uploadUri, + uploadInfo != null ? uploadInfo.getOffset() : "unknown"); + UploadInfo refreshed = uploadStorageService.getUploadInfo(uploadUri, ownerKey); + if (refreshed != null) { + uploadInfo = refreshed; + } + } else { + throw e; + } } } diff --git a/src/main/java/me/desair/tus/server/upload/azure/AzureBlobStorageService.java b/src/main/java/me/desair/tus/server/upload/azure/AzureBlobStorageService.java index 686797a2..ec0f6872 100644 --- a/src/main/java/me/desair/tus/server/upload/azure/AzureBlobStorageService.java +++ b/src/main/java/me/desair/tus/server/upload/azure/AzureBlobStorageService.java @@ -4,6 +4,7 @@ import com.azure.storage.blob.BlobClient; import com.azure.storage.blob.BlobContainerClient; import com.azure.storage.blob.models.BlobItem; +import com.azure.storage.blob.models.BlobProperties; import com.azure.storage.blob.models.BlobStorageException; import com.azure.storage.blob.models.Block; import com.azure.storage.blob.models.BlockList; @@ -47,6 +48,7 @@ import org.apache.commons.io.IOUtils; import org.apache.commons.io.input.BoundedInputStream; import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.Strings; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -233,6 +235,7 @@ public UploadInfo append(UploadInfo upload, InputStream inputStream) int initialBlockCount = blockIds.size(); long totalAppended = 0L; IOException streamException = null; + IOException drainException = null; TusException pendingTusException = null; List plannedBlockIds = new ArrayList<>(); @@ -301,9 +304,18 @@ public UploadInfo append(UploadInfo upload, InputStream inputStream) } } - int confirmedCount = uploader.drainAndComplete(4000); - for (int i = 0; i < confirmedCount; i++) { - blockIds.add(plannedBlockIds.get(i)); + // Drain remaining staged chunks with 60s timeout. + // Catch drainException so any chunks confirmed uploaded before timeout or error + // are committed and the metadata offset is preserved without loss. + try { + uploader.drainAndComplete(); + } catch (IOException e) { + drainException = e; + } finally { + int confirmed = Math.min(uploader.getConfirmedCount(), plannedBlockIds.size()); + for (int i = 0; i < confirmed; i++) { + blockIds.add(plannedBlockIds.get(i)); + } } } @@ -317,7 +329,7 @@ public UploadInfo append(UploadInfo upload, InputStream inputStream) // Batching block commits into a single call at the end eliminates redundant network // round-trips for every chunk, drastically improving performance. In addition, committing // here before throwing any pending stream or limit exception guarantees zero data loss - // if network drops or limits are hit midway. + // if client pauses, network drops, or limits are hit midway. if (blockIds.size() > initialBlockCount) { try { blockBlobClient.commitBlockList(blockIds, true); @@ -343,6 +355,10 @@ public UploadInfo append(UploadInfo upload, InputStream inputStream) } } + if (drainException != null) { + throw drainException; + } + if (streamException != null) { throw streamException; } @@ -492,16 +508,25 @@ public void terminateUpload(UploadInfo uploadInfo) throws UploadNotFoundExceptio containerClient.getBlobClient(checksumKey).deleteIfExists(); } - // 4. Delete lock target and stop signal blobs (handling active lease exceptions gracefully) + // 4. Delete lock target and stop signal blobs try { containerClient.getBlobClient(locksPrefix + id + ".stop").deleteIfExists(); } catch (Exception ignored) { } try { - containerClient.getBlobClient(locksPrefix + id + ".lock").deleteIfExists(); + BlobClient lockBlob = containerClient.getBlobClient(locksPrefix + id + ".lock"); + // Check if lock blob is actively leased by the current request before attempting deletion. + // Attempting deleteIfExists() on a leased blob without the lease ID causes Azure Blob + // Storage to reject the request with HTTP 412 (LeaseIdMissing) and logs an SDK error. + if (Boolean.TRUE.equals(lockBlob.exists())) { + BlobProperties props = lockBlob.getProperties(); + if (props.getLeaseState() != null + && !Strings.CS.equals(props.getLeaseState().toString(), "leased")) { + lockBlob.deleteIfExists(); + } + } } catch (Exception ignored) { - // Lock blob may be actively leased by current request lock (Azure 412 LeaseIdMissing) } log.debug("Terminated upload with ID {}", id); diff --git a/src/main/java/me/desair/tus/server/upload/s3/S3ConcatenationService.java b/src/main/java/me/desair/tus/server/upload/s3/S3ConcatenationService.java index 640cf20a..5b128453 100644 --- a/src/main/java/me/desair/tus/server/upload/s3/S3ConcatenationService.java +++ b/src/main/java/me/desair/tus/server/upload/s3/S3ConcatenationService.java @@ -1,9 +1,7 @@ package me.desair.tus.server.upload.s3; -import io.minio.ComposeObjectArgs; import io.minio.MinioClient; import io.minio.PutObjectArgs; -import io.minio.SourceObject; import java.io.IOException; import java.io.InputStream; import java.io.SequenceInputStream; @@ -48,6 +46,7 @@ public class S3ConcatenationService implements UploadConcatenationService { private final long minPartSize; private final Path temporaryDirectory; private UploadStorageService uploadStorageService; + private S3ServerSideComposeHelper s3ComposeHelper; /** * Basic constructor using default object prefix ("uploads/") and Java temp directory. @@ -112,6 +111,7 @@ public S3ConcatenationService( ? temporaryDirectory : java.nio.file.Paths.get(System.getProperty("java.io.tmpdir")); this.minPartSize = minPartSize; + this.s3ComposeHelper = new S3ServerSideComposeHelper(this.minioClient); } public void setUploadStorageService(UploadStorageService uploadStorageService) { @@ -162,7 +162,7 @@ public void merge(UploadInfo uploadInfo) throws IOException, UploadNotFoundExcep if (canUseServerSideCopy) { // Fast path: Compose S3 objects on cluster server-side without downloading data - mergeUsingServerSideCopy(targetObjectKey, partialUploads); + mergeUsingServerSideCopy(targetObjectKey, partialUploads, totalLength); } else { // Fallback path: Sequential stream re-upload for sub-5MB parts mergeUsingStreamingReupload(targetObjectKey, partialUploads, totalLength); @@ -246,23 +246,34 @@ public List getPartialUploads(UploadInfo info) return output; } - private void mergeUsingServerSideCopy(String targetKey, List partialUploads) - throws IOException { + private void mergeUsingServerSideCopy( + String targetKey, List partialUploads, long totalLength) throws IOException { try { - List sources = new ArrayList<>(); + List partKeys = new ArrayList<>(); for (UploadInfo partial : partialUploads) { - String partKey = partial.getStorageUploadId(); - sources.add(SourceObject.builder().bucket(bucket).object(partKey).build()); + partKeys.add(partial.getStorageUploadId()); } - // Execute S3 server-side object composition - minioClient.composeObject( - ComposeObjectArgs.builder().bucket(bucket).object(targetKey).sources(sources).build()); + // Execute S3 server-side object composition via native S3 multipart copy helper. + // S3ServerSideComposeHelper executes an UploadPartCopy sequence without Content-MD5, + // avoiding MinIO SDK's internal EMPTY_BODY issue on Amazon AWS S3. + s3ComposeHelper.compose(bucket, targetKey, partKeys); } catch (Exception e) { - throw new IOException("Failed server-side S3 composeObject merge for key " + targetKey, e); + // If server-side composition fails for any reason (e.g. S3 permissions, regional policy, + // or unhandled multipart copy restriction), fall back to streaming re-upload as a safety net. + // This ensures concatenated uploads always succeed even if server-side copy is denied. + log.warn( + "Server-side S3 composition failed for target key {}; falling back to streaming re-upload: {}", + targetKey, + e.getMessage()); + mergeUsingStreamingReupload(targetKey, partialUploads, totalLength); } } + void setS3ServerSideComposeHelper(S3ServerSideComposeHelper s3ComposeHelper) { + this.s3ComposeHelper = s3ComposeHelper; + } + private void mergeUsingStreamingReupload( String targetKey, List partialUploads, long totalLength) throws IOException { try (InputStream combinedStream = diff --git a/src/main/java/me/desair/tus/server/upload/s3/S3LockingService.java b/src/main/java/me/desair/tus/server/upload/s3/S3LockingService.java index f37c1278..e9bd22f2 100644 --- a/src/main/java/me/desair/tus/server/upload/s3/S3LockingService.java +++ b/src/main/java/me/desair/tus/server/upload/s3/S3LockingService.java @@ -80,6 +80,56 @@ public S3LockingService(MinioClient minioClient, String bucket) { new UuidUploadIdFactory()); } + /** + * Constructor accepting explicit connection parameters without requiring a pre-existing + * MinioClient. + * + * @param endpoint S3 endpoint URL (e.g. "https://s3.amazonaws.com" or "http://localhost:9000") + * @param region S3 region name (e.g. "eu-central-1", "us-east-1") + * @param accessKey S3 access key / username + * @param secretKey S3 secret key / password + * @param bucket Target S3 bucket name + */ + public S3LockingService( + String endpoint, String region, String accessKey, String secretKey, String bucket) { + this( + buildMinioClient(endpoint, region, accessKey, secretKey), + bucket, + DEFAULT_LOCKS_PREFIX, + DEFAULT_LEASE_DURATION_MS, + DEFAULT_POLL_INTERVAL_MS, + new UuidUploadIdFactory()); + } + + /** + * Constructor accepting a pre-configured {@link MinioClient} along with explicit connection + * parameters. + * + * @param minioClient Pre-configured MinIO Client + * @param endpoint S3 endpoint URL + * @param region S3 region name + * @param accessKey S3 access key / username + * @param secretKey S3 secret key / password + * @param bucket Target S3 bucket name + */ + public S3LockingService( + MinioClient minioClient, + String endpoint, + String region, + String accessKey, + String secretKey, + String bucket) { + this( + minioClient != null + ? minioClient + : buildMinioClient(endpoint, region, accessKey, secretKey), + bucket, + DEFAULT_LOCKS_PREFIX, + DEFAULT_LEASE_DURATION_MS, + DEFAULT_POLL_INTERVAL_MS, + new UuidUploadIdFactory()); + } + /** * Full constructor allowing custom configuration including a custom {@link UploadIdFactory}. * @@ -395,4 +445,14 @@ private String buildLockKey(UploadId uploadId) { private String buildStopKey(UploadId uploadId) { return locksPrefix + uploadId.toString() + ".stop"; } + + private static MinioClient buildMinioClient( + String endpoint, String region, String accessKey, String secretKey) { + String effectiveRegion = (region != null && !region.isEmpty()) ? region : "eu-central-1"; + return MinioClient.builder() + .endpoint(endpoint) + .credentials(accessKey, secretKey) + .region(effectiveRegion) + .build(); + } } diff --git a/src/main/java/me/desair/tus/server/upload/s3/S3ServerSideComposeHelper.java b/src/main/java/me/desair/tus/server/upload/s3/S3ServerSideComposeHelper.java new file mode 100644 index 00000000..8a645173 --- /dev/null +++ b/src/main/java/me/desair/tus/server/upload/s3/S3ServerSideComposeHelper.java @@ -0,0 +1,303 @@ +package me.desair.tus.server.upload.s3; + +import io.minio.AbortMultipartUploadArgs; +import io.minio.Checksum; +import io.minio.CompleteMultipartUploadArgs; +import io.minio.ComposeObjectArgs; +import io.minio.CreateMultipartUploadArgs; +import io.minio.CreateMultipartUploadResponse; +import io.minio.Http; +import io.minio.MinioAsyncClient; +import io.minio.MinioClient; +import io.minio.Signer; +import io.minio.SourceObject; +import io.minio.Time; +import io.minio.Utils; +import io.minio.credentials.Credentials; +import io.minio.credentials.Provider; +import io.minio.credentials.StaticProvider; +import io.minio.messages.Part; +import java.io.IOException; +import java.time.ZonedDateTime; +import java.util.ArrayList; +import java.util.List; +import okhttp3.HttpUrl; +import okhttp3.OkHttpClient; +import okhttp3.Request; +import okhttp3.RequestBody; +import okhttp3.Response; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Helper executing native S3 Multipart Copy operations to combine part objects server-side. + * + *

Why this helper exists: MinIO Java SDK's {@code composeObject()} implementation delegates to + * {@code UploadPartCopy} with an internal {@code EMPTY_BODY} placeholder. In MinIO SDK's {@code + * Http.toRequest()}, that placeholder automatically attaches {@code Content-MD5} and {@code + * Content-Type} headers to the HTTP request. While MinIO server ignores those extra headers, Amazon + * AWS S3 strictly forbids the {@code Content-MD5} header on {@code UploadPartCopy} requests, + * returning HTTP 400 InvalidArgument ("The specified header is not valid in this context"). + * Attempting to remove the header via an OkHttp network interceptor also fails because AWS SigV4 + * signature calculation includes all present headers, resulting in a SignatureDoesNotMatch error. + * + *

Workaround: This helper executes standard S3 Multipart Upload Copy operations directly via + * OkHttp and AWS SigV4 signing without adding {@code Content-MD5}. This allows fast zero-bandwidth + * server-side composition on both Amazon AWS S3 and MinIO/Ceph clusters without needing heavy + * external AWS SDK dependencies. + * + *

Safety Fallback: If native server-side compose fails for any reason (e.g. S3 permissions, + * regional restrictions, or client reflection unavailability), callers (such as {@link + * S3StorageService} and {@link S3ConcatenationService}) catch the exception and fall back to + * sequential streaming concatenation (re-uploading the combined parts). + */ +public class S3ServerSideComposeHelper { + + private static final Logger log = LoggerFactory.getLogger(S3ServerSideComposeHelper.class); + + private final MinioClient minioClient; + private final MinioAsyncClient asyncClient; + private final Http.BaseUrl baseUrl; + private final Provider provider; + private final OkHttpClient httpClient; + private final String explicitRegion; + + /** + * Constructs an instance wrapping the given {@link MinioClient} without reflection. + * + *

If explicit connection parameters are not provided, this helper falls back to calling {@link + * MinioClient#composeObject(ComposeObjectArgs)} directly. + * + * @param minioClient The MinIO client instance + */ + public S3ServerSideComposeHelper(MinioClient minioClient) { + this.minioClient = minioClient; + this.asyncClient = null; + this.baseUrl = null; + this.provider = null; + this.httpClient = null; + this.explicitRegion = null; + } + + /** + * Constructs an instance with an existing {@link MinioClient} and explicit connection parameters. + * + *

This allows native S3 multipart copy without using reflection on {@link MinioClient}. + * + * @param minioClient The existing MinIO client instance + * @param endpoint The S3 endpoint URL (e.g. "https://s3.amazonaws.com" or + * "http://localhost:9000") + * @param region S3 region name (optional, defaults to "eu-central-1" if null or empty) + * @param accessKey S3 access key / username + * @param secretKey S3 secret key / password + */ + public S3ServerSideComposeHelper( + MinioClient minioClient, String endpoint, String region, String accessKey, String secretKey) { + this.minioClient = minioClient; + this.explicitRegion = (region != null && !region.isEmpty()) ? region : "eu-central-1"; + + Http.BaseUrl base = null; + Provider prov = null; + OkHttpClient client = null; + MinioAsyncClient async = null; + + if (endpoint != null && !endpoint.isEmpty()) { + try { + base = new Http.BaseUrl(endpoint); + if (region != null && !region.isEmpty()) { + base.setRegion(region); + } + prov = new StaticProvider(accessKey, secretKey, null); + client = new OkHttpClient(); + + MinioAsyncClient.Builder asyncBuilder = + MinioAsyncClient.builder().endpoint(endpoint).credentials(accessKey, secretKey); + if (region != null && !region.isEmpty()) { + asyncBuilder.region(region); + } + async = asyncBuilder.build(); + } catch (Exception e) { + log.warn( + "Failed to initialize native S3 connection from endpoint {}: {}", + endpoint, + e.getMessage()); + } + } + + this.baseUrl = base; + this.provider = prov; + this.httpClient = client; + this.asyncClient = async; + } + + /** + * Returns whether native S3 multipart copy is available with the extracted client properties. + * + * @return true if all required fields are accessible; false otherwise + */ + public boolean isAvailable() { + return asyncClient != null && baseUrl != null && httpClient != null; + } + + /** + * Composes the given part keys into the target object key entirely server-side on S3. + * + * @param bucket The S3 bucket name + * @param targetKey The destination object key + * @param partKeys The source part object keys to combine in order + * @throws Exception If composition fails + */ + public void compose(String bucket, String targetKey, List partKeys) throws Exception { + if (!isAvailable()) { + // Fallback for mocked MinioClient instances in tests + List sources = new ArrayList<>(); + for (String pk : partKeys) { + sources.add(SourceObject.builder().bucket(bucket).object(pk).build()); + } + minioClient.composeObject( + ComposeObjectArgs.builder().bucket(bucket).object(targetKey).sources(sources).build()); + return; + } + + String region = baseUrl.region(); + if (region == null || region.isEmpty()) { + region = explicitRegion; + } + if (region == null || region.isEmpty()) { + region = "eu-central-1"; + } + + Credentials credentials = (provider != null) ? provider.fetch() : null; + + // 1. Create Multipart Upload + CreateMultipartUploadArgs createArgs = + CreateMultipartUploadArgs.builder().bucket(bucket).object(targetKey).build(); + CreateMultipartUploadResponse createResponse = + asyncClient.createMultipartUpload(createArgs).join(); + String uploadId = createResponse.result().uploadId(); + log.info( + "Started server-side multipart copy upload (ID: {}) for target {} in bucket {} with {} parts", + uploadId, + targetKey, + bucket, + partKeys.size()); + + List parts = new ArrayList<>(); + try { + // 2. UploadPartCopy for each part chunk without Content-MD5 + for (int i = 0; i < partKeys.size(); i++) { + int partNumber = i + 1; + String partKey = partKeys.get(i); + + Http.QueryParameters queryParams = + new Http.QueryParameters( + "partNumber", Integer.toString(partNumber), "uploadId", uploadId); + HttpUrl url = baseUrl.buildUrl(Http.Method.PUT, bucket, targetKey, region, queryParams); + + // x-amz-copy-source must be URL-encoded "/bucket/key" + String copySource = "/" + bucket + "/" + Utils.encodePath(partKey); + String now = ZonedDateTime.now().format(Time.AMZ_DATE_FORMAT); + + Request.Builder requestBuilder = + new Request.Builder() + .url(url) + .put(RequestBody.create(new byte[0], null)) + .header("Host", Utils.getHostHeader(url)) + .header("x-amz-date", now) + .header("x-amz-copy-source", copySource) + .header("x-amz-content-sha256", Checksum.ZERO_SHA256_HASH); + + if (credentials != null + && credentials.sessionToken() != null + && !credentials.sessionToken().isEmpty()) { + requestBuilder.header("x-amz-security-token", credentials.sessionToken()); + } + + Request httpRequest = requestBuilder.build(); + if (credentials != null) { + httpRequest = + Signer.signV4S3( + httpRequest, + region, + credentials.accessKey(), + credentials.secretKey(), + Checksum.ZERO_SHA256_HASH); + } + + try (Response response = httpClient.newCall(httpRequest).execute()) { + if (!response.isSuccessful()) { + String errorBody = response.body() != null ? response.body().string() : ""; + throw new IOException( + "S3 UploadPartCopy failed for part " + + partNumber + + " (" + + partKey + + "): HTTP " + + response.code() + + " " + + errorBody); + } + String responseBody = response.body() != null ? response.body().string() : ""; + String etag = extractEtagFromXml(responseBody); + if (etag == null || etag.isEmpty()) { + etag = response.header("ETag"); + } + if (etag == null || etag.isEmpty()) { + throw new IOException("Missing ETag in UploadPartCopy response for part " + partNumber); + } + parts.add(new Part(partNumber, etag)); + } + } + + // 3. Complete Multipart Upload + CompleteMultipartUploadArgs completeArgs = + CompleteMultipartUploadArgs.builder() + .bucket(bucket) + .object(targetKey) + .uploadId(uploadId) + .parts(parts.toArray(new Part[0])) + .build(); + asyncClient.completeMultipartUpload(completeArgs).join(); + log.info( + "Successfully completed server-side multipart copy upload (ID: {}) for target {} in bucket {}", + uploadId, + targetKey, + bucket); + + } catch (Exception e) { + try { + asyncClient + .abortMultipartUpload( + AbortMultipartUploadArgs.builder() + .bucket(bucket) + .object(targetKey) + .uploadId(uploadId) + .build()) + .join(); + } catch (Exception abortEx) { + log.warn("Failed to abort multipart upload {} for target {}", uploadId, targetKey, abortEx); + } + throw e; + } + } + + /** + * Extracts the ETag value from an S3 {@code CopyPartResult} XML string. + * + * @param xml The XML response from S3 UploadPartCopy + * @return Extracted ETag string, or null if not found + */ + static String extractEtagFromXml(String xml) { + if (xml == null) { + return null; + } + int start = xml.indexOf(""); + int end = xml.indexOf(""); + if (start != -1 && end != -1 && end > start + 6) { + String etag = xml.substring(start + 6, end).trim(); + etag = etag.replace(""", "\""); + return etag; + } + return null; + } +} diff --git a/src/main/java/me/desair/tus/server/upload/s3/S3StorageService.java b/src/main/java/me/desair/tus/server/upload/s3/S3StorageService.java index 6e9c2502..dc89b998 100644 --- a/src/main/java/me/desair/tus/server/upload/s3/S3StorageService.java +++ b/src/main/java/me/desair/tus/server/upload/s3/S3StorageService.java @@ -1,13 +1,11 @@ package me.desair.tus.server.upload.s3; -import io.minio.ComposeObjectArgs; import io.minio.GetObjectArgs; import io.minio.ListObjectsArgs; import io.minio.MinioClient; import io.minio.PutObjectArgs; import io.minio.RemoveObjectArgs; import io.minio.Result; -import io.minio.SourceObject; import io.minio.StatObjectArgs; import io.minio.StatObjectResponse; import io.minio.errors.ErrorResponseException; @@ -121,6 +119,8 @@ public class S3StorageService implements UploadStorageService { private UploadIdFactory idFactory = new UuidUploadIdFactory(); private UploadConcatenationService concatenationService; + private volatile boolean s3ComposeObjectSupported = true; + private S3ServerSideComposeHelper s3ComposeHelper; /** * Basic constructor using default object key prefixes and standard system temp directory. @@ -139,6 +139,64 @@ public S3StorageService(MinioClient minioClient, String bucket) { Paths.get(System.getProperty("java.io.tmpdir"))); } + /** + * Constructor accepting explicit connection parameters without requiring a pre-existing + * MinioClient. + * + *

Builds both {@link MinioClient} and the native {@link S3ServerSideComposeHelper} directly + * using the provided connection properties, avoiding reflection. + * + * @param endpoint S3 endpoint URL (e.g. "https://s3.amazonaws.com" or "http://localhost:9000") + * @param region S3 region name (e.g. "us-east-1", "eu-central-1") + * @param accessKey S3 access key / username + * @param secretKey S3 secret key / password + * @param bucket S3 bucket name + */ + public S3StorageService( + String endpoint, String region, String accessKey, String secretKey, String bucket) { + this( + buildMinioClient(endpoint, region, accessKey, secretKey), + bucket, + DEFAULT_OBJECT_PREFIX, + DEFAULT_METADATA_PREFIX, + DEFAULT_CHECKSUMS_PREFIX, + DEFAULT_LOCKS_PREFIX, + Paths.get(System.getProperty("java.io.tmpdir")), + new S3ServerSideComposeHelper(null, endpoint, region, accessKey, secretKey)); + } + + /** + * Constructor accepting a pre-configured {@link MinioClient} along with the explicit connection + * parameters required by {@link S3ServerSideComposeHelper}. + * + *

This provides full control over {@link MinioClient} configuration while cleanly initializing + * server-side multipart copy without reflection. + * + * @param minioClient Pre-configured MinIO Client + * @param endpoint S3 endpoint URL (e.g. "https://s3.amazonaws.com" or "http://localhost:9000") + * @param region S3 region name (e.g. "us-east-1", "eu-central-1") + * @param accessKey S3 access key / username + * @param secretKey S3 secret key / password + * @param bucket S3 bucket name + */ + public S3StorageService( + MinioClient minioClient, + String endpoint, + String region, + String accessKey, + String secretKey, + String bucket) { + this( + minioClient, + bucket, + DEFAULT_OBJECT_PREFIX, + DEFAULT_METADATA_PREFIX, + DEFAULT_CHECKSUMS_PREFIX, + DEFAULT_LOCKS_PREFIX, + Paths.get(System.getProperty("java.io.tmpdir")), + new S3ServerSideComposeHelper(minioClient, endpoint, region, accessKey, secretKey)); + } + /** * Full constructor allowing full customization of object prefixes and local disk buffer path. * @@ -158,6 +216,27 @@ public S3StorageService( String checksumsPrefix, String locksPrefix, Path temporaryDirectory) { + this( + minioClient, + bucket, + objectPrefix, + metadataPrefix, + checksumsPrefix, + locksPrefix, + temporaryDirectory, + new S3ServerSideComposeHelper(minioClient)); + } + + /** Internal constructor accepting an initialized {@link S3ServerSideComposeHelper}. */ + private S3StorageService( + MinioClient minioClient, + String bucket, + String objectPrefix, + String metadataPrefix, + String checksumsPrefix, + String locksPrefix, + Path temporaryDirectory, + S3ServerSideComposeHelper s3ComposeHelper) { this.minioClient = Objects.requireNonNull(minioClient, "MinioClient must not be null"); this.bucket = Objects.requireNonNull(bucket, "Bucket must not be null"); this.objectPrefix = sanitizePrefix(objectPrefix); @@ -195,6 +274,31 @@ public S3StorageService( this.concatenationService = new S3ConcatenationService( this.minioClient, this.bucket, this.objectPrefix, this, this.temporaryDirectory); + this.s3ComposeHelper = + s3ComposeHelper != null ? s3ComposeHelper : new S3ServerSideComposeHelper(this.minioClient); + if (this.concatenationService instanceof S3ConcatenationService) { + ((S3ConcatenationService) this.concatenationService) + .setS3ServerSideComposeHelper(this.s3ComposeHelper); + } + } + + private static MinioClient buildMinioClient( + String endpoint, String region, String accessKey, String secretKey) { + String effectiveRegion = (region != null && !region.isEmpty()) ? region : "eu-central-1"; + return MinioClient.builder() + .endpoint(endpoint) + .credentials(accessKey, secretKey) + .region(effectiveRegion) + .build(); + } + + /** + * Returns the underlying {@link MinioClient} configured for this storage service. + * + * @return The MinIO client instance + */ + public MinioClient getMinioClient() { + return this.minioClient; } /** @@ -724,6 +828,22 @@ public UploadConcatenationService getUploadConcatenationService() { return concatenationService; } + boolean isS3ComposeObjectSupported() { + return s3ComposeObjectSupported; + } + + void setS3ComposeObjectSupported(boolean s3ComposeObjectSupported) { + this.s3ComposeObjectSupported = s3ComposeObjectSupported; + } + + void setS3ServerSideComposeHelper(S3ServerSideComposeHelper s3ComposeHelper) { + this.s3ComposeHelper = s3ComposeHelper; + } + + S3ServerSideComposeHelper getS3ServerSideComposeHelper() { + return s3ComposeHelper; + } + @Override public void setIdFactory(UploadIdFactory idFactory) { if (idFactory != null) { @@ -993,7 +1113,7 @@ private AppendResult processPayloadChunks( + MAX_PARTS_PER_UPLOAD + " parts."); if (chunkBytesWritten > 0) { - int confirmedCount = uploader.drainAndComplete(4000); + int confirmedCount = uploader.drainAndComplete(); allPartKeys.addAll(plannedPartKeys.subList(0, confirmedCount)); storeIncompletePartToS3(partObjectKey, tempChunkFile, chunkBytesWritten); handedOff = true; @@ -1020,7 +1140,7 @@ private AppendResult processPayloadChunks( } else if (streamFinished && isUploadComplete) { // Sub-5MB final chunk that completes the overall upload: // First drain all preceding parts in the pipeline - int confirmedCount = uploader.drainAndComplete(4000); + int confirmedCount = uploader.drainAndComplete(); allPartKeys.addAll(plannedPartKeys.subList(0, confirmedCount)); String chunkKey = buildChunkPartKey(id, nextPartNumber++); @@ -1031,7 +1151,7 @@ private AppendResult processPayloadChunks( } else { // Sub-5MB incomplete chunk (e.g. upload paused midway or interrupted): // First drain all preceding parts in the pipeline - int confirmedCount = uploader.drainAndComplete(4000); + int confirmedCount = uploader.drainAndComplete(); allPartKeys.addAll(plannedPartKeys.subList(0, confirmedCount)); storeIncompletePartToS3(partObjectKey, tempChunkFile, chunkBytesWritten); @@ -1045,13 +1165,20 @@ private AppendResult processPayloadChunks( } } - // Drain any remaining in-flight chunks in the pipeline - int confirmedCount = uploader.drainAndComplete(4000); - int previouslyConfirmed = allPartKeys.size() - preparedStream.remainingPartKeys.size(); - if (confirmedCount > previouslyConfirmed) { - allPartKeys.clear(); - allPartKeys.addAll(preparedStream.remainingPartKeys); - allPartKeys.addAll(plannedPartKeys.subList(0, confirmedCount)); + // Drain any remaining in-flight chunks in the pipeline with 60s timeout. + // Catch/finally ensures all confirmed parts are retained even if a subsequent chunk times + // out. + int confirmedCount = 0; + try { + confirmedCount = uploader.drainAndComplete(); + } finally { + int confirmed = Math.max(confirmedCount, uploader.getConfirmedCount()); + int previouslyConfirmed = allPartKeys.size() - preparedStream.remainingPartKeys.size(); + if (confirmed > previouslyConfirmed) { + allPartKeys.clear(); + allPartKeys.addAll(preparedStream.remainingPartKeys); + allPartKeys.addAll(plannedPartKeys.subList(0, confirmed)); + } } } @@ -1195,32 +1322,33 @@ private void finalizeCompletedUploadIfFinished( } } - if (canUseServerSideCompose) { + if (canUseServerSideCompose && s3ComposeObjectSupported) { try { - List sources = new ArrayList<>(); - for (String pk : partKeys) { - sources.add(SourceObject.builder().bucket(bucket).object(pk).build()); - } - - // Perform S3 server-side object composition (composeObject) - minioClient.composeObject( - ComposeObjectArgs.builder() - .bucket(bucket) - .object(objectKey) - .sources(sources) - .build()); + // Perform S3 server-side object composition via native S3 multipart copy + s3ComposeHelper.compose(bucket, objectKey, partKeys); } catch (Exception e) { - log.warn( - "S3 composeObject failed for object {}, falling back to streaming concatenation:" - + " {}", + // MinIO Java SDK's composeObject implementation delegates to UploadPartCopy with an + // EMPTY_BODY, which automatically attaches Content-MD5 and Content-Type headers. + // AWS S3 strictly forbids Content-MD5 on UploadPartCopy and rejects it with 400 + // InvalidArgument ("The specified header is not valid in this context"). + // When server-side compose fails on this S3 endpoint, disable it dynamically + // to avoid redundant failing S3 API roundtrips on subsequent uploads, log at INFO, + // and seamlessly merge chunks via streaming part composition. + s3ComposeObjectSupported = false; + log.info( + "S3 server-side compose failed for object {}, falling back to streaming part" + + " composition: {}", objectKey, e.getMessage()); mergeUsingStreamingReupload(objectKey, partKeys, newOffset); } } else { - log.info( - "Detected sub-5MB non-final parts for ID {}. Using streaming concatenation fallback.", - id); + if (!canUseServerSideCompose) { + log.info( + "Detected sub-5MB non-final parts for ID {}. Using streaming part composition" + + " fallback.", + id); + } mergeUsingStreamingReupload(objectKey, partKeys, newOffset); } diff --git a/src/main/java/me/desair/tus/server/upload/util/AsyncChunkUploader.java b/src/main/java/me/desair/tus/server/upload/util/AsyncChunkUploader.java index f523e40c..c091bc1f 100644 --- a/src/main/java/me/desair/tus/server/upload/util/AsyncChunkUploader.java +++ b/src/main/java/me/desair/tus/server/upload/util/AsyncChunkUploader.java @@ -35,6 +35,8 @@ public class AsyncChunkUploader implements AutoCloseable { private static final Logger log = LoggerFactory.getLogger(AsyncChunkUploader.class); + public static final long DEFAULT_DRAIN_TIMEOUT_MS = 300_000L; + @FunctionalInterface public interface ChunkUploadAction { /** @@ -141,6 +143,17 @@ public void submitChunk( } } + /** + * Drains all remaining chunks in the pipeline (Slot 3 and Slot 2) using the default timeout + * ({@link #DEFAULT_DRAIN_TIMEOUT_MS}, 60 seconds). + * + * @return The total number of confirmed successfully uploaded chunks + * @throws IOException If any chunk upload fails or times out + */ + public int drainAndComplete() throws IOException { + return drainAndComplete(DEFAULT_DRAIN_TIMEOUT_MS); + } + /** * Drains all remaining chunks in the pipeline (Slot 3 and Slot 2) up to the specified timeout. * diff --git a/src/test/java/me/desair/tus/server/core/CorePatchRequestHandlerTest.java b/src/test/java/me/desair/tus/server/core/CorePatchRequestHandlerTest.java index 05576a70..1656ae53 100644 --- a/src/test/java/me/desair/tus/server/core/CorePatchRequestHandlerTest.java +++ b/src/test/java/me/desair/tus/server/core/CorePatchRequestHandlerTest.java @@ -13,6 +13,7 @@ import static org.mockito.Mockito.when; import jakarta.servlet.http.HttpServletResponse; +import java.io.IOException; import java.io.InputStream; import java.util.UUID; import me.desair.tus.server.HttpHeader; @@ -226,4 +227,77 @@ public void testProcess7ParamsBranchCoverage() throws Exception { "owner", null); } + + @Test + public void testProcessInterruptedByLockingServiceContentionReturns204WithUpdatedOffset() + throws Exception { + UploadLockingService mockLocking = mock(UploadLockingService.class); + + servletRequest.setRequestURI("/test/upload/123"); + servletRequest.setContent("partial data".getBytes()); + + UploadInfo initialInfo = new UploadInfo(); + initialInfo.setId(new UploadId("123")); + initialInfo.setOffset(0L); + initialInfo.setLength(100L); + + UploadInfo refreshedInfo = new UploadInfo(); + refreshedInfo.setId(new UploadId("123")); + refreshedInfo.setOffset(50L); + refreshedInfo.setLength(100L); + + when(uploadStorageService.getUploadInfo("/test/upload/123", "owner")) + .thenReturn(initialInfo) + .thenReturn(refreshedInfo); + + when(uploadStorageService.append(any(UploadInfo.class), any(InputStream.class))) + .thenAnswer( + invocation -> { + InputStream stream = invocation.getArgument(1); + if (stream instanceof InterruptibleInputStream) { + ((InterruptibleInputStream) stream).interrupt(); + } + throw new IOException( + "Stream was interrupted by the upload locking service watchdog"); + }); + + handler.process( + HttpMethod.PATCH, + new TusServletRequest(servletRequest), + new TusServletResponse(servletResponse), + uploadStorageService, + mockLocking, + "owner", + null); + + assertThat(servletResponse.getStatus(), is(HttpServletResponse.SC_NO_CONTENT)); + assertThat(servletResponse.getHeader(HttpHeader.UPLOAD_OFFSET), is("50")); + } + + @Test(expected = IOException.class) + public void testProcessUninterruptedIoExceptionRethrown() throws Exception { + UploadLockingService mockLocking = mock(UploadLockingService.class); + + servletRequest.setRequestURI("/test/upload/123"); + servletRequest.setContent("data".getBytes()); + + UploadInfo initialInfo = new UploadInfo(); + initialInfo.setId(new UploadId("123")); + initialInfo.setOffset(0L); + initialInfo.setLength(100L); + + when(uploadStorageService.getUploadInfo("/test/upload/123", "owner")).thenReturn(initialInfo); + + when(uploadStorageService.append(any(UploadInfo.class), any(InputStream.class))) + .thenThrow(new IOException("Disk failure")); + + handler.process( + HttpMethod.PATCH, + new TusServletRequest(servletRequest), + new TusServletResponse(servletResponse), + uploadStorageService, + mockLocking, + "owner", + null); + } } diff --git a/src/test/java/me/desair/tus/server/creationwithupload/CreationWithUploadPostRequestHandlerTest.java b/src/test/java/me/desair/tus/server/creationwithupload/CreationWithUploadPostRequestHandlerTest.java index 9001c2c0..bf7848a9 100644 --- a/src/test/java/me/desair/tus/server/creationwithupload/CreationWithUploadPostRequestHandlerTest.java +++ b/src/test/java/me/desair/tus/server/creationwithupload/CreationWithUploadPostRequestHandlerTest.java @@ -10,6 +10,7 @@ import static org.mockito.Mockito.when; import java.io.ByteArrayInputStream; +import java.io.IOException; import java.io.InputStream; import me.desair.tus.server.HttpHeader; import me.desair.tus.server.HttpMethod; @@ -95,4 +96,67 @@ public void testProcessWithoutLockingService() throws Exception { verify(response).setHeader(HttpHeader.UPLOAD_OFFSET, "5"); } + + @Test + public void testProcessInterruptedByLockingServiceContention() throws Exception { + TusServletRequest request = mock(TusServletRequest.class); + TusServletResponse response = mock(TusServletResponse.class); + UploadStorageService storageService = mock(UploadStorageService.class); + UploadLockingService lockingService = mock(UploadLockingService.class); + + when(request.getHeader(HttpHeader.CONTENT_LENGTH)).thenReturn("100"); + when(response.getHeader(HttpHeader.LOCATION)).thenReturn("/files/interrupted-cwu"); + when(request.getContentInputStream()).thenReturn(new ByteArrayInputStream("data".getBytes())); + + UploadInfo uploadInfo = new UploadInfo(); + uploadInfo.setLength(100L); + uploadInfo.setOffset(0L); + + UploadInfo refreshed = new UploadInfo(); + refreshed.setLength(100L); + refreshed.setOffset(50L); + + when(storageService.getUploadInfo("/files/interrupted-cwu", "owner")) + .thenReturn(uploadInfo) + .thenReturn(refreshed); + + when(storageService.append(eq(uploadInfo), any(InputStream.class))) + .thenAnswer( + invocation -> { + Object stream = invocation.getArgument(1); + if (stream instanceof InterruptibleInputStream) { + ((InterruptibleInputStream) stream).interrupt(); + } + throw new IOException( + "Stream was interrupted by the upload locking service watchdog"); + }); + + handler.process( + HttpMethod.POST, request, response, storageService, lockingService, "owner", null); + + verify(response).setHeader(HttpHeader.UPLOAD_OFFSET, "50"); + } + + @Test(expected = IOException.class) + public void testProcessUninterruptedIoExceptionRethrown() throws Exception { + TusServletRequest request = mock(TusServletRequest.class); + TusServletResponse response = mock(TusServletResponse.class); + UploadStorageService storageService = mock(UploadStorageService.class); + UploadLockingService lockingService = mock(UploadLockingService.class); + + when(request.getHeader(HttpHeader.CONTENT_LENGTH)).thenReturn("100"); + when(response.getHeader(HttpHeader.LOCATION)).thenReturn("/files/cwu-error"); + when(request.getContentInputStream()).thenReturn(new ByteArrayInputStream("data".getBytes())); + + UploadInfo uploadInfo = new UploadInfo(); + uploadInfo.setLength(100L); + uploadInfo.setOffset(0L); + + when(storageService.getUploadInfo("/files/cwu-error", "owner")).thenReturn(uploadInfo); + when(storageService.append(eq(uploadInfo), any(InputStream.class))) + .thenThrow(new IOException("S3 network error")); + + handler.process( + HttpMethod.POST, request, response, storageService, lockingService, "owner", null); + } } diff --git a/src/test/java/me/desair/tus/server/rufh/handler/RufhAppendPatchRequestHandlerTest.java b/src/test/java/me/desair/tus/server/rufh/handler/RufhAppendPatchRequestHandlerTest.java index 609f1a96..ff2fd00e 100644 --- a/src/test/java/me/desair/tus/server/rufh/handler/RufhAppendPatchRequestHandlerTest.java +++ b/src/test/java/me/desair/tus/server/rufh/handler/RufhAppendPatchRequestHandlerTest.java @@ -10,6 +10,7 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import java.io.IOException; import me.desair.tus.server.HttpHeader; import me.desair.tus.server.HttpMethod; import me.desair.tus.server.upload.UploadId; @@ -305,4 +306,82 @@ public void testProcessCompletingAppendOnDeferredLengthUploadWithNullOffsetSetsL assertThat(response.getHeader(HttpHeader.UPLOAD_OFFSET), is(String.valueOf(content.length))); assertThat(response.getHeader(HttpHeader.UPLOAD_COMPLETE), is("?1")); } + + @Test + public void testProcessInterruptedByLockingServiceContention() throws Exception { + request.setMethod("PATCH"); + request.setRequestURI("/files/append-id"); + request.addHeader(HttpHeader.CONTENT_TYPE, HttpHeader.CONTENT_TYPE_PARTIAL_UPLOAD); + request.addHeader(HttpHeader.UPLOAD_OFFSET, "0"); + request.addHeader(HttpHeader.UPLOAD_COMPLETE, "?0"); + byte[] content = "partial-data".getBytes(); + request.setContent(content); + + UploadInfo info = new UploadInfo(); + info.setId(new UploadId("append-id")); + info.setOffset(0L); + info.setLength(1000L); + + UploadInfo refreshed = new UploadInfo(); + refreshed.setId(new UploadId("append-id")); + refreshed.setOffset(500L); + refreshed.setLength(1000L); + + when(storageService.getUploadInfo("/files/append-id", "owner")) + .thenReturn(info) + .thenReturn(refreshed); + + when(storageService.append(any(UploadInfo.class), any())) + .thenAnswer( + invocation -> { + Object stream = invocation.getArgument(1); + if (stream instanceof InterruptibleInputStream) { + ((InterruptibleInputStream) stream).interrupt(); + } + throw new IOException( + "Stream was interrupted by the upload locking service watchdog"); + }); + + handler.process( + HttpMethod.PATCH, + new TusServletRequest(request), + new TusServletResponse(response), + storageService, + lockingService, + "owner", + null); + + assertThat(response.getStatus(), is(204)); + assertThat(response.getHeader(HttpHeader.UPLOAD_OFFSET), is("500")); + assertThat(response.getHeader(HttpHeader.UPLOAD_COMPLETE), is("?0")); + } + + @Test(expected = IOException.class) + public void testProcessUninterruptedIoExceptionRethrown() throws Exception { + request.setMethod("PATCH"); + request.setRequestURI("/files/append-id"); + request.addHeader(HttpHeader.CONTENT_TYPE, HttpHeader.CONTENT_TYPE_PARTIAL_UPLOAD); + request.addHeader(HttpHeader.UPLOAD_OFFSET, "0"); + request.addHeader(HttpHeader.UPLOAD_COMPLETE, "?0"); + byte[] content = "data".getBytes(); + request.setContent(content); + + UploadInfo info = new UploadInfo(); + info.setId(new UploadId("append-id")); + info.setOffset(0L); + info.setLength(1000L); + + when(storageService.getUploadInfo("/files/append-id", "owner")).thenReturn(info); + when(storageService.append(any(UploadInfo.class), any())) + .thenThrow(new IOException("Storage failure")); + + handler.process( + HttpMethod.PATCH, + new TusServletRequest(request), + new TusServletResponse(response), + storageService, + lockingService, + "owner", + null); + } } diff --git a/src/test/java/me/desair/tus/server/rufh/handler/RufhCreationPostRequestHandlerTest.java b/src/test/java/me/desair/tus/server/rufh/handler/RufhCreationPostRequestHandlerTest.java index e4e9b8c4..3863559f 100644 --- a/src/test/java/me/desair/tus/server/rufh/handler/RufhCreationPostRequestHandlerTest.java +++ b/src/test/java/me/desair/tus/server/rufh/handler/RufhCreationPostRequestHandlerTest.java @@ -11,6 +11,7 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import java.io.IOException; import java.io.InputStream; import me.desair.tus.server.HttpHeader; import me.desair.tus.server.HttpMethod; @@ -567,4 +568,81 @@ public InputStream getContentInputStream() { assertThat(response.getHeader(HttpHeader.UPLOAD_OFFSET), is(String.valueOf(content.length))); assertThat(response.getHeader(HttpHeader.UPLOAD_COMPLETE), is("?0")); } + + @Test + public void testProcessCreationWithBodyInterrupted() throws Exception { + byte[] content = "partial-creation-data".getBytes(); + request.setMethod("POST"); + request.setRequestURI("/files"); + request.addHeader(HttpHeader.UPLOAD_LENGTH, "1000"); + request.setContent(content); + + UploadInfo createdInfo = new UploadInfo(); + createdInfo.setId(new UploadId("interrupted-create-id")); + createdInfo.setOffset(0L); + createdInfo.setLength(1000L); + + UploadInfo refreshedInfo = new UploadInfo(); + refreshedInfo.setId(new UploadId("interrupted-create-id")); + refreshedInfo.setOffset(500L); + refreshedInfo.setLength(1000L); + + when(storageService.create(any(UploadInfo.class), nullable(String.class))) + .thenReturn(createdInfo); + when(storageService.getUploadInfo("/files/interrupted-create-id", "owner")) + .thenReturn(refreshedInfo); + + when(storageService.append(any(UploadInfo.class), any())) + .thenAnswer( + invocation -> { + Object stream = invocation.getArgument(1); + if (stream instanceof InterruptibleInputStream) { + ((InterruptibleInputStream) stream).interrupt(); + } + throw new IOException( + "Stream was interrupted by the upload locking service watchdog"); + }); + + handler.process( + HttpMethod.POST, + new TusServletRequest(request), + new TusServletResponse(response), + storageService, + lockingService, + "owner", + null); + + assertThat(response.getStatus(), is(201)); + assertThat(response.getHeader(HttpHeader.UPLOAD_OFFSET), is("500")); + assertThat(response.getHeader(HttpHeader.UPLOAD_COMPLETE), is("?0")); + } + + @Test(expected = IOException.class) + public void testProcessCreationWithBodyUninterruptedIoException() throws Exception { + byte[] content = "creation-data".getBytes(); + request.setMethod("POST"); + request.setRequestURI("/files"); + request.addHeader(HttpHeader.UPLOAD_LENGTH, "1000"); + request.setContent(content); + + UploadInfo createdInfo = new UploadInfo(); + createdInfo.setId(new UploadId("fail-create-id")); + createdInfo.setOffset(0L); + createdInfo.setLength(1000L); + + when(storageService.create(any(UploadInfo.class), nullable(String.class))) + .thenReturn(createdInfo); + + when(storageService.append(any(UploadInfo.class), any())) + .thenThrow(new IOException("Disk write failure")); + + handler.process( + HttpMethod.POST, + new TusServletRequest(request), + new TusServletResponse(response), + storageService, + lockingService, + "owner", + null); + } } diff --git a/src/test/java/me/desair/tus/server/upload/azure/ITAzureBlobStorageService.java b/src/test/java/me/desair/tus/server/upload/azure/ITAzureBlobStorageService.java index 44505494..1273e67e 100644 --- a/src/test/java/me/desair/tus/server/upload/azure/ITAzureBlobStorageService.java +++ b/src/test/java/me/desair/tus/server/upload/azure/ITAzureBlobStorageService.java @@ -258,6 +258,40 @@ public void terminateUploadShouldDeleteBlobs() throws Exception { assertNull(storageService.getUploadInfo(created.getId())); } + @Test + public void terminateUploadShouldHandleLeasedAndUnleasedLockBlobs() throws Exception { + UploadInfo info = new UploadInfo(); + info.setLength(10L); + UploadInfo created = storageService.create(info, "owner1"); + + storageService.append(created, new ByteArrayInputStream("0123456789".getBytes())); + assertNotNull(storageService.getUploadInfo(created.getId())); + + // 1. Verify unleased lock blob deletion: create an unleased lock blob and verify it is removed + BlobClient lockBlob = containerClient.getBlobClient("locks/" + created.getId() + ".lock"); + lockBlob.upload(BinaryData.fromBytes("lock".getBytes(StandardCharsets.UTF_8)), true); + assertTrue(Boolean.TRUE.equals(lockBlob.exists())); + + storageService.terminateUpload(created); + assertNull(storageService.getUploadInfo(created.getId())); + assertFalse(Boolean.TRUE.equals(lockBlob.exists())); + + // 2. Verify leased lock blob: create upload, hold active lease, and verify terminateUpload + // skips deletion so Azure HTTP 412 (LeaseIdMissing) error is never triggered or logged + UploadInfo info2 = new UploadInfo(); + info2.setLength(10L); + UploadInfo created2 = storageService.create(info2, "owner1"); + + AzureBlobLockingService lockingService = new AzureBlobLockingService(containerClient); + UploadLock lock = lockingService.lockUploadByUri("/test/upload/" + created2.getId()); + try { + storageService.terminateUpload(created2); + assertNull(storageService.getUploadInfo(created2.getId())); + } finally { + lock.release(); + } + } + @Test public void removeLastNumberOfBytesPartBlobOnly() throws Exception { storageService.setPreferredBlockSize(4L * 1024 * 1024); diff --git a/src/test/java/me/desair/tus/server/upload/s3/S3ConcatenationServiceTest.java b/src/test/java/me/desair/tus/server/upload/s3/S3ConcatenationServiceTest.java index f9970131..7f3d3470 100644 --- a/src/test/java/me/desair/tus/server/upload/s3/S3ConcatenationServiceTest.java +++ b/src/test/java/me/desair/tus/server/upload/s3/S3ConcatenationServiceTest.java @@ -263,8 +263,8 @@ public void testMergeMultiplePartsWithSub5MbNonFinalPartFallsBackToStreaming() t Mockito.verify(minioClient).putObject(Mockito.any(PutObjectArgs.class)); } - @Test(expected = IOException.class) - public void testMergeServerSideCopyFails() throws Exception { + @Test + public void testMergeServerSideCopyFailsFallsBackToStreaming() throws Exception { UploadInfo p1 = new UploadInfo(); p1.setId(new UploadId("part-1")); p1.setOwnerKey("owner-1"); @@ -273,6 +273,8 @@ public void testMergeServerSideCopyFails() throws Exception { p1.setStorageUploadId("uploads/part-1"); Mockito.when(storageService.getUploadInfo("/part-1", "owner-1")).thenReturn(p1); + Mockito.when(storageService.getUploadedBytes(new UploadId("part-1"))) + .thenReturn(new ByteArrayInputStream(new byte[10])); Mockito.when(minioClient.composeObject(Mockito.any(ComposeObjectArgs.class))) .thenThrow(new RuntimeException("Compose error")); @@ -282,6 +284,10 @@ public void testMergeServerSideCopyFails() throws Exception { finalUpload.setConcatenationPartIds(Arrays.asList("/part-1")); concatenationService.merge(finalUpload); + + // Verify fallback to streaming putObject + Mockito.verify(minioClient).putObject(Mockito.any(PutObjectArgs.class)); + assertEquals(Long.valueOf(10L * 1024 * 1024), finalUpload.getLength()); } @Test(expected = IOException.class) diff --git a/src/test/java/me/desair/tus/server/upload/s3/S3LockingServiceTest.java b/src/test/java/me/desair/tus/server/upload/s3/S3LockingServiceTest.java index 832220d6..cff254f3 100644 --- a/src/test/java/me/desair/tus/server/upload/s3/S3LockingServiceTest.java +++ b/src/test/java/me/desair/tus/server/upload/s3/S3LockingServiceTest.java @@ -137,6 +137,24 @@ public void testLockUploadByUriInvalidUri() throws Exception { assertNull(lock); } + @Test + public void testExplicitConnectionParametersConstructors() { + S3LockingService serviceWithParams = + new S3LockingService( + "https://s3.amazonaws.com", "eu-central-1", "accessKey", "secretKey", "test-bucket"); + assertNotNull(serviceWithParams); + + S3LockingService serviceWithClientAndParams = + new S3LockingService( + minioClient, + "https://s3.amazonaws.com", + "eu-central-1", + "accessKey", + "secretKey", + "test-bucket"); + assertNotNull(serviceWithClientAndParams); + } + @Test public void testIsLocked() throws Exception { assertFalse(lockingService.isLocked((UploadId) null)); diff --git a/src/test/java/me/desair/tus/server/upload/s3/S3ServerSideComposeHelperTest.java b/src/test/java/me/desair/tus/server/upload/s3/S3ServerSideComposeHelperTest.java new file mode 100644 index 00000000..c0afd7ae --- /dev/null +++ b/src/test/java/me/desair/tus/server/upload/s3/S3ServerSideComposeHelperTest.java @@ -0,0 +1,362 @@ +package me.desair.tus.server.upload.s3; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import io.minio.AbortMultipartUploadArgs; +import io.minio.CompleteMultipartUploadArgs; +import io.minio.ComposeObjectArgs; +import io.minio.CreateMultipartUploadArgs; +import io.minio.CreateMultipartUploadResponse; +import io.minio.Http; +import io.minio.MinioAsyncClient; +import io.minio.MinioClient; +import io.minio.ObjectWriteResponse; +import io.minio.credentials.Credentials; +import io.minio.credentials.Provider; +import io.minio.messages.InitiateMultipartUploadResult; +import java.io.IOException; +import java.lang.reflect.Field; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import okhttp3.Call; +import okhttp3.HttpUrl; +import okhttp3.MediaType; +import okhttp3.OkHttpClient; +import okhttp3.Protocol; +import okhttp3.Request; +import okhttp3.Response; +import okhttp3.ResponseBody; +import org.junit.Test; + +/** Unit tests for {@link S3ServerSideComposeHelper}. */ +public class S3ServerSideComposeHelperTest { + + @Test + public void testExtractEtagFromXml() { + // Standard XML response + String xml = + "" + + "2026-10-03T12:00:00.000Z" + + "\"1234567890abcdef\"" + + ""; + assertEquals("\"1234567890abcdef\"", S3ServerSideComposeHelper.extractEtagFromXml(xml)); + + // Entity escaped XML quotes + String escapedXml = + ""9876543210fedcba""; + assertEquals("\"9876543210fedcba\"", S3ServerSideComposeHelper.extractEtagFromXml(escapedXml)); + + // Whitespace inside tag + String whitespaceXml = " \"xyz123\" "; + assertEquals("\"xyz123\"", S3ServerSideComposeHelper.extractEtagFromXml(whitespaceXml)); + + // Null or missing tags + assertNull(S3ServerSideComposeHelper.extractEtagFromXml(null)); + assertNull(S3ServerSideComposeHelper.extractEtagFromXml("")); + assertNull(S3ServerSideComposeHelper.extractEtagFromXml("")); + assertNull(S3ServerSideComposeHelper.extractEtagFromXml("")); + } + + @Test + public void testFallbackWhenNotAvailable() throws Exception { + MinioClient mockMinioClient = mock(MinioClient.class); + // Since mock MinioClient does not have initialized asyncClient/baseUrl/httpClient fields, + // isAvailable() will return false, triggering the composeObject() fallback. + S3ServerSideComposeHelper helper = new S3ServerSideComposeHelper(mockMinioClient); + assertFalse(helper.isAvailable()); + + List partKeys = Arrays.asList("upload/part-1", "upload/part-2"); + helper.compose("my-bucket", "upload/target", partKeys); + + verify(mockMinioClient).composeObject(any(ComposeObjectArgs.class)); + } + + @Test + public void testNullMinioClient() { + S3ServerSideComposeHelper helper = new S3ServerSideComposeHelper(null); + assertFalse(helper.isAvailable()); + } + + @Test + public void testNativeComposeSuccess() throws Exception { + MinioClient minioClient = mock(MinioClient.class); + MinioAsyncClient asyncClient = mock(MinioAsyncClient.class); + OkHttpClient httpClient = mock(OkHttpClient.class); + Provider provider = mock(Provider.class); + + Credentials creds = new Credentials("testAccessKey", "testSecretKey", "testSessionToken", null); + when(provider.fetch()).thenReturn(creds); + + Http.BaseUrl baseUrl = new Http.BaseUrl(HttpUrl.parse("https://s3.us-west-2.amazonaws.com")); + + // Prepare createMultipartUpload response + InitiateMultipartUploadResult initResult = mock(InitiateMultipartUploadResult.class); + when(initResult.uploadId()).thenReturn("mock-upload-id"); + CreateMultipartUploadResponse createResponse = + new CreateMultipartUploadResponse( + null, "my-bucket", "us-west-2", "upload/target", initResult); + when(asyncClient.createMultipartUpload(any(CreateMultipartUploadArgs.class))) + .thenReturn(CompletableFuture.completedFuture(createResponse)); + + // Prepare completeMultipartUpload response + ObjectWriteResponse completeResponse = + new ObjectWriteResponse( + null, "my-bucket", "us-west-2", "upload/target", "etag-final", "v1"); + when(asyncClient.completeMultipartUpload(any(CompleteMultipartUploadArgs.class))) + .thenReturn(CompletableFuture.completedFuture(completeResponse)); + + // Mock OkHttp execution for uploadPartCopy + Call call = mock(Call.class); + when(httpClient.newCall(any(Request.class))).thenReturn(call); + + String partXml = "\"part-etag-1\""; + ResponseBody body = ResponseBody.create(partXml, MediaType.parse("application/xml")); + Response response = + new Response.Builder() + .request(new Request.Builder().url("https://s3.us-west-2.amazonaws.com").build()) + .protocol(Protocol.HTTP_1_1) + .code(200) + .message("OK") + .body(body) + .build(); + when(call.execute()).thenReturn(response); + + // Assemble helper with reflection fields populated + S3ServerSideComposeHelper helper = new S3ServerSideComposeHelper(minioClient); + setField(helper, "asyncClient", asyncClient); + setField(helper, "baseUrl", baseUrl); + setField(helper, "httpClient", httpClient); + setField(helper, "provider", provider); + + assertTrue(helper.isAvailable()); + + List partKeys = Collections.singletonList("upload/part-1"); + helper.compose("my-bucket", "upload/target", partKeys); + + verify(asyncClient).createMultipartUpload(any(CreateMultipartUploadArgs.class)); + verify(httpClient).newCall(any(Request.class)); + verify(asyncClient).completeMultipartUpload(any(CompleteMultipartUploadArgs.class)); + } + + @Test + public void testNativeComposeHttpErrorAborts() throws Exception { + MinioClient minioClient = mock(MinioClient.class); + MinioAsyncClient asyncClient = mock(MinioAsyncClient.class); + OkHttpClient httpClient = mock(OkHttpClient.class); + Provider provider = mock(Provider.class); + + Credentials creds = new Credentials("testAccessKey", "testSecretKey", null, null); + when(provider.fetch()).thenReturn(creds); + + Http.BaseUrl baseUrl = new Http.BaseUrl(HttpUrl.parse("https://s3.amazonaws.com")); + + // Prepare createMultipartUpload response + InitiateMultipartUploadResult initResult = mock(InitiateMultipartUploadResult.class); + when(initResult.uploadId()).thenReturn("mock-upload-id"); + CreateMultipartUploadResponse createResponse = + new CreateMultipartUploadResponse(null, "my-bucket", "", "upload/target", initResult); + when(asyncClient.createMultipartUpload(any(CreateMultipartUploadArgs.class))) + .thenReturn(CompletableFuture.completedFuture(createResponse)); + + when(asyncClient.abortMultipartUpload(any(AbortMultipartUploadArgs.class))) + .thenReturn(CompletableFuture.completedFuture(null)); + + // Mock OkHttp returning HTTP 400 + Call call = mock(Call.class); + when(httpClient.newCall(any(Request.class))).thenReturn(call); + + ResponseBody body = + ResponseBody.create( + "Test Error", MediaType.parse("application/xml")); + Response response = + new Response.Builder() + .request(new Request.Builder().url("https://s3.amazonaws.com").build()) + .protocol(Protocol.HTTP_1_1) + .code(400) + .message("Bad Request") + .body(body) + .build(); + when(call.execute()).thenReturn(response); + + S3ServerSideComposeHelper helper = new S3ServerSideComposeHelper(minioClient); + setField(helper, "asyncClient", asyncClient); + setField(helper, "baseUrl", baseUrl); + setField(helper, "httpClient", httpClient); + setField(helper, "provider", provider); + + try { + helper.compose("my-bucket", "upload/target", Collections.singletonList("upload/part-1")); + fail("Expected IOException to be thrown"); + } catch (IOException e) { + assertTrue(e.getMessage().contains("HTTP 400")); + } + + verify(asyncClient).abortMultipartUpload(any(AbortMultipartUploadArgs.class)); + } + + @Test + public void testNativeComposeMissingEtagHeaderFallback() throws Exception { + MinioClient minioClient = mock(MinioClient.class); + MinioAsyncClient asyncClient = mock(MinioAsyncClient.class); + OkHttpClient httpClient = mock(OkHttpClient.class); + + Http.BaseUrl baseUrl = new Http.BaseUrl(HttpUrl.parse("https://s3.amazonaws.com")); + + InitiateMultipartUploadResult initResult = mock(InitiateMultipartUploadResult.class); + when(initResult.uploadId()).thenReturn("mock-upload-id"); + CreateMultipartUploadResponse createResponse = + new CreateMultipartUploadResponse(null, "my-bucket", "", "upload/target", initResult); + when(asyncClient.createMultipartUpload(any(CreateMultipartUploadArgs.class))) + .thenReturn(CompletableFuture.completedFuture(createResponse)); + + ObjectWriteResponse completeResponse = + new ObjectWriteResponse(null, "my-bucket", "", "upload/target", "etag-final", "v1"); + when(asyncClient.completeMultipartUpload(any(CompleteMultipartUploadArgs.class))) + .thenReturn(CompletableFuture.completedFuture(completeResponse)); + + Call call = mock(Call.class); + when(httpClient.newCall(any(Request.class))).thenReturn(call); + + // Response body has no XML ETag tag, but response header has ETag + ResponseBody body = ResponseBody.create("", MediaType.parse("application/xml")); + Response response = + new Response.Builder() + .request(new Request.Builder().url("https://s3.amazonaws.com").build()) + .protocol(Protocol.HTTP_1_1) + .code(200) + .message("OK") + .header("ETag", "\"header-etag\"") + .body(body) + .build(); + when(call.execute()).thenReturn(response); + + S3ServerSideComposeHelper helper = new S3ServerSideComposeHelper(minioClient); + setField(helper, "asyncClient", asyncClient); + setField(helper, "baseUrl", baseUrl); + setField(helper, "httpClient", httpClient); + + helper.compose("my-bucket", "upload/target", Collections.singletonList("upload/part-1")); + + verify(asyncClient).completeMultipartUpload(any(CompleteMultipartUploadArgs.class)); + } + + @Test + public void testExplicitConnectionParametersConstructor() { + MinioClient realClient = + MinioClient.builder() + .endpoint("https://s3.amazonaws.com") + .credentials("testKey", "testSecret") + .region("us-east-1") + .build(); + + S3ServerSideComposeHelper helper = + new S3ServerSideComposeHelper( + realClient, "https://s3.amazonaws.com", "us-east-1", "testKey", "testSecret"); + assertTrue(helper.isAvailable()); + + S3ServerSideComposeHelper minioOnlyHelper = new S3ServerSideComposeHelper(realClient); + assertFalse(minioOnlyHelper.isAvailable()); + } + + @Test(expected = IOException.class) + public void testNativeComposeMissingEtagThrowsException() throws Exception { + MinioClient minioClient = mock(MinioClient.class); + MinioAsyncClient asyncClient = mock(MinioAsyncClient.class); + OkHttpClient httpClient = mock(OkHttpClient.class); + + Http.BaseUrl baseUrl = new Http.BaseUrl(HttpUrl.parse("https://s3.amazonaws.com")); + + InitiateMultipartUploadResult initResult = mock(InitiateMultipartUploadResult.class); + when(initResult.uploadId()).thenReturn("mock-upload-id"); + CreateMultipartUploadResponse createResponse = + new CreateMultipartUploadResponse(null, "my-bucket", "", "upload/target", initResult); + when(asyncClient.createMultipartUpload(any(CreateMultipartUploadArgs.class))) + .thenReturn(CompletableFuture.completedFuture(createResponse)); + + Call call = mock(Call.class); + when(httpClient.newCall(any(Request.class))).thenReturn(call); + + // Response body has no XML ETag tag, and response header also has no ETag + ResponseBody body = ResponseBody.create("", MediaType.parse("application/xml")); + Response response = + new Response.Builder() + .request(new Request.Builder().url("https://s3.amazonaws.com").build()) + .protocol(Protocol.HTTP_1_1) + .code(200) + .message("OK") + .body(body) + .build(); + when(call.execute()).thenReturn(response); + + S3ServerSideComposeHelper helper = new S3ServerSideComposeHelper(minioClient); + setField(helper, "asyncClient", asyncClient); + setField(helper, "baseUrl", baseUrl); + setField(helper, "httpClient", httpClient); + + helper.compose("my-bucket", "upload/target", Collections.singletonList("upload/part-1")); + } + + @Test + public void testNativeComposeAbortFailureHandledQuietly() throws Exception { + MinioClient minioClient = mock(MinioClient.class); + MinioAsyncClient asyncClient = mock(MinioAsyncClient.class); + OkHttpClient httpClient = mock(OkHttpClient.class); + + Http.BaseUrl baseUrl = new Http.BaseUrl(HttpUrl.parse("https://s3.amazonaws.com")); + + InitiateMultipartUploadResult initResult = mock(InitiateMultipartUploadResult.class); + when(initResult.uploadId()).thenReturn("mock-upload-id"); + CreateMultipartUploadResponse createResponse = + new CreateMultipartUploadResponse(null, "my-bucket", "", "upload/target", initResult); + when(asyncClient.createMultipartUpload(any(CreateMultipartUploadArgs.class))) + .thenReturn(CompletableFuture.completedFuture(createResponse)); + + // Abort itself throws an exception to test the log.warn catch block + CompletableFuture failedAbort = + new CompletableFuture<>(); + failedAbort.completeExceptionally(new RuntimeException("Abort S3 error")); + when(asyncClient.abortMultipartUpload(any(AbortMultipartUploadArgs.class))) + .thenReturn(failedAbort); + + Call call = mock(Call.class); + when(httpClient.newCall(any(Request.class))).thenReturn(call); + + Response response = + new Response.Builder() + .request(new Request.Builder().url("https://s3.amazonaws.com").build()) + .protocol(Protocol.HTTP_1_1) + .code(500) + .message("Internal Server Error") + .body(ResponseBody.create("error", MediaType.parse("text/plain"))) + .build(); + when(call.execute()).thenReturn(response); + + S3ServerSideComposeHelper helper = new S3ServerSideComposeHelper(minioClient); + setField(helper, "asyncClient", asyncClient); + setField(helper, "baseUrl", baseUrl); + setField(helper, "httpClient", httpClient); + + try { + helper.compose("my-bucket", "upload/target", Collections.singletonList("upload/part-1")); + fail("Expected IOException"); + } catch (IOException e) { + assertTrue(e.getMessage().contains("HTTP 500")); + } + } + + private static void setField(Object target, String fieldName, Object value) throws Exception { + Field field = S3ServerSideComposeHelper.class.getDeclaredField(fieldName); + field.setAccessible(true); + field.set(target, value); + } +} diff --git a/src/test/java/me/desair/tus/server/upload/s3/S3StorageServiceTest.java b/src/test/java/me/desair/tus/server/upload/s3/S3StorageServiceTest.java index 1b43b0d8..d795775c 100644 --- a/src/test/java/me/desair/tus/server/upload/s3/S3StorageServiceTest.java +++ b/src/test/java/me/desair/tus/server/upload/s3/S3StorageServiceTest.java @@ -79,6 +79,26 @@ public void testNullTemporaryDirectoryConstructor() { assertNotNull(serviceWithNullTmp); } + @Test + public void testExplicitConnectionParametersConstructors() { + // 1. Constructor taking connection parameters directly (Option A) + S3StorageService serviceWithParams = + new S3StorageService( + "https://s3.amazonaws.com", "us-east-1", "accessKey", "secretKey", "test-bucket"); + assertNotNull(serviceWithParams); + + // 2. Constructor taking MinioClient and connection parameters (Option B) + S3StorageService serviceWithClientAndParams = + new S3StorageService( + minioClient, + "https://s3.amazonaws.com", + "us-east-1", + "accessKey", + "secretKey", + "test-bucket"); + assertNotNull(serviceWithClientAndParams); + } + @Test public void testGetS3ObjectKeyByUri() throws Exception { UploadInfo info = new UploadInfo(); @@ -1696,6 +1716,118 @@ public void testFinalizeCompletedUploadComposeObjectWhenAllIntermediatePartsAreA .composeObject(any(ComposeObjectArgs.class)); } + @Test + public void testFinalizeCompletedUploadComposeObjectFailsFallsBackToStreamingAndDisablesCompose() + throws Exception { + assertTrue(storageService.isS3ComposeObjectSupported()); + + UploadInfo info = new UploadInfo(); + UploadId id = new UploadId("compose-fail-test-123"); + long fiveMb = 5L * 1024L * 1024L; + info.setId(id); + info.setLength(fiveMb + 100L); + info.setOffset(0L); + + String json = UploadInfoJsonSerializer.serialize(info); + + Item item1 = mock(Item.class); + when(item1.objectName()).thenReturn("uploads/compose-fail-test-123.part.00001"); + when(item1.size()).thenReturn(fiveMb); + + Item item2 = mock(Item.class); + when(item2.objectName()).thenReturn("uploads/compose-fail-test-123.part.00002"); + when(item2.size()).thenReturn(100L); + + when(minioClient.listObjects(any(ListObjectsArgs.class))) + .thenReturn(Arrays.asList(new Result<>(item1), new Result<>(item2))); + + StatObjectResponse stat1 = mock(StatObjectResponse.class); + when(stat1.size()).thenReturn(fiveMb); + StatObjectResponse stat2 = mock(StatObjectResponse.class); + when(stat2.size()).thenReturn(100L); + + when(minioClient.statObject(any(StatObjectArgs.class))) + .thenAnswer( + invocation -> { + StatObjectArgs args = invocation.getArgument(0); + if (args.object().endsWith(".part.00001")) { + return stat1; + } else if (args.object().endsWith(".part.00002")) { + return stat2; + } + ErrorResponse err = mock(ErrorResponse.class); + when(err.code()).thenReturn("NoSuchKey"); + throw new ErrorResponseException(err, null, null); + }); + + when(minioClient.getObject(any(GetObjectArgs.class))) + .thenAnswer( + invocation -> { + GetObjectArgs args = invocation.getArgument(0); + if (args.object().endsWith(".info")) { + return mockGetObjectResponse(json.getBytes()); + } + return mockGetObjectResponse(new byte[100]); + }); + + when(minioClient.composeObject(any(ComposeObjectArgs.class))) + .thenThrow(new RuntimeException("The specified header is not valid in this context")); + + UploadInfo result = + storageService.append(info, new ByteArrayInputStream(new byte[(int) (fiveMb + 100L)])); + assertNotNull(result); + + // Verify composeObject was attempted and failed + org.mockito.Mockito.verify(minioClient, org.mockito.Mockito.times(1)) + .composeObject(any(ComposeObjectArgs.class)); + // Verify fallback to streaming putObject was executed + org.mockito.Mockito.verify(minioClient, org.mockito.Mockito.atLeastOnce()) + .putObject(any(PutObjectArgs.class)); + // Verify s3ComposeObjectSupported is now disabled + assertFalse(storageService.isS3ComposeObjectSupported()); + + // Subsequent upload should directly use streaming without calling composeObject + UploadInfo info2 = new UploadInfo(); + UploadId id2 = new UploadId("compose-skip-test-456"); + info2.setId(id2); + info2.setLength(fiveMb + 100L); + info2.setOffset(0L); + + String json2 = UploadInfoJsonSerializer.serialize(info2); + Item item21 = mock(Item.class); + when(item21.objectName()).thenReturn("uploads/compose-skip-test-456.part.00001"); + when(item21.size()).thenReturn(fiveMb); + + Item item22 = mock(Item.class); + when(item22.objectName()).thenReturn("uploads/compose-skip-test-456.part.00002"); + when(item22.size()).thenReturn(100L); + + when(minioClient.listObjects(any(ListObjectsArgs.class))) + .thenReturn(Arrays.asList(new Result<>(item21), new Result<>(item22))); + + when(minioClient.getObject(any(GetObjectArgs.class))) + .thenAnswer( + invocation -> { + GetObjectArgs args = invocation.getArgument(0); + if (args.object().endsWith(".info")) { + return mockGetObjectResponse(json2.getBytes()); + } + return mockGetObjectResponse(new byte[100]); + }); + + UploadInfo result2 = + storageService.append(info2, new ByteArrayInputStream(new byte[(int) (fiveMb + 100L)])); + assertNotNull(result2); + + // composeObject count should still be 1 (never called for the second upload) + org.mockito.Mockito.verify(minioClient, org.mockito.Mockito.times(1)) + .composeObject(any(ComposeObjectArgs.class)); + + // Reset flag for other tests + storageService.setS3ComposeObjectSupported(true); + assertTrue(storageService.isS3ComposeObjectSupported()); + } + @Test public void testPrepareStreamRollsBackSub5MbNumberedPart() throws Exception { UploadInfo info = new UploadInfo(); diff --git a/src/test/java/me/desair/tus/server/upload/util/AsyncChunkUploaderTest.java b/src/test/java/me/desair/tus/server/upload/util/AsyncChunkUploaderTest.java index 53a0de9f..86a430b5 100644 --- a/src/test/java/me/desair/tus/server/upload/util/AsyncChunkUploaderTest.java +++ b/src/test/java/me/desair/tus/server/upload/util/AsyncChunkUploaderTest.java @@ -573,4 +573,18 @@ public void testSubmitChunkBackpressureInterruptedThrowsIOException() throws Exc "Interrupted backpressure wait should throw IOException", gotBackpressureInterruptedException.get()); } + + @Test + public void testDrainAndCompleteDefaultTimeout() throws Exception { + File chunk = new File(tempDir, "chunk-default-drain.tmp"); + Files.write(chunk.toPath(), "test-data".getBytes()); + + try (AsyncChunkUploader uploader = new AsyncChunkUploader(executor)) { + uploader.submitChunk(chunk, 9, "part-default", () -> {}); + int confirmed = uploader.drainAndComplete(); + assertEquals("One chunk confirmed uploaded using default timeout", 1, confirmed); + } + + assertFalse("Chunk file should be cleaned up", chunk.exists()); + } } From 762f39c1ba22835d5088eaf10aa6ca9f1f45b8a8 Mon Sep 17 00:00:00 2001 From: Tom Desair Date: Sat, 3 Oct 2026 17:19:25 +0200 Subject: [PATCH 6/7] feat: unify lock wait timeout and cloud chunk drain timeout - Add withLockWaitTimeout(Duration) and getLockWaitTimeout() to TusFileUploadService (default 60s) - Derive max lock retries (timeout / 200ms, default 300) and cloud upload chunk drain timeout (timeout - 5s, default 55s) - Update AsyncChunkUploader, UploadStorageService, S3StorageService, AzureBlobStorageService, DiskStorageService, and ThreadLocalCachedStorageAndLockingService to configure and propagate drain timeout - Update documentation in README.md, CHANGELOG.md, docs/LOCKING.md, docs/S3_STORAGE.md, and docs/AZURE_BLOB_STORAGE.md with lock wait terminology - Add unit tests for timeout derivation, validation, storage synchronization, and cache delegation --- CHANGELOG.md | 5 +- README.md | 2 +- docs/AZURE_BLOB_STORAGE.md | 7 +- docs/LOCKING.md | 12 ++-- docs/S3_STORAGE.md | 8 +++ .../tus/server/TusFileUploadService.java | 67 +++++++++++++++---- .../server/upload/UploadStorageService.java | 20 ++++++ .../upload/azure/AzureBlobStorageService.java | 19 +++++- ...adLocalCachedStorageAndLockingService.java | 11 +++ .../upload/disk/DiskStorageService.java | 14 ++++ .../server/upload/s3/S3LockingService.java | 2 +- .../server/upload/s3/S3StorageService.java | 20 +++++- .../upload/util/AsyncChunkUploader.java | 31 +++++++-- .../tus/server/TusFileUploadServiceTest.java | 56 +++++++++++++--- .../upload/UploadStorageServiceTest.java | 4 ++ .../azure/AzureBlobStorageServiceTest.java | 22 ++++++ ...calCachedStorageAndLockingServiceTest.java | 7 ++ .../upload/disk/DiskStorageServiceTest.java | 12 ++++ .../upload/s3/S3StorageServiceTest.java | 22 ++++++ .../upload/util/AsyncChunkUploaderTest.java | 15 +++++ 20 files changed, 313 insertions(+), 43 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c906a2f2..108ff851 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,11 +16,12 @@ All notable changes to this project will be documented in this file. - **Dedicated Compliance Test Suites**: Added comprehensive, spec-quoted end-to-end tests using a dedicated Python script `scripts/rufh_conformity_test.py` with documentation on how to run the tests in `docs/CONFORMITY_TESTING.md`. - **User Migration & Interim Responses Documentation**: Added `docs/MIGRATION.md` and `docs/INTERIM_RESPONSES.md` detailing migration strategies, HTTP 104 status frames under IETF RUFH, Tomcat/Servlet container limitations, cached reflection optimizations, and Spring Boot Tomcat Valve integration. - **Server-Side Upload Completion Listeners (`UploadCompletionListener`)**: Added a functional interface callback mechanism allowing developers to register post-upload listeners via `withUploadCompletionListener(UploadCompletionListener)` or `addUploadCompletionListener(UploadCompletionListener)`. Listeners receive the completed `UploadInfo` and `TusFileUploadService` instance after lock release, allowing immediate byte streaming and deletion without contention. Added helper overloads `TusFileUploadService.getUploadedBytes(UploadInfo)` and `TusFileUploadService.deleteUpload(UploadInfo)`. +- **Unified Lock Wait & Cloud Drain Timeout**: Added `TusFileUploadService.withLockWaitTimeout(Duration)` (default 60 seconds) to configure the maximum wait duration for requests contending for an active upload lock. Automatically derives the background cloud upload chunk drain timeout (`lockWaitTimeout - 5 seconds`, default 55s) and the retry polling budget (`lockWaitTimeout / 200ms`, default 300 retries). - **JSON Serialization**: Support storing `UploadInfo` objects as JSON files in the storage backend using `TusFileUploadService.withJsonSerialization(true)`. ### Changed - **Default Disk-Based Locking**: `TusFileUploadService.withStoragePath(String)` now defaults to `LeaseFileLockingService` instead of `DiskLockingService` for out-of-the-box Kubernetes, container, and shared network storage compatibility. See `docs/DISK_BASED_LOCKING.md` for legacy opt-out instructions. -- **Calibrated Retry Budget**: Extended `TusFileUploadService` lock acquisition retry budget to 8.0 seconds (40 retries x 200ms) to ensure reliable contention resolution over network storage. +- **Unified Lock Wait Budget & Cloud Chunk Drain Calibration**: Replaced unreleased `withMaxLockRetries` with `withLockWaitTimeout(Duration)` (default 60s / 300 retries), synchronizing the lock wait timeout budget across storage backends and automatically deriving the cloud chunk drain timeout (`lockWaitTimeout - 5s`, default 55s) to guarantee in-flight chunks are cleanly flushed to cloud storage before releasing upload locks. - **Absolute Base URL & Location Header Support**: Extended `withUploadUri(String)` to accept absolute base URLs (e.g. `https://upload.example.com/files`), returning full URLs in `Location` response headers for upload creation across both Tus 1.0.0 and RUFH protocols while preserving backward compatibility for relative paths. - **`process()` Return Value (`UploadInfo`)**: `TusFileUploadService.process(...)` now returns the created or updated `UploadInfo` instance (or `null` on errors or `OPTIONS` preflight requests), enabling applications to track and store upload IDs directly into user sessions or database repositories. - **Jackson Bundled in Compile Scope**: Promoted Jackson dependencies (`jackson-databind`, `jackson-annotations`, `jackson-core`) to `compile` scope, eliminating `NoClassDefFoundError` when enabling JSON serialization or using cloud storage. @@ -32,7 +33,7 @@ All notable changes to this project will be documented in this file. - **Clear Content-Length on Error Responses**: Cleared `Content-Length` response header prior to invoking `HttpServletResponse.sendError(...)` during exception handling, resolving buffer conflicts and exceptions in Undertow and other servlet containers ([#40](https://github.com/tomdesair/tus-java-server/issues/40)). - **Prevent Disk Truncate Underflow**: Guarded file truncate logic in `DiskStorageService` against underflow when removing bytes (`Math.max(0L, file.size() - byteCount)`). - **File Channel Leak Prevention in FileBasedLock**: Guaranteed `FileChannel` is closed immediately upon lock acquisition errors to avoid file descriptor leaks. -- **Cloud Upload Pause & Drain Timeout Resilience**: Extended `AsyncChunkUploader` default drain timeout from 4 seconds to 60 seconds and wrapped chunk draining in error-resilient recovery logic across `AzureBlobStorageService` and `S3StorageService`. Confirmed uploaded chunks and staged Azure blocks are committed and metadata (`UploadInfo` offset) is persisted to storage before throwing stream or drain exceptions, ensuring paused uploads (such as Uppy client pause/resume) preserve their progress and resume from the exact byte offset instead of restarting from 0. +- **Cloud Upload Pause & Drain Timeout Resilience**: Integrated `AsyncChunkUploader` default drain timeout (55 seconds, calibrated to `lockWaitTimeout - 5s`) with error-resilient recovery logic across `AzureBlobStorageService` and `S3StorageService`. Confirmed uploaded chunks and staged Azure blocks are committed and metadata (`UploadInfo` offset) is persisted to storage before throwing stream or drain exceptions, ensuring paused uploads (such as Uppy client pause/resume) preserve their progress and resume from the exact byte offset instead of restarting from 0. - **Azure Blob Storage Upload Cancellation Lease Safety**: Checked lock blob lease state before calling `deleteIfExists()` in `AzureBlobStorageService.terminateUpload()`. When an active lease is held on the lock blob by an ongoing request (e.g. `DELETE` cancellation), attempting deletion without specifying the lease ID triggered an Azure SDK error log (`HTTP 412 LeaseIdMissing`). Skipping deletion for actively leased blobs prevents this error and allows normal lease release upon request completion. - **Lock Contention Stream Interruption Handling**: Handled `IOException` from `InterruptibleInputStream` across upload request handlers (`CorePatchRequestHandler`, `RufhAppendPatchRequestHandler`, `RufhCreationPostRequestHandler`, `CreationWithUploadPostRequestHandler`). When an upload stream is interrupted by the locking service watchdog during lock contention (such as concurrent `HEAD` progress checks or `DELETE` cancellation requests), the storage backend commits all buffered bytes received so far and updates the offset in storage. Request handlers now reload the refreshed `UploadInfo` and return a clean HTTP 204/201 response with the updated offset rather than bubbling an unhandled `IOException` / HTTP 500 to the servlet container. - **S3 Server-Side Part Composition & Native Multipart Copy**: Implemented `S3ServerSideComposeHelper` to resolve AWS S3 header rejection during server-side part composition. MinIO Java SDK 9.0.3's `composeObject` delegates to `UploadPartCopy` with an empty body placeholder that automatically injects `Content-MD5` and `Content-Type` headers, which Amazon AWS S3 strictly forbids on part copy requests and rejects with HTTP 400 (`The specified header is not valid in this context`). `S3ServerSideComposeHelper` executes native S3 multipart copy requests directly with SigV4 signing omitting `Content-MD5`, allowing fast zero-bandwidth server-side composition on both AWS S3 and MinIO/Ceph clusters without external AWS SDK dependencies. Added reflection-free constructors to `S3StorageService` and `S3LockingService` accepting connection parameters directly, with `eu-central-1` as the default fallback region, while preserving multi-tier streaming concatenation fallbacks. diff --git a/README.md b/README.md index 2fe648d2..b240e1fd 100644 --- a/README.md +++ b/README.md @@ -181,7 +181,7 @@ After creating the object, you can configure it using the following methods: | `withStoragePath(String)` | `${java.io.tmpdir}/tus` | Path on the filesystem or shared drive where uploaded bytes and metadata are stored when using `DiskStorageService`. | | `withSupportedProtocolVersions(ProtocolVersion)` | `ProtocolVersion.AUTO` | Configures protocol handling: `AUTO` (header-based auto-detection), `TUS_1_0_0` (Tus 1.0.0 only), or `RUFH` (IETF draft-12 only). | | `withMaxUploadSize(Long)` | `Long.MAX_VALUE` | Maximum allowed total upload size in bytes per upload resource. | -| `withMaxLockRetries(int)` | `40` | Maximum lock acquisition retries during lock contention resolution (200ms sleep, resulting in an 8.0s timeout budget). | +| `withLockWaitTimeout(Duration)` | `Duration.ofSeconds(60)` | Maximum duration a request waits to acquire an upload lock held by an in-flight transfer (retrying every 200ms, resulting in 300 retries). Automatically configures cloud background chunk drain timeout to 5s less than this value (default 55s). | | `withChunkedTransferDecoding(Boolean)` | `false` | Enables manual chunked HTTP decoding for servlet containers that do not decode chunked requests natively. | | `withThreadLocalCache(Boolean)` | `false` | Enables in-memory thread-local caching of upload request data to reduce storage backend I/O load. | | `withUploadExpirationPeriod(Long)` | `null` (disabled) | Expiration period in milliseconds after which incomplete/expired uploads become eligible for cleanup. | diff --git a/docs/AZURE_BLOB_STORAGE.md b/docs/AZURE_BLOB_STORAGE.md index 7a1fb930..f3ba0ec2 100644 --- a/docs/AZURE_BLOB_STORAGE.md +++ b/docs/AZURE_BLOB_STORAGE.md @@ -182,11 +182,16 @@ Block sizes auto-calibrate based on total upload size: ### Lease Renewal Rationale `AzureBlobLockingService` uses native Azure Blob Leases (30-second duration) for distributed locking. Because large file uploads can stream over several minutes or hours, `AzureBlobUploadLock` runs a background daemon thread that renews the lease every 10 seconds. If an application server crashes unexpectedly, the lease auto-expires after 30 seconds without requiring manual lock cleanup sweeps. -### Lock Contention Resolution +### Lock Wait & Contention Resolution Lock contention resolution operates on two levels: 1. **JVM-local**: Active `InterruptibleInputStream` instances are registered in a concurrent map and interrupted directly if a concurrent lock request arrives in the same JVM. 2. **Cross-replica**: A `.stop` signal blob (`locks/.stop`) is written to Azure Storage. A background watchdog thread polls for `.stop` blobs and interrupts active streams on other cluster nodes. +When a client resumes an interrupted transfer by sending a `HEAD` request (or cancels it with `DELETE`), the request enters a **lock wait** retry loop in `TusFileUploadService`: +- **Lock Wait Timeout (`withLockWaitTimeout(Duration)`)**: Governs how long the incoming request waits (polling every 200ms; default 60 seconds / 300 retries). +- **Background Chunk Draining (`drainTimeout`)**: The interrupted upload is automatically allotted up to `lockWaitTimeout - 5 seconds` (default 55 seconds) to drain and stage in-flight blocks to Azure Blob Storage via `AsyncChunkUploader` before committing the block list and persisting the updated offset. +- **Lock Handover**: Because the drain timeout leaves a 5-second buffer before the lock wait budget expires, the active upload cleanly finishes staging blocks and releases the Azure Blob Lease within the caller's wait window, enabling seamless resumption without `UploadAlreadyLockedException`. + --- ## 8. Troubleshooting Guide diff --git a/docs/LOCKING.md b/docs/LOCKING.md index 6416525e..6d73d8ab 100644 --- a/docs/LOCKING.md +++ b/docs/LOCKING.md @@ -1,6 +1,6 @@ -# Upload Locking & Lock Contention Resolution +# Upload Locking, Lock Wait & Contention Resolution -This document describes why locking is necessary in the `tus-java-server` library, how the core `UploadLockingService` interface is structured, how lock contention resolution works across replicas, and where to find detailed documentation for each concrete locking mechanism implementation. +This document describes why locking is necessary in the `tus-java-server` library, how the core `UploadLockingService` interface is structured, how lock wait and contention resolution works across replicas, and where to find detailed documentation for each concrete locking mechanism implementation. --- @@ -8,14 +8,14 @@ This document describes why locking is necessary in the `tus-java-server` librar In the `tus` protocol (and IETF Resumable Uploads for HTTP specification), client uploads can be interrupted and resumed across multiple HTTP requests. Multiple concurrent requests targeting the same upload resource must be strictly prevented to avoid data corruption (such as out-of-order byte writes or overlapping file offsets). -### Stalled Uploads & Lock Contention Handling +### Stalled Uploads, Lock Wait & Contention Handling 1. **Active Streaming**: When a client sends upload bytes via a `PATCH` (or RUFH `POST`/`PATCH`) request, the server acquires an exclusive lock on that upload. 2. **Network Interruption**: If the client's network drops, the original `PATCH` connection may remain open on the server in a "half-open" state (a stalled socket read waiting for client bytes). 3. **Resume Attempt**: The client, recognizing the disconnect, attempts to resume by sending a `HEAD` request to query the current offset (or a `DELETE` request to terminate the upload). -4. **Lock Conflict**: The stalled `PATCH` request is still running on the server and holding the lock, which would block the client's `HEAD` or `DELETE` request indefinitely if not resolved. - -To solve this, `tus-java-server` includes a **lock contention resolution mechanism** where an incoming `HEAD` or `DELETE` request signals the server to interrupt the stalled `PATCH` byte stream cleanly, releasing the lock for immediate resumption. +4. **Lock Conflict & Lock Wait**: The stalled `PATCH` request is still running on the server and holding the lock. Rather than failing immediately, the server enters a **lock wait** retry loop (retrying every 200ms up to `withLockWaitTimeout(Duration)`, default 60s / 300 retries). +5. **Contention Resolution**: The waiting request calls `lockingService.requestLockRelease(requestUri)`, which signals the active upload to interrupt its input stream. +6. **Drain & Handover**: For cloud backends (S3, Azure), the interrupted upload is allotted a background drain timeout (`lockWaitTimeout - 5 seconds`, default 55s) to safely flush in-flight chunks and persist updated byte offsets before releasing the lock. Once freed, the waiting request acquires the lock and immediately returns the accurate offset to the client. --- diff --git a/docs/S3_STORAGE.md b/docs/S3_STORAGE.md index 4328c739..3e4b46f2 100644 --- a/docs/S3_STORAGE.md +++ b/docs/S3_STORAGE.md @@ -204,6 +204,14 @@ S3StorageService s3Storage = new S3StorageService( - If lock contention occurs across replicas, `S3LockingService` writes a `.stop` signal object in S3, signaling the active request on another pod to interrupt its input stream cleanly. - No external database or Redis cache is required for distributed locking. +### Lock Wait & Background Chunk Draining + +When a client resumes an interrupted transfer by sending a `HEAD` request (or cancels it with `DELETE`), the request enters a **lock wait** retry loop in `TusFileUploadService`: +- **Lock Wait Timeout (`withLockWaitTimeout(Duration)`)**: Governs how long the incoming request waits (polling every 200ms; default 60 seconds / 300 retries). +- **Contention Interruption**: The waiting pod writes a `.stop` object in S3. A background watchdog on the pod running the active `PATCH` detects this within 2 seconds and cleanly interrupts the stream. +- **Background Chunk Draining (`drainTimeout`)**: The active upload is automatically allotted up to `lockWaitTimeout - 5 seconds` (default 55 seconds) to drain and persist any in-flight 5MB+ chunks to S3 via `AsyncChunkUploader` before committing the new offset. +- **Lock Handover**: Because the drain timeout leaves a 5-second buffer before the lock wait budget expires, the active upload finishes persisting data and releases the lock well within the caller's wait window, enabling clean resumption without race conditions. + ### Why Lease Renewal is Required (`S3Lock` vs `FileBasedLock`) Understanding the architectural distinction between disk/file locking and S3 distributed locking is essential: diff --git a/src/main/java/me/desair/tus/server/TusFileUploadService.java b/src/main/java/me/desair/tus/server/TusFileUploadService.java index 8f203f23..159d7a7a 100644 --- a/src/main/java/me/desair/tus/server/TusFileUploadService.java +++ b/src/main/java/me/desair/tus/server/TusFileUploadService.java @@ -7,6 +7,7 @@ import java.io.FilterInputStream; import java.io.IOException; import java.io.InputStream; +import java.time.Duration; import java.util.EnumSet; import java.util.LinkedHashMap; import java.util.LinkedHashSet; @@ -58,7 +59,10 @@ public class TusFileUploadService implements Closeable { private static final Logger log = LoggerFactory.getLogger(TusFileUploadService.class); - public static final int DEFAULT_MAX_LOCK_RETRIES = 40; + public static final Duration DEFAULT_LOCK_WAIT_TIMEOUT = Duration.ofSeconds(60); + public static final long LOCK_RETRY_INTERVAL_MS = 200L; + public static final int DEFAULT_MAX_LOCK_RETRIES = + (int) (DEFAULT_LOCK_WAIT_TIMEOUT.toMillis() / LOCK_RETRY_INTERVAL_MS); private UploadStorageService uploadStorageService; private UploadLockingService uploadLockingService; @@ -68,6 +72,7 @@ public class TusFileUploadService implements Closeable { private boolean isThreadLocalCacheEnabled = false; private boolean isChunkedTransferDecodingEnabled = false; private ProtocolVersion supportedProtocolVersion = ProtocolVersion.AUTO; + private Duration lockWaitTimeout = DEFAULT_LOCK_WAIT_TIMEOUT; private int maxLockRetries = DEFAULT_MAX_LOCK_RETRIES; private final List uploadCompletionListeners = new CopyOnWriteArrayList<>(); @@ -77,9 +82,18 @@ public TusFileUploadService() { String storagePath = FileUtils.getTempDirectoryPath() + File.separator + "tus"; this.uploadStorageService = new DiskStorageService(idFactory, storagePath); this.uploadLockingService = new LeaseFileLockingService(idFactory, storagePath); + this.uploadStorageService.setDrainTimeout(calculateDrainTimeout(DEFAULT_LOCK_WAIT_TIMEOUT)); initFeatures(); } + private static Duration calculateDrainTimeout(Duration timeout) { + return timeout.minus(Duration.ofSeconds(5)); + } + + private static int calculateMaxLockRetries(Duration timeout) { + return (int) (timeout.toMillis() / LOCK_RETRY_INTERVAL_MS); + } + protected void initFeatures() { // The order of the features is important addTusExtension(new CoreProtocol()); @@ -290,6 +304,7 @@ public TusFileUploadService withUploadStorageService(UploadStorageService upload this.uploadStorageService.isUploadDeduplicationEnabled()); uploadStorageService.setJsonSerializationEnabled( this.uploadStorageService.isJsonSerializationEnabled()); + uploadStorageService.setDrainTimeout(this.uploadStorageService.getDrainTimeout()); uploadStorageService.setIdFactory(this.idFactory); // Update the upload storage service this.uploadStorageService = uploadStorageService; @@ -345,21 +360,47 @@ public TusFileUploadService withUploadLockingService(UploadLockingService upload } /** - * Specify the maximum number of retries the service will attempt to acquire an upload lock before - * failing with an {@link UploadAlreadyLockedException} during lock contention resolution (e.g. - * for HEAD or DELETE requests). Default is {@value #DEFAULT_MAX_LOCK_RETRIES} retries. + * Specify the maximum duration a request will wait to acquire an upload lock held by another + * in-flight request before failing with an {@link + * me.desair.tus.server.exception.UploadAlreadyLockedException} during lock wait and contention + * resolution (e.g. for HEAD or DELETE requests). + * + *

Configuring this setting automatically derives: + * + *

    + *
  • The background cloud upload chunk drain timeout for cloud storage backends (S3, Azure), + * set to {@code lockWaitTimeout - 5 seconds}. + *
  • The maximum number of lock acquisition retries (polling every {@value + * #LOCK_RETRY_INTERVAL_MS}ms). + *
+ * + *

The timeout must be greater than 5 seconds to provide a positive drain window for in-flight + * chunks. Default is 60 seconds (yielding a 55-second drain timeout and 300 retries). * - * @param maxLockRetries The maximum number of lock acquisition retries (must be 0 or greater) + * @param lockWaitTimeout The maximum duration to wait for an upload lock (must be > 5 seconds) * @return The current service */ - public TusFileUploadService withMaxLockRetries(int maxLockRetries) { - Validate.isTrue(maxLockRetries >= 0, "The max lock retries must be 0 or greater"); - this.maxLockRetries = maxLockRetries; + public TusFileUploadService withLockWaitTimeout(Duration lockWaitTimeout) { + Validate.notNull(lockWaitTimeout, "The lock wait timeout cannot be null"); + Validate.isTrue( + lockWaitTimeout.toMillis() > 5000L, "The lock wait timeout must be greater than 5 seconds"); + this.lockWaitTimeout = lockWaitTimeout; + this.maxLockRetries = calculateMaxLockRetries(lockWaitTimeout); + this.uploadStorageService.setDrainTimeout(calculateDrainTimeout(lockWaitTimeout)); return this; } /** - * Get the maximum number of lock acquisition retries. + * Get the maximum duration a request will wait to acquire an upload lock. + * + * @return The current lock wait timeout + */ + public Duration getLockWaitTimeout() { + return lockWaitTimeout; + } + + /** + * Get the maximum number of lock acquisition retries, derived from {@link #getLockWaitTimeout()}. * * @return The maximum number of lock acquisition retries */ @@ -571,9 +612,9 @@ protected UploadLock acquireUploadLock(HttpMethod method, String requestUri) throws TusException, IOException { UploadLock lock = null; int retries = 0; - // Retry budget calibrated by default to 40 retries x 200ms = 8.0 seconds to accommodate - // NFS/network storage attribute cache propagation (actimeo=3s), watchdog polling - // interval (1.5s), and socket stream interruption and cleanup overhead. + // Lock wait loop: retries every LOCK_RETRY_INTERVAL_MS ms up to the configured lockWaitTimeout + // budget (default 60s / 300 retries). When an in-flight upload is active, requestLockRelease + // signals it to interrupt and cleanly drain in-flight chunks before releasing the lock. while (retries < maxLockRetries) { try { lock = uploadLockingService.lockUploadByUri(requestUri); @@ -583,7 +624,7 @@ protected UploadLock acquireUploadLock(HttpMethod method, String requestUri) uploadLockingService.requestLockRelease(requestUri); retries++; try { - Thread.sleep(200L); + Thread.sleep(LOCK_RETRY_INTERVAL_MS); } catch (InterruptedException ie) { Thread.currentThread().interrupt(); throw new IOException("Lock acquisition retry interrupted", ie); diff --git a/src/main/java/me/desair/tus/server/upload/UploadStorageService.java b/src/main/java/me/desair/tus/server/upload/UploadStorageService.java index bdce0298..5f168ba4 100644 --- a/src/main/java/me/desair/tus/server/upload/UploadStorageService.java +++ b/src/main/java/me/desair/tus/server/upload/UploadStorageService.java @@ -3,6 +3,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.time.Duration; import me.desair.tus.server.checksum.ChecksumAlgorithm; import me.desair.tus.server.exception.TusException; import me.desair.tus.server.exception.UploadNotFoundException; @@ -297,6 +298,25 @@ default int getCloudUploadThreadPoolSize() { return 10; } + /** + * Set the timeout for draining in-flight chunks during upload completion or interruption. + * + * @param drainTimeout Drain timeout duration + */ + default void setDrainTimeout(Duration drainTimeout) { + // Default no-op for non-cloud implementations + } + + /** + * Get the timeout for draining in-flight chunks during upload completion or interruption. + * Defaults to 55 seconds (derived from the 60-second default lock wait timeout minus 5 seconds). + * + * @return Drain timeout duration, defaults to 55 seconds + */ + default Duration getDrainTimeout() { + return Duration.ofSeconds(55); + } + /** * Closes any underlying storage resources. * diff --git a/src/main/java/me/desair/tus/server/upload/azure/AzureBlobStorageService.java b/src/main/java/me/desair/tus/server/upload/azure/AzureBlobStorageService.java index ec0f6872..2f62320e 100644 --- a/src/main/java/me/desair/tus/server/upload/azure/AzureBlobStorageService.java +++ b/src/main/java/me/desair/tus/server/upload/azure/AzureBlobStorageService.java @@ -22,6 +22,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.time.Duration; import java.util.ArrayList; import java.util.Base64; import java.util.List; @@ -94,6 +95,7 @@ public class AzureBlobStorageService implements UploadStorageService { private long preferredBlockSize = DEFAULT_PREFERRED_BLOCK_SIZE; private int cloudUploadThreadPoolSize = 10; + private Duration drainTimeout = Duration.ofSeconds(55); private final ThreadPoolExecutor uploadExecutor; private Long maxUploadSize; @@ -241,7 +243,8 @@ public UploadInfo append(UploadInfo upload, InputStream inputStream) List plannedBlockIds = new ArrayList<>(); List plannedChunkSizes = new ArrayList<>(); - try (AsyncChunkUploader uploader = new AsyncChunkUploader(uploadExecutor)) { + try (AsyncChunkUploader uploader = + new AsyncChunkUploader(uploadExecutor, drainTimeout.toMillis())) { // 4. Read incoming stream in chunks, staging blocks directly to Azure Block Blob while (true) { File chunkFile = null; @@ -304,7 +307,7 @@ public UploadInfo append(UploadInfo upload, InputStream inputStream) } } - // Drain remaining staged chunks with 60s timeout. + // Drain remaining staged chunks with configured timeout (defaults to 55s, lock wait - 5s). // Catch drainException so any chunks confirmed uploaded before timeout or error // are committed and the metadata offset is preserved without loss. try { @@ -759,6 +762,18 @@ public int getCloudUploadThreadPoolSize() { return cloudUploadThreadPoolSize; } + @Override + public void setDrainTimeout(Duration drainTimeout) { + if (drainTimeout != null) { + this.drainTimeout = drainTimeout; + } + } + + @Override + public Duration getDrainTimeout() { + return drainTimeout; + } + @Override public void close() throws IOException { uploadExecutor.shutdown(); diff --git a/src/main/java/me/desair/tus/server/upload/cache/ThreadLocalCachedStorageAndLockingService.java b/src/main/java/me/desair/tus/server/upload/cache/ThreadLocalCachedStorageAndLockingService.java index ee83f592..5658f808 100644 --- a/src/main/java/me/desair/tus/server/upload/cache/ThreadLocalCachedStorageAndLockingService.java +++ b/src/main/java/me/desair/tus/server/upload/cache/ThreadLocalCachedStorageAndLockingService.java @@ -4,6 +4,7 @@ import java.io.InputStream; import java.io.OutputStream; import java.lang.ref.WeakReference; +import java.time.Duration; import java.util.Objects; import me.desair.tus.server.checksum.ChecksumAlgorithm; import me.desair.tus.server.exception.TusException; @@ -243,6 +244,16 @@ public int getCloudUploadThreadPoolSize() { return storageServiceDelegate.getCloudUploadThreadPoolSize(); } + @Override + public void setDrainTimeout(Duration drainTimeout) { + storageServiceDelegate.setDrainTimeout(drainTimeout); + } + + @Override + public Duration getDrainTimeout() { + return storageServiceDelegate.getDrainTimeout(); + } + @Override public UploadLock lockUploadByUri(String requestUri) throws TusException, IOException { UploadLock uploadLock = lockingServiceDelegate.lockUploadByUri(requestUri); diff --git a/src/main/java/me/desair/tus/server/upload/disk/DiskStorageService.java b/src/main/java/me/desair/tus/server/upload/disk/DiskStorageService.java index 5a428699..ca5c853e 100644 --- a/src/main/java/me/desair/tus/server/upload/disk/DiskStorageService.java +++ b/src/main/java/me/desair/tus/server/upload/disk/DiskStorageService.java @@ -15,6 +15,7 @@ import java.nio.file.DirectoryStream; import java.nio.file.Files; import java.nio.file.Path; +import java.time.Duration; import java.util.Collections; import java.util.List; import java.util.Objects; @@ -52,6 +53,7 @@ public class DiskStorageService extends AbstractDiskBasedService implements Uplo private Long minAppendSize = null; private Long minSize = null; private int cloudUploadThreadPoolSize = 10; + private Duration drainTimeout = Duration.ofSeconds(55); private Long uploadExpirationPeriod = null; private UploadIdFactory idFactory; private UploadConcatenationService uploadConcatenationService; @@ -130,6 +132,18 @@ public int getCloudUploadThreadPoolSize() { return cloudUploadThreadPoolSize; } + @Override + public void setDrainTimeout(Duration drainTimeout) { + if (drainTimeout != null) { + this.drainTimeout = drainTimeout; + } + } + + @Override + public Duration getDrainTimeout() { + return drainTimeout; + } + @Override public void setUploadDeduplicationEnabled(boolean enabled) { this.isUploadDeduplicationEnabled = enabled; diff --git a/src/main/java/me/desair/tus/server/upload/s3/S3LockingService.java b/src/main/java/me/desair/tus/server/upload/s3/S3LockingService.java index e9bd22f2..84245e97 100644 --- a/src/main/java/me/desair/tus/server/upload/s3/S3LockingService.java +++ b/src/main/java/me/desair/tus/server/upload/s3/S3LockingService.java @@ -297,7 +297,7 @@ protected UploadLock tryAcquireLock(UploadId uploadId, LeaseData leaseData) { } catch (ErrorResponseException e) { S3ErrorType errorType = S3Utils.parseErrorResponse(e); if (errorType == S3ErrorType.PRECONDITION_FAILED || errorType == S3ErrorType.CONFLICT) { - log.info("Lock contention for key {}: S3 conditional write precondition failed", lockKey); + log.debug("Lock contention for key {}: S3 conditional write precondition failed", lockKey); return null; } log.warn("Unexpected S3 error response acquiring lock for key {}", lockKey, e); diff --git a/src/main/java/me/desair/tus/server/upload/s3/S3StorageService.java b/src/main/java/me/desair/tus/server/upload/s3/S3StorageService.java index dc89b998..01766dcb 100644 --- a/src/main/java/me/desair/tus/server/upload/s3/S3StorageService.java +++ b/src/main/java/me/desair/tus/server/upload/s3/S3StorageService.java @@ -22,6 +22,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.time.Duration; import java.util.ArrayList; import java.util.Arrays; import java.util.Enumeration; @@ -108,6 +109,7 @@ public class S3StorageService implements UploadStorageService { private long preferredPartSize = DEFAULT_PREFERRED_PART_SIZE; private int cloudUploadThreadPoolSize = 10; + private Duration drainTimeout = Duration.ofSeconds(55); private final ThreadPoolExecutor uploadExecutor; private Long maxUploadSize; @@ -872,6 +874,18 @@ public int getCloudUploadThreadPoolSize() { return cloudUploadThreadPoolSize; } + @Override + public void setDrainTimeout(Duration drainTimeout) { + if (drainTimeout != null) { + this.drainTimeout = drainTimeout; + } + } + + @Override + public Duration getDrainTimeout() { + return drainTimeout; + } + /** * Set the preferred chunk part size in bytes used when buffering and uploading parts to S3. * @@ -1042,7 +1056,8 @@ private AppendResult processPayloadChunks( List plannedPartKeys = new ArrayList<>(); - try (AsyncChunkUploader uploader = new AsyncChunkUploader(uploadExecutor)) { + try (AsyncChunkUploader uploader = + new AsyncChunkUploader(uploadExecutor, drainTimeout.toMillis())) { while (!streamFinished) { File tempChunkFile = Files.createTempFile(temporaryDirectory, "tus-s3-chunk-", ".tmp").toFile(); @@ -1165,7 +1180,8 @@ private AppendResult processPayloadChunks( } } - // Drain any remaining in-flight chunks in the pipeline with 60s timeout. + // Drain any remaining in-flight chunks in the pipeline with configured drain timeout + // (defaults to 55s, lock wait - 5s). // Catch/finally ensures all confirmed parts are retained even if a subsequent chunk times // out. int confirmedCount = 0; diff --git a/src/main/java/me/desair/tus/server/upload/util/AsyncChunkUploader.java b/src/main/java/me/desair/tus/server/upload/util/AsyncChunkUploader.java index c091bc1f..b143fded 100644 --- a/src/main/java/me/desair/tus/server/upload/util/AsyncChunkUploader.java +++ b/src/main/java/me/desair/tus/server/upload/util/AsyncChunkUploader.java @@ -35,7 +35,7 @@ public class AsyncChunkUploader implements AutoCloseable { private static final Logger log = LoggerFactory.getLogger(AsyncChunkUploader.class); - public static final long DEFAULT_DRAIN_TIMEOUT_MS = 300_000L; + public static final long DEFAULT_DRAIN_TIMEOUT_MS = 55_000L; @FunctionalInterface public interface ChunkUploadAction { @@ -48,6 +48,7 @@ public interface ChunkUploadAction { } private final ExecutorService executor; + private final long drainTimeoutMs; // Slot 3: In-flight upload task and its associated local file and key private Future inFlightUpload; @@ -67,12 +68,23 @@ public interface ChunkUploadAction { private boolean completed; /** - * Constructs an uploader using the given shared executor. + * Constructs an uploader using the given shared executor and default 55-second drain timeout. * * @param executor Shared thread pool executor for background chunk uploads */ public AsyncChunkUploader(ExecutorService executor) { + this(executor, DEFAULT_DRAIN_TIMEOUT_MS); + } + + /** + * Constructs an uploader using the given shared executor and explicit drain timeout. + * + * @param executor Shared thread pool executor for background chunk uploads + * @param drainTimeoutMs Maximum duration in milliseconds to drain in-flight chunks + */ + public AsyncChunkUploader(ExecutorService executor, long drainTimeoutMs) { this.executor = Objects.requireNonNull(executor, "ExecutorService must not be null"); + this.drainTimeoutMs = drainTimeoutMs; } /** @@ -144,14 +156,23 @@ public void submitChunk( } /** - * Drains all remaining chunks in the pipeline (Slot 3 and Slot 2) using the default timeout - * ({@link #DEFAULT_DRAIN_TIMEOUT_MS}, 60 seconds). + * Drains all remaining chunks in the pipeline (Slot 3 and Slot 2) using the configured drain + * timeout (defaults to {@link #DEFAULT_DRAIN_TIMEOUT_MS}, 55 seconds). * * @return The total number of confirmed successfully uploaded chunks * @throws IOException If any chunk upload fails or times out */ public int drainAndComplete() throws IOException { - return drainAndComplete(DEFAULT_DRAIN_TIMEOUT_MS); + return drainAndComplete(drainTimeoutMs); + } + + /** + * Returns the configured drain timeout in milliseconds. + * + * @return Drain timeout in milliseconds + */ + public long getDrainTimeoutMs() { + return drainTimeoutMs; } /** diff --git a/src/test/java/me/desair/tus/server/TusFileUploadServiceTest.java b/src/test/java/me/desair/tus/server/TusFileUploadServiceTest.java index 981ebace..33cf0a29 100644 --- a/src/test/java/me/desair/tus/server/TusFileUploadServiceTest.java +++ b/src/test/java/me/desair/tus/server/TusFileUploadServiceTest.java @@ -14,6 +14,7 @@ import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; +import java.time.Duration; import me.desair.tus.server.exception.UploadAlreadyLockedException; import me.desair.tus.server.upload.UploadId; import me.desair.tus.server.upload.UploadInfo; @@ -62,7 +63,7 @@ public void testAcquireUploadLockFallback() throws Exception { TusFileUploadService service = new TusFileUploadService() .withUploadLockingService(mockLockingService) - .withMaxLockRetries(5); + .withLockWaitTimeout(Duration.ofSeconds(6)); UploadLock lock = service.acquireUploadLock(HttpMethod.HEAD, "/files/test"); assertNotNull(lock); @@ -343,6 +344,7 @@ public void testWithUploadStorageServicePreservesConfiguration() { verify(newStorage).setMinSize(2048L); verify(newStorage).setCloudUploadThreadPoolSize(18); verify(newStorage).setUploadDeduplicationEnabled(true); + verify(newStorage).setDrainTimeout(Duration.ofSeconds(55)); } @Test @@ -357,6 +359,7 @@ public void testThreadLocalCacheDelegatesAppendAndMinSizes() { assertThat(service.getUploadStorageService().getMaxAppendSize(), is(1024L)); assertThat(service.getUploadStorageService().getMinAppendSize(), is(512L)); assertThat(service.getUploadStorageService().getMinSize(), is(2048L)); + assertThat(service.getUploadStorageService().getDrainTimeout(), is(Duration.ofSeconds(55))); assertThat(service.getUploadStorageService().getCloudUploadThreadPoolSize(), is(16)); } @@ -629,20 +632,53 @@ public void testDeleteUploadSingleArg() throws Exception { } @Test - public void testWithMaxLockRetries() { + public void testWithLockWaitTimeout() { TusFileUploadService service = new TusFileUploadService(); - assertEquals(40, service.getMaxLockRetries()); + assertEquals(Duration.ofSeconds(60), service.getLockWaitTimeout()); + assertEquals(300, service.getMaxLockRetries()); + assertEquals(Duration.ofSeconds(55), service.getUploadStorageService().getDrainTimeout()); - service.withMaxLockRetries(5); - assertEquals(5, service.getMaxLockRetries()); + service.withLockWaitTimeout(Duration.ofSeconds(30)); + assertEquals(Duration.ofSeconds(30), service.getLockWaitTimeout()); + assertEquals(150, service.getMaxLockRetries()); + assertEquals(Duration.ofSeconds(25), service.getUploadStorageService().getDrainTimeout()); - service.withMaxLockRetries(0); - assertEquals(0, service.getMaxLockRetries()); + service.withLockWaitTimeout(Duration.ofMillis(6000)); + assertEquals(Duration.ofMillis(6000), service.getLockWaitTimeout()); + assertEquals(30, service.getMaxLockRetries()); + assertEquals(Duration.ofMillis(1000), service.getUploadStorageService().getDrainTimeout()); + } + + @Test(expected = NullPointerException.class) + public void testWithLockWaitTimeoutNullThrows() { + new TusFileUploadService().withLockWaitTimeout(null); + } + + @Test(expected = IllegalArgumentException.class) + public void testWithLockWaitTimeoutFiveSecondsOrLessThrows() { + new TusFileUploadService().withLockWaitTimeout(Duration.ofSeconds(5)); + } + + @Test(expected = IllegalArgumentException.class) + public void testWithLockWaitTimeoutZeroThrows() { + new TusFileUploadService().withLockWaitTimeout(Duration.ZERO); } @Test(expected = IllegalArgumentException.class) - public void testWithMaxLockRetriesNegativeThrows() { - new TusFileUploadService().withMaxLockRetries(-1); + public void testWithLockWaitTimeoutNegativeThrows() { + new TusFileUploadService().withLockWaitTimeout(Duration.ofSeconds(-1)); + } + + @Test + public void testWithUploadStorageServiceCopiesDrainTimeout() { + TusFileUploadService service = + new TusFileUploadService().withLockWaitTimeout(Duration.ofSeconds(20)); + + UploadStorageService newStorageService = mock(UploadStorageService.class); + when(newStorageService.isUploadDeduplicationEnabled()).thenReturn(false); + + service.withUploadStorageService(newStorageService); + verify(newStorageService).setDrainTimeout(Duration.ofSeconds(15)); } @Test @@ -659,7 +695,7 @@ public void testAcquireUploadLockWithConfiguredRetries() throws Exception { TusFileUploadService service = new TusFileUploadService() .withUploadLockingService(mockLockingService) - .withMaxLockRetries(2); + .withLockWaitTimeout(Duration.ofMillis(5200)); UploadLock lock = service.acquireUploadLock(HttpMethod.HEAD, "/files/test"); assertNotNull(lock); diff --git a/src/test/java/me/desair/tus/server/upload/UploadStorageServiceTest.java b/src/test/java/me/desair/tus/server/upload/UploadStorageServiceTest.java index 1933b822..151db73e 100644 --- a/src/test/java/me/desair/tus/server/upload/UploadStorageServiceTest.java +++ b/src/test/java/me/desair/tus/server/upload/UploadStorageServiceTest.java @@ -7,6 +7,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.time.Duration; import me.desair.tus.server.checksum.ChecksumAlgorithm; import me.desair.tus.server.exception.TusException; import me.desair.tus.server.exception.UploadNotFoundException; @@ -145,6 +146,9 @@ public void testDefaultAppendAndSizeConfigurationMethods() throws Exception { dummyStorageService.setCloudUploadThreadPoolSize(15); assertThat(dummyStorageService.getCloudUploadThreadPoolSize(), is(10)); + dummyStorageService.setDrainTimeout(Duration.ofSeconds(20)); + assertThat(dummyStorageService.getDrainTimeout(), is(Duration.ofSeconds(55))); + // Default close is a no-op dummyStorageService.close(); } diff --git a/src/test/java/me/desair/tus/server/upload/azure/AzureBlobStorageServiceTest.java b/src/test/java/me/desair/tus/server/upload/azure/AzureBlobStorageServiceTest.java index 9f4c052e..8c2f33da 100644 --- a/src/test/java/me/desair/tus/server/upload/azure/AzureBlobStorageServiceTest.java +++ b/src/test/java/me/desair/tus/server/upload/azure/AzureBlobStorageServiceTest.java @@ -11,6 +11,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.time.Duration; import me.desair.tus.server.checksum.ChecksumAlgorithm; import me.desair.tus.server.exception.MaxAppendSizeExceededException; import me.desair.tus.server.upload.TimeBasedUploadIdFactory; @@ -261,6 +262,27 @@ public void testCloudUploadThreadPoolSizeConfiguration() { } } + @Test + public void testDrainTimeoutConfiguration() { + assertEquals( + "Default drain timeout is 55 seconds", + Duration.ofSeconds(55), + storageService.getDrainTimeout()); + + storageService.setDrainTimeout(Duration.ofSeconds(25)); + assertEquals( + "Updated drain timeout is 25 seconds", + Duration.ofSeconds(25), + storageService.getDrainTimeout()); + + // Null is ignored preserving current value + storageService.setDrainTimeout(null); + assertEquals( + "Null drain timeout preserves previous value", + Duration.ofSeconds(25), + storageService.getDrainTimeout()); + } + @Test public void testCloseGracefulShutdown() throws Exception { // Verify closing storage service cleanly terminates background upload executor without error diff --git a/src/test/java/me/desair/tus/server/upload/cache/ThreadLocalCachedStorageAndLockingServiceTest.java b/src/test/java/me/desair/tus/server/upload/cache/ThreadLocalCachedStorageAndLockingServiceTest.java index d27e22cb..be72ffec 100644 --- a/src/test/java/me/desair/tus/server/upload/cache/ThreadLocalCachedStorageAndLockingServiceTest.java +++ b/src/test/java/me/desair/tus/server/upload/cache/ThreadLocalCachedStorageAndLockingServiceTest.java @@ -12,6 +12,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.time.Duration; import java.util.UUID; import me.desair.tus.server.checksum.ChecksumAlgorithm; import me.desair.tus.server.upload.UploadId; @@ -185,6 +186,12 @@ public void testDelegateMethods() throws Exception { service.getCloudUploadThreadPoolSize(); verify(mockStorage, times(1)).getCloudUploadThreadPoolSize(); + service.setDrainTimeout(Duration.ofSeconds(25)); + verify(mockStorage, times(1)).setDrainTimeout(Duration.ofSeconds(25)); + + service.getDrainTimeout(); + verify(mockStorage, times(1)).getDrainTimeout(); + when(mockStorage.create(info, "owner")).thenReturn(info); assertEquals(info, service.create(info, "owner")); verify(mockStorage, times(1)).create(info, "owner"); diff --git a/src/test/java/me/desair/tus/server/upload/disk/DiskStorageServiceTest.java b/src/test/java/me/desair/tus/server/upload/disk/DiskStorageServiceTest.java index 486be745..d24195ed 100644 --- a/src/test/java/me/desair/tus/server/upload/disk/DiskStorageServiceTest.java +++ b/src/test/java/me/desair/tus/server/upload/disk/DiskStorageServiceTest.java @@ -25,6 +25,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.time.Duration; import java.util.Arrays; import java.util.List; import java.util.UUID; @@ -1123,4 +1124,15 @@ public void testSetCloudUploadThreadPoolSizeZeroThrowsIllegalArgumentException() public void testSetCloudUploadThreadPoolSizeNegativeThrowsIllegalArgumentException() { storageService.setCloudUploadThreadPoolSize(-1); } + + @Test + public void testDefaultDrainTimeoutOnDiskStorageService() { + assertThat(storageService.getDrainTimeout(), is(Duration.ofSeconds(55))); + storageService.setDrainTimeout(Duration.ofSeconds(20)); + assertThat(storageService.getDrainTimeout(), is(Duration.ofSeconds(20))); + + // Null is ignored preserving current value + storageService.setDrainTimeout(null); + assertThat(storageService.getDrainTimeout(), is(Duration.ofSeconds(20))); + } } diff --git a/src/test/java/me/desair/tus/server/upload/s3/S3StorageServiceTest.java b/src/test/java/me/desair/tus/server/upload/s3/S3StorageServiceTest.java index d795775c..78008706 100644 --- a/src/test/java/me/desair/tus/server/upload/s3/S3StorageServiceTest.java +++ b/src/test/java/me/desair/tus/server/upload/s3/S3StorageServiceTest.java @@ -32,6 +32,7 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; +import java.time.Duration; import java.util.Arrays; import java.util.Collections; import me.desair.tus.server.checksum.ChecksumAlgorithm; @@ -2062,6 +2063,27 @@ public void testCloudUploadThreadPoolSizeConfiguration() { } } + @Test + public void testDrainTimeoutConfiguration() { + assertEquals( + "Default drain timeout is 55 seconds", + Duration.ofSeconds(55), + storageService.getDrainTimeout()); + + storageService.setDrainTimeout(Duration.ofSeconds(20)); + assertEquals( + "Updated drain timeout is 20 seconds", + Duration.ofSeconds(20), + storageService.getDrainTimeout()); + + // Null is ignored preserving current value + storageService.setDrainTimeout(null); + assertEquals( + "Null drain timeout preserves previous value", + Duration.ofSeconds(20), + storageService.getDrainTimeout()); + } + @Test public void testCloseGracefulShutdown() throws Exception { // Verify closing storage service cleanly terminates background upload executor without error diff --git a/src/test/java/me/desair/tus/server/upload/util/AsyncChunkUploaderTest.java b/src/test/java/me/desair/tus/server/upload/util/AsyncChunkUploaderTest.java index 86a430b5..85cc42ce 100644 --- a/src/test/java/me/desair/tus/server/upload/util/AsyncChunkUploaderTest.java +++ b/src/test/java/me/desair/tus/server/upload/util/AsyncChunkUploaderTest.java @@ -587,4 +587,19 @@ public void testDrainAndCompleteDefaultTimeout() throws Exception { assertFalse("Chunk file should be cleaned up", chunk.exists()); } + + @Test + public void testCustomDrainTimeoutConstructorAndGetter() { + try (AsyncChunkUploader uploader = new AsyncChunkUploader(executor, 12_345L)) { + assertEquals(12_345L, uploader.getDrainTimeoutMs()); + } + } + + @Test + public void testDefaultDrainTimeoutConstant() { + assertEquals(55_000L, AsyncChunkUploader.DEFAULT_DRAIN_TIMEOUT_MS); + try (AsyncChunkUploader uploader = new AsyncChunkUploader(executor)) { + assertEquals(55_000L, uploader.getDrainTimeoutMs()); + } + } } From 7b843fdb48020327387d423e3f8d0363331f7df1 Mon Sep 17 00:00:00 2001 From: Tom Desair Date: Sun, 4 Oct 2026 18:50:03 +0200 Subject: [PATCH 7/7] feat(s3): decouple S3 storage, locking, and concatenation services from MinioClient --- CHANGELOG.md | 2 +- docs/S3_STORAGE.md | 84 +++++++---- .../upload/s3/S3ConcatenationService.java | 132 +++++++++++------- .../server/upload/s3/S3LockingService.java | 70 +++++----- .../upload/s3/S3ServerSideComposeHelper.java | 43 +++--- .../server/upload/s3/S3StorageService.java | 107 +++++--------- .../tus/server/upload/s3/S3UploadLock.java | 2 +- .../java/me/desair/tus/server/TestUtils.java | 12 +- .../server/upload/s3/ITS3LockingService.java | 3 +- .../server/upload/s3/ITS3RufhProtocol.java | 10 +- .../server/upload/s3/ITS3StorageService.java | 3 +- .../upload/s3/ITS3TusFileUploadService.java | 10 +- .../upload/s3/S3ConcatenationServiceTest.java | 81 ++++++++--- .../upload/s3/S3LockingServiceTest.java | 47 +++++-- .../s3/S3ServerSideComposeHelperTest.java | 10 +- .../upload/s3/S3StorageServiceTest.java | 72 ++++++++-- 16 files changed, 428 insertions(+), 260 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 108ff851..fa9dee4f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,7 @@ All notable changes to this project will be documented in this file. ### New - **NFS- & SMB-Safe Lease Locking (`LeaseFileLockingService` & `LeaseFileMutex`)**: Added distributed, container-safe filesystem locking using atomic sibling mutex directories (`.mutex/`), in-place expired lock takeover, and TTL-based JSON lease files with background heartbeat renewal and ownership fencing. Operates reliably across multi-server replicas on NFS (v3/v4), AWS EFS, Azure Files, Windows SMB/CIFS, and local disks without requiring Redis, ZooKeeper, or OS-level `FileLock` daemons. Comprehensive guide and legacy opt-out instructions available in `docs/DISK_BASED_LOCKING.md`. -- **S3-Compatible Storage & Distributed Locking**: Added native S3 storage support via `S3StorageService` (MinIO SDK), distributed locking via `S3LockingService` (S3 conditional writes with TTL leases and interrupt signals for multi-replica container deployments), S3-native concatenation via `S3ConcatenationService`, and complete documentation in `docs/S3_STORAGE.md`. +- **S3-Compatible Storage & Distributed Locking**: Added native S3 storage support via `S3StorageService`, distributed locking via `S3LockingService` (S3 conditional writes with TTL leases and interrupt signals for multi-replica container deployments), S3-native concatenation via `S3ConcatenationService`, configured entirely via standard S3 connection parameters, and complete documentation in `docs/S3_STORAGE.md`. - **Azure Blob Storage & Distributed Leases**: Added native Azure Blob Storage support via `AzureBlobStorageService` (Block Blob staging with streaming appends, sub-threshold buffering, truncation, and deduplication), distributed locking via `AzureBlobLockingService` (Azure Blob Leases with auto-renewal, JVM interruption, cross-replica `.stop` signals, and clean shutdown), zero-copy server-side concatenation via `AzureBlobConcatenationService` (`stageBlockFromUrl`), and comprehensive documentation in `docs/AZURE_BLOB_STORAGE.md`. - **IETF Resumable Uploads for HTTP (RUFH) Protocol**: Implemented full support for the official IETF Resumable Uploads for HTTP specification (`draft-ietf-httpbis-resumable-upload-12`). - **Dual Protocol Auto-Detection**: Added transparent protocol routing in `TusFileUploadService` supporting both legacy `TUS_1_0_0` (`Tus-Resumable: 1.0.0`) and `RUFH` (`ProtocolVersion.RUFH`) clients concurrently on the same endpoint. diff --git a/docs/S3_STORAGE.md b/docs/S3_STORAGE.md index 3e4b46f2..c87c67e5 100644 --- a/docs/S3_STORAGE.md +++ b/docs/S3_STORAGE.md @@ -29,38 +29,35 @@ Jackson dependencies (`jackson-databind`, `jackson-annotations`, `jackson-core`) ### Step 2: Configure `TusFileUploadService` ```java -import io.minio.MinioClient; import me.desair.tus.server.TusFileUploadService; import me.desair.tus.server.upload.s3.S3StorageService; import me.desair.tus.server.upload.s3.S3LockingService; -// 1. Production Configuration: Load S3 parameters securely from environment variables +// Option A: Local S3-compatible backend (e.g. MinIO, RustFS, Ceph) +// The region parameter can be omitted (defaults to "local"); local backends ignore region in SigV4 validation. +S3StorageService localS3Storage = new S3StorageService( + "http://localhost:9000", "minioadmin", "minioadmin", "my-upload-bucket"); +S3LockingService localS3Locking = new S3LockingService( + "http://localhost:9000", "minioadmin", "minioadmin", "my-upload-bucket"); + +// Option B: AWS S3 with explicit region +// AWS strictly enforces region validation for bucket routing and SigV4 authentication. String endpoint = System.getenv().getOrDefault("S3_ENDPOINT", "https://s3.eu-central-1.amazonaws.com"); String bucketName = System.getenv().getOrDefault("S3_BUCKET_NAME", "my-upload-bucket"); String accessKey = System.getenv("AWS_ACCESS_KEY_ID"); String secretKey = System.getenv("AWS_SECRET_ACCESS_KEY"); -MinioClient minioClient = MinioClient.builder() - .endpoint(endpoint) - .credentials(accessKey, secretKey) - .build(); - -// 2. Instantiate S3 Storage and Distributed Locking services -// Option A: Direct connection parameters (recommended, builds internal client with server-side compose helper) -S3StorageService s3StorageService = new S3StorageService(endpoint, "eu-central-1", accessKey, secretKey, bucketName); -S3LockingService s3LockingService = new S3LockingService(endpoint, "eu-central-1", accessKey, secretKey, bucketName); - -// Option B: Using a pre-configured MinIO Client -// S3StorageService s3StorageService = new S3StorageService(minioClient, endpoint, "eu-central-1", accessKey, secretKey, bucketName); -// S3LockingService s3LockingService = new S3LockingService(minioClient, bucketName); +S3StorageService awsS3Storage = new S3StorageService( + endpoint, "eu-central-1", accessKey, secretKey, bucketName); +S3LockingService awsS3Locking = new S3LockingService( + endpoint, "eu-central-1", accessKey, secretKey, bucketName); -// 3. Configure TusFileUploadService with S3 storage and locking +// Configure TusFileUploadService with S3 storage and locking // Note: Automatic JVM shutdown hooks are built-in by default to terminate watchdog threads on pod exit. -// Manual call to tusService.close() or s3LockingService.close() is optional for custom container lifecycles. TusFileUploadService tusService = new TusFileUploadService() .withUploadUri("/files/upload") - .withUploadStorageService(s3StorageService) - .withUploadLockingService(s3LockingService); + .withUploadStorageService(awsS3Storage) + .withUploadLockingService(awsS3Locking); ``` --- @@ -144,19 +141,41 @@ try (InputStream stream = minioClient.getObject( --- -## 5. Configuring Custom S3 Endpoints (MinIO, RustFS, R2, Ceph, GCS) +## 5. Configuring Custom S3 Endpoints & Regions (AWS vs. Local S3 Backends) + +`S3StorageService`, `S3LockingService`, and `S3ConcatenationService` accept any S3 endpoint and handle regional authentication flexibly: + +- **Local S3-Compatible Backends (MinIO, RustFS, Ceph, SeaweedFS)**: + Self-hosted and local S3-compatible storage engines do not strictly validate AWS regions during Signature Version 4 (SigV4) authentication. You can omit the `region` parameter completely; it automatically defaults to `"local"`. +- **AWS S3 & Cloud Providers (AWS S3, Cloudflare R2, Google Cloud Storage)**: + AWS strictly validates the region parameter against bucket locations during SigV4 authentication and request routing. Always supply the target region (e.g., `"eu-central-1"`, `"us-east-1"`, or `"auto"` for Cloudflare R2). + +### Example 1: Local S3-Compatible Backend (Without Region) + +```java +// Omit region parameter; defaults to "local" +S3StorageService localS3Storage = + new S3StorageService("http://minio.local:9000", "minioadmin", "minioadmin", "my-bucket"); + +S3LockingService localS3Locking = + new S3LockingService("http://minio.local:9000", "minioadmin", "minioadmin", "my-bucket"); + +S3ConcatenationService localS3Concat = + new S3ConcatenationService("http://minio.local:9000", "minioadmin", "minioadmin", "my-bucket", localS3Storage); +``` -`S3StorageService` accepts any pre-configured `MinioClient`. To connect to an S3-compatible backend (such as local MinIO, RustFS, or Cloudflare R2), override the endpoint when building the `MinioClient`: +### Example 2: AWS S3 (With Explicit Region) ```java -import io.minio.MinioClient; +// Provide explicit AWS region (e.g. "eu-central-1", "us-east-1") +S3StorageService awsS3Storage = + new S3StorageService("https://s3.eu-central-1.amazonaws.com", "eu-central-1", accessKey, secretKey, "my-bucket"); -MinioClient minioClient = MinioClient.builder() - .endpoint("http://minio.local:9000") - .credentials("minioadmin", "minioadmin") - .build(); +S3LockingService awsS3Locking = + new S3LockingService("https://s3.eu-central-1.amazonaws.com", "eu-central-1", accessKey, secretKey, "my-bucket"); -S3StorageService s3Storage = new S3StorageService(minioClient, "my-bucket"); +S3ConcatenationService awsS3Concat = + new S3ConcatenationService("https://s3.eu-central-1.amazonaws.com", "eu-central-1", accessKey, secretKey, "my-bucket", awsS3Storage); ``` --- @@ -184,7 +203,10 @@ S3 requires every part chunk of a multipart upload to be at least 5 MB (except t Path customTempDir = Paths.get("/var/tmp/tus-buffer"); S3StorageService s3Storage = new S3StorageService( - minioClient, + endpoint, + "eu-central-1", + accessKey, + secretKey, "my-bucket", "uploads/", "metadata/", @@ -259,19 +281,19 @@ The jitter window can be customized via `.withJitter(minMs, maxMs)` on `S3Lockin // On backends with atomic conditional writes, jitter is not needed. // Passing (0L, 0L) completely bypasses Thread.sleep for fastest lock acquisition: S3LockingService s3LockingService = - new S3LockingService(minioClient, bucketName) + new S3LockingService(endpoint, "eu-central-1", accessKey, secretKey, bucketName) .withJitter(0L, 0L); // Option B: High-Latency or Distributed Backends (Ceph, multi-datacenter MinIO) // High-latency backends or geo-replicated clusters may need a wider window to settle writes: S3LockingService s3LockingService = - new S3LockingService(minioClient, bucketName) + new S3LockingService(endpoint, "eu-central-1", accessKey, secretKey, bucketName) .withJitter(50L, 200L); // Option C: Explicit Non-CAS Mode (e.g. Wasabi) // Pre-checks existing locks before writing unconditionally: S3LockingService s3LockingService = - new S3LockingService(minioClient, bucketName) + new S3LockingService(endpoint, "eu-central-1", accessKey, secretKey, bucketName) .withS3ConditionalWritesSupported(false); ``` diff --git a/src/main/java/me/desair/tus/server/upload/s3/S3ConcatenationService.java b/src/main/java/me/desair/tus/server/upload/s3/S3ConcatenationService.java index 5b128453..b6207758 100644 --- a/src/main/java/me/desair/tus/server/upload/s3/S3ConcatenationService.java +++ b/src/main/java/me/desair/tus/server/upload/s3/S3ConcatenationService.java @@ -45,77 +45,126 @@ public class S3ConcatenationService implements UploadConcatenationService { private final String objectPrefix; private final long minPartSize; private final Path temporaryDirectory; - private UploadStorageService uploadStorageService; + private final UploadStorageService uploadStorageService; private S3ServerSideComposeHelper s3ComposeHelper; /** - * Basic constructor using default object prefix ("uploads/") and Java temp directory. + * Convenience constructor for local S3-compatible backends taking backing {@link + * UploadStorageService}. Defaults the region to "local". * - * @param minioClient The MinIO client - * @param bucket The S3 bucket name + * @param endpoint S3 endpoint URL (e.g. "http://localhost:9000") + * @param accessKey S3 access key / username + * @param secretKey S3 secret key / password + * @param bucket S3 bucket name + * @param uploadStorageService Underlying storage service */ - public S3ConcatenationService(MinioClient minioClient, String bucket) { - this(minioClient, bucket, "uploads/", null, null); + public S3ConcatenationService( + String endpoint, + String accessKey, + String secretKey, + String bucket, + UploadStorageService uploadStorageService) { + this(endpoint, "local", accessKey, secretKey, bucket, uploadStorageService); } /** - * Convenient constructor taking MinioClient, bucket, and UploadStorageService. + * Convenient constructor taking connection parameters and backing {@link UploadStorageService}. * - * @param minioClient The MinIO client - * @param bucket The S3 bucket name + * @param endpoint S3 endpoint URL (e.g. "https://s3.amazonaws.com" or "http://localhost:9000") + * @param region S3 region name (e.g. "us-east-1", "eu-central-1") + * @param accessKey S3 access key / username + * @param secretKey S3 secret key / password + * @param bucket S3 bucket name * @param uploadStorageService Underlying storage service */ public S3ConcatenationService( - MinioClient minioClient, String bucket, UploadStorageService uploadStorageService) { - this(minioClient, bucket, "uploads/", uploadStorageService, null); + String endpoint, + String region, + String accessKey, + String secretKey, + String bucket, + UploadStorageService uploadStorageService) { + this( + endpoint, + region, + accessKey, + secretKey, + bucket, + "uploads/", + uploadStorageService, + null, + DEFAULT_MIN_PART_SIZE); } /** - * Constructs an S3ConcatenationService. + * Full constructor allowing custom object prefix, temporary directory, and minimum part size. + * + *

Delegates to the internal package-private constructor accepting {@link MinioClient}. * - * @param minioClient The MinIO client - * @param bucket The S3 bucket name + * @param endpoint S3 endpoint URL + * @param region S3 region name + * @param accessKey S3 access key / username + * @param secretKey S3 secret key / password + * @param bucket S3 bucket name * @param objectPrefix Key prefix for data objects * @param uploadStorageService Underlying storage service * @param temporaryDirectory Directory for temporary buffer files + * @param minPartSize Minimum part chunk size for server-side composition */ public S3ConcatenationService( - MinioClient minioClient, + String endpoint, + String region, + String accessKey, + String secretKey, String bucket, String objectPrefix, UploadStorageService uploadStorageService, - Path temporaryDirectory) { + Path temporaryDirectory, + long minPartSize) { this( - minioClient, + buildMinioClient(endpoint, region, accessKey, secretKey), bucket, objectPrefix, uploadStorageService, temporaryDirectory, - DEFAULT_MIN_PART_SIZE); + minPartSize, + new S3ServerSideComposeHelper(endpoint, region, accessKey, secretKey)); } - /** Full constructor allowing custom minimum part size. */ - public S3ConcatenationService( + /** + * Internal package-private constructor accepting {@link MinioClient} where all parameter + * configuration is concentrated. + */ + S3ConcatenationService( MinioClient minioClient, String bucket, String objectPrefix, UploadStorageService uploadStorageService, Path temporaryDirectory, - long minPartSize) { + long minPartSize, + S3ServerSideComposeHelper s3ComposeHelper) { this.minioClient = Objects.requireNonNull(minioClient, "MinioClient must not be null"); this.bucket = Objects.requireNonNull(bucket, "Bucket must not be null"); this.objectPrefix = objectPrefix != null ? objectPrefix : ""; - this.uploadStorageService = uploadStorageService; + this.uploadStorageService = + Objects.requireNonNull(uploadStorageService, "UploadStorageService must not be null"); this.temporaryDirectory = temporaryDirectory != null ? temporaryDirectory : java.nio.file.Paths.get(System.getProperty("java.io.tmpdir")); this.minPartSize = minPartSize; - this.s3ComposeHelper = new S3ServerSideComposeHelper(this.minioClient); + this.s3ComposeHelper = + s3ComposeHelper != null ? s3ComposeHelper : new S3ServerSideComposeHelper(this.minioClient); } - public void setUploadStorageService(UploadStorageService uploadStorageService) { - this.uploadStorageService = uploadStorageService; + private static MinioClient buildMinioClient( + String endpoint, String region, String accessKey, String secretKey) { + String effectiveRegion = (region != null && !region.isEmpty()) ? region : "local"; + return MinioClient.builder() + .endpoint(endpoint) + .credentials(accessKey, secretKey) + .region(effectiveRegion) + .build(); } @Override @@ -126,8 +175,7 @@ public void merge(UploadInfo uploadInfo) throws IOException, UploadNotFoundExcep return; } - Long expirationPeriod = - uploadStorageService != null ? uploadStorageService.getUploadExpirationPeriod() : null; + Long expirationPeriod = uploadStorageService.getUploadExpirationPeriod(); List partialUploads = getPartialUploads(uploadInfo); Long totalLength = calculateTotalLength(partialUploads); @@ -175,12 +223,10 @@ public void merge(UploadInfo uploadInfo) throws IOException, UploadNotFoundExcep } uploadInfo.setStorageUploadId(targetObjectKey); - if (uploadStorageService != null) { - try { - uploadStorageService.update(uploadInfo); - } catch (UploadNotFoundException e) { - log.warn("Failed to update concatenated upload info for " + uploadInfo.getId(), e); - } + try { + uploadStorageService.update(uploadInfo); + } catch (UploadNotFoundException e) { + log.warn("Failed to update concatenated upload info for " + uploadInfo.getId(), e); } } } @@ -192,11 +238,6 @@ public InputStream getConcatenatedBytes(UploadInfo uploadInfo) return null; } - if (uploadStorageService == null) { - throw new IOException( - "UploadStorageService must be configured to retrieve concatenated upload bytes"); - } - if (uploadInfo.isUploadInProgress()) { merge(uploadInfo); } @@ -219,10 +260,7 @@ public List getPartialUploads(UploadInfo info) List output = new ArrayList<>(concatenationParts.size()); for (String childUri : concatenationParts) { - UploadInfo childInfo = - uploadStorageService != null - ? uploadStorageService.getUploadInfo(childUri, info.getOwnerKey()) - : null; + UploadInfo childInfo = uploadStorageService.getUploadInfo(childUri, info.getOwnerKey()); if (childInfo == null) { throw new UploadNotFoundException( "Upload with URI " + childUri + " was not found for owner " + info.getOwnerKey()); @@ -308,12 +346,10 @@ private boolean checkAllCompleted(Long expirationPeriod, List partia completed = false; } else if (expirationPeriod != null) { childInfo.updateExpiration(expirationPeriod); - if (uploadStorageService != null) { - try { - uploadStorageService.update(childInfo); - } catch (UploadNotFoundException e) { - log.debug("Failed to update child upload expiration for " + childInfo.getId(), e); - } + try { + uploadStorageService.update(childInfo); + } catch (UploadNotFoundException e) { + log.debug("Failed to update child upload expiration for " + childInfo.getId(), e); } } } diff --git a/src/main/java/me/desair/tus/server/upload/s3/S3LockingService.java b/src/main/java/me/desair/tus/server/upload/s3/S3LockingService.java index 84245e97..1def9dc0 100644 --- a/src/main/java/me/desair/tus/server/upload/s3/S3LockingService.java +++ b/src/main/java/me/desair/tus/server/upload/s3/S3LockingService.java @@ -64,25 +64,21 @@ public class S3LockingService extends AbstractLeaseLockingService { private volatile boolean s3ConditionalWritesSupported = true; /** - * Basic constructor using default lock prefix ("locks/"), 30s lease duration, and 2s polling - * interval. + * Convenience constructor for local S3-compatible backends where region is omitted. Defaults the + * region to "local". * - * @param minioClient Pre-configured MinIO Client + * @param endpoint S3 endpoint URL (e.g. "http://localhost:9000") + * @param accessKey S3 access key / username + * @param secretKey S3 secret key / password * @param bucket Target S3 bucket name */ - public S3LockingService(MinioClient minioClient, String bucket) { - this( - minioClient, - bucket, - DEFAULT_LOCKS_PREFIX, - DEFAULT_LEASE_DURATION_MS, - DEFAULT_POLL_INTERVAL_MS, - new UuidUploadIdFactory()); + public S3LockingService(String endpoint, String accessKey, String secretKey, String bucket) { + this(endpoint, "local", accessKey, secretKey, bucket); } /** - * Constructor accepting explicit connection parameters without requiring a pre-existing - * MinioClient. + * Basic constructor accepting explicit connection parameters without exposing underlying client + * libraries. * * @param endpoint S3 endpoint URL (e.g. "https://s3.amazonaws.com" or "http://localhost:9000") * @param region S3 region name (e.g. "eu-central-1", "us-east-1") @@ -93,7 +89,10 @@ public S3LockingService(MinioClient minioClient, String bucket) { public S3LockingService( String endpoint, String region, String accessKey, String secretKey, String bucket) { this( - buildMinioClient(endpoint, region, accessKey, secretKey), + endpoint, + region, + accessKey, + secretKey, bucket, DEFAULT_LOCKS_PREFIX, DEFAULT_LEASE_DURATION_MS, @@ -102,45 +101,44 @@ public S3LockingService( } /** - * Constructor accepting a pre-configured {@link MinioClient} along with explicit connection - * parameters. + * Full constructor allowing custom configuration including a custom {@link UploadIdFactory}. + * + *

Delegates to the internal package-private constructor that accepts {@link MinioClient}. * - * @param minioClient Pre-configured MinIO Client * @param endpoint S3 endpoint URL * @param region S3 region name * @param accessKey S3 access key / username * @param secretKey S3 secret key / password * @param bucket Target S3 bucket name + * @param locksPrefix Object key prefix for locks and stop signals + * @param leaseDurationMs Lock lease duration in milliseconds + * @param pollIntervalMs Watchdog poll interval for lock contention interrupt signals + * @param idFactory Custom {@link UploadIdFactory} */ public S3LockingService( - MinioClient minioClient, String endpoint, String region, String accessKey, String secretKey, - String bucket) { + String bucket, + String locksPrefix, + long leaseDurationMs, + long pollIntervalMs, + UploadIdFactory idFactory) { this( - minioClient != null - ? minioClient - : buildMinioClient(endpoint, region, accessKey, secretKey), + buildMinioClient(endpoint, region, accessKey, secretKey), bucket, - DEFAULT_LOCKS_PREFIX, - DEFAULT_LEASE_DURATION_MS, - DEFAULT_POLL_INTERVAL_MS, - new UuidUploadIdFactory()); + locksPrefix, + leaseDurationMs, + pollIntervalMs, + idFactory); } /** - * Full constructor allowing custom configuration including a custom {@link UploadIdFactory}. - * - * @param minioClient Pre-configured MinIO Client - * @param bucket Target S3 bucket name - * @param locksPrefix Object key prefix for locks and stop signals - * @param leaseDurationMs Lock lease duration in milliseconds - * @param pollIntervalMs Watchdog poll interval for lock contention interrupt signals - * @param idFactory Custom {@link UploadIdFactory} + * Package-private constructor accepting {@link MinioClient} where all parameter configuration is + * concentrated. */ - public S3LockingService( + S3LockingService( MinioClient minioClient, String bucket, String locksPrefix, @@ -448,7 +446,7 @@ private String buildStopKey(UploadId uploadId) { private static MinioClient buildMinioClient( String endpoint, String region, String accessKey, String secretKey) { - String effectiveRegion = (region != null && !region.isEmpty()) ? region : "eu-central-1"; + String effectiveRegion = (region != null && !region.isEmpty()) ? region : "local"; return MinioClient.builder() .endpoint(endpoint) .credentials(accessKey, secretKey) diff --git a/src/main/java/me/desair/tus/server/upload/s3/S3ServerSideComposeHelper.java b/src/main/java/me/desair/tus/server/upload/s3/S3ServerSideComposeHelper.java index 8a645173..a805391d 100644 --- a/src/main/java/me/desair/tus/server/upload/s3/S3ServerSideComposeHelper.java +++ b/src/main/java/me/desair/tus/server/upload/s3/S3ServerSideComposeHelper.java @@ -63,38 +63,35 @@ public class S3ServerSideComposeHelper { private final String explicitRegion; /** - * Constructs an instance wrapping the given {@link MinioClient} without reflection. - * - *

If explicit connection parameters are not provided, this helper falls back to calling {@link - * MinioClient#composeObject(ComposeObjectArgs)} directly. + * Constructs an instance with explicit connection parameters without requiring a {@link + * MinioClient}. * - * @param minioClient The MinIO client instance + * @param endpoint The S3 endpoint URL (e.g. "https://s3.amazonaws.com" or + * "http://localhost:9000") + * @param region S3 region name (optional, defaults to "local" if null or empty) + * @param accessKey S3 access key / username + * @param secretKey S3 secret key / password */ - public S3ServerSideComposeHelper(MinioClient minioClient) { - this.minioClient = minioClient; - this.asyncClient = null; - this.baseUrl = null; - this.provider = null; - this.httpClient = null; - this.explicitRegion = null; + public S3ServerSideComposeHelper( + String endpoint, String region, String accessKey, String secretKey) { + this(null, endpoint, region, accessKey, secretKey); } /** - * Constructs an instance with an existing {@link MinioClient} and explicit connection parameters. + * Constructs an instance wrapping the given {@link MinioClient} without reflection. * - *

This allows native S3 multipart copy without using reflection on {@link MinioClient}. + *

Package-private constructor for internal testing. * - * @param minioClient The existing MinIO client instance - * @param endpoint The S3 endpoint URL (e.g. "https://s3.amazonaws.com" or - * "http://localhost:9000") - * @param region S3 region name (optional, defaults to "eu-central-1" if null or empty) - * @param accessKey S3 access key / username - * @param secretKey S3 secret key / password + * @param minioClient The MinIO client instance */ - public S3ServerSideComposeHelper( + S3ServerSideComposeHelper(MinioClient minioClient) { + this(minioClient, null, null, null, null); + } + + private S3ServerSideComposeHelper( MinioClient minioClient, String endpoint, String region, String accessKey, String secretKey) { this.minioClient = minioClient; - this.explicitRegion = (region != null && !region.isEmpty()) ? region : "eu-central-1"; + this.explicitRegion = (region != null && !region.isEmpty()) ? region : "local"; Http.BaseUrl base = null; Provider prov = null; @@ -164,7 +161,7 @@ public void compose(String bucket, String targetKey, List partKeys) thro region = explicitRegion; } if (region == null || region.isEmpty()) { - region = "eu-central-1"; + region = "local"; } Credentials credentials = (provider != null) ? provider.fetch() : null; diff --git a/src/main/java/me/desair/tus/server/upload/s3/S3StorageService.java b/src/main/java/me/desair/tus/server/upload/s3/S3StorageService.java index 01766dcb..1f1513d1 100644 --- a/src/main/java/me/desair/tus/server/upload/s3/S3StorageService.java +++ b/src/main/java/me/desair/tus/server/upload/s3/S3StorageService.java @@ -125,28 +125,21 @@ public class S3StorageService implements UploadStorageService { private S3ServerSideComposeHelper s3ComposeHelper; /** - * Basic constructor using default object key prefixes and standard system temp directory. + * Convenience constructor for local S3-compatible backends (e.g., MinIO, RustFS, Ceph) where + * region is omitted. Defaults the region to "local". * - * @param minioClient Pre-configured MinIO Client + * @param endpoint S3 endpoint URL (e.g. "http://localhost:9000") + * @param accessKey S3 access key / username + * @param secretKey S3 secret key / password * @param bucket S3 bucket name */ - public S3StorageService(MinioClient minioClient, String bucket) { - this( - minioClient, - bucket, - DEFAULT_OBJECT_PREFIX, - DEFAULT_METADATA_PREFIX, - DEFAULT_CHECKSUMS_PREFIX, - DEFAULT_LOCKS_PREFIX, - Paths.get(System.getProperty("java.io.tmpdir"))); + public S3StorageService(String endpoint, String accessKey, String secretKey, String bucket) { + this(endpoint, "local", accessKey, secretKey, bucket); } /** - * Constructor accepting explicit connection parameters without requiring a pre-existing - * MinioClient. - * - *

Builds both {@link MinioClient} and the native {@link S3ServerSideComposeHelper} directly - * using the provided connection properties, avoiding reflection. + * Basic constructor accepting explicit connection parameters without exposing underlying client + * libraries. * * @param endpoint S3 endpoint URL (e.g. "https://s3.amazonaws.com" or "http://localhost:9000") * @param region S3 region name (e.g. "us-east-1", "eu-central-1") @@ -157,53 +150,28 @@ public S3StorageService(MinioClient minioClient, String bucket) { public S3StorageService( String endpoint, String region, String accessKey, String secretKey, String bucket) { this( - buildMinioClient(endpoint, region, accessKey, secretKey), + endpoint, + region, + accessKey, + secretKey, bucket, DEFAULT_OBJECT_PREFIX, DEFAULT_METADATA_PREFIX, DEFAULT_CHECKSUMS_PREFIX, DEFAULT_LOCKS_PREFIX, - Paths.get(System.getProperty("java.io.tmpdir")), - new S3ServerSideComposeHelper(null, endpoint, region, accessKey, secretKey)); + Paths.get(System.getProperty("java.io.tmpdir"))); } /** - * Constructor accepting a pre-configured {@link MinioClient} along with the explicit connection - * parameters required by {@link S3ServerSideComposeHelper}. + * Full constructor accepting explicit connection parameters and prefix/buffer customization. * - *

This provides full control over {@link MinioClient} configuration while cleanly initializing - * server-side multipart copy without reflection. + *

Delegates to the internal package-private constructor accepting {@link MinioClient}. * - * @param minioClient Pre-configured MinIO Client * @param endpoint S3 endpoint URL (e.g. "https://s3.amazonaws.com" or "http://localhost:9000") * @param region S3 region name (e.g. "us-east-1", "eu-central-1") * @param accessKey S3 access key / username * @param secretKey S3 secret key / password * @param bucket S3 bucket name - */ - public S3StorageService( - MinioClient minioClient, - String endpoint, - String region, - String accessKey, - String secretKey, - String bucket) { - this( - minioClient, - bucket, - DEFAULT_OBJECT_PREFIX, - DEFAULT_METADATA_PREFIX, - DEFAULT_CHECKSUMS_PREFIX, - DEFAULT_LOCKS_PREFIX, - Paths.get(System.getProperty("java.io.tmpdir")), - new S3ServerSideComposeHelper(minioClient, endpoint, region, accessKey, secretKey)); - } - - /** - * Full constructor allowing full customization of object prefixes and local disk buffer path. - * - * @param minioClient Pre-configured MinIO Client - * @param bucket S3 bucket name * @param objectPrefix Key prefix for final completed file objects * @param metadataPrefix Key prefix for metadata (.info JSON and .part buffer) objects * @param checksumsPrefix Key prefix for checksum deduplication index objects @@ -211,7 +179,10 @@ public S3StorageService( * @param temporaryDirectory Local directory path for staging chunks before S3 upload */ public S3StorageService( - MinioClient minioClient, + String endpoint, + String region, + String accessKey, + String secretKey, String bucket, String objectPrefix, String metadataPrefix, @@ -219,18 +190,21 @@ public S3StorageService( String locksPrefix, Path temporaryDirectory) { this( - minioClient, + buildMinioClient(endpoint, region, accessKey, secretKey), bucket, objectPrefix, metadataPrefix, checksumsPrefix, locksPrefix, temporaryDirectory, - new S3ServerSideComposeHelper(minioClient)); + new S3ServerSideComposeHelper(endpoint, region, accessKey, secretKey)); } - /** Internal constructor accepting an initialized {@link S3ServerSideComposeHelper}. */ - private S3StorageService( + /** + * Internal package-private constructor accepting {@link MinioClient} where all parameter + * configuration and initialization logic is concentrated. + */ + S3StorageService( MinioClient minioClient, String bucket, String objectPrefix, @@ -273,20 +247,22 @@ private S3StorageService( }, new ThreadPoolExecutor.CallerRunsPolicy()); - this.concatenationService = - new S3ConcatenationService( - this.minioClient, this.bucket, this.objectPrefix, this, this.temporaryDirectory); this.s3ComposeHelper = s3ComposeHelper != null ? s3ComposeHelper : new S3ServerSideComposeHelper(this.minioClient); - if (this.concatenationService instanceof S3ConcatenationService) { - ((S3ConcatenationService) this.concatenationService) - .setS3ServerSideComposeHelper(this.s3ComposeHelper); - } + this.concatenationService = + new S3ConcatenationService( + this.minioClient, + this.bucket, + this.objectPrefix, + this, + this.temporaryDirectory, + DEFAULT_MIN_PART_SIZE, + this.s3ComposeHelper); } private static MinioClient buildMinioClient( String endpoint, String region, String accessKey, String secretKey) { - String effectiveRegion = (region != null && !region.isEmpty()) ? region : "eu-central-1"; + String effectiveRegion = (region != null && !region.isEmpty()) ? region : "local"; return MinioClient.builder() .endpoint(endpoint) .credentials(accessKey, secretKey) @@ -294,15 +270,6 @@ private static MinioClient buildMinioClient( .build(); } - /** - * Returns the underlying {@link MinioClient} configured for this storage service. - * - * @return The MinIO client instance - */ - public MinioClient getMinioClient() { - return this.minioClient; - } - /** * Returns the S3 object key for the completed upload data of the given upload info. If the upload * was deduplicated, this returns the parent upload's physical S3 object key. diff --git a/src/main/java/me/desair/tus/server/upload/s3/S3UploadLock.java b/src/main/java/me/desair/tus/server/upload/s3/S3UploadLock.java index 5b3a18b3..a0519af7 100644 --- a/src/main/java/me/desair/tus/server/upload/s3/S3UploadLock.java +++ b/src/main/java/me/desair/tus/server/upload/s3/S3UploadLock.java @@ -51,7 +51,7 @@ public class S3UploadLock extends AbstractLeaseLock { * @param stopKey The S3 object key for the interrupt stop signal * @param inputStreamMap Map of active request input streams */ - public S3UploadLock( + S3UploadLock( LeaseData leaseData, MinioClient minioClient, String bucket, diff --git a/src/test/java/me/desair/tus/server/TestUtils.java b/src/test/java/me/desair/tus/server/TestUtils.java index 55f1f6fe..a8358eea 100644 --- a/src/test/java/me/desair/tus/server/TestUtils.java +++ b/src/test/java/me/desair/tus/server/TestUtils.java @@ -77,6 +77,16 @@ public static GenericContainer createRustFsContainer() { .withEnv("RUSTFS_SECRET_KEY", "rustfsadmin"); } + /** + * Returns the S3 endpoint URL for the active S3 Testcontainer (RustFS). + * + * @param s3Container The active S3 Testcontainer (RustFS) + * @return S3 endpoint URL + */ + public static String getS3Endpoint(GenericContainer s3Container) { + return "http://" + s3Container.getHost() + ":" + s3Container.getMappedPort(9000); + } + /** * Create a {@link MinioClient} configured to connect to the given S3/RustFS container. * @@ -84,7 +94,7 @@ public static GenericContainer createRustFsContainer() { * @return Pre-configured MinioClient */ public static MinioClient createMinioClient(GenericContainer s3Container) { - String s3Url = "http://" + s3Container.getHost() + ":" + s3Container.getMappedPort(9000); + String s3Url = getS3Endpoint(s3Container); return MinioClient.builder().endpoint(s3Url).credentials("rustfsadmin", "rustfsadmin").build(); } diff --git a/src/test/java/me/desair/tus/server/upload/s3/ITS3LockingService.java b/src/test/java/me/desair/tus/server/upload/s3/ITS3LockingService.java index 916ff8e2..02724796 100644 --- a/src/test/java/me/desair/tus/server/upload/s3/ITS3LockingService.java +++ b/src/test/java/me/desair/tus/server/upload/s3/ITS3LockingService.java @@ -46,7 +46,8 @@ public static void tearDownClass() { @Before public void setUp() { org.junit.Assume.assumeTrue(TestUtils.isContainerRuntimeAvailable()); - lockingService = new S3LockingService(minioClient, BUCKET); + String endpoint = TestUtils.getS3Endpoint(rustfsContainer); + lockingService = new S3LockingService(endpoint, "rustfsadmin", "rustfsadmin", BUCKET); } @Test diff --git a/src/test/java/me/desair/tus/server/upload/s3/ITS3RufhProtocol.java b/src/test/java/me/desair/tus/server/upload/s3/ITS3RufhProtocol.java index 60b21270..97872e3c 100644 --- a/src/test/java/me/desair/tus/server/upload/s3/ITS3RufhProtocol.java +++ b/src/test/java/me/desair/tus/server/upload/s3/ITS3RufhProtocol.java @@ -47,9 +47,13 @@ protected TusFileUploadService createTusFileUploadService() { protected TusFileUploadService createTusFileUploadService(String uploadUri) { org.junit.Assume.assumeTrue(TestUtils.isContainerRuntimeAvailable()); - S3StorageService s3Storage = new S3StorageService(minioClient, BUCKET); - S3LockingService s3Locking = new S3LockingService(minioClient, BUCKET); - S3ConcatenationService s3Concat = new S3ConcatenationService(minioClient, BUCKET, s3Storage); + String endpoint = TestUtils.getS3Endpoint(rustfsContainer); + S3StorageService s3Storage = + new S3StorageService(endpoint, "rustfsadmin", "rustfsadmin", BUCKET); + S3LockingService s3Locking = + new S3LockingService(endpoint, "rustfsadmin", "rustfsadmin", BUCKET); + S3ConcatenationService s3Concat = + new S3ConcatenationService(endpoint, "rustfsadmin", "rustfsadmin", BUCKET, s3Storage); s3Storage.setUploadConcatenationService(s3Concat); return new TusFileUploadService() diff --git a/src/test/java/me/desair/tus/server/upload/s3/ITS3StorageService.java b/src/test/java/me/desair/tus/server/upload/s3/ITS3StorageService.java index c2f10768..b230dca9 100644 --- a/src/test/java/me/desair/tus/server/upload/s3/ITS3StorageService.java +++ b/src/test/java/me/desair/tus/server/upload/s3/ITS3StorageService.java @@ -50,7 +50,8 @@ public static void tearDownClass() { @Before public void setUp() { org.junit.Assume.assumeTrue(TestUtils.isContainerRuntimeAvailable()); - storageService = new S3StorageService(minioClient, BUCKET); + String endpoint = TestUtils.getS3Endpoint(rustfsContainer); + storageService = new S3StorageService(endpoint, "rustfsadmin", "rustfsadmin", BUCKET); } @Test diff --git a/src/test/java/me/desair/tus/server/upload/s3/ITS3TusFileUploadService.java b/src/test/java/me/desair/tus/server/upload/s3/ITS3TusFileUploadService.java index c3a0a008..6d8da536 100644 --- a/src/test/java/me/desair/tus/server/upload/s3/ITS3TusFileUploadService.java +++ b/src/test/java/me/desair/tus/server/upload/s3/ITS3TusFileUploadService.java @@ -49,9 +49,13 @@ protected TusFileUploadService createTusFileUploadService() { protected TusFileUploadService createTusFileUploadService(String uploadUri) { org.junit.Assume.assumeTrue(TestUtils.isContainerRuntimeAvailable()); - S3StorageService s3Storage = new S3StorageService(minioClient, BUCKET); - S3LockingService s3Locking = new S3LockingService(minioClient, BUCKET); - S3ConcatenationService s3Concat = new S3ConcatenationService(minioClient, BUCKET, s3Storage); + String endpoint = TestUtils.getS3Endpoint(rustfsContainer); + S3StorageService s3Storage = + new S3StorageService(endpoint, "rustfsadmin", "rustfsadmin", BUCKET); + S3LockingService s3Locking = + new S3LockingService(endpoint, "rustfsadmin", "rustfsadmin", BUCKET); + S3ConcatenationService s3Concat = + new S3ConcatenationService(endpoint, "rustfsadmin", "rustfsadmin", BUCKET, s3Storage); s3Storage.setUploadConcatenationService(s3Concat); return new TusFileUploadService() diff --git a/src/test/java/me/desair/tus/server/upload/s3/S3ConcatenationServiceTest.java b/src/test/java/me/desair/tus/server/upload/s3/S3ConcatenationServiceTest.java index 7f3d3470..df8255c6 100644 --- a/src/test/java/me/desair/tus/server/upload/s3/S3ConcatenationServiceTest.java +++ b/src/test/java/me/desair/tus/server/upload/s3/S3ConcatenationServiceTest.java @@ -39,18 +39,63 @@ public void setUp() { "test-bucket", "uploads/", storageService, - Paths.get(System.getProperty("java.io.tmpdir"))); + Paths.get(System.getProperty("java.io.tmpdir")), + 5242880L, + null); } @Test - public void testConstructorsAndSetters() { - S3ConcatenationService service1 = new S3ConcatenationService(minioClient, "test-bucket"); - S3ConcatenationService service2 = - new S3ConcatenationService(minioClient, "test-bucket", storageService); - service1.setUploadStorageService(storageService); - - assertNotNull(service1); - assertNotNull(service2); + public void testConstructors() { + S3ConcatenationService service = + new S3ConcatenationService( + minioClient, + "test-bucket", + "uploads/", + storageService, + Paths.get(System.getProperty("java.io.tmpdir")), + 5242880L, + null); + assertNotNull(service); + + // Public connection parameter constructors without region (defaults to "local") + S3ConcatenationService serviceParamsNoRegionWithStorage = + new S3ConcatenationService( + "https://s3.amazonaws.com", "accessKey", "secretKey", "test-bucket", storageService); + assertNotNull(serviceParamsNoRegionWithStorage); + + // Public connection parameter constructors with region + S3ConcatenationService serviceParamsWithRegion = + new S3ConcatenationService( + "https://s3.amazonaws.com", + "us-east-1", + "accessKey", + "secretKey", + "test-bucket", + storageService); + assertNotNull(serviceParamsWithRegion); + + S3ConcatenationService serviceParamsFull = + new S3ConcatenationService( + "https://s3.amazonaws.com", + "us-east-1", + "accessKey", + "secretKey", + "test-bucket", + "uploads/", + storageService, + Paths.get(System.getProperty("java.io.tmpdir")), + 5242880L); + assertNotNull(serviceParamsFull); + + S3ConcatenationService defaultRegionService = + new S3ConcatenationService( + "https://s3.amazonaws.com", + null, + "accessKey", + "secretKey", + "test-bucket", + storageService); + assertNotNull(defaultRegionService); } @Test @@ -354,14 +399,16 @@ public void testGetConcatenatedBytesTriggersMergeWhenStorageUploadIdIsNull() thr assertNotNull(result); } - @Test(expected = IOException.class) - public void testGetConcatenatedBytesWithoutStorageService() throws Exception { - S3ConcatenationService standalone = new S3ConcatenationService(minioClient, "test-bucket"); - UploadInfo info = new UploadInfo(); - info.setId(new UploadId("concat-1")); - info.setStorageUploadId("uploads/concat-1"); - - standalone.getConcatenatedBytes(info); + @Test(expected = NullPointerException.class) + public void testConstructorNullStorageServiceThrowsException() { + new S3ConcatenationService( + minioClient, + "test-bucket", + "uploads/", + null, + Paths.get(System.getProperty("java.io.tmpdir")), + 5242880L, + null); } @Test diff --git a/src/test/java/me/desair/tus/server/upload/s3/S3LockingServiceTest.java b/src/test/java/me/desair/tus/server/upload/s3/S3LockingServiceTest.java index cff254f3..f067aa38 100644 --- a/src/test/java/me/desair/tus/server/upload/s3/S3LockingServiceTest.java +++ b/src/test/java/me/desair/tus/server/upload/s3/S3LockingServiceTest.java @@ -45,7 +45,14 @@ public class S3LockingServiceTest { @Before public void setUp() throws Exception { minioClient = Mockito.mock(MinioClient.class); - lockingService = new S3LockingService(minioClient, "test-bucket"); + lockingService = + new S3LockingService( + minioClient, + "test-bucket", + S3LockingService.DEFAULT_LOCKS_PREFIX, + S3LockingService.DEFAULT_LEASE_DURATION_MS, + S3LockingService.DEFAULT_POLL_INTERVAL_MS, + new UuidUploadIdFactory()); s3StorageMap.clear(); Mockito.when(minioClient.putObject(Mockito.any(PutObjectArgs.class))) @@ -139,20 +146,32 @@ public void testLockUploadByUriInvalidUri() throws Exception { @Test public void testExplicitConnectionParametersConstructors() { + S3LockingService serviceWithoutRegion = + new S3LockingService("https://s3.amazonaws.com", "accessKey", "secretKey", "test-bucket"); + assertNotNull(serviceWithoutRegion); + S3LockingService serviceWithParams = new S3LockingService( "https://s3.amazonaws.com", "eu-central-1", "accessKey", "secretKey", "test-bucket"); assertNotNull(serviceWithParams); - S3LockingService serviceWithClientAndParams = + S3LockingService fullServiceWithParams = new S3LockingService( - minioClient, "https://s3.amazonaws.com", "eu-central-1", "accessKey", "secretKey", - "test-bucket"); - assertNotNull(serviceWithClientAndParams); + "test-bucket", + "locks/", + 30000L, + 2000L, + new me.desair.tus.server.upload.UuidUploadIdFactory()); + assertNotNull(fullServiceWithParams); + + S3LockingService defaultRegionService = + new S3LockingService( + "https://s3.amazonaws.com", null, "accessKey", "secretKey", "test-bucket"); + assertNotNull(defaultRegionService); } @Test @@ -368,6 +387,16 @@ public void testClose() throws Exception { lockingService.close(); } + private S3LockingService createLockingService(MinioClient client, String bucket) { + return new S3LockingService( + client, + bucket, + S3LockingService.DEFAULT_LOCKS_PREFIX, + S3LockingService.DEFAULT_LEASE_DURATION_MS, + S3LockingService.DEFAULT_POLL_INTERVAL_MS, + new UuidUploadIdFactory()); + } + @Test public void testCheckStopSignalForEntryExceptionAndNullId() throws Exception { lockingService.setIdFactory(new me.desair.tus.server.upload.TimeBasedUploadIdFactory()); @@ -378,7 +407,7 @@ public void testCheckStopSignalForEntryExceptionAndNullId() throws Exception { .when(mockClient) .removeObject(Mockito.any(io.minio.RemoveObjectArgs.class)); - S3LockingService service = new S3LockingService(mockClient, "test-bucket"); + S3LockingService service = createLockingService(mockClient, "test-bucket"); me.desair.tus.server.upload.TimeBasedUploadIdFactory idFactory = new me.desair.tus.server.upload.TimeBasedUploadIdFactory(); idFactory.setUploadUri("/files/upload"); @@ -512,7 +541,7 @@ public void testCleanupStaleLocksWithExpiredAndNonExpiredLocks() throws Exceptio .thenReturn(expiredStream) .thenReturn(validStream); - S3LockingService service = new S3LockingService(mockClient, "test-bucket"); + S3LockingService service = createLockingService(mockClient, "test-bucket"); service.cleanupStaleLocks(); // Expired lock object is deleted @@ -531,7 +560,7 @@ public void testWriteStopSignalException() throws Exception { Mockito.when(mockClient.putObject(Mockito.any(PutObjectArgs.class))) .thenThrow(new RuntimeException("S3 Put error")); - S3LockingService service = new S3LockingService(mockClient, "test-bucket"); + S3LockingService service = createLockingService(mockClient, "test-bucket"); me.desair.tus.server.upload.TimeBasedUploadIdFactory idFactory = new me.desair.tus.server.upload.TimeBasedUploadIdFactory(); idFactory.setUploadUri("/files/upload"); @@ -544,7 +573,7 @@ public void testWriteStopSignalException() throws Exception { @Test public void testCloseInterruptsActiveStreams() throws Exception { MinioClient mockClient = Mockito.mock(MinioClient.class); - S3LockingService service = new S3LockingService(mockClient, "test-bucket"); + S3LockingService service = createLockingService(mockClient, "test-bucket"); ByteArrayInputStream bis = new ByteArrayInputStream(new byte[] {1, 2, 3}); InterruptibleInputStream iis = new InterruptibleInputStream(bis); diff --git a/src/test/java/me/desair/tus/server/upload/s3/S3ServerSideComposeHelperTest.java b/src/test/java/me/desair/tus/server/upload/s3/S3ServerSideComposeHelperTest.java index c0afd7ae..3dff378f 100644 --- a/src/test/java/me/desair/tus/server/upload/s3/S3ServerSideComposeHelperTest.java +++ b/src/test/java/me/desair/tus/server/upload/s3/S3ServerSideComposeHelperTest.java @@ -259,10 +259,14 @@ public void testExplicitConnectionParametersConstructor() { .region("us-east-1") .build(); - S3ServerSideComposeHelper helper = + S3ServerSideComposeHelper helperWithoutClient = new S3ServerSideComposeHelper( - realClient, "https://s3.amazonaws.com", "us-east-1", "testKey", "testSecret"); - assertTrue(helper.isAvailable()); + "https://s3.amazonaws.com", "us-east-1", "testKey", "testSecret"); + assertTrue(helperWithoutClient.isAvailable()); + + S3ServerSideComposeHelper helperWithNullRegion = + new S3ServerSideComposeHelper("https://s3.amazonaws.com", null, "testKey", "testSecret"); + assertTrue(helperWithNullRegion.isAvailable()); S3ServerSideComposeHelper minioOnlyHelper = new S3ServerSideComposeHelper(realClient); assertFalse(minioOnlyHelper.isAvailable()); diff --git a/src/test/java/me/desair/tus/server/upload/s3/S3StorageServiceTest.java b/src/test/java/me/desair/tus/server/upload/s3/S3StorageServiceTest.java index 78008706..f0d5dc29 100644 --- a/src/test/java/me/desair/tus/server/upload/s3/S3StorageServiceTest.java +++ b/src/test/java/me/desair/tus/server/upload/s3/S3StorageServiceTest.java @@ -32,6 +32,7 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; +import java.nio.file.Paths; import java.time.Duration; import java.util.Arrays; import java.util.Collections; @@ -53,7 +54,16 @@ public class S3StorageServiceTest { @Before public void setUp() { minioClient = mock(MinioClient.class); - storageService = new S3StorageService(minioClient, "test-bucket"); + storageService = + new S3StorageService( + minioClient, + "test-bucket", + S3StorageService.DEFAULT_OBJECT_PREFIX, + S3StorageService.DEFAULT_METADATA_PREFIX, + S3StorageService.DEFAULT_CHECKSUMS_PREFIX, + S3StorageService.DEFAULT_LOCKS_PREFIX, + Paths.get(System.getProperty("java.io.tmpdir")), + null); } @Test @@ -76,28 +86,51 @@ public void testNullTemporaryDirectoryConstructor() { java.nio.file.Path nullPath = null; S3StorageService serviceWithNullTmp = new S3StorageService( - minioClient, "test-bucket", "uploads/", "uploads/", "checksums/", "locks/", nullPath); + minioClient, + "test-bucket", + "uploads/", + "uploads/", + "checksums/", + "locks/", + nullPath, + null); assertNotNull(serviceWithNullTmp); } @Test public void testExplicitConnectionParametersConstructors() { - // 1. Constructor taking connection parameters directly (Option A) + // 1. Constructor taking connection parameters directly without region (defaults to "local") + S3StorageService serviceWithoutRegion = + new S3StorageService("https://s3.amazonaws.com", "accessKey", "secretKey", "test-bucket"); + assertNotNull(serviceWithoutRegion); + + // 2. Constructor taking connection parameters directly with region S3StorageService serviceWithParams = new S3StorageService( "https://s3.amazonaws.com", "us-east-1", "accessKey", "secretKey", "test-bucket"); assertNotNull(serviceWithParams); - // 2. Constructor taking MinioClient and connection parameters (Option B) - S3StorageService serviceWithClientAndParams = + // 3. Full constructor taking connection parameters with prefix and temporary directory + // configuration + S3StorageService fullServiceWithParams = new S3StorageService( - minioClient, "https://s3.amazonaws.com", "us-east-1", "accessKey", "secretKey", - "test-bucket"); - assertNotNull(serviceWithClientAndParams); + "test-bucket", + "uploads/", + "metadata/", + "checksums/", + "locks/", + java.nio.file.Paths.get(System.getProperty("java.io.tmpdir"))); + assertNotNull(fullServiceWithParams); + + // 4. Null/empty region defaults cleanly + S3StorageService defaultRegionService = + new S3StorageService( + "https://s3.amazonaws.com", null, "accessKey", "secretKey", "test-bucket"); + assertNotNull(defaultRegionService); } @Test @@ -769,7 +802,15 @@ public void testConfigurationSettersAndGetters() { storageService.setIdFactory(new me.desair.tus.server.upload.UuidUploadIdFactory()); - S3ConcatenationService concat = new S3ConcatenationService(minioClient, "test-bucket"); + S3ConcatenationService concat = + new S3ConcatenationService( + minioClient, + "test-bucket", + "uploads/", + storageService, + Paths.get(System.getProperty("java.io.tmpdir")), + 5242880L, + null); storageService.setUploadConcatenationService(concat); assertEquals(concat, storageService.getUploadConcatenationService()); @@ -1591,11 +1632,11 @@ public void testDeleteObjectQuietlyNullAndException() throws Exception { public void testSanitizePrefixNullOrEmptyInS3StorageService() throws Exception { java.nio.file.Path tmpDir = java.nio.file.Paths.get(System.getProperty("java.io.tmpdir")); - S3StorageService s1 = new S3StorageService(minioClient, "bucket", "", "", "", "", tmpDir); + S3StorageService s1 = new S3StorageService(minioClient, "bucket", "", "", "", "", tmpDir, null); assertNotNull(s1); S3StorageService s2 = - new S3StorageService(minioClient, "bucket", null, null, null, null, tmpDir); + new S3StorageService(minioClient, "bucket", null, null, null, null, tmpDir, null); assertNotNull(s2); } @@ -1998,7 +2039,14 @@ public void testCleanupExpiredUploadsPrunesStaleTempFilesAndChecksumIndices() th S3StorageService customService = new S3StorageService( - minioClient, "test-bucket", "uploads/", "metadata/", "checksums/", "locks/", tempDir); + minioClient, + "test-bucket", + "uploads/", + "metadata/", + "checksums/", + "locks/", + tempDir, + null); customService.setUploadDeduplicationEnabled(true); Item checksumItem = mock(Item.class);