diff --git a/.claude/tools/fetch_ci_report.js b/.claude/tools/fetch_ci_report.js index 09b4cab3d82d..4803a71b7c7b 100755 --- a/.claude/tools/fetch_ci_report.js +++ b/.claude/tools/fetch_ci_report.js @@ -574,14 +574,16 @@ async function getCIReportsFromPR(prUrl) { console.log(`Fetching CI reports for PR #${prNumber}...\n`); - // Fetch PR comments to find CI bot comment. + // Report URLs are the signal — do not filter by bot login. + // Altinity posts via github-actions[bot] with the virtual-hosted S3 URL; + // older comments used clickhouse-gh[bot] and path-style S3. // Drop GH_CONFIG_DIR before spawning gh: some agent/runner checkouts set it to a poisoned // config dir (no/expired auth) that makes `gh api` fail, while the default config is fine. // Other repo tooling (patch-release-check) does the same via `env -u GH_CONFIG_DIR gh`. const ghEnv = { ...process.env }; delete ghEnv.GH_CONFIG_DIR; try { - const commentsJson = execSync(`gh api repos/Altinity/ClickHouse/issues/${prNumber}/comments --paginate --jq '.[] | select(.user.login == "clickhouse-gh[bot]") | {body, created_at}'`, { + const commentsJson = execSync(`gh api repos/Altinity/ClickHouse/issues/${prNumber}/comments --paginate --jq '.[] | {body, created_at}'`, { encoding: 'utf8', stdio: ['pipe', 'pipe', 'pipe'], env: ghEnv @@ -589,15 +591,12 @@ async function getCIReportsFromPR(prUrl) { const comments = commentsJson.trim().split('\n').filter(l => l.trim()).map(l => JSON.parse(l)); comments.sort((a, b) => (b.created_at || '').localeCompare(a.created_at || '')); - if (!comments || comments.length === 0) { - throw new Error('No CI bot comment found'); - } - // Search through all bot comments for CI report URLs (not just the latest). Exclude backtick and + // Search through all comments for CI report URLs (not just the latest). Exclude backtick and // quote chars so a URL quoted in markdown (e.g. inside the AI-review text) is not captured with // trailing junk, strip trailing punctuation, and dedupe -- otherwise the same report is fetched - // twice and the summary is doubled. - const reportUrlPattern = /https:\/\/s3\.amazonaws\.com\/altinity-build-artifacts\/json\.html\?[^\s)`'"]+/g; + // twice and the summary is doubled. Match both path-style and virtual-hosted S3 URLs. + const reportUrlPattern = /https:\/\/(?:s3\.amazonaws\.com\/altinity-build-artifacts|altinity-build-artifacts\.s3\.amazonaws\.com)\/json\.html\?[^\s)`'"]+/g; for (const comment of comments) { if (!comment.body) continue; let urls = comment.body.match(reportUrlPattern); @@ -607,9 +606,9 @@ async function getCIReportsFromPR(prUrl) { } } - throw new Error('No CI report URLs found in bot comments'); + throw new Error('No CI report URLs found in PR comments'); } catch (error) { - if (error.message.includes('No CI bot comment found') || error.message.includes('No CI report URLs found')) { + if (error.message.includes('No CI report URLs found')) { throw error; } throw new Error(`Failed to fetch PR comments: ${error.message}`); diff --git a/.github/actions/create_workflow_report/create_workflow_report.py b/.github/actions/create_workflow_report/create_workflow_report.py index 4cad8779a914..dde3813ea7dd 100755 --- a/.github/actions/create_workflow_report/create_workflow_report.py +++ b/.github/actions/create_workflow_report/create_workflow_report.py @@ -192,6 +192,9 @@ def _enrich_prs_in_release_merge_prs(df: pd.DataFrame, repo: str) -> pd.DataFram f"https://api.github.com/repos/{repo}/pulls/{pr_number}", headers=headers, ) + if response.status_code == 404: + # NOTE (strtgbb): not in this repo — upstream PR merged from a fork + continue if response.status_code != 200: raise Exception( f"Failed to fetch pull request info: {response.status_code} {response.text}" @@ -207,6 +210,8 @@ def _enrich_prs_in_release_merge_prs(df: pd.DataFrame, repo: str) -> pd.DataFram "pr_labels": html.escape(", ".join(sorted(label_names)), quote=True), } ) + if not rows: + return pd.DataFrame(columns=["pr_number", "pr_name", "pr_labels"]) return pd.DataFrame(rows) diff --git a/.gitmodules b/.gitmodules index b7e34d05214d..2f3dd2930a0a 100644 --- a/.gitmodules +++ b/.gitmodules @@ -329,7 +329,7 @@ url = https://github.com/ClickHouse/double-conversion.git [submodule "contrib/mongo-cxx-driver"] path = contrib/mongo-cxx-driver - url = https://github.com/mongodb/mongo-cxx-driver.git + url = https://github.com/ClickHouse/mongo-cxx-driver.git [submodule "contrib/mongo-c-driver"] path = contrib/mongo-c-driver url = https://github.com/ClickHouse/mongo-c-driver.git diff --git a/base/poco/NetSSL_OpenSSL/src/SSLManager.cpp b/base/poco/NetSSL_OpenSSL/src/SSLManager.cpp index 94b66cdcf5f1..80d85855fe3c 100644 --- a/base/poco/NetSSL_OpenSSL/src/SSLManager.cpp +++ b/base/poco/NetSSL_OpenSSL/src/SSLManager.cpp @@ -340,7 +340,9 @@ void SSLManager::initDefaultContext(bool server) { _ptrDefaultClientContext->enableSessionCache(cacheSessions); } - bool extendedVerification = config.getBool(prefix + CFG_EXTENDED_VERIFICATION, false); + /// Only an outbound connection has a requested host name to check: on an accepted socket this + /// would run against the client's address, which a client certificate does not name. + bool extendedVerification = config.getBool(prefix + CFG_EXTENDED_VERIFICATION, !server); if (server) _ptrDefaultServerContext->enableExtendedCertificateVerification(extendedVerification); else diff --git a/ci/jobs/integration_test_job.py b/ci/jobs/integration_test_job.py index df0a298ed775..4ccac05dc242 100644 --- a/ci/jobs/integration_test_job.py +++ b/ci/jobs/integration_test_job.py @@ -1329,7 +1329,7 @@ def main(): # hard subprocess backstop). Used below to keep an empty flaky/targeted result a # best-effort SKIPPED only when a timeout actually exhausted the budget. timed_out = False - session_timeout_parallel = 3600 * 2 + session_timeout_parallel = 3600 * 2.5 session_timeout_sequential = 3600 if is_llvm_coverage: diff --git a/ci/jobs/scripts/check_style/experimental_settings_ignore.txt b/ci/jobs/scripts/check_style/experimental_settings_ignore.txt index 8cd361e36c13..4c4a0ec48a10 100644 --- a/ci/jobs/scripts/check_style/experimental_settings_ignore.txt +++ b/ci/jobs/scripts/check_style/experimental_settings_ignore.txt @@ -87,3 +87,5 @@ enable_vector_similarity_index allow_nullable_tuple_in_extracted_subcolumns allow_lossy_numeric_supertype allow_metadata_only_named_tuple_alter +allow_experimental_cleanup_old_data_files_compaction +allow_experimental_iceberg_compaction diff --git a/ci/jobs/scripts/check_style/various_checks.sh b/ci/jobs/scripts/check_style/various_checks.sh index b2ee8fa4015a..82acf5a93c28 100755 --- a/ci/jobs/scripts/check_style/various_checks.sh +++ b/ci/jobs/scripts/check_style/various_checks.sh @@ -210,6 +210,11 @@ tests_with_system_drop=( $( sort -u ) ) for test_case in "${tests_with_system_drop[@]}"; do + # `SYSTEM DROP FILESYSTEM CACHE ''` affects only the named cache, not the + # server-wide state, so a test which drops the cache it created itself can run in + # parallel. Skip a test if every `SYSTEM DROP` in it drops a cache by name. + grep -oiP "system\s+drop(\s+filesystem\s+cache\s+'[^']+')?" "$test_case" | + grep -qivP "filesystem\s+cache\s+'" || continue grep -qP '(--|#)\s*[Tt]ags:.*no-parallel' "$test_case" || echo "Test with SYSTEM DROP should have no-parallel tag: $test_case" done diff --git a/ci/jobs/scripts/stress/stress.py b/ci/jobs/scripts/stress/stress.py index b4e8b47141fd..793e71d1d8d7 100755 --- a/ci/jobs/scripts/stress/stress.py +++ b/ci/jobs/scripts/stress/stress.py @@ -400,7 +400,13 @@ def install_thread_pool_fault_injection() -> None: logging.info("Installing thread-pool fault-injection config: %s -> %s", src, dst) subprocess.run(["ln", "-sf", src, dst], check=True) - if not call_with_retry(make_query_command("SYSTEM RELOAD CONFIG"), timeout=30, retry_count=5): + # NOTE (strtgbb): ARM debug + ThreadFuzzer config reload is ~18-23s; the + # default 15s receive_timeout loses the race and fails the job. + if not call_with_retry( + make_query_command("SYSTEM RELOAD CONFIG", receive_timeout=60), + timeout=90, + retry_count=5, + ): # Fail-close before the verify query: a stale non-zero probability left # over from an earlier reload would otherwise mask the reload failure. raise RuntimeError( @@ -655,9 +661,9 @@ def execute_bash(full_command, timeout=120): raise -def make_query_command(query: str) -> str: +def make_query_command(query: str, receive_timeout: int = 15) -> str: return ( - f'clickhouse client -q "{query}" --receive_timeout=15 --max_untracked_memory=1Gi ' + f'clickhouse client -q "{query}" --receive_timeout={receive_timeout} --max_untracked_memory=1Gi ' "--memory_profiler_step=1Gi --max_memory_usage_for_user=0 --max_memory_usage_in_client=1000000000 " "--enable-progress-table-toggle=0 " "--ast_fuzzer_runs=0", diff --git a/ci/praktika/native_jobs.py b/ci/praktika/native_jobs.py index 804b2805a904..e6c9a673f43c 100644 --- a/ci/praktika/native_jobs.py +++ b/ci/praktika/native_jobs.py @@ -292,6 +292,8 @@ def _prepare_submodule_cache(workflow, workflow_config: RunConfig) -> Result: no_strict=True, ) Shell.check(f"rm -f {archive_path}") + if not created and not S3.head_object(s3_path): + raise RuntimeError(f"failed to upload submodule cache {s3_path}") info = ( f"cache miss, created: {cache_hash}" if created @@ -302,10 +304,12 @@ def _prepare_submodule_cache(workflow, workflow_config: RunConfig) -> Result: workflow_config.dump() status = Result.Status.OK except Exception as e: - print(f"WARNING: Submodule cache failed: {e}") + print(f"ERROR: Submodule cache failed: {e}") traceback.print_exc() info = f"{e}\n{traceback.format_exc()}" - status = Result.Status.OK # non-fatal, jobs fall back to GitHub clone + # Do not continue with an empty submodule_cache_hash. Builds would + # skip the restore and clone the same pins themselves. + status = Result.Status.FAIL return Result.create_from( name="Submodule Cache", diff --git a/ci/settings/altinity_overrides.py b/ci/settings/altinity_overrides.py index 4221ccf17b87..9e965980b8ef 100644 --- a/ci/settings/altinity_overrides.py +++ b/ci/settings/altinity_overrides.py @@ -59,6 +59,10 @@ class RunnerLabels: INSTALL_PYTHON_REQS_FOR_NATIVE_JOBS = "" +# NOTE (strtgbb): anonymous submodule fetches of non-tip pins get refused +# after the burst of ~150 clones; send the ambient gh token instead. +ENABLE_SUBMODULE_CLONE_AUTH = True + DISABLED_WORKFLOWS = [ "backport_branches.py", "custom_build_praktika.py", diff --git a/cmake/autogenerated_versions.txt b/cmake/autogenerated_versions.txt index 581fe1859c69..95803bd73fbb 100644 --- a/cmake/autogenerated_versions.txt +++ b/cmake/autogenerated_versions.txt @@ -2,13 +2,13 @@ # NOTE: VERSION_REVISION has nothing common with DBMS_TCP_PROTOCOL_VERSION, # only DBMS_TCP_PROTOCOL_VERSION should be incremented on protocol changes. -SET(VERSION_REVISION 54518) +SET(VERSION_REVISION 54523) SET(VERSION_MAJOR 26) SET(VERSION_MINOR 8) -SET(VERSION_PATCH 6) -SET(VERSION_GITHASH ec5605431dacfc812affc406cc81ca398ca68174) -SET(VERSION_DESCRIBE v26.8.6.10001.altinitytest) -SET(VERSION_STRING 26.8.6.10001.altinitytest) +SET(VERSION_PATCH 11) +SET(VERSION_GITHASH bedf2ab54b8a0c34afbf2d907eaf933324f43cd6) +SET(VERSION_DESCRIBE v26.8.11.10001.altinitytest) +SET(VERSION_STRING 26.8.11.10001.altinitytest) # end of autochange SET(VERSION_TWEAK 10001) diff --git a/contrib/cctz b/contrib/cctz index c2ba12b73531..8e694da054a9 160000 --- a/contrib/cctz +++ b/contrib/cctz @@ -1 +1 @@ -Subproject commit c2ba12b73531a7dbc3ac45b5007649e35760f6d8 +Subproject commit 8e694da054a9a31d98392bf03ee188b04f810d0a diff --git a/contrib/libdeflate b/contrib/libdeflate index ec0718b8e06d..43f4fc23b7ab 160000 --- a/contrib/libdeflate +++ b/contrib/libdeflate @@ -1 +1 @@ -Subproject commit ec0718b8e06dc172eb87ede6a493865ccf7610ec +Subproject commit 43f4fc23b7abeb9bd078d54cab63271b3980fce7 diff --git a/contrib/mongo-c-driver b/contrib/mongo-c-driver index 6b6b676bdbd4..a00f0b91bd24 160000 --- a/contrib/mongo-c-driver +++ b/contrib/mongo-c-driver @@ -1 +1 @@ -Subproject commit 6b6b676bdbd46fdb954ed1535893020ef91cce5b +Subproject commit a00f0b91bd24f8ddbe62ec3864cb3704385670f5 diff --git a/contrib/mongo-c-driver-cmake/CMakeLists.txt b/contrib/mongo-c-driver-cmake/CMakeLists.txt index 74cd35eef8aa..a79ef97c8fe2 100644 --- a/contrib/mongo-c-driver-cmake/CMakeLists.txt +++ b/contrib/mongo-c-driver-cmake/CMakeLists.txt @@ -5,13 +5,13 @@ if(NOT USE_MONGODB) endif() set(libbson_VERSION_MAJOR 2) -set(libbson_VERSION_MINOR 3) -set(libbson_VERSION_PATCH 0) -set(libbson_VERSION 2.3.0) +set(libbson_VERSION_MINOR 5) +set(libbson_VERSION_PATCH 4) +set(libbson_VERSION 2.5.4) set(libmongoc_VERSION_MAJOR 2) -set(libmongoc_VERSION_MINOR 3) -set(libmongoc_VERSION_PATCH 0) -set(libmongoc_VERSION 2.3.0) +set(libmongoc_VERSION_MINOR 5) +set(libmongoc_VERSION_PATCH 4) +set(libmongoc_VERSION 2.5.4) set(LIBBSON_SOURCES_ROOT "${ClickHouse_SOURCE_DIR}/contrib/mongo-c-driver/src") set(LIBBSON_SOURCE_DIR "${LIBBSON_SOURCES_ROOT}/libbson/src") @@ -135,7 +135,7 @@ endif() set(LIBMONGOC_SOURCE_DIR "${LIBBSON_SOURCES_ROOT}/libmongoc/src") set(LIBMONGOC_BINARY_DIR "${LIBBSON_BINARY_ROOT}/libmongoc/src") file(GLOB_RECURSE LIBMONGOC_SOURCES "${LIBMONGOC_SOURCE_DIR}/*.c") -set(UTF8PROC_SOURCE_DIR "${LIBBSON_SOURCES_ROOT}/utf8proc-2.8.0") +set(UTF8PROC_SOURCE_DIR "${LIBBSON_SOURCES_ROOT}/utf8proc-2.11.3") set(UTF8PROC_SOURCES "${UTF8PROC_SOURCE_DIR}/utf8proc.c") set(UTHASH_SOURCE_DIR "${LIBBSON_SOURCES_ROOT}/uthash") diff --git a/contrib/mongo-cxx-driver b/contrib/mongo-cxx-driver index 9b0c13260590..fa9ee6ed008e 160000 --- a/contrib/mongo-cxx-driver +++ b/contrib/mongo-cxx-driver @@ -1 +1 @@ -Subproject commit 9b0c13260590bb6485fa57c20f61098870b99ab6 +Subproject commit fa9ee6ed008e84b9562a7a55d70681de8a6bc1c6 diff --git a/contrib/mongo-cxx-driver-cmake/CMakeLists.txt b/contrib/mongo-cxx-driver-cmake/CMakeLists.txt index 6fd3cc70adb2..a859c5d8488a 100644 --- a/contrib/mongo-cxx-driver-cmake/CMakeLists.txt +++ b/contrib/mongo-cxx-driver-cmake/CMakeLists.txt @@ -54,10 +54,17 @@ set(BSONCXX_SOURCES set(BSONCXX_POLY_USE_IMPLS ON) +# ClickHouse uses the `v_noabi` API, whose symbols are gated behind the "unstable ABI" switch since r4.4.0. +set(BSONCXX_ENABLE_UNSTABLE_ABI ON) + configure_file( ${BSONCXX_SOURCES_DIR}/lib/bsoncxx/v1/config/config.hpp.in ${BSONCXX_BINARY_DIR}/lib/bsoncxx/v1/config/config.hpp ) +configure_file( + ${BSONCXX_SOURCES_DIR}/lib/bsoncxx/v_noabi/bsoncxx/config/config.hpp.in + ${BSONCXX_BINARY_DIR}/lib/bsoncxx/v_noabi/bsoncxx/config/config.hpp +) configure_file( ${BSONCXX_SOURCES_DIR}/lib/bsoncxx/v1/config/version.hpp.in ${BSONCXX_BINARY_DIR}/lib/bsoncxx/v1/config/version.hpp @@ -152,6 +159,7 @@ set(MONGOCXX_SOURCES ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/bulk_write.cpp ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/change_stream.cpp ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/client.cpp + ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/client_bulk_write.cpp ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/client_encryption.cpp ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/client_session.cpp ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/collection.cpp @@ -203,6 +211,9 @@ set(MONGOCXX_SOURCES ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/insert_one_result.cpp ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/instance.cpp ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/logger.cpp + ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/oidc_callback.cpp + ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/oidc_callback_params.cpp + ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/oidc_credential.cpp ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/pipeline.cpp ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/pool.cpp ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/range_options.cpp @@ -216,6 +227,7 @@ set(MONGOCXX_SOURCES ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/search_indexes.cpp ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/server_api.cpp ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/server_error.cpp + ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/string_options.cpp ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/text_options.cpp ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/tls.cpp ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/transaction_options.cpp @@ -336,10 +348,17 @@ endif() set(BSONCXX_STATIC 1) set(MONGOCXX_STATIC 1) +# ClickHouse uses the `v_noabi` API, whose symbols are gated behind the "unstable ABI" switch since r4.4.0. +set(MONGOCXX_ENABLE_UNSTABLE_ABI ON) + configure_file( ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/config/config.hpp.in ${MONGOCXX_BINARY_DIR}/lib/mongocxx/v1/config/config.hpp ) +configure_file( + ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v_noabi/mongocxx/config/config.hpp.in + ${MONGOCXX_BINARY_DIR}/lib/mongocxx/v_noabi/mongocxx/config/config.hpp +) configure_file( ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/config/version.hpp.in ${MONGOCXX_BINARY_DIR}/lib/mongocxx/v1/config/version.hpp diff --git a/docs/reference/interfaces/specs/NativeFormat.mdx b/docs/reference/interfaces/specs/NativeFormat.mdx index 4bc0faf1b9f9..9a5cf1bfe687 100644 --- a/docs/reference/interfaces/specs/NativeFormat.mdx +++ b/docs/reference/interfaces/specs/NativeFormat.mdx @@ -196,7 +196,14 @@ The byte values differ from the compact codes: `REPLICATED` is `0x03` in this ne The `count` is the full stack length **including the leading `DEFAULT` entry** that begins every stack. The compact codes already cover every one- and two-entry stack, so a `COMBINATION` always has a `count` of at least three. -**Recursive `kind_stack` for `Tuple` columns.** The `kind_stack` payload above is the byte (or `COMBINATION` sequence) for one column's own serialization info. A `Tuple` carries a `SerializationInfoTuple`, which first writes the tuple's *own* kind-stack payload and then writes one full kind-stack payload for *each* element, in order; a decoder reads back the same recursive structure. So for `Tuple(A, B, C)` the field-4 bytes are `[tuple_kind][A_kind][B_kind][C_kind]`, and each element payload is itself recursive if that element is again a composite. The `has_custom_serialization` byte (field 3) is set whenever the tuple's own info *or any element's* info is non-default, so a `Tuple` whose only special element is sparse, replicated, or detached still triggers the kind-stack payload. A decoder that reads only the single leading enum byte for a `Tuple` will stop too early and misread the remaining element-kind bytes as column data. +**Kind stack validity.** A decoder rejects a malformed stack with `INCORRECT_DATA` rather than building the serialization it describes: + +- the stack starts with `DEFAULT`; +- the stack is a subsequence of `DEFAULT`, `SPARSE`, `REPLICATED`, `DETACHED` — the order in which the kinds wrap each other. `SPARSE` can sit inside `REPLICATED` but not the other way round, and `DETACHED` is always last because a `ColumnBLOB` holds the serialized form of everything below it. This also means no kind occurs twice, which bounds `count` by the number of kinds. + +Note that this order is not the order of the kind byte values above. Any other stack describes a column layout no writer builds and that no materialization step unwraps back to a full column of the declared type — a `ColumnSparse` whose values column is itself sparse or replicated, for instance, stays wrapped. + +**Recursive `kind_stack` for `Tuple` columns.** The `kind_stack` payload above is the byte (or `COMBINATION` sequence) for one column's own serialization info. A `Tuple` carries a `SerializationInfoTuple`, which first writes the tuple's *own* kind-stack payload and then writes one full kind-stack payload for *each* element, in order; a decoder reads back the same recursive structure. So for `Tuple(A, B, C)` the field-4 bytes are `[tuple_kind][A_kind][B_kind][C_kind]`, and each element payload is itself recursive if that element is again a composite. The `has_custom_serialization` byte (field 3) is set whenever the tuple's own info *or any element's* info is non-default, so a `Tuple` whose only special element is sparse or replicated still triggers the kind-stack payload. A decoder that reads only the single leading enum byte for a `Tuple` will stop too early and misread the remaining element-kind bytes as column data. **Sparse wire format.** When `kind_stack = 0x01`, the column `data` is two streams written back-to-back in the single shared TCP stream: @@ -230,6 +237,8 @@ A decoder reconstructs a dense column by selecting `elements[indexes[i]]` for ea If the wrapped column was sparse, its stack is `{DEFAULT, SPARSE, DETACHED}`, which serializes as `DETACHED_OVER_SPARSE`. A client decoding such a column reads the blob length and bytes, then decompresses the blob to recover the inner column payload (see the [`ColumnBLOB` note](#compression-negotiation) under compression). +`DETACHED` is directional: it may appear only in result blocks sent by a server, and only for a whole top-level column — never as one of the element kinds of a `Tuple`. Unlike the other kinds, it makes the runtime column a `ColumnBLOB` instead of the column of the declared type, so a receiver accepts it only when its own pipeline converts the blob back. A server therefore rejects `DETACHED` in any block a client sends — external table data, scalars and insert data alike — with `INCORRECT_DATA`. + ### Block variants {#block-variants} All Data-family packets share the same Block wire format. The variants differ only in their column and row counts: diff --git a/docs/reference/settings/server-settings/_server_settings_outside_source.mdx b/docs/reference/settings/server-settings/_server_settings_outside_source.mdx index 88f55cc2d0d9..03357b3470b0 100644 --- a/docs/reference/settings/server-settings/_server_settings_outside_source.mdx +++ b/docs/reference/settings/server-settings/_server_settings_outside_source.mdx @@ -1168,7 +1168,7 @@ Keys for server/client settings: | `certificateFile` | Path to the client/server certificate file in PEM format. You can omit it if `privateKeyFile` contains the certificate. | | | `cipherList` | Supported OpenSSL encryptions. | `ALL:!ADH:!LOW:!EXP:!MD5:!3DES:@STRENGTH` | | `disableProtocols` | Protocols that are not allowed to be used. | | -| `extendedVerification` | If enabled, verify that the certificate CN or SAN matches the peer hostname. | `false` | +| `extendedVerification` | If enabled, verify that the certificate CN or SAN matches the peer hostname. | `true` for `openSSL.client`, `false` for `openSSL.server` | | `fips` | Activates OpenSSL FIPS mode. Supported if the library's OpenSSL version supports FIPS. | `false` | | `invalidCertificateHandler` | Class (a subclass of CertificateHandler) for verifying invalid certificates. For example: ` RejectCertificateHandler ` . | `RejectCertificateHandler` | | `loadDefaultCAFile` | Wether built-in CA certificates for OpenSSL will be used. ClickHouse assumes that builtin CA certificates are in the file `/etc/ssl/cert.pem` (resp. the directory `/etc/ssl/certs`) or in file (resp. directory) specified by the environment variable `SSL_CERT_FILE` (resp. `SSL_CERT_DIR`). | `true` | diff --git a/programs/local/LocalServer.cpp b/programs/local/LocalServer.cpp index 822087044f05..1b8e9a8912fa 100644 --- a/programs/local/LocalServer.cpp +++ b/programs/local/LocalServer.cpp @@ -216,6 +216,9 @@ namespace ServerSetting extern const ServerSettingsUInt64 max_format_parsing_thread_pool_size; extern const ServerSettingsUInt64 max_format_parsing_thread_pool_free_size; extern const ServerSettingsUInt64 format_parsing_thread_pool_queue_size; + extern const ServerSettingsUInt64 max_iceberg_manifest_decode_thread_pool_size; + extern const ServerSettingsUInt64 max_iceberg_manifest_decode_thread_pool_free_size; + extern const ServerSettingsUInt64 iceberg_manifest_decode_thread_pool_queue_size; extern const ServerSettingsUInt64 page_cache_history_window_ms; extern const ServerSettingsString page_cache_policy; extern const ServerSettingsDouble page_cache_size_ratio; @@ -466,6 +469,11 @@ void LocalServer::initialize(Poco::Util::Application & self) server_settings[ServerSetting::max_format_parsing_thread_pool_size], server_settings[ServerSetting::max_format_parsing_thread_pool_free_size], server_settings[ServerSetting::format_parsing_thread_pool_queue_size]); + + getIcebergManifestDecodeThreadPool().initialize( + server_settings[ServerSetting::max_iceberg_manifest_decode_thread_pool_size], + server_settings[ServerSetting::max_iceberg_manifest_decode_thread_pool_free_size], + server_settings[ServerSetting::iceberg_manifest_decode_thread_pool_queue_size]); } diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index c8fcfd0a50e8..ee8ff640e6ce 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -430,6 +430,9 @@ namespace ServerSetting extern const ServerSettingsUInt64 max_format_parsing_thread_pool_size; extern const ServerSettingsUInt64 max_format_parsing_thread_pool_free_size; extern const ServerSettingsUInt64 format_parsing_thread_pool_queue_size; + extern const ServerSettingsUInt64 max_iceberg_manifest_decode_thread_pool_size; + extern const ServerSettingsUInt64 max_iceberg_manifest_decode_thread_pool_free_size; + extern const ServerSettingsUInt64 iceberg_manifest_decode_thread_pool_queue_size; extern const ServerSettingsUInt64 page_cache_history_window_ms; extern const ServerSettingsString page_cache_policy; extern const ServerSettingsDouble page_cache_size_ratio; @@ -2000,6 +2003,11 @@ try server_settings[ServerSetting::max_format_parsing_thread_pool_free_size], server_settings[ServerSetting::format_parsing_thread_pool_queue_size]); + getIcebergManifestDecodeThreadPool().initialize( + server_settings[ServerSetting::max_iceberg_manifest_decode_thread_pool_size], + server_settings[ServerSetting::max_iceberg_manifest_decode_thread_pool_free_size], + server_settings[ServerSetting::iceberg_manifest_decode_thread_pool_queue_size]); + std::string path_str = getCanonicalPath(String(server_settings[ServerSetting::path]), original_working_directory); fs::path path = path_str; @@ -3058,6 +3066,11 @@ try new_server_settings[ServerSetting::max_format_parsing_thread_pool_free_size], new_server_settings[ServerSetting::format_parsing_thread_pool_queue_size]); + getIcebergManifestDecodeThreadPool().reloadConfiguration( + new_server_settings[ServerSetting::max_iceberg_manifest_decode_thread_pool_size], + new_server_settings[ServerSetting::max_iceberg_manifest_decode_thread_pool_free_size], + new_server_settings[ServerSetting::iceberg_manifest_decode_thread_pool_queue_size]); + global_context->setMergeWorkload(new_server_settings[ServerSetting::merge_workload]); global_context->setMutationWorkload(new_server_settings[ServerSetting::mutation_workload]); global_context->setThrowOnUnknownWorkload(new_server_settings[ServerSetting::throw_on_unknown_workload]); diff --git a/src/AggregateFunctions/Moments.h b/src/AggregateFunctions/Moments.h index ead359f40e7e..7e77f0f7ee4e 100644 --- a/src/AggregateFunctions/Moments.h +++ b/src/AggregateFunctions/Moments.h @@ -21,6 +21,7 @@ namespace ErrorCodes { extern const int BAD_ARGUMENTS; extern const int LOGICAL_ERROR; + extern const int INCORRECT_DATA; } /// Zeroes the value's bit pattern when the flag is not set. Unlike multiplying by the @@ -906,9 +907,23 @@ struct AnalysisOfVarianceMoments void read(ReadBuffer & buf) { - readVectorBinary(xs1, buf); - readVectorBinary(xs2, buf); - readVectorBinary(ns, buf); + /// add and merge cap the group count at MAX_GROUPS_NUMBER via resizeIfNeeded, but read + /// trusts the serialized lengths. Enforce the same bound here, before the vectors are + /// resized, so a crafted state cannot exceed the aggregate's own invariant. + readVectorBinary(xs1, buf, MAX_GROUPS_NUMBER); + readVectorBinary(xs2, buf, MAX_GROUPS_NUMBER); + readVectorBinary(ns, buf, MAX_GROUPS_NUMBER); + + /// The three vectors hold one entry per group and must stay equal in length. + /// The finalize path iterates up to xs1.size() and indexes xs2 and ns with the + /// same position, so a state with mismatched lengths reads out of bounds. + if (xs1.size() != xs2.size() || xs1.size() != ns.size()) + throw Exception( + ErrorCodes::INCORRECT_DATA, + "Sizes of nested arrays in analysisOfVariance state do not match: {}, {}, {}", + xs1.size(), + xs2.size(), + ns.size()); } Float64 getMeanAll() const diff --git a/src/Analyzer/Resolve/QueryAnalyzer.cpp b/src/Analyzer/Resolve/QueryAnalyzer.cpp index 7840ebc21b98..2c1d2e6ce2b5 100644 --- a/src/Analyzer/Resolve/QueryAnalyzer.cpp +++ b/src/Analyzer/Resolve/QueryAnalyzer.cpp @@ -108,6 +108,7 @@ namespace Setting extern const SettingsUInt64 use_structure_from_insertion_table_in_table_functions; extern const SettingsBool allow_suspicious_types_in_group_by; extern const SettingsBool allow_suspicious_types_in_order_by; + extern const SettingsBool validate_group_by_all_key_types; extern const SettingsBool allow_experimental_correlated_subqueries; extern const SettingsString implicit_table_at_top_level; extern const SettingsBool parallel_replicas_for_cluster_engines; @@ -904,6 +905,11 @@ void QueryAnalyzer::validateJoinTableExpressionWithoutAlias(const QueryTreeNodeP if ((query_node && !query_node->getCTEName().empty()) || (union_node && !union_node->getCTEName().empty())) return; + /// A parameterized view has a name to qualify its columns with, so it is exempt like the plain table below. + if (const auto * table_function_node = table_expression_node->as(); + table_function_node && table_function_node->isParameterizedView()) + return; + auto table_expression_node_type = table_expression_node->getNodeType(); if (table_expression_node_type == QueryTreeNodeType::TABLE_FUNCTION || @@ -1649,6 +1655,13 @@ void QueryAnalyzer::qualifyColumnNodesWithProjectionNames(const QueryTreeNodes & if (table_node->isMaterializedCTE()) additional_column_qualification_parts = {table_node->getMaterializedCTE()->cte_name}; } + else if (auto * table_function_node = table_expression_node->as(); + table_function_node && table_function_node->isParameterizedView()) + { + /// A parameterized view has a name of its own, qualify with it exactly like for a `TableNode`. + const auto & table_storage_id = table_function_node->getStorageID(); + additional_column_qualification_parts = {table_storage_id.getDatabaseName(), table_storage_id.getTableName()}; + } else if (auto * query_node = table_expression_node->as(); query_node && query_node->isCTE()) additional_column_qualification_parts = {query_node->getCTEName()}; else if (auto * union_node = table_expression_node->as(); union_node && union_node->isCTE()) @@ -1684,6 +1697,9 @@ void QueryAnalyzer::qualifyColumnNodesWithProjectionNames(const QueryTreeNodes & else forced_qualifier = table_node->getStorageID().getTableName(); } + else if (auto * table_function_node = table_expression_node->as(); + table_function_node && table_function_node->isParameterizedView()) + forced_qualifier = table_function_node->getStorageID().getTableName(); else if (auto * query_node = table_expression_node->as(); query_node && query_node->isCTE()) forced_qualifier = query_node->getCTEName(); else if (auto * union_node = table_expression_node->as(); union_node && union_node->isCTE()) @@ -4076,7 +4092,7 @@ void registerNullableGroupByKeys(const QueryTreeNodes & group_by_keys, Identifie /** Resolve GROUP BY clause. */ -void QueryAnalyzer::resolveGroupByNode(QueryNode & query_node_typed, IdentifierResolveScope & scope) +void QueryAnalyzer::resolveGroupByNode(QueryNode & query_node_typed, IdentifierResolveScope & scope, bool validate_key_types) { QueryTreeNodes nullable_group_by_keys; @@ -4098,7 +4114,10 @@ void QueryAnalyzer::resolveGroupByNode(QueryNode & query_node_typed, IdentifierR { for (const auto & group_by_elem : grouping_set->as()->getNodes()) { - validateGroupByKeyType(group_by_elem->getResultType(), scope); + if (validate_key_types) + validateGroupByKeyType(group_by_elem->getResultType(), scope); + /// Outside the guard: the promotion to Nullable is what `group_by_use_nulls` asks for, + /// independently of whether the key types are validated. if (scope.group_by_use_nulls) nullable_group_by_keys.push_back(group_by_elem); } @@ -4117,7 +4136,9 @@ void QueryAnalyzer::resolveGroupByNode(QueryNode & query_node_typed, IdentifierR for (const auto & group_by_elem : query_node_typed.getGroupBy().getNodes()) { - validateGroupByKeyType(group_by_elem->getResultType(), scope); + if (validate_key_types) + validateGroupByKeyType(group_by_elem->getResultType(), scope); + /// Outside the guard, for the same reason as in the grouping-sets branch above. if (scope.group_by_use_nulls) nullable_group_by_keys.push_back(group_by_elem); } @@ -4426,6 +4447,15 @@ void QueryAnalyzer::initializeTableExpressionData(const TableExpressionNodePtr & else if (table_function_node) { table_expression_data.table_expression_description = "table_function"; + + /// A parameterized view has a name of its own, expose it exactly like a `TableNode` does. + if (table_function_node->isParameterizedView()) + { + const auto & table_storage_id = table_function_node->getStorageID(); + table_expression_data.database_name = table_storage_id.database_name; + table_expression_data.table_name = table_storage_id.table_name; + table_expression_data.table_expression_name = table_storage_id.getFullNameNotQuoted(); + } } if (table_expression_node->hasAlias()) @@ -6528,6 +6558,10 @@ void QueryAnalyzer::resolveQuery(const QueryTreeNodePtr & query_node, Identifier NamesAndTypes projection_columns; + /// `expandGroupByAll` clears the flag, and under `group_by_use_nulls` it runs before the grouping keys + /// are resolved, so the ALL-ness has to be remembered here to still be known at either validation site. + const bool query_is_group_by_all = query_node_typed.isGroupByAll(); + if (!scope.group_by_use_nulls) { projection_columns = resolveProjectionExpressionNodeList(query_node_typed.getProjectionNode(), scope); @@ -6583,7 +6617,11 @@ void QueryAnalyzer::resolveQuery(const QueryTreeNodePtr & query_node, Identifier resolveExpressionNode(query_node_typed.getWhere(), scope, false /*allow_lambda_expression*/, false /*allow_table_expression*/); if (query_node_typed.hasGroupBy()) - resolveGroupByNode(query_node_typed, scope); + resolveGroupByNode( + query_node_typed, + scope, + /* validate_key_types */ !query_is_group_by_all + || scope.context->getSettingsRef()[Setting::validate_group_by_all_key_types]); if (query_node_typed.hasHaving()) resolveExpressionNode(query_node_typed.getHaving(), scope, false /*allow_lambda_expression*/, false /*allow_table_expression*/); @@ -6743,8 +6781,12 @@ void QueryAnalyzer::resolveQuery(const QueryTreeNodePtr & query_node, Identifier { expandTuplesInList(query_node_typed.getGroupBy().getNodes()); - for (const auto & group_by_elem : query_node_typed.getGroupBy().getNodes()) - validateGroupByKeyType(group_by_elem->getResultType(), scope); + /// Only the acceptance check is optional; the tuple expansion above is not. + if (scope.context->getSettingsRef()[Setting::validate_group_by_all_key_types]) + { + for (const auto & group_by_elem : query_node_typed.getGroupBy().getNodes()) + validateGroupByKeyType(group_by_elem->getResultType(), scope); + } } tryMoveNonAggregateHavingPredicatesToWhere(query_node, scope); diff --git a/src/Analyzer/Resolve/QueryAnalyzer.h b/src/Analyzer/Resolve/QueryAnalyzer.h index d3f1db45ca8e..129c38311cd7 100644 --- a/src/Analyzer/Resolve/QueryAnalyzer.h +++ b/src/Analyzer/Resolve/QueryAnalyzer.h @@ -270,7 +270,7 @@ class QueryAnalyzer void validateSortingKeyType(const DataTypePtr & sorting_key_type, const IdentifierResolveScope & scope) const; - void resolveGroupByNode(QueryNode & query_node_typed, IdentifierResolveScope & scope); + void resolveGroupByNode(QueryNode & query_node_typed, IdentifierResolveScope & scope, bool validate_key_types); void validateGroupByKeyType(const DataTypePtr & group_by_key_type, const IdentifierResolveScope & scope) const; diff --git a/src/Analyzer/TableFunctionNode.cpp b/src/Analyzer/TableFunctionNode.cpp index a4aeb95a021d..dcd62f670a17 100644 --- a/src/Analyzer/TableFunctionNode.cpp +++ b/src/Analyzer/TableFunctionNode.cpp @@ -71,6 +71,12 @@ const StorageSnapshotPtr & TableFunctionNode::getStorageSnapshot() const return storage_snapshot; } +bool TableFunctionNode::isParameterizedView() const +{ + const auto * storage_view = storage ? storage->as() : nullptr; + return storage_view && storage_view->isParameterizedView(); +} + void TableFunctionNode::dumpTreeImpl(WriteBuffer & buffer, FormatState & format_state, size_t indent) const { buffer << std::string(indent, ' ') << "TABLE_FUNCTION id: " << format_state.getNodeId(this); @@ -167,8 +173,7 @@ ASTPtr TableFunctionNode::toASTImpl(const ConvertToASTOptions & options) const /// An unqualified parameterized-view name re-resolves against the receiving server's default /// database, so qualify it from `storage_id`. Only a 2-part result is resolvable as a /// parameterized view, so a dotted database name is left alone. - if (const auto * storage_view = storage ? storage->as() : nullptr; - storage_view && storage_view->isParameterizedView() && storage_id.hasDatabase() + if (isParameterizedView() && storage_id.hasDatabase() && Identifier{table_function_name}.getPartsSize() == 1) { const auto database_name = storage_id.getDatabaseName(); diff --git a/src/Analyzer/TableFunctionNode.h b/src/Analyzer/TableFunctionNode.h index 8548b5d0c26e..d8f150e1e727 100644 --- a/src/Analyzer/TableFunctionNode.h +++ b/src/Analyzer/TableFunctionNode.h @@ -101,6 +101,9 @@ class TableFunctionNode : public ITableExpressionNode return storage; } + /// True for a parameterized view call resolved into its `StorageView` (no real table function behind it) + bool isParameterizedView() const; + /// Resolve table function with table function, storage and context void resolve(TableFunctionPtr table_function_value, StoragePtr storage_value, ContextPtr context, VectorWithMemoryTracking unresolved_arguments_indexes_); diff --git a/src/Client/BuzzHouse/Generator/SessionSettings.cpp b/src/Client/BuzzHouse/Generator/SessionSettings.cpp index 7d29a36c07e6..1c35fcd3ae56 100644 --- a/src/Client/BuzzHouse/Generator/SessionSettings.cpp +++ b/src/Client/BuzzHouse/Generator/SessionSettings.cpp @@ -2022,6 +2022,7 @@ void loadFuzzerServerSettings(const FuzzConfig & fc) "min_insert_block_size_bytes", "min_insert_block_size_bytes_for_materialized_views", "min_joined_block_size_bytes", + "output_format_arrow_record_batch_size_bytes", "output_format_parquet_row_group_size_bytes", "page_cache_block_size", "page_cache_max_coalesced_bytes", @@ -2080,6 +2081,7 @@ void loadFuzzerServerSettings(const FuzzConfig & fc) "min_table_rows_to_use_projection_index", "number_of_mutations_to_delay", "number_of_mutations_to_throw", + "output_format_arrow_record_batch_size", "output_format_parquet_data_page_size", "output_format_parquet_row_group_size", "output_format_pretty_max_rows", diff --git a/src/Client/Connection.cpp b/src/Client/Connection.cpp index 901899ebcbab..74f0273acba3 100644 --- a/src/Client/Connection.cpp +++ b/src/Client/Connection.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -1717,7 +1718,9 @@ void Connection::initBlockInput() if (!block_in) { initMaybeCompressedInput(); - block_in = std::make_unique(*maybe_compressed_in, server_revision, format_settings); + /// The server may send marshalled result blocks; their consumers convert them back. + block_in = std::make_unique( + *maybe_compressed_in, server_revision, format_settings, ISerialization::KindSet::all()); } } diff --git a/src/Columns/ColumnBLOB.h b/src/Columns/ColumnBLOB.h index cd1ab67703e4..2778db7aff5d 100644 --- a/src/Columns/ColumnBLOB.h +++ b/src/Columns/ColumnBLOB.h @@ -112,6 +112,8 @@ class ColumnBLOB final : public COWHelper, ColumnBLOB> return from_blob_task(blob); } + ColumnPtr convertToFullColumnIfDetached() const override { return convertFrom(); } + /// Creates serialized and compressed blob from the source column. static void toBLOB( BLOB & blob, diff --git a/src/Columns/ColumnSparse.cpp b/src/Columns/ColumnSparse.cpp index e749675070d6..5df6b0586425 100644 --- a/src/Columns/ColumnSparse.cpp +++ b/src/Columns/ColumnSparse.cpp @@ -1012,8 +1012,9 @@ ColumnPtr removeSpecialRepresentations(const ColumnPtr & column) if (!column) return column; - /// We can have only Replicated(Sparse) but not Sparse(Replicated). - return recursiveRemoveSparse(column->convertToFullColumnIfReplicated()); + /// Order matters: the BLOB holds the serialized form of everything below it, and we can have + /// only Replicated(Sparse) but not Sparse(Replicated). + return recursiveRemoveSparse(column->convertToFullColumnIfDetached()->convertToFullColumnIfReplicated()); } } diff --git a/src/Columns/IColumn.h b/src/Columns/IColumn.h index 44a7409b5abe..bf36f6191c98 100644 --- a/src/Columns/IColumn.h +++ b/src/Columns/IColumn.h @@ -132,14 +132,20 @@ class IColumn : public COW /// If column is ColumnReplicated, transforms it to full column. [[nodiscard]] virtual Ptr convertToFullColumnIfReplicated() const { return getPtr(); } - /// Recursively strip internal representation wrappers (Const, Replicated, Sparse) + /// If column isn't ColumnBLOB, return itself. + /// If column is ColumnBLOB, deserializes the BLOB back into the column it holds. + [[nodiscard]] virtual Ptr convertToFullColumnIfDetached() const { return getPtr(); } + + /// Recursively strip internal representation wrappers (Const, Detached, Replicated, Sparse) /// from this column and all its subcolumns. Does NOT strip LowCardinality — that is /// a semantic type, not a representation wrapper. Callers that also need LowCardinality /// removed should chain ->convertToFullColumnIfLowCardinality() for top-level removal, /// or use recursiveRemoveLowCardinality for recursive removal. [[nodiscard]] virtual Ptr convertToFullIfWrapped() const { - Ptr converted = convertToFullColumnIfConst() + /// Detached goes first: the BLOB holds the serialized form of everything below it. + Ptr converted = convertToFullColumnIfDetached() + ->convertToFullColumnIfConst() ->convertToFullColumnIfReplicated() ->convertToFullColumnIfSparse(); diff --git a/src/Columns/tests/gtest_column_blob.cpp b/src/Columns/tests/gtest_column_blob.cpp new file mode 100644 index 000000000000..e9b34cfecac9 --- /dev/null +++ b/src/Columns/tests/gtest_column_blob.cpp @@ -0,0 +1,59 @@ +#include + +#include +#include +#include + +using namespace DB; + +namespace +{ + +/// The real task deserializes the BLOB; handing back `inner` exercises the same unwrapping plumbing. +ColumnPtr makeBLOB(const ColumnPtr & inner) +{ + auto blob = ColumnBLOB::create(inner->cloneEmpty()); + blob->setFromBLOBTask([inner](const ColumnBLOB::BLOB &) { return inner; }); + return blob; +} + +} + +/// `ColumnBLOB` is a transport representation that must be stripped by the generic wrapper-removal +/// helpers: their callers go on to `assert_cast` the column to the one the declared type produces. +TEST(ColumnBLOB, ConvertToFullIfWrappedRemovesBLOB) +{ + auto inner = ColumnUInt64::create(); + inner->insertValue(42); + ColumnPtr inner_ptr = std::move(inner); + + auto full = makeBLOB(inner_ptr)->convertToFullIfWrapped(); + + EXPECT_EQ(typeid_cast(full.get()), nullptr); + EXPECT_EQ(full.get(), inner_ptr.get()); +} + +TEST(ColumnBLOB, RemoveSpecialRepresentationsRemovesBLOB) +{ + auto inner = ColumnUInt64::create(); + inner->insertValue(42); + ColumnPtr inner_ptr = std::move(inner); + + auto full = removeSpecialRepresentations(makeBLOB(inner_ptr)); + + EXPECT_EQ(typeid_cast(full.get()), nullptr); + EXPECT_EQ(full.get(), inner_ptr.get()); +} + +/// A BLOB can wrap a sparse column (kind stack `{Default, Sparse, Detached}`), so removing it has to +/// leave the column it uncovers to the rest of the chain. +TEST(ColumnBLOB, RemovalContinuesIntoTheUncoveredColumn) +{ + auto values = ColumnUInt64::create(); + ColumnPtr sparse = ColumnSparse::create(std::move(values)); + + auto full = removeSpecialRepresentations(makeBLOB(sparse)); + + EXPECT_EQ(typeid_cast(full.get()), nullptr); + EXPECT_EQ(typeid_cast(full.get()), nullptr); +} diff --git a/src/Common/CurrentMetrics.cpp b/src/Common/CurrentMetrics.cpp index be7058025c05..0f8dc57b9972 100644 --- a/src/Common/CurrentMetrics.cpp +++ b/src/Common/CurrentMetrics.cpp @@ -288,6 +288,9 @@ M(FormatParsingThreads, "Number of threads in the thread pool used for parsing input.") \ M(FormatParsingThreadsActive, "Number of threads in the thread pool used for parsing input running a task.") \ M(FormatParsingThreadsScheduled, "Number of queued or active jobs in the thread pool used for parsing input.") \ + M(IcebergManifestDecodeThreads, "Number of threads in the thread pool used for decoding Iceberg data manifest files.") \ + M(IcebergManifestDecodeThreadsActive, "Number of threads in the thread pool used for decoding Iceberg data manifest files running a task.") \ + M(IcebergManifestDecodeThreadsScheduled, "Number of queued or active jobs in the thread pool used for decoding Iceberg data manifest files.") \ M(OutdatedPartsLoadingThreads, "Number of threads in the threadpool for loading Outdated data parts.") \ M(OutdatedPartsLoadingThreadsActive, "Number of active threads in the threadpool for loading Outdated data parts.") \ M(OutdatedPartsLoadingThreadsScheduled, "Number of queued or active jobs in the threadpool for loading Outdated data parts.") \ diff --git a/src/Common/FailPoint.cpp b/src/Common/FailPoint.cpp index d55047e98b14..a457bcbc6ba4 100644 --- a/src/Common/FailPoint.cpp +++ b/src/Common/FailPoint.cpp @@ -292,6 +292,8 @@ static struct InitFiu REGULAR(rmt_merge_selecting_task_no_free_threads) \ REGULAR(rmt_merge_selecting_task_max_part_size) \ REGULAR(merge_tree_load_statistics_throw) \ + REGULAR(merge_tree_load_outdated_parts_retryable_error) \ + PAUSEABLE(merge_tree_load_outdated_parts_pause) \ PAUSEABLE(smt_mutate_task_pause_in_prepare) \ PAUSEABLE(smt_merge_selecting_task_pause_when_scheduled) \ REGULAR(smt_merge_selecting_task_reach_memory_limit) \ @@ -353,6 +355,7 @@ static struct InitFiu PAUSEABLE(keeper_changelog_readahead_serve_wait) \ PAUSEABLE(keeper_changelog_readahead_park_armed) \ PAUSEABLE(keeper_changelog_readahead_pre_drain) \ + PAUSEABLE(object_storage_source_pause_before_virtual_columns) \ REGULAR(keeper_changelog_readahead_fill_exception) \ REGULAR(distributed_plan_record_failure_while_starting_tasks) \ ONCE(zk_send_thread_request_window_throw) \ diff --git a/src/Common/ZooKeeper/ZooKeeperImpl.cpp b/src/Common/ZooKeeper/ZooKeeperImpl.cpp index 90d6c1bed91a..0e61bf81933b 100644 --- a/src/Common/ZooKeeper/ZooKeeperImpl.cpp +++ b/src/Common/ZooKeeper/ZooKeeperImpl.cpp @@ -372,6 +372,25 @@ void triggerWatchCallback( } } +#if USE_SSL +/// The host part of ":", split lexically at the same character Poco::Net::SocketAddress +/// splits it at, so that a port spelled as a service name still resolves. The result is matched +/// against a certificate, so an IPv6 literal loses its brackets; a shape naming no host is empty. +std::string peerHostName(const std::string & host_and_port) +{ + if (host_and_port.starts_with('/')) + return {}; + + if (host_and_port.starts_with('[')) + { + size_t closing_bracket = host_and_port.find(']'); + return closing_bracket == std::string::npos ? std::string{} : host_and_port.substr(1, closing_bracket - 1); + } + + return host_and_port.substr(0, host_and_port.find(':')); +} +#endif + } template @@ -617,7 +636,13 @@ void ZooKeeper::connect( if (node.secure) { #if USE_SSL - socket = Poco::Net::SecureStreamSocket(); + auto secure_socket = Poco::Net::SecureStreamSocket(); + /// The certificate names the configured host while the socket connects to the + /// address it resolved to, so the name has to be carried explicitly. This is + /// also what puts the host into the SNI extension. + if (const auto peer_host_name = peerHostName(node.host); !peer_host_name.empty()) + secure_socket.setPeerHostName(peer_host_name); + socket = secure_socket; #else throw Poco::Exception( "Communication with ZooKeeper over SSL is disabled because poco library was built without NetSSL support."); diff --git a/src/Common/filesystemHelpers.cpp b/src/Common/filesystemHelpers.cpp index 94c132db1b67..fb0624911e94 100644 --- a/src/Common/filesystemHelpers.cpp +++ b/src/Common/filesystemHelpers.cpp @@ -217,8 +217,22 @@ String getFilesystemName([[maybe_unused]] const String & mount_point) #endif } +/// A path with an embedded NUL is malformed, and, more importantly, it cannot be validated: the +/// comparisons below see the whole value, while every syscall the path is later passed to (`open`, +/// `mkdir`, `stat`) stops at the first NUL. A path shaped as `\0/` would therefore be reported as contained in the prefix while it addresses ``, +/// anywhere on the filesystem. Report such a path as not contained, so that every containment check +/// fails closed. +static bool containsEmbeddedNul(const std::filesystem::path & path) +{ + return path.native().contains('\0'); +} + bool pathStartsWith(const std::filesystem::path & path, const std::filesystem::path & prefix_path) { + if (containsEmbeddedNul(path) || containsEmbeddedNul(prefix_path)) + return false; + auto rel = fs::relative(path, prefix_path); if (rel.empty() || rel == "..") return false; @@ -240,6 +254,9 @@ static bool fileOrSymlinkPathStartsWith(const std::filesystem::path & path, cons /// `.` and `..` and extra `/`. Path is not canonized because otherwise path will /// not be a path of a symlink itself. + if (containsEmbeddedNul(path) || containsEmbeddedNul(prefix_path)) + return false; + auto rel = fs::absolute(path).lexically_normal().lexically_relative(fs::absolute(prefix_path).lexically_normal()); if (rel.empty() || rel == "..") diff --git a/src/Common/setThreadName.h b/src/Common/setThreadName.h index fbd355262586..a8403b643569 100644 --- a/src/Common/setThreadName.h +++ b/src/Common/setThreadName.h @@ -72,6 +72,7 @@ namespace DB M(HASHED_DICT_LOAD, "HashedDictLoad") \ M(HTTP_HANDLER, "HTTPHandler") \ M(HTTP_SERVER_CONN, "HTTPSrvConn") \ + M(ICEBERG_DELETE_DECODE, "IcebergDelDec") \ M(ICEBERG_ITERATOR, "IcebergIter") \ M(ICEBERG_SCHEDULE_POOL, "IcebergSchPool") \ M(INTERSERVER_HANDLER, "IntersrvHandler") \ diff --git a/src/Common/tests/gtest_path_starts_with_embedded_nul.cpp b/src/Common/tests/gtest_path_starts_with_embedded_nul.cpp new file mode 100644 index 000000000000..3a9373ab3f46 --- /dev/null +++ b/src/Common/tests/gtest_path_starts_with_embedded_nul.cpp @@ -0,0 +1,78 @@ +#include + +#include + +#include +#include + +#include /// for ::getpid + +namespace fs = std::filesystem; + +namespace +{ + +/// The containment helpers compare the whole string, while every syscall a path is later passed to stops at +/// the first NUL byte. The tests below build the shape that abuses the difference: as a whole the path +/// normalizes back into the prefix, truncated at the NUL it addresses `probe` next to the prefix. +struct EmbeddedNulPaths +{ + fs::path prefix; + std::string contained; + std::string escaping_through_nul; + + EmbeddedNulPaths() + : prefix(fs::temp_directory_path() / ("path_starts_with_embedded_nul_" + std::to_string(::getpid()))) + { + fs::create_directories(prefix); + std::string back_into_prefix = "/../" + prefix.filename().string() + "/file"; + contained = (prefix / ".." / "probe").string() + back_into_prefix; + escaping_through_nul = (prefix / ".." / "probe").string() + std::string(1, '\0') + back_into_prefix; + } + + ~EmbeddedNulPaths() + { + std::error_code ec; + fs::remove_all(prefix, ec); + } +}; + +std::string withEmbeddedNul(const fs::path & path) +{ + return path.string() + std::string(1, '\0') + "suffix"; +} + +} + +TEST(PathStartsWithEmbeddedNul, PathStartsWith) +{ + EmbeddedNulPaths paths; + const std::string prefix = paths.prefix.string(); + + /// Positive control: the same traversal without a NUL is contained, so it is the NUL that flips the result. + EXPECT_TRUE(DB::pathStartsWith(paths.contained, prefix)); + EXPECT_TRUE(DB::pathStartsWith(fs::path(paths.contained), paths.prefix)); + EXPECT_TRUE(DB::pathStartsWith((paths.prefix / "file").string(), prefix)); + + EXPECT_FALSE(DB::pathStartsWith(paths.escaping_through_nul, prefix)); + EXPECT_FALSE(DB::pathStartsWith(fs::path(paths.escaping_through_nul), paths.prefix)); + + /// A NUL anywhere in the path, even where it cannot escape, and a NUL in the prefix are rejected too. + EXPECT_FALSE(DB::pathStartsWith(withEmbeddedNul(paths.prefix / "file"), prefix)); + EXPECT_FALSE(DB::pathStartsWith((paths.prefix / "file").string(), withEmbeddedNul(paths.prefix))); + EXPECT_FALSE(DB::pathStartsWith(fs::path((paths.prefix / "file").string()), fs::path(withEmbeddedNul(paths.prefix)))); +} + +TEST(PathStartsWithEmbeddedNul, FileOrSymlinkPathStartsWith) +{ + EmbeddedNulPaths paths; + const std::string prefix = paths.prefix.string(); + + EXPECT_TRUE(DB::fileOrSymlinkPathStartsWith(paths.contained, prefix)); + EXPECT_TRUE(DB::fileOrSymlinkPathStartsWith((paths.prefix / "file").string(), prefix)); + + EXPECT_FALSE(DB::fileOrSymlinkPathStartsWith(paths.escaping_through_nul, prefix)); + + EXPECT_FALSE(DB::fileOrSymlinkPathStartsWith(withEmbeddedNul(paths.prefix / "file"), prefix)); + EXPECT_FALSE(DB::fileOrSymlinkPathStartsWith((paths.prefix / "file").string(), withEmbeddedNul(paths.prefix))); +} diff --git a/src/Coordination/KeeperStorageImpl.cpp b/src/Coordination/KeeperStorageImpl.cpp index 2c007d237e58..92b5405c561c 100644 --- a/src/Coordination/KeeperStorageImpl.cpp +++ b/src/Coordination/KeeperStorageImpl.cpp @@ -1532,15 +1532,26 @@ static Coordination::Error preprocess( return Coordination::Error::ZOK; } +/// Cuts the deltas of the next subrequest, up to its `SubDeltaEnd` marker, off the front of `deltas` +/// and drops the marker. `preprocess` appends the marker after every subrequest, so a range without +/// it does not match the request that is being processed: the markers were lost, and stepping past +/// the end of the range to look for them is undefined behavior. This runs on the raft commit and +/// replay threads, so treat it like every other mismatch between a request and its deltas. +/// +/// `FailedMultiDelta` is the other marker `preprocess` emits, and the callers handle it before they +/// get here: it is the sole delta of a failed multi request. Inside a subrequest slice it is out of +/// place, and `commit` would ignore it and report the subrequest as successful, so the walk stops on +/// both markers and rejects the failure marker instead of passing it on as an ordinary delta. static KeeperStorage::DeltaRange extractSubdeltas(KeeperStorage::DeltaRange & deltas) { - auto it = deltas.begin(); - - for (; it != deltas.end(); ++it) - { - if (std::holds_alternative(it->operation)) - break; - } + auto it = std::ranges::find_if( + deltas, + [](const auto & delta) + { return std::holds_alternative(delta.operation) || std::holds_alternative(delta.operation); }); + if (it == deltas.end()) + onStorageInconsistency("Missing SubDeltaEnd marker for a Multi subrequest"); + if (std::holds_alternative(it->operation)) + onStorageInconsistency("Unexpected failure marker inside a subrequest of a Multi request"); KeeperStorage::DeltaRange result{deltas.begin(), it}; ++it; @@ -1564,10 +1575,34 @@ process(const Coordination::ZooKeeperMultiRequest & zk_request, Storage & storag const auto & subrequests = zk_request.requests; - // the deltas will have at least SubDeltaEnd or FailedMultiDelta - chassert(!deltas.empty()); + /// `preprocess` appends at least `SubDeltaEnd` or `FailedMultiDelta` for every subrequest, so the + /// range is empty only for a multi request that has no subrequests. Such a request is accepted - + /// ZooKeeper answers it with an empty successful response, and a client that builds a transaction + /// from a list that turns out to be empty sends exactly that - so answer it the same way here. + /// `processWatches` below already handles the empty range, and this runs on the raft commit + /// thread, where an exception terminates the process. + /// + /// The success return is reserved for the true zero-subrequest case: a multi request with + /// subrequests but without deltas means that the markers of the preprocessing were lost, and + /// answering it with an empty success would silently drop every suboperation, so it goes through + /// the storage inconsistency path like every other request whose deltas do not match. + if (deltas.empty()) + { + if (!subrequests.empty()) + onStorageInconsistency("Unexpected empty deltas for Multi request with subrequests"); + + response->error = Coordination::Error::ZOK; + return response; + } + if (const auto * failed_multi = std::get_if(&deltas.front().operation)) { + /// `preprocess` puts the failure marker last and the caller rolls back everything before it, + /// so the marker is the only delta of a failed multi request. Anything else in the range is + /// a delta that no subrequest response would account for, so it cannot be dropped silently. + if (std::next(deltas.begin()) != deltas.end()) + onStorageInconsistency("Unexpected deltas after the failure marker of a Multi request"); + const size_t subrequests_count = subrequests.size(); for (size_t i = 0; i < subrequests_count; ++i) @@ -1591,6 +1626,12 @@ process(const Coordination::ZooKeeperMultiRequest & zk_request, Storage & storag *multi_subrequest, [&](const auto & subrequest) { return process(subrequest, storage, std::move(subdeltas), session_id); })); } + /// Every delta of the transaction belongs to one of the subrequests above. Deltas left after the + /// last marker belong to no subrequest: they are already applied to the storage, and no response + /// would account for them, so they cannot be silently ignored either. + if (!deltas.empty()) + onStorageInconsistency("Unexpected deltas after the last subrequest of a Multi request"); + response->error = Coordination::Error::ZOK; return response; } diff --git a/src/Core/FormatFactorySettings.h b/src/Core/FormatFactorySettings.h index c37da84008e3..5bc1e8766ec4 100644 --- a/src/Core/FormatFactorySettings.h +++ b/src/Core/FormatFactorySettings.h @@ -1501,6 +1501,24 @@ Write Date values as plain 16-bit numbers (read back as UInt16), instead of conv )", 0) \ DECLARE(Bool, output_format_arrow_unsupported_types_as_binary, true, R"( Output types having no conversion as raw binary data. If false - such types would raise UNKNOWN_TYPE exception. +)", 0) \ + DECLARE(UInt64, output_format_arrow_record_batch_size, 0, R"( +Target number of rows per record batch for the `Arrow` and `ArrowStream` output formats. Combining small blocks reduces metadata and buffer-padding overhead, particularly for queries with selective filters. + +Blocks accumulate until this target or [output_format_arrow_record_batch_size_bytes](#output_format_arrow_record_batch_size_bytes) is reached. A block that already meets the row or byte target is written separately, without splitting. If you set a row target, combined batches contain fewer than twice that many rows, but a single input block can be larger. + +Buffering blocks can increase memory use and delay output. If the result never reaches either target, `ArrowStream` writes the first record batch only when the query finishes, though it can write the schema earlier. Leave both targets at `0` to write record batches as blocks arrive. + +`0` (the default) disables the row target. Try `65409` as a starting value. +)", 0) \ + DECLARE(UInt64, output_format_arrow_record_batch_size_bytes, 0, R"( +Target record batch size for the `Arrow` and `ArrowStream` output formats, measured in bytes of accumulated block data. This uses the same measure as [min_insert_block_size_bytes](/reference/settings/settings#min_insert_block_size_bytes). A batch is written when either this target or [output_format_arrow_record_batch_size](#output_format_arrow_record_batch_size) is reached. + +Note that `LowCardinality` columns can produce Arrow batches much larger or smaller than this byte target. Repeated values expand in the output unless [output_format_arrow_low_cardinality_as_dictionary](#output_format_arrow_low_cardinality_as_dictionary) is enabled. Filtered blocks can also retain large dictionaries, so even a block with very few rows can reach the target and be written separately. For these columns, use [output_format_arrow_record_batch_size](#output_format_arrow_record_batch_size) to control the row count and set the byte target to `0`. + +Buffering blocks can increase memory use and delay the first record batch until the query finishes. Leave both targets at `0` to write record batches as blocks arrive. + +`0` (the default) disables the byte target. Try `1048576` (1 MiB) as a starting value. )", 0) \ \ DECLARE(Bool, output_format_orc_string_as_string, true, R"( diff --git a/src/Core/PostgreSQLProtocol.h b/src/Core/PostgreSQLProtocol.h index 706465d5e20e..0adea1b15bb5 100644 --- a/src/Core/PostgreSQLProtocol.h +++ b/src/Core/PostgreSQLProtocol.h @@ -173,6 +173,25 @@ class ColumnTypeSpec ColumnTypeSpec convertDataTypeToPostgresColumnTypeSpec(const DataTypePtr & data_type); +/// Reads exactly `size` bytes into `s`. The size is declared by the client and the payload may +/// never arrive, so the string grows as the bytes are received instead of being resized to the +/// declared size up front: otherwise a tiny packet declaring a huge field makes the server +/// allocate that much and then wait for data that never comes. +inline void readStringOfDeclaredSize(String & s, size_t size, ReadBuffer & in) +{ + s.clear(); + while (s.size() < size) + { + if (in.eof()) + throw Exception(ErrorCodes::UNKNOWN_PACKET_FROM_CLIENT, + "Message from client declares a field of {} bytes, but contains only {}", size, s.size()); + + const size_t bytes_to_copy = std::min(size - s.size(), in.available()); + s.append(in.position(), bytes_to_copy); + in.position() += bytes_to_copy; + } +} + class MessageTransport { private: @@ -286,6 +305,40 @@ class FrontMessage : public IMessage * (if type is provided for the message by the protocol). */ virtual void deserialize(ReadBuffer & in) = 0; + +protected: + template + static void deserializePayload(ReadBuffer & in, std::string_view message_name, F && deserialize_payload) + { + Int32 size = 0; + readBinaryBigEndian(size, in); + if (size < 4) + throw Exception(ErrorCodes::UNKNOWN_PACKET_FROM_CLIENT, + "Wrong message length {} in {}, it must be at least 4", size, message_name); + + const size_t payload_size = static_cast(size - 4); + LimitReadBuffer payload_in(in, {.read_no_less = payload_size, .read_no_more = payload_size}); + try + { + deserialize_payload(payload_in); + } + catch (...) + { + /// Keep the stream aligned before the handler starts discarding messages through `Sync`. + /// When the frame itself could not be read (the client closed the connection before sending + /// the declared bytes), the buffer is canceled and there is nothing left to align. + if (!payload_in.isCanceled()) + payload_in.ignore(payload_size - payload_in.count()); + throw; + } + + const size_t unread_payload_bytes = payload_size - payload_in.count(); + payload_in.ignore(unread_payload_bytes); + if (unread_payload_bytes != 0) + throw Exception(ErrorCodes::UNKNOWN_PACKET_FROM_CLIENT, + "Wrong message length {} in {}, it has {} unexpected trailing payload bytes", + size, message_name, unread_payload_bytes); + } }; class BackendMessage : public IMessage, public ISerializable @@ -419,7 +472,7 @@ class Terminate : FrontMessage public: void deserialize(ReadBuffer & in) override { - in.ignore(4); + deserializePayload(in, "Terminate message", [](ReadBuffer &) {}); } MessageType getMessageType() const override @@ -534,22 +587,18 @@ class SASLInitialResponse : public Messaging::FrontMessage void deserialize(ReadBuffer & in) override { - UInt8 message_type = 0; - readBinaryBigEndian(message_type, in); - Int32 size = 0; - readBinaryBigEndian(size, in); - readNullTerminated(auth_method, in); - Int32 size_sasl_mechanism = 0; - readBinaryBigEndian(size_sasl_mechanism, in); - /// -1 is the protocol sentinel for "no initial response"; any other negative value is malformed. - if (size_sasl_mechanism < -1) - throw Exception(ErrorCodes::UNKNOWN_PACKET_FROM_CLIENT, - "Wrong SASL mechanism length {} in SASLInitialResponse, it must not be less than -1", size_sasl_mechanism); - if (size_sasl_mechanism > 0) + deserializePayload(in, "SASLInitialResponse message", [this](ReadBuffer & payload_in) { - sasl_mechanism.resize(size_sasl_mechanism); - in.readStrict(sasl_mechanism.data(), size_sasl_mechanism); - } + readNullTerminated(auth_method, payload_in); + Int32 size_sasl_mechanism = 0; + readBinaryBigEndian(size_sasl_mechanism, payload_in); + /// -1 is the protocol sentinel for "no initial response"; any other negative value is malformed. + if (size_sasl_mechanism < -1) + throw Exception(ErrorCodes::UNKNOWN_PACKET_FROM_CLIENT, + "Wrong SASL mechanism length {} in SASLInitialResponse, it must not be less than -1", size_sasl_mechanism); + if (size_sasl_mechanism > 0) + readStringOfDeclaredSize(sasl_mechanism, size_sasl_mechanism, payload_in); + }); } MessageType getMessageType() const override @@ -594,15 +643,10 @@ class SASLResponse : public Messaging::FrontMessage void deserialize(ReadBuffer & in) override { - UInt8 message_type = 0; - readBinaryBigEndian(message_type, in); - Int32 size = 0; - readBinaryBigEndian(size, in); - if (size < 4) - throw Exception(ErrorCodes::UNKNOWN_PACKET_FROM_CLIENT, - "Wrong message length {} in SASLResponse, it must be at least 4", size); - sasl_mechanism.resize(size - 4); - in.readStrict(sasl_mechanism.data(), size - 4); + deserializePayload(in, "SASLResponse message", [this](ReadBuffer & payload_in) + { + readStringUntilEOF(sasl_mechanism, payload_in); + }); } MessageType getMessageType() const override @@ -640,9 +684,10 @@ class PasswordMessage : FrontMessage void deserialize(ReadBuffer & in) override { - Int32 sz = 0; - readBinaryBigEndian(sz, in); - readNullTerminated(password, in); + deserializePayload(in, "PasswordMessage", [this](ReadBuffer & payload_in) + { + readNullTerminated(password, payload_in); + }); } MessageType getMessageType() const override @@ -720,9 +765,10 @@ class Query : FrontMessage void deserialize(ReadBuffer & in) override { - Int32 sz = 0; - readBinaryBigEndian(sz, in); - readNullTerminated(query, in); + deserializePayload(in, "Query message", [this](ReadBuffer & payload_in) + { + readNullTerminated(query, payload_in); + }); } MessageType getMessageType() const override @@ -836,8 +882,8 @@ class BindQuery : FrontMessage parameters.emplace_back(std::nullopt); continue; } - String current_param(sz_param, 0); - in.readStrict(current_param.data(), sz_param); + String current_param; + readStringOfDeclaredSize(current_param, sz_param, in); parameters.push_back(std::move(current_param)); } @@ -1000,8 +1046,11 @@ class SyncQuery : FrontMessage public: void deserialize(ReadBuffer & in) override { - Int32 sz = 0; - readBinaryBigEndian(sz, in); + Int32 size = 0; + readBinaryBigEndian(size, in); + if (size != 4) + throw Exception(ErrorCodes::UNKNOWN_PACKET_FROM_CLIENT, + "Wrong message length {} in Sync message, it must be 4", size); } MessageType getMessageType() const override @@ -1149,9 +1198,10 @@ class CopyDataQuery : FrontMessage void deserialize(ReadBuffer & in) override { - Int32 sz = 0; - readBinaryBigEndian(sz, in); - readNullTerminated(query, in); + deserializePayload(in, "CopyData message", [this](ReadBuffer & payload_in) + { + readNullTerminated(query, payload_in); + }); } MessageType getMessageType() const override @@ -1219,18 +1269,10 @@ class CopyInData : FrontMessage void deserialize(ReadBuffer & in) override { - Int32 sz = 0; - readBinaryBigEndian(sz, in); - if (sz < static_cast(sizeof(Int32))) - throw Exception(ErrorCodes::UNKNOWN_PACKET_FROM_CLIENT, - "Wrong message length {} in CopyData, it must be at least 4", sz); - query.reserve(sz - sizeof(Int32)); - for (size_t i = 0; i < sz - sizeof(Int32); ++i) + deserializePayload(in, "CopyData message", [this](ReadBuffer & payload_in) { - char byte = 0; - readBinary(byte, in); - query.push_back(byte); - } + readStringUntilEOF(query, payload_in); + }); } MessageType getMessageType() const override @@ -1557,6 +1599,16 @@ class CleartextPasswordAuth : public AuthenticationMethod class ScrambleSHA256Auth : public AuthenticationMethod { + /// Both SASL messages of the SCRAM exchange are sent with the `PasswordMessage` type byte. + static void expectPasswordMessage(Messaging::MessageTransport & mt) + { + Messaging::FrontMessageType type = mt.receiveMessageType(); + if (type != Messaging::FrontMessageType::PASSWORD_MESSAGE) + throw Exception(ErrorCodes::UNEXPECTED_PACKET_FROM_CLIENT, + "Client sent wrong message or closed the connection. Message byte was {}.", + static_cast(type)); + } + static size_t findPatternPosition(const String & key, const String & pattern) { size_t pos = key.size(); @@ -1668,6 +1720,7 @@ class ScrambleSHA256Auth : public AuthenticationMethod String auth_message; mt.send(Messaging::AuthenticationSASL(), true); + expectPasswordMessage(mt); auto rsp = mt.receive(); auto server_nonce = generateNonce(); @@ -1690,6 +1743,7 @@ class ScrambleSHA256Auth : public AuthenticationMethod auto sasl_continue_message = fmt::format("r={},s={},i={}", nonce, salt, num_iterations); mt.send(Messaging::AuthenticationSASLContinue(sasl_continue_message), true); auth_message += "," + sasl_continue_message; + expectPasswordMessage(mt); auto rsp_continue = mt.receive(); auto proof = parseProof(rsp_continue->sasl_mechanism); auto proof_position = findProofPosition(rsp_continue->sasl_mechanism); diff --git a/src/Core/ServerSettings.cpp b/src/Core/ServerSettings.cpp index 2134d5b63904..dbafaf84a46b 100644 --- a/src/Core/ServerSettings.cpp +++ b/src/Core/ServerSettings.cpp @@ -183,6 +183,17 @@ A value of `0` means unlimited. )", 0) \ DECLARE(UInt64, max_format_parsing_thread_pool_size, 100, R"( Maximum total number of threads to use for parsing input. +)", 0) \ + DECLARE(UInt64, max_iceberg_manifest_decode_thread_pool_size, 100, R"( +Maximum total number of threads to use for decoding Iceberg data manifest files. + +The pool is separate from the IO pool on purpose: a decode task can block until the query consumes the entries it has produced, while the delete manifest decode waits for its tasks on the IO pool before any entry is consumed - sharing one pool could deadlock. +)", 0) \ + DECLARE(UInt64, max_iceberg_manifest_decode_thread_pool_free_size, 0, R"( +Maximum number of idle standby threads to keep in the thread pool for decoding Iceberg data manifest files. +)", 0) \ + DECLARE(UInt64, iceberg_manifest_decode_thread_pool_queue_size, 10000, R"( +The maximum number of jobs that can be scheduled on the thread pool for decoding Iceberg data manifest files. )", 0) \ DECLARE(UInt64, max_format_parsing_thread_pool_free_size, 0, R"( Maximum number of idle standby threads to keep in the thread pool for parsing input. @@ -1920,7 +1931,7 @@ Configured as `named_collections_storage.type` (`` (resp. the directory ``) or in file (resp. directory) specified by the environment variable `` (resp. ``).)", 0, "openSSL.client.loadDefaultCAFile") \ DECLARE(String, openssl_client_chipher_list, "ALL:!ADH:!LOW:!EXP:!MD5:!3DES:@STRENGTH", R"(Supported OpenSSL encryptions.)", 0, "openSSL.client.cipherList") \ DECLARE(Bool, openssl_client_cache_sessions, false, R"(Enables or disables caching sessions. Must be used in combination with ``. Acceptable values: ``, ``.)", 0, "openSSL.client.cacheSessions") \ - DECLARE(Bool, openssl_client_extended_verification, false, R"(If enabled, verify that the certificate CN or SAN matches the peer hostname.)", 0, "openSSL.client.extendedVerification") \ + DECLARE(Bool, openssl_client_extended_verification, true, R"(If enabled, verify that the certificate CN or SAN matches the peer hostname.)", 0, "openSSL.client.extendedVerification") \ DECLARE(Bool, openssl_client_required_tls_v1, false, R"(Require a TLSv1 connection. Acceptable values: ``, ``.)", 0, "openSSL.client.requireTLSv1") \ DECLARE(Bool, openssl_client_required_tls_v1_1, false, R"(Require a TLSv1.1 connection. Acceptable values: ``, ``.)", 0, "openSSL.client.requireTLSv1_1") \ DECLARE(Bool, openssl_client_required_tls_v1_2, false, R"(Require a TLSv1.2 connection. Acceptable values: ``, ``.)", 0, "openSSL.client.requireTLSv1_2") \ @@ -3665,6 +3676,12 @@ ChangeableSettingsMap collectChangeableServerSettings(ContextPtr context) {getFormatParsingThreadPool().isInitialized() ? std::to_string(getFormatParsingThreadPool().get().getMaxFreeThreads()) : "0", ChangeableWithoutRestart::Yes}}, {"format_parsing_thread_pool_queue_size", {getFormatParsingThreadPool().isInitialized() ? std::to_string(getFormatParsingThreadPool().get().getQueueSize()) : "0", ChangeableWithoutRestart::Yes}}, + {"max_iceberg_manifest_decode_thread_pool_size", + {getIcebergManifestDecodeThreadPool().isInitialized() ? std::to_string(getIcebergManifestDecodeThreadPool().get().getMaxThreads()) : "0", ChangeableWithoutRestart::Yes}}, + {"max_iceberg_manifest_decode_thread_pool_free_size", + {getIcebergManifestDecodeThreadPool().isInitialized() ? std::to_string(getIcebergManifestDecodeThreadPool().get().getMaxFreeThreads()) : "0", ChangeableWithoutRestart::Yes}}, + {"iceberg_manifest_decode_thread_pool_queue_size", + {getIcebergManifestDecodeThreadPool().isInitialized() ? std::to_string(getIcebergManifestDecodeThreadPool().get().getQueueSize()) : "0", ChangeableWithoutRestart::Yes}}, {"abort_on_logical_error", {std::to_string(DB::abort_on_logical_error), ChangeableWithoutRestart::Yes}}, diff --git a/src/Core/Settings.cpp b/src/Core/Settings.cpp index 881dd7186dd6..79b5100b5010 100644 --- a/src/Core/Settings.cpp +++ b/src/Core/Settings.cpp @@ -1210,6 +1210,14 @@ Allows or restricts using [Variant](/reference/data-types/variant) and [Dynamic] )", 0) \ DECLARE(Bool, allow_suspicious_types_in_order_by, false, R"( Allows or restricts using [Variant](/reference/data-types/variant) and [Dynamic](/reference/data-types/dynamic) types in ORDER BY keys. +)", 0) \ + DECLARE(Bool, validate_group_by_all_key_types, true, R"( +Controls whether the grouping keys that `GROUP BY ALL` expands the `SELECT` expressions into are checked against [allow_suspicious_types_in_group_by](#allow_suspicious_types_in_group_by). Disable it to restore the behavior of versions before 26.7, which accepted a [Variant](/reference/data-types/variant) or [Dynamic](/reference/data-types/dynamic) key written as `GROUP BY ALL`, for example a grouping key that is an untyped JSON subpath. Takes effect only when the analyzer is enabled (`enable_analyzer = 1`, the default); with the old analyzer such a key was rejected before 26.7 as well, so there is nothing to restore there. An explicit `GROUP BY` is unaffected and keeps rejecting such a key either way, so this is narrower than setting `allow_suspicious_types_in_group_by`, which also permits them in an explicit `GROUP BY`. + +Possible values: + +- 0 - The key types `GROUP BY ALL` expands into are not validated. +- 1 - They are validated, as an explicit `GROUP BY` validates its own keys. )", 0) \ DECLARE(Bool, use_variant_default_implementation_for_comparisons, true, R"( Enables or disables default implementation for Variant type in comparison functions. @@ -6028,14 +6036,21 @@ Possible values: DECLARE(UInt64, iceberg_metadata_staleness_ms, 0, R"( If non-zero, skip fetching iceberg metadata from remote catalog if there is a cached metadata snapshot, more recent than the given staleness window. Zero means to always fetch the latest metadata version from the remote catalog. Setting this a non-zero trades staleness to a lower latency of read operations. )", 0) \ - DECLARE(NonZeroUInt64, iceberg_delete_manifest_decode_concurrency, 4, R"( -Maximum number of Iceberg delete manifest files decoded concurrently during query execution before any data file is read. + DECLARE_WITH_ALIAS(NonZeroUInt64, iceberg_manifest_decode_concurrency, 4, R"( +Maximum number of Iceberg manifest files decoded concurrently while reading a table. -All delete manifests must be decoded before any data file is read, so this work sits on the critical path before the first row is returned. Decoding several at a time overlaps both the object storage round-trips and the per-row pruning work. +Delete manifests are all decoded before any data file is read; data manifests are decoded while the list of data files for the query is produced, and new ones are decoded only as the query consumes already decoded entries. Decoding several manifests at a time overlaps the object storage round-trips and the per-entry pruning work. -Higher values raise peak memory during query initialization when the Iceberg metadata files cache is disabled or full, since each in-flight manifest then holds its own decoded contents. +Higher values raise peak memory when the Iceberg metadata files cache is disabled or full, since each in-flight manifest then holds its own decoded contents. Must be greater than zero; `1` decodes the manifests one at a time. +)", 0, iceberg_delete_manifest_decode_concurrency) \ + DECLARE(NonZeroUInt64, iceberg_file_entries_queue_size, 100, R"( +Capacity of the queue between the Iceberg data manifest decode tasks and the query, in data file entries. + +The decode tasks pause once the queue is full and the query is not consuming, so this also bounds the read-ahead. + +Must be greater than zero. )", 0) \ DECLARE(Bool, use_parquet_metadata_cache, true, R"( If turned on, parquet format may utilize the parquet metadata cache. @@ -7261,7 +7276,7 @@ Maximum time to wait for a file segment which is being downloaded to the filesys Prefer bigger buffer size if filesystem cache is enabled to avoid writing small file segments which deteriorate cache performance. On the other hand, enabling this setting might increase memory usage. )", 0) \ DECLARE(UInt64, filesystem_cache_boundary_alignment, 0, R"( -Filesystem cache boundary alignment. This setting is applied only for non-disk read (e.g. for cache of remote table engines / table functions, but not for storage configuration of MergeTree tables). Value 0 means no alignment. +Filesystem cache boundary alignment. For non-disk read (e.g. for cache of remote table engines / table functions) value 0 means no alignment. For disk read (e.g. for MergeTree tables on a disk with cache) value 0 means that `boundary_alignment` from the cache configuration is used. )", 0) \ DECLARE(UInt64, temporary_data_in_cache_reserve_space_wait_lock_timeout_milliseconds, (10 * 60 * 1000), R"( Wait time to lock cache for space reservation for temporary data in filesystem cache @@ -9203,6 +9218,7 @@ Experimental dictionary source for integration with YTsaurus. )", EXPERIMENTAL) \ DECLARE(Bool, distributed_plan_force_shuffle_aggregation, false, R"( Use Shuffle aggregation strategy instead of PartialAggregation + Merge in distributed query plan. +Ignored where the Shuffle strategy cannot produce a correct result, for example for `GROUPING SETS` or when the aggregation must produce results in bucket order. )", EXPERIMENTAL) \ DECLARE(Bool, enable_cascades_optimizer, false, R"( Enable the Cascades cost-based optimizer for distributed query plans. diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp index 441325235ea7..f85167d810fd 100644 --- a/src/Core/SettingsChangesHistory.cpp +++ b/src/Core/SettingsChangesHistory.cpp @@ -43,6 +43,7 @@ const VersionToSettingsChangesMap & getSettingsChangesHistory() /// Note: please check if the key already exists to prevent duplicate entries. addSettingsChanges(settings_changes_history, "26.8", { + {"validate_group_by_all_key_types", true, true, "The validation of the key types that `GROUP BY ALL` expands the `SELECT` expressions into is kept under `compatibility` with 26.7: the previous value is deliberately equal to the new one, because 26.7 already rejected such a key and only a version before 26.7 restores the earlier acceptance."}, {"allow_experimental_ai_functions", false, false, "The setting is obsolete, AI functions are beta now and enabled by default."}, {"ai_function_max_retries", 0, 1, "Retry a transient API error once by default, so a single 429 or 5xx from the provider does not fail the query."}, {"adaptive_aggregator_freeze_threshold_bytes", 4194304, 4194304, "New setting bounding the adaptive aggregator's frozen local tables in bytes, whichever of it and the key-count threshold is reached first; 0 disables the byte bound."}, @@ -139,6 +140,8 @@ const VersionToSettingsChangesMap & getSettingsChangesHistory() {"output_format_arrow_use_native_writer", true, true, "Obsolete setting, the native ClickHouse writer is now always used for the `Arrow` and `ArrowStream` formats (the Apache Arrow library-based writer has been removed)."}, {"distributed_cache_min_inflight_bytes_to_discard_connection_on_seek", 0, 4 * 1024 * 1024, "New setting to drop and reopen a distributed cache connection on a seek when too many in-flight bytes would otherwise be discarded. Defaults to 4 MiB; 0 restores the previous behavior (always reuse the connection via the read range id)."}, {"iceberg_delete_manifest_decode_concurrency", 2, 4, "New setting bounding how many Iceberg delete manifest files are decoded concurrently before the first row is read. Before 26.8 one manifest was decoded at a time with the next one's fetch already in flight, so `2` is the closest equivalent of the previous behavior, which put the sum of their object storage round-trips on the critical path before the first row."}, + {"iceberg_file_entries_queue_size", 100, 100, "New setting for the previously hardcoded capacity of the queue between the Iceberg data manifest decode tasks and the query."}, + {"iceberg_manifest_decode_concurrency", 2, 4, "New setting bounding how many Iceberg manifest files are decoded concurrently, for delete and data manifests alike. It replaces `iceberg_delete_manifest_decode_concurrency` (kept as an alias). `2` approximates the data path before this change, which decoded one manifest at a time with the next one's fetch already in flight."}, {"run_query_in_background", false, false, "New setting to run a query in the background, detached from the connection that submitted it, discarding the result."}, {"enable_cascades_optimizer", false, false, "New experimental setting."}, {"merge_tree_min_bytes_per_read_stream", 0, (64 * 1024), "New setting to cap the number of streams for ordinary local unordered `MergeTree` narrow-column scans using a sqrt cost model, reducing per-stream overhead on high-core-count machines. previous_value=0 (disabled) so `compatibility` with versions before 26.8 restores the pre-existing stream count."}, @@ -152,9 +155,12 @@ const VersionToSettingsChangesMap & getSettingsChangesHistory() {"enable_function_early_short_circuit", false, false, "New setting"}, {"merge_tree_prefetch_json_shared_data_substreams", true, true, "New setting to control prefetching of JSON shared data substreams that are read by seeking to a mark in Wide parts."}, {"iceberg_compaction_commit_batch_size", 100, 100, "New setting"}, + {"output_format_arrow_record_batch_size", 0, 0, "New setting to combine small blocks in `Arrow` and `ArrowStream` output using a target row count. The default `0` preserves one record batch per block."}, + {"output_format_arrow_record_batch_size_bytes", 0, 0, "New setting to combine small blocks in `Arrow` and `ArrowStream` output using a target size in bytes of accumulated data. The default `0` preserves one record batch per block."}, }); addSettingsChanges(settings_changes_history, "26.7", { + {"validate_group_by_all_key_types", false, true, "New setting gating the validation of the key types that `GROUP BY ALL` expands the `SELECT` expressions into. 26.7 started rejecting a `Variant`/`Dynamic` key there, which earlier versions accepted with the analyzer enabled (`enable_analyzer = 1`, the default; the old analyzer rejected such a key before 26.7 as well), so the previous value is `false` and `compatibility` with a version before 26.7 restores the earlier acceptance."}, {"analyzer_compatibility_allow_non_aggregate_in_having", false, false, "New compatibility setting. When enabled, the analyzer mimics the legacy `HAVING`-to-`WHERE` rewrite for non-aggregate AND-conjuncts instead of raising `NOT_AN_AGGREGATE`."}, {"query_plan_optimize_lazy_materialization_for_object_storage", false, true, "New setting to use lazy materialization for `ORDER BY ... LIMIT n` queries reading Parquet files from object storage (including Iceberg tables)."}, {"iceberg_compaction_max_rows_in_data_file", std::numeric_limits::max(), std::numeric_limits::max(), "New setting for the max rows of an iceberg data file produced by compaction, separate from the insert-time limit."}, diff --git a/src/Core/tests/gtest_postgresql_protocol.cpp b/src/Core/tests/gtest_postgresql_protocol.cpp index 0e1e95596835..719a4fe8567f 100644 --- a/src/Core/tests/gtest_postgresql_protocol.cpp +++ b/src/Core/tests/gtest_postgresql_protocol.cpp @@ -28,11 +28,6 @@ namespace PreparedStatements = DB::PostgreSQLProtocol::PostgresPreparedStatement namespace { -void putUInt8(std::string & s, UInt8 v) -{ - s.push_back(static_cast(v)); -} - void putInt16(std::string & s, Int16 v) { s.push_back(static_cast((v >> 8) & 0xFF)); @@ -45,6 +40,14 @@ void putInt32(std::string & s, Int32 v) s.push_back(static_cast((v >> (8 * i)) & 0xFF)); } +std::string framePayload(std::string payload) +{ + std::string bytes; + putInt32(bytes, static_cast(4 + payload.size())); + bytes += payload; + return bytes; +} + /// Run `body` over the bytes and report whether it threw UNKNOWN_PACKET_FROM_CLIENT. template bool throwsUnknownPacket(const std::string & bytes, F && body) @@ -62,6 +65,31 @@ bool throwsUnknownPacket(const std::string & bytes, F && body) } } +template +void expectTrailingPayloadIsRejectedAndAligned(std::string payload) +{ + /// These bytes look like a `Sync` frame, but they are trailing bytes in the current message. + payload.append("S\0\0\0\4", 5); + std::string bytes = framePayload(std::move(payload)); + bytes.push_back('X'); + + ReadBufferFromMemory in(bytes.data(), bytes.size()); + TMessage msg; + try + { + msg.deserialize(in); + FAIL() << "Expected UNKNOWN_PACKET_FROM_CLIENT"; + } + catch (const Exception & e) + { + EXPECT_EQ(e.code(), ErrorCodes::UNKNOWN_PACKET_FROM_CLIENT); + } + + char marker = 0; + in.readStrict(marker); + EXPECT_EQ(marker, 'X'); +} + } TEST(PostgreSQLProtocol, DropMessageRejectsLengthBelowFour) @@ -91,10 +119,10 @@ TEST(PostgreSQLProtocol, DropMessageRejectsLengthBelowFour) TEST(PostgreSQLProtocol, SASLResponseRejectsLengthBelowFour) { + /// The message type byte is read by `receiveMessageType`, so the frame starts at the length. for (Int32 size = 0; size < 4; ++size) { std::string bytes; - putUInt8(bytes, 'p'); putInt32(bytes, size); EXPECT_TRUE(throwsUnknownPacket(bytes, [](ReadBuffer & in) { @@ -104,27 +132,38 @@ TEST(PostgreSQLProtocol, SASLResponseRejectsLengthBelowFour) } /// size == 4 means an empty SASL payload. - std::string bytes; - putUInt8(bytes, 'p'); - putInt32(bytes, 4); - ReadBufferFromMemory in(bytes.data(), bytes.size()); - Messaging::SASLResponse msg; - EXPECT_NO_THROW(msg.deserialize(in)); - EXPECT_TRUE(msg.sasl_mechanism.empty()); + { + std::string bytes = framePayload(""); + ReadBufferFromMemory in(bytes.data(), bytes.size()); + Messaging::SASLResponse msg; + EXPECT_NO_THROW(msg.deserialize(in)); + EXPECT_TRUE(msg.sasl_mechanism.empty()); + } + + /// The payload is the rest of the frame, with no terminator, and nothing beyond it. + { + std::string bytes = framePayload("c=biws,r=nonce,p=proof"); + bytes.push_back('X'); + ReadBufferFromMemory in(bytes.data(), bytes.size()); + Messaging::SASLResponse msg; + EXPECT_NO_THROW(msg.deserialize(in)); + EXPECT_EQ(msg.sasl_mechanism, "c=biws,r=nonce,p=proof"); + + char marker = 0; + in.readStrict(marker); + EXPECT_EQ(marker, 'X'); + } } TEST(PostgreSQLProtocol, SASLInitialResponseHandlesMechanismLength) { auto build = [](Int32 size_sasl_mechanism, const std::string & data) { - std::string bytes; - putUInt8(bytes, 'p'); - putInt32(bytes, 0); /// the outer size field is not used for bounds here - bytes += "SCRAM-SHA-256"; - bytes.push_back('\0'); - putInt32(bytes, size_sasl_mechanism); - bytes += data; - return bytes; + std::string payload = "SCRAM-SHA-256"; + payload.push_back('\0'); + putInt32(payload, size_sasl_mechanism); + payload += data; + return framePayload(std::move(payload)); }; /// Below -1 is malformed. @@ -151,6 +190,14 @@ TEST(PostgreSQLProtocol, SASLInitialResponseHandlesMechanismLength) EXPECT_NO_THROW(msg.deserialize(in)); EXPECT_EQ(msg.sasl_mechanism, "abc"); } + + /// A mechanism declaring more bytes than the frame carries is rejected on the frame boundary, + /// instead of the server allocating the declared size and waiting for bytes that never come. + EXPECT_TRUE(throwsUnknownPacket(build(1000000, "abc"), [](ReadBuffer & in) + { + Messaging::SASLInitialResponse msg; + msg.deserialize(in); + })); } TEST(PostgreSQLProtocol, BindHandlesParameterLength) @@ -195,6 +242,14 @@ TEST(PostgreSQLProtocol, BindHandlesParameterLength) ASSERT_EQ(msg.parameters.size(), 1u); EXPECT_EQ(msg.parameters[0], "hi"); } + + /// A parameter declaring more bytes than the frame carries is rejected on the frame boundary, + /// instead of the server allocating the declared size up front. + EXPECT_TRUE(throwsUnknownPacket(build(1000000, "hi"), [](ReadBuffer & in) + { + Messaging::BindQuery msg; + msg.deserialize(in); + })); } TEST(PostgreSQLProtocol, BindRejectsNegativeCounts) @@ -980,3 +1035,64 @@ TEST(PostgreSQLProtocol, CopyDataRejectsLengthBelowFour) EXPECT_NO_THROW(msg.deserialize(in)); EXPECT_EQ(msg.query, "ab"); } + +TEST(PostgreSQLProtocol, MessagesRejectFrameShorterThanDeclared) +{ + /// The declared length is a frame boundary in both directions: a frame that ends before it - + /// the client declared more bytes than it sent and then closed the write side - must be + /// rejected, not parsed from the bytes that did arrive. This holds whether the parser stops on + /// its own terminator (`Query`, `PasswordMessage`) or reads to the end of the frame + /// (`SASLResponse`, `CopyInData`). + auto truncated = [](const std::string & payload) + { + std::string bytes; + /// A thousand bytes of the declared frame are never sent. + putInt32(bytes, static_cast(payload.size() + sizeof(Int32) + 1000)); + bytes += payload; + return bytes; + }; + + auto expect_throws = [&](const std::string & payload, auto && parse) + { + std::string bytes = truncated(payload); + ReadBufferFromMemory in(bytes.data(), bytes.size()); + EXPECT_THROW(parse(in), Exception); + }; + + std::string query_payload = "SELECT 1"; + query_payload.push_back('\0'); + expect_throws(query_payload, [](ReadBuffer & in) + { + Messaging::Query msg; + msg.deserialize(in); + }); + + std::string password_payload = "x"; + password_payload.push_back('\0'); + expect_throws(password_payload, [](ReadBuffer & in) + { + Messaging::PasswordMessage msg; + msg.deserialize(in); + }); + + expect_throws("c=biws,r=nonce,p=proof", [](ReadBuffer & in) + { + Messaging::SASLResponse msg; + msg.deserialize(in); + }); + + expect_throws("ab", [](ReadBuffer & in) + { + Messaging::CopyInData msg; + msg.deserialize(in); + }); +} + +TEST(PostgreSQLProtocol, PasswordMessageRejectsTrailingPayload) +{ + /// A `PasswordMessage` whose password ends before the declared boundary must not let the + /// remainder be taken for the next message. + std::string password_payload = "hunter2"; + password_payload.push_back('\0'); + expectTrailingPayloadIsRejectedAndAligned(std::move(password_payload)); +} diff --git a/src/DataTypes/IDataType.cpp b/src/DataTypes/IDataType.cpp index ccb9ccad65a6..ccc8e24295be 100644 --- a/src/DataTypes/IDataType.cpp +++ b/src/DataTypes/IDataType.cpp @@ -163,7 +163,7 @@ std::unique_ptr IDataType::getSubcolumnData( size_t prefix_len = i + 1; if (!subpath[i].visited && ISerialization::hasSubcolumnForPath(subpath, prefix_len)) { - auto name = ISerialization::getSubcolumnNameForStream(subpath, prefix_len, false, initial_array_level); + auto name = ISerialization::getSubcolumnNameForStream(subpath, prefix_len, initial_array_level); /// Create data from path only if it's requested subcolumn. /// Use the first exact match to be consistent with ColumnsDescription::addSubcolumns /// which also keeps the first subcolumn when there are name collisions diff --git a/src/DataTypes/Serializations/ISerialization.cpp b/src/DataTypes/Serializations/ISerialization.cpp index 7b1602de58b7..7194ada0ee5c 100644 --- a/src/DataTypes/Serializations/ISerialization.cpp +++ b/src/DataTypes/Serializations/ISerialization.cpp @@ -97,7 +97,7 @@ ISerialization::KindStack ISerialization::getKindStack(const IColumn & column) return {Kind::DEFAULT}; } -static String kindToString(ISerialization::Kind kind) +String ISerialization::kindToString(ISerialization::Kind kind) { switch (kind) { @@ -471,20 +471,26 @@ String ISerialization::getFileNameForRenamedColumnStream(const NameAndTypePair & return getFileNameForRenamedColumnStream(column_from.getNameInStorage(), column_to.getNameInStorage(), file_name); } -String ISerialization::getSubcolumnNameForStream(const SubstreamPath & path, bool encode_sparse_stream, size_t initial_array_level) +String ISerialization::getSubcolumnNameForStream(const SubstreamPath & path) { - return getSubcolumnNameForStream(path, path.size(), encode_sparse_stream, initial_array_level); + return getSubcolumnNameForStream(path, path.size()); } -String ISerialization::getSubcolumnNameForStream(const SubstreamPath & path, size_t prefix_len, bool encode_sparse_stream, size_t initial_array_level) +String ISerialization::getSubcolumnNameForStream(const SubstreamPath & path, size_t prefix_len, size_t initial_array_level) { - auto subcolumn_name = getNameForSubstreamPath("", path.begin(), path.begin() + prefix_len, false, encode_sparse_stream, false, initial_array_level); + auto subcolumn_name = getNameForSubstreamPath("", path.begin(), path.begin() + prefix_len, false, false, false, initial_array_level); if (!subcolumn_name.empty()) subcolumn_name = subcolumn_name.substr(1); // It starts with a dot. return subcolumn_name; } +String ISerialization::getSubstreamsCacheKeyForStream(const SubstreamPath & path) +{ + /// Unlike the subcolumn name, this rendering is injective, so two substreams never share a cache slot. + return getNameForSubstreamPath("", path.begin(), path.end(), /*escape_for_file_name=*/true, /*encode_sparse_stream=*/true, /*escape_variant_substreams=*/true); +} + namespace { @@ -530,7 +536,7 @@ void ISerialization::addElementToSubstreamsCache(ISerialization::SubstreamsCache if (!cache) return; - cache->insert_or_assign(getSubcolumnNameForStream(path, true), std::move(element)); + cache->insert_or_assign(getSubstreamsCacheKeyForStream(path), std::move(element)); } ISerialization::ISubstreamsCacheElement * ISerialization::getElementFromSubstreamsCache(ISerialization::SubstreamsCache * cache, const ISerialization::SubstreamPath & path) @@ -538,7 +544,7 @@ ISerialization::ISubstreamsCacheElement * ISerialization::getElementFromSubstrea if (!cache) return nullptr; - auto it = cache->find(getSubcolumnNameForStream(path, true)); + auto it = cache->find(getSubstreamsCacheKeyForStream(path)); return it == cache->end() ? nullptr : it->second.get(); } @@ -547,7 +553,7 @@ void ISerialization::addToSubstreamsDeserializeStatesCache(SubstreamsDeserialize if (!cache) return; - cache->emplace(getSubcolumnNameForStream(path, true), state); + cache->emplace(getSubstreamsCacheKeyForStream(path), state); } ISerialization::DeserializeBinaryBulkStatePtr ISerialization::getFromSubstreamsDeserializeStatesCache(SubstreamsDeserializeStatesCache * cache, const SubstreamPath & path) @@ -555,7 +561,7 @@ ISerialization::DeserializeBinaryBulkStatePtr ISerialization::getFromSubstreamsD if (!cache) return nullptr; - auto it = cache->find(getSubcolumnNameForStream(path, true)); + auto it = cache->find(getSubstreamsCacheKeyForStream(path)); return it == cache->end() ? nullptr : it->second; } diff --git a/src/DataTypes/Serializations/ISerialization.h b/src/DataTypes/Serializations/ISerialization.h index 938c3ef9a526..9ec18aafc98d 100644 --- a/src/DataTypes/Serializations/ISerialization.h +++ b/src/DataTypes/Serializations/ISerialization.h @@ -94,6 +94,30 @@ class ISerialization : private boost::noncopyable, public std::enable_shared_fro /// - etc using KindStack = std::vector; + /// The kind of a column in the Native format is chosen by the peer that sends the data, so a + /// reader declares with such a set which kinds the peer is allowed to select. + class KindSet + { + public: + constexpr KindSet(std::initializer_list kinds) /// NOLINT(google-explicit-constructor) + { + for (auto kind : kinds) + bits |= maskOf(kind); + } + + static constexpr KindSet all() { return KindSet(~UInt32(0)); } + + constexpr bool contains(Kind kind) const { return (bits & maskOf(kind)) != 0; } + constexpr KindSet without(Kind kind) const { return KindSet(bits & ~maskOf(kind)); } + + private: + explicit constexpr KindSet(UInt32 bits_) : bits(bits_) { } + + static constexpr UInt32 maskOf(Kind kind) { return UInt32(1) << static_cast(kind); } + + UInt32 bits = 0; + }; + virtual KindStack getKindStack() const { return {Kind::DEFAULT}; } SerializationPtr getPtr() const { return shared_from_this(); } @@ -104,6 +128,7 @@ class ISerialization : private boost::noncopyable, public std::enable_shared_fro virtual MutableColumnPtr wrapColumnForDeserialization(MutableColumnPtr column) const { return column; } static KindStack getKindStack(const IColumn & column); + static String kindToString(Kind kind); static String kindStackToString(const KindStack & kind); static KindStack stringToKindStack(const String & str); /// Check if provided kind stack contains specific kind. @@ -699,8 +724,13 @@ class ISerialization : private boost::noncopyable, public std::enable_shared_fro static String getFileNameForRenamedColumnStream(const NameAndTypePair & column_from, const NameAndTypePair & column_to, const String & file_name); static String getFileNameForRenamedColumnStream(const String & name_from, const String & name_to, const String & file_name); - static String getSubcolumnNameForStream(const SubstreamPath & path, bool encode_sparse_stream = false, size_t initial_array_level = 0); - static String getSubcolumnNameForStream(const SubstreamPath & path, size_t prefix_len, bool encode_sparse_stream = false, size_t initial_array_level = 0); + static String getSubcolumnNameForStream(const SubstreamPath & path); + static String getSubcolumnNameForStream(const SubstreamPath & path, size_t prefix_len, size_t initial_array_level = 0); + /// Rejects a stale `(path, true)` call, which would otherwise silently bind `true` to `prefix_len`. + static String getSubcolumnNameForStream(const SubstreamPath & path, bool) = delete; + + /// Key of a stream in SubstreamsCache and SubstreamsDeserializeStatesCache. + static String getSubstreamsCacheKeyForStream(const SubstreamPath & path); static void addColumnWithNumReadRowsToSubstreamsCache(SubstreamsCache * cache, const SubstreamPath & path, ColumnPtr column, size_t num_read_rows); static std::optional> getColumnWithNumReadRowsFromSubstreamsCache(SubstreamsCache * cache, const SubstreamPath & path); diff --git a/src/DataTypes/Serializations/SerializationInfo.cpp b/src/DataTypes/Serializations/SerializationInfo.cpp index 49caa07fcd5e..4a83ab59a5ab 100644 --- a/src/DataTypes/Serializations/SerializationInfo.cpp +++ b/src/DataTypes/Serializations/SerializationInfo.cpp @@ -1,5 +1,8 @@ #include +#include +#include + #include #include #include @@ -20,6 +23,7 @@ namespace DB namespace ErrorCodes { extern const int CORRUPTED_DATA; + extern const int INCORRECT_DATA; } namespace @@ -254,7 +258,46 @@ void SerializationInfo::serialializeKindStackBinary(WriteBuffer & out) const } } -void SerializationInfo::deserializeFromKindsBinary(ReadBuffer & in) +/// The order in which the kinds wrap each other, innermost first: `ColumnSparse` can sit inside +/// `ColumnReplicated` but not the other way round (see `removeSpecialRepresentations`), and nothing +/// wraps a `ColumnBLOB`. Not the order of the enum, whose values are part of the Native format. +static constexpr std::array canonical_kind_order +{ + ISerialization::Kind::DEFAULT, + ISerialization::Kind::SPARSE, + ISerialization::Kind::REPLICATED, + ISerialization::Kind::DETACHED, +}; + +void SerializationInfo::checkKindStack(ISerialization::KindSet allowed_kinds) const +{ + if (kind_stack.empty() || kind_stack.front() != ISerialization::Kind::DEFAULT) + throw Exception(ErrorCodes::INCORRECT_DATA, "Serialization kind stack must start with Default"); + + /// A stack describes nested wrappers, so it must be a subsequence of the canonical order — and + /// therefore free of repeats. Any other stack is a layout no writer builds and nothing unwraps. + auto expected = canonical_kind_order.begin(); + + for (auto kind : kind_stack) + { + if (!allowed_kinds.contains(kind)) + throw Exception( + ErrorCodes::INCORRECT_DATA, + "Unexpected serialization kind {} in the received data", + ISerialization::kindToString(kind)); + + expected = std::find(expected, canonical_kind_order.end(), kind); + if (expected == canonical_kind_order.end()) + throw Exception( + ErrorCodes::INCORRECT_DATA, + "Serialization kind {} is out of order in a kind stack", + ISerialization::kindToString(kind)); + + ++expected; + } +} + +void SerializationInfo::deserializeFromKindsBinary(ReadBuffer & in, ISerialization::KindSet allowed_kinds) { UInt8 type = 0; readBinary(type, in); @@ -283,6 +326,12 @@ void SerializationInfo::deserializeFromKindsBinary(ReadBuffer & in) { size_t num_kinds = 0; readVarUInt(num_kinds, in); + /// Refuse an impossible peer-declared count before reading that many kinds; + /// `checkKindStack` rejects the same stacks afterwards by their shape. + if (num_kinds > magic_enum::enum_count()) + throw Exception(ErrorCodes::INCORRECT_DATA, "Too many serialization kinds in a kind stack: {}", num_kinds); + + kind_stack.clear(); for (size_t i = 0; i != num_kinds; ++i) { UInt8 kind = 0; @@ -296,6 +345,8 @@ void SerializationInfo::deserializeFromKindsBinary(ReadBuffer & in) break; } } + + checkKindStack(allowed_kinds); } void SerializationInfo::writeJSONFields(WriteBuffer & out, const String * name) const diff --git a/src/DataTypes/Serializations/SerializationInfo.h b/src/DataTypes/Serializations/SerializationInfo.h index 977a25a2a8de..f120bd62d778 100644 --- a/src/DataTypes/Serializations/SerializationInfo.h +++ b/src/DataTypes/Serializations/SerializationInfo.h @@ -76,7 +76,9 @@ class SerializationInfo const SerializationInfoSettings & new_settings) const; virtual void serialializeKindStackBinary(WriteBuffer & out) const; - virtual void deserializeFromKindsBinary(ReadBuffer & in); + + /// Rejects a kind outside of `allowed_kinds` as invalid data. + virtual void deserializeFromKindsBinary(ReadBuffer & in, ISerialization::KindSet allowed_kinds); virtual void writeJSON(WriteBuffer & out, const String * name) const; virtual void toJSON(Poco::JSON::Object & object) const; @@ -93,6 +95,9 @@ class SerializationInfo protected: virtual void writeJSONFields(WriteBuffer & out, const String * name) const; + /// Rejects a kind stack that no writer can produce, or that selects a kind the reader does not accept. + void checkKindStack(ISerialization::KindSet allowed_kinds) const; + const SerializationInfoSettings settings; ISerialization::KindStack kind_stack; diff --git a/src/DataTypes/Serializations/SerializationInfoTuple.cpp b/src/DataTypes/Serializations/SerializationInfoTuple.cpp index 35c4d20d80e4..f572c95c9800 100644 --- a/src/DataTypes/Serializations/SerializationInfoTuple.cpp +++ b/src/DataTypes/Serializations/SerializationInfoTuple.cpp @@ -161,11 +161,14 @@ void SerializationInfoTuple::serialializeKindStackBinary(WriteBuffer & out) cons elem->serialializeKindStackBinary(out); } -void SerializationInfoTuple::deserializeFromKindsBinary(ReadBuffer & in) +void SerializationInfoTuple::deserializeFromKindsBinary(ReadBuffer & in, ISerialization::KindSet allowed_kinds) { - SerializationInfo::deserializeFromKindsBinary(in); + SerializationInfo::deserializeFromKindsBinary(in, allowed_kinds); + + /// A detached blob always covers a whole column, never a single tuple element. + auto elements_allowed_kinds = allowed_kinds.without(ISerialization::Kind::DETACHED); for (const auto & elem : elems) - elem->deserializeFromKindsBinary(in); + elem->deserializeFromKindsBinary(in, elements_allowed_kinds); } void SerializationInfoTuple::writeJSONFields(WriteBuffer & out, const String * name) const diff --git a/src/DataTypes/Serializations/SerializationInfoTuple.h b/src/DataTypes/Serializations/SerializationInfoTuple.h index ea826527d1c1..9c5a1401260b 100644 --- a/src/DataTypes/Serializations/SerializationInfoTuple.h +++ b/src/DataTypes/Serializations/SerializationInfoTuple.h @@ -27,7 +27,7 @@ class SerializationInfoTuple : public SerializationInfo const Settings & new_settings) const override; void serialializeKindStackBinary(WriteBuffer & out) const override; - void deserializeFromKindsBinary(ReadBuffer & in) override; + void deserializeFromKindsBinary(ReadBuffer & in, ISerialization::KindSet allowed_kinds) override; void toJSON(Poco::JSON::Object & object) const override; void fromJSON(const Poco::JSON::Object & object) override; diff --git a/src/DataTypes/Serializations/SerializationVariant.cpp b/src/DataTypes/Serializations/SerializationVariant.cpp index 8628d0c5373b..9e9728a4d44f 100644 --- a/src/DataTypes/Serializations/SerializationVariant.cpp +++ b/src/DataTypes/Serializations/SerializationVariant.cpp @@ -32,6 +32,7 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; extern const int NOT_IMPLEMENTED; extern const int INCORRECT_DATA; + extern const int CANNOT_READ_ALL_DATA; } /// Validate that a discriminator value is within bounds (< num_variants) or is NULL_DISCRIMINATOR. @@ -685,8 +686,7 @@ std::vector SerializationVariant::deserializeCompactDiscriminators( DeserializeBinaryBulkStateVariantDiscriminators & state, const DeserializeBinaryBulkSettings & settings) const { - auto & discriminators = assert_cast(discriminators_column); - auto & discriminators_data = discriminators.getData(); + auto & discriminators_data = assert_cast(discriminators_column).getData(); /// Reset state if we are reading from the start of the granule and not from the previous position in the file. if (!continuous_reading) @@ -709,16 +709,19 @@ std::vector SerializationVariant::deserializeCompactDiscriminators( size_t limit_in_granule = std::min(limit, state.remaining_rows_in_granule); if (state.granule_format == CompactDiscriminatorsGranuleFormat::COMPACT) { - auto & data = discriminators.getData(); - data.resize_fill(data.size() + limit_in_granule, state.compact_discr); + discriminators_data.resize_fill(discriminators_data.size() + limit_in_granule, state.compact_discr); if (state.compact_discr != ColumnVariant::NULL_DISCRIMINATOR) variant_limits[state.compact_discr] += limit_in_granule; } else { - SerializationNumber::create()->deserializeBinaryBulk(discriminators, *stream, limit_in_granule, 0); - size_t start = discriminators_data.size() - limit_in_granule; + size_t start = discriminators_data.size(); + SerializationNumber::deserializeBinaryBulk(discriminators_data, *stream, limit_in_granule); + size_t num_read = discriminators_data.size() - start; + if (num_read != limit_in_granule) + throw Exception(ErrorCodes::CANNOT_READ_ALL_DATA, + "Cannot read all discriminators in Variant granule. Expected: {}, got: {}", limit_in_granule, num_read); for (size_t i = start; i != discriminators_data.size(); ++i) { diff --git a/src/DataTypes/Serializations/SerializationVariantElement.cpp b/src/DataTypes/Serializations/SerializationVariantElement.cpp index 9526595dcdfe..48af83d42da9 100644 --- a/src/DataTypes/Serializations/SerializationVariantElement.cpp +++ b/src/DataTypes/Serializations/SerializationVariantElement.cpp @@ -14,6 +14,7 @@ namespace ErrorCodes { extern const int NOT_IMPLEMENTED; extern const int LOGICAL_ERROR; + extern const int CANNOT_READ_ALL_DATA; } UInt128 SerializationVariantElement::getHash(const SerializationPtr & nested_, const String & variant_element_name_, ColumnVariant::Discriminator variant_discriminator_, size_t num_variants_, bool nullable_added_by_extraction_) @@ -290,8 +291,7 @@ size_t SerializationVariantElement::deserializeCompactDiscriminators( const ISerialization * serialization) { auto * discriminators_state = checkAndGetState(discriminators_state_, serialization); - auto & discriminators = assert_cast(discriminators_column); - auto & discriminators_data = discriminators.getData(); + auto & discriminators_data = assert_cast(discriminators_column).getData(); /// Reset state if we are reading from the start of the granule and not from the previous position in the file. if (!continuous_reading) @@ -315,16 +315,19 @@ size_t SerializationVariantElement::deserializeCompactDiscriminators( size_t limit_in_granule = std::min(limit, discriminators_state->remaining_rows_in_granule); if (discriminators_state->granule_format == SerializationVariant::CompactDiscriminatorsGranuleFormat::COMPACT) { - auto & data = discriminators.getData(); - data.resize_fill(data.size() + limit_in_granule, discriminators_state->compact_discr); + discriminators_data.resize_fill(discriminators_data.size() + limit_in_granule, discriminators_state->compact_discr); if (discriminators_state->compact_discr == variant_discriminator) variant_limit += limit_in_granule; } else { - SerializationNumber::create()->deserializeBinaryBulk(discriminators, *stream, limit_in_granule, 0); - size_t start = discriminators_data.size() - limit_in_granule; + size_t start = discriminators_data.size(); + SerializationNumber::deserializeBinaryBulk(discriminators_data, *stream, limit_in_granule); + size_t num_read = discriminators_data.size() - start; + if (num_read != limit_in_granule) + throw Exception(ErrorCodes::CANNOT_READ_ALL_DATA, + "Cannot read all discriminators in Variant granule. Expected: {}, got: {}", limit_in_granule, num_read); for (size_t i = start; i != discriminators_data.size(); ++i) variant_limit += (discriminators_data[i] == variant_discriminator); diff --git a/src/DataTypes/Serializations/tests/gtest_serialization_info.cpp b/src/DataTypes/Serializations/tests/gtest_serialization_info.cpp index 10fa60a14d59..bb648614bb18 100644 --- a/src/DataTypes/Serializations/tests/gtest_serialization_info.cpp +++ b/src/DataTypes/Serializations/tests/gtest_serialization_info.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -312,4 +313,47 @@ TEST(SerializationInfoJSON, ChooseKindStackZeroRows) EXPECT_EQ(kind_stack, expected); } +/// Only a reader that accepts `Kind::DETACHED` at all reaches this check, and no such reader is exposed +/// to a client, so it cannot be covered by a functional test. +TEST(SerializationInfoBinary, RejectsDetachedThatIsNotOutermost) +{ + /// COMBINATION encoding of {Default, Detached, Sparse}. + const char kinds[] = {5, 3, 0, 2, 1}; + ReadBufferFromMemory in(kinds, sizeof(kinds)); + + SerializationInfo info({ISerialization::Kind::DEFAULT}, defaultSettings()); + EXPECT_THROW(info.deserializeFromKindsBinary(in, ISerialization::KindSet::all()), Exception); +} + +TEST(SerializationInfoBinary, AcceptsDetachedOverSparse) +{ + /// COMBINATION encoding of {Default, Sparse, Detached}. + const char kinds[] = {5, 3, 0, 1, 2}; + ReadBufferFromMemory in(kinds, sizeof(kinds)); + + SerializationInfo info({ISerialization::Kind::DEFAULT}, defaultSettings()); + info.deserializeFromKindsBinary(in, ISerialization::KindSet::all()); + + ISerialization::KindStack expected{ISerialization::Kind::DEFAULT, ISerialization::Kind::SPARSE, ISerialization::Kind::DETACHED}; + EXPECT_EQ(info.getKindStack(), expected); +} + +/// The full stack a writer can build, with every kind in its canonical position. +TEST(SerializationInfoBinary, AcceptsDetachedOverReplicatedOverSparse) +{ + /// COMBINATION encoding of {Default, Sparse, Replicated, Detached}. + const char kinds[] = {5, 4, 0, 1, 3, 2}; + ReadBufferFromMemory in(kinds, sizeof(kinds)); + + SerializationInfo info({ISerialization::Kind::DEFAULT}, defaultSettings()); + info.deserializeFromKindsBinary(in, ISerialization::KindSet::all()); + + ISerialization::KindStack expected{ + ISerialization::Kind::DEFAULT, + ISerialization::Kind::SPARSE, + ISerialization::Kind::REPLICATED, + ISerialization::Kind::DETACHED}; + EXPECT_EQ(info.getKindStack(), expected); +} + } diff --git a/src/Databases/DataLake/DatabaseDataLake.cpp b/src/Databases/DataLake/DatabaseDataLake.cpp index d0c25a2cb838..15d962e17b55 100644 --- a/src/Databases/DataLake/DatabaseDataLake.cpp +++ b/src/Databases/DataLake/DatabaseDataLake.cpp @@ -97,6 +97,7 @@ namespace DatabaseDataLakeSetting extern const DatabaseDataLakeSettingsString google_adc_quota_project_id; extern const DatabaseDataLakeSettingsString google_adc_credentials_file; extern const DatabaseDataLakeSettingsBool force_add_bucket; + extern const DatabaseDataLakeSettingsBool flat_namespaces; } namespace Setting @@ -267,6 +268,7 @@ void DatabaseDataLake::initialize() const settings[DatabaseDataLakeSetting::auth_header], settings[DatabaseDataLakeSetting::oauth_server_uri].value, settings[DatabaseDataLakeSetting::oauth_server_use_request_body].value, + settings[DatabaseDataLakeSetting::flat_namespaces].value, settings[DatabaseDataLakeSetting::namespaces].value, Context::getGlobalContextInstance()); break; @@ -283,6 +285,7 @@ void DatabaseDataLake::initialize() const settings[DatabaseDataLakeSetting::auth_header], settings[DatabaseDataLakeSetting::oauth_server_uri].value, settings[DatabaseDataLakeSetting::oauth_server_use_request_body].value, + settings[DatabaseDataLakeSetting::flat_namespaces].value, settings[DatabaseDataLakeSetting::namespaces].value, Context::getGlobalContextInstance()); break; @@ -300,6 +303,7 @@ void DatabaseDataLake::initialize() const settings[DatabaseDataLakeSetting::auth_header], settings[DatabaseDataLakeSetting::oauth_server_uri].value, settings[DatabaseDataLakeSetting::oauth_server_use_request_body].value, + settings[DatabaseDataLakeSetting::flat_namespaces].value, settings[DatabaseDataLakeSetting::namespaces].value, Context::getGlobalContextInstance()); break; @@ -322,6 +326,7 @@ void DatabaseDataLake::initialize() const onelake_auth_scope, settings[DatabaseDataLakeSetting::oauth_server_uri].value, settings[DatabaseDataLakeSetting::oauth_server_use_request_body].value, + settings[DatabaseDataLakeSetting::flat_namespaces].value, settings[DatabaseDataLakeSetting::namespaces].value, Context::getGlobalContextInstance()); break; diff --git a/src/Databases/DataLake/DatabaseDataLakeSettings.cpp b/src/Databases/DataLake/DatabaseDataLakeSettings.cpp index 866d124fbe0f..cb5e467f749c 100644 --- a/src/Databases/DataLake/DatabaseDataLakeSettings.cpp +++ b/src/Databases/DataLake/DatabaseDataLakeSettings.cpp @@ -51,6 +51,7 @@ namespace ErrorCodes DECLARE(String, dlf_access_key_id, "", "Access id of DLF token for Paimon REST Catalog", 0) \ DECLARE(String, dlf_access_key_secret, "", "Access secret of DLF token for Paimon REST Catalog", 0) \ DECLARE(Bool, force_add_bucket, false, "When constructing object-storage URLs from the catalog-provided table location and storage_endpoint, prepend the bucket/container name even if the endpoint already contains it. Useful for catalogs that hand back paths without the bucket and expect it to be added at URL construction (Polaris-style paths).", 0) \ + DECLARE(Bool, flat_namespaces, false, "The catalog supports only single-level namespaces, so only top-level namespaces are listed and sub-namespaces are not requested. Set this for Iceberg REST catalogs that reject the `parent` query parameter of the list-namespaces endpoint, such as Apache Polaris federated to AWS Glue, which answers `Glue dataCatalog does not support multipart namespace` with HTTP 400. Some catalogs instead ignore `parent` and echo top-level namespaces back for every parent; without this setting those are listed as fake nested namespaces. Not needed for catalog types that are always flat (`delta_sharing`, BigLake, S3 Tables).", 0) \ DECLARE(String, namespaces, "*", "Comma-separated list of allowed namespaces", 0) \ #define LIST_OF_DATABASE_ICEBERG_SETTINGS(M, ALIAS) \ diff --git a/src/Databases/DataLake/RestCatalog.cpp b/src/Databases/DataLake/RestCatalog.cpp index ccd48047d39e..e53e969f1f0c 100644 --- a/src/Databases/DataLake/RestCatalog.cpp +++ b/src/Databases/DataLake/RestCatalog.cpp @@ -18,7 +18,6 @@ #include #include #include - #include #include #include @@ -226,6 +225,7 @@ RestCatalog::RestCatalog( const std::string & auth_header_, const std::string & oauth_server_uri_, bool oauth_server_use_request_body_, + bool flat_namespaces_, const std::string & namespaces_, DB::ContextPtr context_) : ICatalog(warehouse_) @@ -235,6 +235,7 @@ RestCatalog::RestCatalog( , auth_scope(auth_scope_) , oauth_server_uri(oauth_server_uri_) , oauth_server_use_request_body(oauth_server_use_request_body_) + , flat_namespaces(flat_namespaces_) , allowed_namespaces(namespaces_) { CatalogState initial_state; @@ -262,6 +263,7 @@ RestCatalog::RestCatalog( const std::string & auth_scope_, const std::string & oauth_server_uri_, bool oauth_server_use_request_body_, + bool flat_namespaces_, const std::string & namespaces_, DB::ContextPtr context_) : ICatalog(warehouse_) @@ -271,6 +273,7 @@ RestCatalog::RestCatalog( , auth_scope(auth_scope_) , oauth_server_uri(oauth_server_uri_) , oauth_server_use_request_body(oauth_server_use_request_body_) + , flat_namespaces(flat_namespaces_) , allowed_namespaces(namespaces_) { } @@ -389,9 +392,10 @@ OneLakeCatalog::OneLakeCatalog( const std::string & auth_scope_, const std::string & oauth_server_uri_, bool oauth_server_use_request_body_, + bool flat_namespaces_, const std::string & namespaces_, DB::ContextPtr context_) - : RestCatalog(warehouse_, base_url_, auth_scope_, oauth_server_uri_, oauth_server_use_request_body_, namespaces_, context_) + : RestCatalog(warehouse_, base_url_, auth_scope_, oauth_server_uri_, oauth_server_use_request_body_, flat_namespaces_, namespaces_, context_) { CatalogState initial_state; initial_state.tenant_id = onelake_tenant_id; @@ -679,9 +683,10 @@ HorizonCatalog::HorizonCatalog( const std::string & auth_header_, const std::string & oauth_server_uri_, bool oauth_server_use_request_body_, + bool flat_namespaces_, const std::string & namespaces_, DB::ContextPtr context_) - : RestCatalog(warehouse_, base_url_, auth_scope_, oauth_server_uri_, oauth_server_use_request_body_, namespaces_, context_) + : RestCatalog(warehouse_, base_url_, auth_scope_, oauth_server_uri_, oauth_server_use_request_body_, flat_namespaces_, namespaces_, context_) { CatalogState initial_state; if (!catalog_credential_.empty()) @@ -1031,7 +1036,7 @@ BigLakeCatalog::BigLakeCatalog( const std::string & namespaces_, DB::ContextPtr context_, bool allow_server_credentials_in_user_queries_) - : RestCatalog(warehouse_, base_url_, "", "", false, namespaces_, context_) + : RestCatalog(warehouse_, base_url_, "", "", false, /* flat_namespaces */false, namespaces_, context_) , google_project_id(google_project_id_) , google_service_account(google_service_account_) , google_metadata_service(google_metadata_service_) @@ -1415,6 +1420,9 @@ void RestCatalog::getNamespacesRecursive( } } + if (hasFlatNamespaces()) + continue; + if (allowed_namespaces.isNamespaceAllowed(current_namespace, /*nested*/ true)) getNamespacesRecursive(current_namespace, result, stop_condition, func); else @@ -1442,9 +1450,12 @@ Poco::URI::QueryParameters RestCatalog::createParentNamespaceParams(const std::s bool RestCatalog::hasFlatNamespaces() const { - /// Catalogs whose namespaces are single-level and which ignore the `parent` filter when listing - /// namespaces. For these, sub-namespace listing is skipped (see `parseNamespaces`) so that an echo - /// of the parent is not turned into a fake child, which would otherwise recurse without bound. + /// Catalogs whose namespaces are single-level and which ignore or reject the `parent` filter when + /// listing namespaces. For these, sub-namespace listing is skipped so that an echo of the parent is + /// not turned into a fake child, which would otherwise recurse without bound. + if (flat_namespaces) + return true; + const auto type = getCatalogType(); return type == DB::DatabaseDataLakeCatalogType::ICEBERG_BIGLAKE || type == DB::DatabaseDataLakeCatalogType::ICEBERG_DELTA_SHARING @@ -1517,12 +1528,21 @@ RestCatalog::Namespaces RestCatalog::listChildNamespaces(const std::string & bas "Received error while fetching list of namespaces from iceberg catalog `{}`. ", warehouse); - if (e.code() == Poco::Net::HTTPResponse::HTTPStatus::HTTP_NOT_FOUND) + if (!base_namespace.empty() && e.getHTTPStatus() == Poco::Net::HTTPResponse::HTTPStatus::HTTP_NOT_FOUND) message += "Namespace provided in the `parent` query parameter is not found. "; + if (!base_namespace.empty() + && (e.getHTTPStatus() == Poco::Net::HTTPResponse::HTTPStatus::HTTP_BAD_REQUEST + || e.getHTTPStatus() == Poco::Net::HTTPResponse::HTTPStatus::HTTP_NOT_IMPLEMENTED)) + message += fmt::format( + "The catalog refused to list sub-namespaces of `{}`. If it supports only single-level " + "namespaces, recreate the database with `SETTINGS flat_namespaces = 1` so that only " + "top-level namespaces are listed. ", + base_namespace); + message += fmt::format( - "Code: {}, status: {}, message: {}", - e.code(), e.getHTTPStatus(), e.displayText()); + "Code: {}, HTTP status: {}, message: {}", + e.code(), static_cast(e.getHTTPStatus()), e.displayText()); throw DB::Exception(DB::ErrorCodes::DATALAKE_DATABASE_ERROR, "{}", message); } diff --git a/src/Databases/DataLake/RestCatalog.h b/src/Databases/DataLake/RestCatalog.h index 077f186395ca..12e53bdf617a 100644 --- a/src/Databases/DataLake/RestCatalog.h +++ b/src/Databases/DataLake/RestCatalog.h @@ -44,6 +44,7 @@ class RestCatalog : public ICatalog, public DB::WithContext const std::string & auth_header_, const std::string & oauth_server_uri_, bool oauth_server_use_request_body_, + bool flat_namespaces_, const std::string & namespaces_, DB::ContextPtr context_); @@ -143,6 +144,7 @@ class RestCatalog : public ICatalog, public DB::WithContext const std::string & auth_scope_, const std::string & oauth_server_uri_, bool oauth_server_use_request_body_, + bool flat_namespaces_, const std::string & namespaces_, DB::ContextPtr context_); @@ -158,6 +160,7 @@ class RestCatalog : public ICatalog, public DB::WithContext std::string auth_scope; std::string oauth_server_uri; bool oauth_server_use_request_body; + bool flat_namespaces = false; mutable MultiVersion access_token; public: @@ -204,10 +207,10 @@ class RestCatalog : public ICatalog, public DB::WithContext StopCondition stop_condition, ExecuteFunc func) const; - /// Whether this catalog has flat (single-level) namespaces and ignores the `parent` filter when - /// listing namespaces. Such catalogs (BigLake, Databricks Delta Sharing) echo the same namespaces - /// for any parent; treating those echoes as children would recurse without bound, so sub-namespace - /// listing is skipped for them (see `parseNamespaces`). + /// Whether this catalog has flat (single-level) namespaces, either because its type is always flat + /// (BigLake, Databricks Delta Sharing, S3 Tables) or because of the `flat_namespaces` database + /// setting. Such catalogs are never asked for sub-namespaces (see `getNamespacesRecursive`): they + /// either echo the parent back for any `parent` (which would recurse without bound) or reject it. bool hasFlatNamespaces() const; /// List the immediate child namespaces directly under `base_namespace` @@ -300,6 +303,7 @@ class OneLakeCatalog : public RestCatalog const std::string & auth_scope_, const std::string & oauth_server_uri_, bool oauth_server_use_request_body_, + bool flat_namespaces_, const std::string & namespaces_, DB::ContextPtr context_); @@ -422,6 +426,7 @@ class HorizonCatalog : public RestCatalog const std::string & auth_header_, const std::string & oauth_server_uri_, bool oauth_server_use_request_body_, + bool flat_namespaces_, const std::string & namespaces_, DB::ContextPtr context_); diff --git a/src/Databases/DataLake/S3TablesCatalog.cpp b/src/Databases/DataLake/S3TablesCatalog.cpp index 7b1d5ff57a3a..280b81ab812e 100644 --- a/src/Databases/DataLake/S3TablesCatalog.cpp +++ b/src/Databases/DataLake/S3TablesCatalog.cpp @@ -64,7 +64,7 @@ S3TablesCatalog::S3TablesCatalog( const CatalogSettings & catalog_settings_, DB::ContextPtr context_, bool allow_server_credentials_in_user_queries_) - : RestCatalog(warehouse_, base_url_, "", "", false, catalog_settings_.namespaces, context_) + : RestCatalog(warehouse_, base_url_, "", "", false, /* flat_namespaces */false, catalog_settings_.namespaces, context_) , region(region_) , storage_endpoint(catalog_settings_.storage_endpoint) , signing_service("s3tables") diff --git a/src/Databases/DataLake/tests/gtest_rest_catalog.cpp b/src/Databases/DataLake/tests/gtest_rest_catalog.cpp index 7cec11446624..72d1a9f3e910 100644 --- a/src/Databases/DataLake/tests/gtest_rest_catalog.cpp +++ b/src/Databases/DataLake/tests/gtest_rest_catalog.cpp @@ -63,6 +63,11 @@ enum class CatalogShape /// echoes the same top-level namespace for every parent. A REST catalog would recurse on this /// forever (gold -> gold.gold -> ...); a flat-namespace catalog must list the top level only. ParentIgnoringEcho, + /// Flat-namespace catalog (Apache Polaris federated to AWS Glue) that rejects `parent` with + /// HTTP 400 instead of ignoring it, and holds a table in its only top-level namespace. + ParentRejecting, + /// Rejects the client credentials at the OAuth token endpoint. + OAuthTokenRejected, }; void writeJSON(Poco::Net::HTTPServerResponse & response, const std::string & body, Poco::Net::HTTPResponse::HTTPStatus status = Poco::Net::HTTPResponse::HTTP_OK) @@ -136,6 +141,14 @@ class RestCatalogRequestHandler final : public Poco::Net::HTTPRequestHandler { const std::string request_body(std::istreambuf_iterator(request.stream()), {}); ++token_requests; + if (shape == CatalogShape::OAuthTokenRejected) + { + writeError( + response, + Poco::Net::HTTPResponse::HTTP_UNAUTHORIZED, + R"({"error":"invalid_client","error_description":"Client secret does not match"})"); + return; + } /// Horizon secret-only credentials omit client_id; standard REST includes it. if (request_body.contains("client_secret=") && !request_body.contains("client_id=")) { @@ -165,6 +178,12 @@ class RestCatalogRequestHandler final : public Poco::Net::HTTPRequestHandler else if (shape == CatalogShape::ParentIgnoringEcho) /// Ignores `parent` and echoes the top-level namespace back for any parent. writeJSON(response, R"({"namespaces":[["gold"]]})"); + else if (shape == CatalogShape::ParentRejecting) + writeError( + response, + Poco::Net::HTTPResponse::HTTP_BAD_REQUEST, + R"({"error":{"message":"Malformed request: Glue dataCatalog does not support multipart namespace.",)" + R"("type":"BadRequestException","code":400}})"); else writeJSON(response, R"({"namespaces":[]})"); return; @@ -172,7 +191,7 @@ class RestCatalogRequestHandler final : public Poco::Net::HTTPRequestHandler if (path == "/v1/namespaces/namespace/tables") { - if (shape == CatalogShape::TopLevelTable) + if (shape == CatalogShape::TopLevelTable || shape == CatalogShape::ParentRejecting) writeJSON(response, R"({"identifiers":[{"name":"table_a"}]})"); else writeJSON(response, R"({"identifiers":[]})"); @@ -313,7 +332,7 @@ void expectThrowsCode(std::function fn, int expected_code) } } -bool restCatalogEmpty(CatalogShape shape) +bool restCatalogEmpty(CatalogShape shape, bool flat_namespaces = false) { RestCatalogTestServer server(shape); auto context = DB::Context::createCopy(getContext().context); @@ -327,12 +346,33 @@ bool restCatalogEmpty(CatalogShape shape) /* auth_header */"", /* oauth_server_uri */"", /* oauth_server_use_request_body */false, + flat_namespaces, /* namespaces */"*", context); return catalog.empty(); } +DataLake::ICatalog::Namespaces restCatalogNamespaces(CatalogShape shape, bool flat_namespaces) +{ + RestCatalogTestServer server(shape); + auto context = DB::Context::createCopy(getContext().context); + context->makeQueryContext(); + + RestCatalog catalog( + "warehouse", + server.getUrl(), + /* catalog_credential */"", + /* auth_scope */"", + /* auth_header */"", + /* oauth_server_uri */"", + /* oauth_server_use_request_body */false, + flat_namespaces, + context); + + return catalog.getNamespaces(); +} + bool deltaSharingCatalogEmpty(CatalogShape shape) { RestCatalogTestServer server(shape); @@ -347,6 +387,7 @@ bool deltaSharingCatalogEmpty(CatalogShape shape) /* auth_header */"", /* oauth_server_uri */"", /* oauth_server_use_request_body */false, + /* flat_namespaces */false, /* namespaces */"*", context); @@ -379,6 +420,86 @@ TEST(RestCatalog, EmptyReturnsTrueWhenNoTablesExist) EXPECT_TRUE(restCatalogEmpty(CatalogShape::Empty)); } +TEST(RestCatalog, RejectedClientCredentialsReportTheOAuthError) +{ + RestCatalogTestServer server(CatalogShape::OAuthTokenRejected); + auto context = DB::Context::createCopy(getContext().context); + context->makeQueryContext(); + + try + { + RestCatalog catalog( + "warehouse", + server.getUrl(), + /* catalog_credential */"client-1:wrong-secret", + /* auth_scope */"PRINCIPAL_ROLE:ALL", + /* auth_header */"", + /* oauth_server_uri */"", + /* oauth_server_use_request_body */true, + /* flat_namespaces */false, + context); + FAIL() << "expected the rejected client credentials to fail the catalog construction"; + } + catch (const DB::Exception & e) + { + EXPECT_EQ(e.code(), DB::ErrorCodes::DATALAKE_DATABASE_ERROR); + EXPECT_TRUE(e.message().contains("invalid_client")) << e.message(); + EXPECT_TRUE(e.message().contains("Client secret does not match")) << e.message(); + EXPECT_TRUE(e.message().contains("status 401")) << e.message(); + EXPECT_FALSE(e.message().contains("wrong-secret")) << e.message(); + } +} + +TEST(RestCatalog, ParentFilterRejectionReportsFlatNamespacesSetting) +{ + try + { + restCatalogNamespaces(CatalogShape::ParentRejecting, /* flat_namespaces */false); + FAIL() << "expected the rejected `parent` filter to fail the namespace listing"; + } + catch (const DB::Exception & e) + { + EXPECT_EQ(e.code(), DB::ErrorCodes::DATALAKE_DATABASE_ERROR); + EXPECT_TRUE(e.message().contains("flat_namespaces")) << e.message(); + EXPECT_TRUE(e.message().contains("HTTP status: 400")) << e.message(); + } +} + +TEST(RestCatalog, FlatNamespacesSettingSkipsSubNamespaceListing) +{ + EXPECT_FALSE(restCatalogEmpty(CatalogShape::ParentRejecting, /* flat_namespaces */true)); + EXPECT_EQ(restCatalogNamespaces(CatalogShape::ParentRejecting, /* flat_namespaces */true), ICatalog::Namespaces{"namespace"}); +} + +TEST(RestCatalog, FlatNamespacesSettingIgnoresEchoedParent) +{ + EXPECT_EQ(restCatalogNamespaces(CatalogShape::ParentIgnoringEcho, /* flat_namespaces */true), ICatalog::Namespaces{"gold"}); +} + +TEST(RestCatalog, OneLakeFlatNamespacesSettingSkipsSubNamespaceListing) +{ + RestCatalogTestServer server(CatalogShape::ParentRejecting); + auto context = DB::Context::createCopy(getContext().context); + context->makeQueryContext(); + + OneLakeCatalog catalog( + "warehouse", + server.getUrl(), + /* onelake_tenant_id */"tenant-1", + /* onelake_client_id */"", + /* onelake_client_secret */"", + /* bearer_token */"token-1", + /* refresh_token */"", + /* auth_scope */"", + /* oauth_server_uri */"", + /* oauth_server_use_request_body */false, + /* flat_namespaces */true, + context); + + EXPECT_FALSE(catalog.empty()); + EXPECT_EQ(catalog.getNamespaces(), ICatalog::Namespaces{"namespace"}); +} + TEST(RestCatalog, TryGetTableMetadataDistinguishesMissingTableFromOtherErrors) { RestCatalogTestServer server(CatalogShape::TopLevelTable); @@ -393,6 +514,7 @@ TEST(RestCatalog, TryGetTableMetadataDistinguishesMissingTableFromOtherErrors) /* auth_header */"", /* oauth_server_uri */"", /* oauth_server_use_request_body */false, + /* flat_namespaces */false, /* namespaces */"*", context); @@ -430,6 +552,7 @@ TEST(RestCatalog, TryGetTableMetadataAuthErrorPropagates) /* auth_scope */"", /* oauth_server_uri */"", /* oauth_server_use_request_body */false, + /* flat_namespaces */false, /* namespaces */"*", context); @@ -462,6 +585,7 @@ TEST(RestCatalog, ApplySettingsChangesWithoutAuthenticationRejected) /* auth_header */"", /* oauth_server_uri */"", /* oauth_server_use_request_body */false, + /* flat_namespaces */false, /* namespaces */"*", context); @@ -484,6 +608,7 @@ TEST(RestCatalog, ApplySettingsChangesCredentialMode) /* auth_header */"", /* oauth_server_uri */"", /* oauth_server_use_request_body */false, + /* flat_namespaces */false, /* namespaces */"*", context); @@ -526,6 +651,7 @@ TEST(RestCatalog, ApplySettingsChangesAuthHeaderMode) /* auth_header */"Authorization: Bearer token-1", /* oauth_server_uri */"", /* oauth_server_use_request_body */false, + /* flat_namespaces */false, /* namespaces */"*", context); @@ -559,6 +685,7 @@ TEST(RestCatalog, OneLakeApplySettingsChangesBearerMode) /* auth_scope */"", /* oauth_server_uri */"", /* oauth_server_use_request_body */false, + /* flat_namespaces */false, /* namespaces */"*", context); @@ -620,6 +747,7 @@ TEST(RestCatalog, OneLakeRejectsMalformedBearerToken) /* auth_scope */ "", /* oauth_server_uri */ "", /* oauth_server_use_request_body */ false, + /* flat_namespaces */ false, /* namespaces */"*", context); }, @@ -661,6 +789,7 @@ TEST(RestCatalog, OneLakeRefreshTokenTransparentRenewal) /* auth_scope */"https://storage.azure.com/.default", /* oauth_server_uri */server.getUrl() + "/token", /* oauth_server_use_request_body */true, + /* flat_namespaces */false, /* namespaces */"*", context); @@ -703,6 +832,7 @@ TEST(RestCatalog, OneLakeRefreshTokenExpiredThrowsWithAlterHint) /* auth_scope */"https://storage.azure.com/.default", /* oauth_server_uri */server.getUrl() + "/token", /* oauth_server_use_request_body */true, + /* flat_namespaces */false, /* namespaces */"*", context); /// ADD_FAILURE (rather than FAIL) does not return from the test, so the @@ -738,6 +868,7 @@ TEST(RestCatalog, OneLakeApplySettingsChangesRefreshMode) /* auth_scope */"https://storage.azure.com/.default", /* oauth_server_uri */server.getUrl() + "/token", /* oauth_server_use_request_body */true, + /* flat_namespaces */false, /* namespaces */"*", context); @@ -792,6 +923,7 @@ TEST(RestCatalog, HorizonCatalogAuthenticatesWithBarePAT) /* auth_header */"", /* oauth_server_uri */"", /* oauth_server_use_request_body */true, + /* flat_namespaces */false, /* namespaces */"*", context); @@ -824,6 +956,7 @@ TEST(RestCatalog, HorizonCatalogRequiresCredentialOrAuthHeader) /* auth_header */"", /* oauth_server_uri */"", /* oauth_server_use_request_body */true, + /* flat_namespaces */false, /* namespaces */"*", context); }, @@ -844,6 +977,7 @@ TEST(RestCatalog, HorizonApplySettingsChangesBarePAT) /* auth_header */"", /* oauth_server_uri */"", /* oauth_server_use_request_body */true, + /* flat_namespaces */false, /* namespaces */"*", context); diff --git a/src/Databases/DatabaseHDFS.cpp b/src/Databases/DatabaseHDFS.cpp index 525a562a8f43..726aa1282b76 100644 --- a/src/Databases/DatabaseHDFS.cpp +++ b/src/Databases/DatabaseHDFS.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -105,15 +106,29 @@ bool DatabaseHDFS::checkUrl(const std::string & url, ContextPtr context_, bool t bool DatabaseHDFS::isTableExist(const String & name, ContextPtr context_) const { - std::lock_guard lock(mutex); - if (loaded_tables.contains(name)) - return true; + /// A name exists when it forms a URL this database may use, which needs no HDFS request. The cache + /// must not answer it: that reports which names other callers resolved, past any filter tightening. + if (source.empty() && !name.starts_with("hdfs://")) + return false; - return checkUrl(name, context_, false); + return checkUrl(getTablePath(name), context_, false); } StoragePtr DatabaseHDFS::getTableImpl(const String & name, ContextPtr context_) const { + auto url = getTablePath(name); + auto args = makeASTFunction("hdfs", make_intrusive(url)); + + auto table_function = TableFunctionFactory::instance().get(args, context_); + if (!table_function) + return nullptr; + + /// The cache is keyed on the name alone, so what authorizes a resolution is checked above it. The + /// grant is the table function's to check: a filtered grant matches the URI it reports, not the path. + table_function->checkSourceAccess(context_, /* is_insert_query */ false); + + checkUrl(url, context_, true); + /// Check if the table exists in the loaded tables map. { std::lock_guard lock(mutex); @@ -122,16 +137,6 @@ StoragePtr DatabaseHDFS::getTableImpl(const String & name, ContextPtr context_) return it->second; } - auto url = getTablePath(name); - - checkUrl(url, context_, true); - - auto args = makeASTFunction("hdfs", make_intrusive(url)); - - auto table_function = TableFunctionFactory::instance().get(args, context_); - if (!table_function) - return nullptr; - /// TableFunctionHDFS throws exceptions, if table cannot be created. auto table_storage = table_function->execute(args, context_, name); if (table_storage) diff --git a/src/Databases/DatabaseOrdinary.cpp b/src/Databases/DatabaseOrdinary.cpp index 199f3eccaf55..7bb6c4ac0100 100644 --- a/src/Databases/DatabaseOrdinary.cpp +++ b/src/Databases/DatabaseOrdinary.cpp @@ -32,6 +32,7 @@ #include #include #include +#include #include #include #include @@ -141,6 +142,22 @@ static void checkReplicaPathExists(ASTCreateQuery & create_query, ContextPtr loc ); } +void DatabaseOrdinary::checkReplicaPathIsSafe(const ASTCreateQuery & create_query, ContextPtr local_context) +{ + /// A conversion mints a path the table never had, so the substituted name is validated as strictly + /// as a CREATE validates it -- but one level below CREATE, because the requirement that a path start + /// with '/' applies to a genuinely new table, not to a template this server has long been expanding. + const auto & server_settings = local_context->getServerSettings(); + TableZnodeInfo::resolve( + server_settings[ServerSetting::default_replica_path], + server_settings[ServerSetting::default_replica_name], + StorageID(create_query.getDatabase(), create_query.getTable(), create_query.uuid), + create_query, + LoadingStrictnessLevel::SECONDARY_CREATE, + local_context, + /*validate_substitutions=*/true); +} + void DatabaseOrdinary::setMergeTreeEngine(ASTCreateQuery & create_query, ContextPtr local_context, bool replicated) { auto * storage = create_query.storage; @@ -232,6 +249,7 @@ void DatabaseOrdinary::convertMergeTreeToReplicatedIfNeeded(ASTPtr ast, const Qu LOG_INFO(log, "Found {} flag for table {}. Will try to change it's engine in metadata to replicated.", CONVERT_TO_REPLICATED_FLAG_NAME, backQuote(qualified_name.getFullName())); + checkReplicaPathIsSafe(create_query, getContext()); checkReplicaPathExists(create_query, getContext()); setMergeTreeEngine(create_query, getContext(), /*replicated*/ true); diff --git a/src/Databases/DatabaseOrdinary.h b/src/Databases/DatabaseOrdinary.h index 72893830f53d..73956b751a10 100644 --- a/src/Databases/DatabaseOrdinary.h +++ b/src/Databases/DatabaseOrdinary.h @@ -88,6 +88,10 @@ class DatabaseOrdinary : public DatabaseOnDisk static void setMergeTreeEngine(ASTCreateQuery & create_query, ContextPtr context, bool replicated); + /// Rejects a conversion to a replicated engine whose Keeper path would not be a safe one. + /// Contacts nothing and mutates nothing, so a caller can run it before its own side effects. + static void checkReplicaPathIsSafe(const ASTCreateQuery & create_query, ContextPtr context); + protected: /// Erase pending async load/startup task references for a table. Must hold `mutex`. /// Shared by detachTableUnlocked and the Atomic rename detach path (issue #91777). diff --git a/src/Databases/DatabaseURL.cpp b/src/Databases/DatabaseURL.cpp index e48390e7254e..b08e1332830f 100644 --- a/src/Databases/DatabaseURL.cpp +++ b/src/Databases/DatabaseURL.cpp @@ -264,9 +264,10 @@ class StorageURLDatabaseTable final : public StorageProxy DatabaseURL::DatabaseURL(const String & name_, const String & base_url_, ContextPtr context_) : IDatabase(name_), WithContext(context_->getGlobalContext()), base_url(base_url_) { + /// Not echoed back: password masking anchors on the `://` this value lacks, so it would log the password. if (!base_url.empty() && !hasURLScheme(base_url)) throw Exception(ErrorCodes::BAD_ARGUMENTS, - "The base URL of a URL database must contain a scheme (e.g. https://), got: {}", base_url); + "The base URL of a URL database must contain a scheme (e.g. https://)"); } String DatabaseURL::getTableURL(const String & name) const diff --git a/src/Databases/enableAllExperimentalSettings.cpp b/src/Databases/enableAllExperimentalSettings.cpp index a7ca1b28bda6..1f2e027b50b3 100644 --- a/src/Databases/enableAllExperimentalSettings.cpp +++ b/src/Databases/enableAllExperimentalSettings.cpp @@ -62,8 +62,6 @@ void enableAllExperimentalSettings(ContextMutablePtr context) context->setSetting("allow_experimental_time_series_aggregate_functions", 1); context->setSetting("allow_experimental_lightweight_update", 1); context->setSetting("allow_insert_into_iceberg", 1); - context->setSetting("allow_experimental_iceberg_compaction", 1); - context->setSetting("allow_experimental_cleanup_old_data_files_compaction", 1); context->setSetting("allow_iceberg_remove_orphan_files", 1); context->setSetting("allow_experimental_expire_snapshots", 1); context->setSetting("allow_experimental_delta_lake_writes", 1); diff --git a/src/Dictionaries/ClickHouseDictionarySource.cpp b/src/Dictionaries/ClickHouseDictionarySource.cpp index 8a8fcf27c1ec..250a3a504f3d 100644 --- a/src/Dictionaries/ClickHouseDictionarySource.cpp +++ b/src/Dictionaries/ClickHouseDictionarySource.cpp @@ -162,6 +162,26 @@ std::string ClickHouseDictionarySource::toString() const return "ClickHouse: " + configuration.db + '.' + configuration.table + (where.empty() ? "" : ", where: " + where); } +namespace +{ + +/// The query text comes from the dictionary definition (possibly a `CREATE DICTIONARY` written by a user +/// who has no other privileges), and for a local source it is executed as an `internal` query on behalf +/// of the configured user. So only a `SELECT` is allowed: any other statement (`CREATE TABLE`, ...) +/// would run with the access checks of `internal` queries skipped. +void checkQueryIsSelect(const String & query, const char * description, const char * error_message) +{ + const char * query_begin = query.data(); + const char * query_end = query.data() + query.size(); + ParserQuery parser(query_end); + ASTPtr ast = parseQuery(parser, query_begin, query_end, description, 0, DBMS_DEFAULT_MAX_PARSER_DEPTH, DBMS_DEFAULT_MAX_PARSER_BACKTRACKS); + + if (!ast || ast->getQueryKind() != IAST::QueryKind::Select) + throw Exception(ErrorCodes::INCORRECT_QUERY, "{}", error_message); +} + +} + BlockIO ClickHouseDictionarySource::createStreamForQuery(const String & query) { BlockIO io; @@ -173,13 +193,7 @@ BlockIO ClickHouseDictionarySource::createStreamForQuery(const String & query) auto context_copy = Context::createCopy(context); context_copy->makeQueryContext(); - const char * query_begin = query.data(); - const char * query_end = query.data() + query.size(); - ParserQuery parser(query_end); - ASTPtr ast = parseQuery(parser, query_begin, query_end, "Query for ClickHouse dictionary", 0, DBMS_DEFAULT_MAX_PARSER_DEPTH, DBMS_DEFAULT_MAX_PARSER_BACKTRACKS); - - if (!ast || ast->getQueryKind() != IAST::QueryKind::Select) - throw Exception(ErrorCodes::INCORRECT_QUERY, "Only SELECT query can be used as a dictionary source"); + checkQueryIsSelect(query, "Query for ClickHouse dictionary", "Only SELECT query can be used as a dictionary source"); if (configuration.is_local) { @@ -205,6 +219,8 @@ std::string ClickHouseDictionarySource::doInvalidateQuery(const std::string & re { LOG_TRACE(log, "Performing invalidate query"); + checkQueryIsSelect(request, "Invalidate query for ClickHouse dictionary", "Only SELECT query can be used as a dictionary invalidate query"); + /// Copy context because results of scalar subqueries potentially could be cached auto context_copy = Context::createCopy(context); context_copy->makeQueryContext(); diff --git a/src/Disks/DiskObjectStorage/ObjectStorages/Local/LocalObjectStorage.cpp b/src/Disks/DiskObjectStorage/ObjectStorages/Local/LocalObjectStorage.cpp index 8d29f6a9a85d..41ffed349e10 100644 --- a/src/Disks/DiskObjectStorage/ObjectStorages/Local/LocalObjectStorage.cpp +++ b/src/Disks/DiskObjectStorage/ObjectStorages/Local/LocalObjectStorage.cpp @@ -138,6 +138,14 @@ LocalObjectStorage::LocalObjectStorage(LocalObjectStorageSettings settings_) String resolvePathRelativelyToBase(const String & path, const String & base_path) { + /// A path with an embedded NUL cannot be validated: `std::string` and `fs::path` compare the whole + /// value, while every syscall the resolved path is later passed to (`open`, `mkdir`, `stat`) stops at + /// the NUL. A path shaped as `\0/` would therefore + /// pass the containment check below and still make the kernel operate on ``, anywhere on the + /// filesystem. `listObjects` rejects such a path for its own reason - keep both checks. + if (path.contains('\0')) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Path contains an embedded NUL byte"); + auto configured_base = fs::path(base_path).lexically_normal(); auto is_inside = [&](const String & candidate) diff --git a/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp b/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp index 5fdd13cef073..9cf9e364d53a 100644 --- a/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp +++ b/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp @@ -721,18 +721,21 @@ void S3ObjectStorage::applyNewSettings( auto modified_settings = std::make_unique(*s3_settings.get()); + /// Static configurations keep their resolved authentication settings when a session change rebuilds the client. auto apply_endpoint_settings = [&] { if (auto endpoint_settings = context->getStorageS3Settings().getSettings(uri.uri.toString(), context->getUserName())) { - modified_settings->auth_settings.updateIfChanged(endpoint_settings->auth_settings); + if (options.allow_client_change) + modified_settings->auth_settings.updateIfChanged(endpoint_settings->auth_settings); modified_settings->request_settings.updateIfChanged(endpoint_settings->request_settings); } }; auto apply_config_settings = [&] { - modified_settings->auth_settings.updateIfChanged(settings_from_config->auth_settings); + if (options.allow_client_change) + modified_settings->auth_settings.updateIfChanged(settings_from_config->auth_settings); modified_settings->request_settings.updateIfChanged(settings_from_config->request_settings); }; diff --git a/src/Disks/tests/gtest_resolve_path_relatively_to_base_embedded_nul.cpp b/src/Disks/tests/gtest_resolve_path_relatively_to_base_embedded_nul.cpp new file mode 100644 index 000000000000..4786b1ef090f --- /dev/null +++ b/src/Disks/tests/gtest_resolve_path_relatively_to_base_embedded_nul.cpp @@ -0,0 +1,76 @@ +#include + +#include +#include + +#include +#include + +#include /// for ::getpid + +namespace fs = std::filesystem; + +namespace DB::ErrorCodes +{ + extern const int BAD_ARGUMENTS; + extern const int PATH_ACCESS_DENIED; +} + +namespace +{ + +struct ScopedBaseDir +{ + fs::path path; + + ScopedBaseDir() + : path(fs::temp_directory_path() / ("resolve_path_relatively_to_base_embedded_nul_" + std::to_string(::getpid()))) + { + fs::create_directories(path); + } + + ~ScopedBaseDir() + { + std::error_code ec; + fs::remove_all(path, ec); + } +}; + +int errorCodeOf(const std::string & path, const std::string & base) +{ + try + { + DB::resolvePathRelativelyToBase(path, base); + } + catch (const DB::Exception & e) + { + return e.code(); + } + return 0; +} + +} + +TEST(ResolvePathRelativelyToBaseEmbeddedNul, RejectsEmbeddedNul) +{ + ScopedBaseDir base; + const std::string base_path = base.path.string(); + const std::string back_into_base = "/../" + base.path.filename().string() + "/file"; + + /// Positive controls: the resolver still accepts a relative path and an absolute traversal that ends up inside. + EXPECT_EQ(DB::resolvePathRelativelyToBase("file", base_path), (base.path / "file").string()); + const std::string contained = (base.path / ".." / "probe").string() + back_into_base; + EXPECT_EQ(DB::resolvePathRelativelyToBase(contained, base_path), contained); + + /// Negative control: a plain traversal outside is denied by the containment check. + EXPECT_EQ(errorCodeOf((base.path / ".." / "probe").string(), base_path), DB::ErrorCodes::PATH_ACCESS_DENIED); + + /// As a whole string, this path normalizes into the base directory; truncated at the NUL, as every syscall + /// would see it, it addresses `probe` next to it. It must be rejected before any containment comparison. + const std::string escaping_through_nul = (base.path / ".." / "probe").string() + std::string(1, '\0') + back_into_base; + EXPECT_EQ(errorCodeOf(escaping_through_nul, base_path), DB::ErrorCodes::BAD_ARGUMENTS); + + /// A NUL that cannot escape is rejected all the same, in a relative and in an absolute path. + EXPECT_EQ(errorCodeOf(std::string("file\0suffix", 11), base_path), DB::ErrorCodes::BAD_ARGUMENTS); + EXPECT_EQ(errorCodeOf((base.path / "file").string() + std::string(1, '\0') + "suffix", base_path), DB::ErrorCodes::BAD_ARGUMENTS); +} diff --git a/src/Formats/FormatFactory.cpp b/src/Formats/FormatFactory.cpp index 17befc8e3313..a36d3af0f3e3 100644 --- a/src/Formats/FormatFactory.cpp +++ b/src/Formats/FormatFactory.cpp @@ -348,6 +348,8 @@ FormatSettings getFormatSettings(const ContextPtr & context, const Settings & se format_settings.arrow.output_compression_method = settings[Setting::output_format_arrow_compression_method]; format_settings.arrow.output_date_as_uint16 = settings[Setting::output_format_arrow_date_as_uint16]; format_settings.arrow.output_unsupported_types_as_binary = settings[Setting::output_format_arrow_unsupported_types_as_binary]; + format_settings.arrow.output_record_batch_rows = settings[Setting::output_format_arrow_record_batch_size]; + format_settings.arrow.output_record_batch_bytes = settings[Setting::output_format_arrow_record_batch_size_bytes]; format_settings.orc.allow_missing_columns = settings[Setting::input_format_orc_allow_missing_columns]; format_settings.orc.row_batch_size = settings[Setting::input_format_orc_row_batch_size]; format_settings.orc.skip_columns_with_unsupported_types_in_schema_inference = settings[Setting::input_format_orc_skip_columns_with_unsupported_types_in_schema_inference]; diff --git a/src/Formats/FormatSettings.h b/src/Formats/FormatSettings.h index 52ac7ff55317..42f62e34fb22 100644 --- a/src/Formats/FormatSettings.h +++ b/src/Formats/FormatSettings.h @@ -199,6 +199,8 @@ struct FormatSettings ArrowCompression output_compression_method = ArrowCompression::NONE; bool output_date_as_uint16 = false; bool output_unsupported_types_as_binary = true; + UInt64 output_record_batch_rows = 0; + UInt64 output_record_batch_bytes = 0; } arrow{}; struct AvroSchemaRegistryTimeouts diff --git a/src/Formats/NativeReader.cpp b/src/Formats/NativeReader.cpp index 9d26ddb52a4f..53f7f911f0ba 100644 --- a/src/Formats/NativeReader.cpp +++ b/src/Formats/NativeReader.cpp @@ -36,8 +36,12 @@ namespace ErrorCodes } -NativeReader::NativeReader(ReadBuffer & istr_, UInt64 server_revision_, std::optional format_settings_) - : istr(istr_), server_revision(server_revision_), format_settings(format_settings_) +NativeReader::NativeReader( + ReadBuffer & istr_, + UInt64 server_revision_, + std::optional format_settings_, + ISerialization::KindSet allowed_kinds_) + : istr(istr_), server_revision(server_revision_), format_settings(format_settings_), allowed_kinds(allowed_kinds_) { } @@ -225,7 +229,7 @@ Block NativeReader::read() UInt8 has_custom = 0; readBinary(has_custom, istr); if (has_custom) - info->deserializeFromKindsBinary(istr); + info->deserializeFromKindsBinary(istr, allowed_kinds); serialization = column.type->getSerialization(*info); auto new_column = column.type->createColumn(*serialization); diff --git a/src/Formats/NativeReader.h b/src/Formats/NativeReader.h index ddf4b2142564..390efaed1592 100644 --- a/src/Formats/NativeReader.h +++ b/src/Formats/NativeReader.h @@ -5,6 +5,7 @@ #include #include #include +#include #include @@ -24,8 +25,17 @@ class CompressedReadBufferFromFile; class NativeReader { public: + /// Kinds accepted from the peer unless the caller declares its own set. `Detached` is excluded + /// because it yields a column that does not match the declared type, which few callers handle. + static constexpr ISerialization::KindSet default_allowed_kinds + = ISerialization::KindSet::all().without(ISerialization::Kind::DETACHED); + /// If a non-zero server_revision is specified, additional block information may be expected and read. - NativeReader(ReadBuffer & istr_, UInt64 server_revision_, std::optional format_settings_ = std::nullopt); + NativeReader( + ReadBuffer & istr_, + UInt64 server_revision_, + std::optional format_settings_ = std::nullopt, + ISerialization::KindSet allowed_kinds_ = default_allowed_kinds); /// For cases when data structure (header) is known in advance. /// NOTE We may use header for data validation and/or type conversions. It is not implemented. @@ -62,6 +72,7 @@ class NativeReader UInt64 server_revision; std::optional format_settings = std::nullopt; BlockMissingValues * block_missing_values = nullptr; + ISerialization::KindSet allowed_kinds = default_allowed_kinds; bool use_index = false; IndexForNativeFormat::Blocks::const_iterator index_block_it; diff --git a/src/Functions/FunctionsConversion.cpp b/src/Functions/FunctionsConversion.cpp index c71f20f6ab38..726781d7ea8c 100644 --- a/src/Functions/FunctionsConversion.cpp +++ b/src/Functions/FunctionsConversion.cpp @@ -842,21 +842,42 @@ FunctionCast::WrapperType FunctionCast::createAggregateFunctionWrapper(const Dat namespace { -/// `Map(K, V)` is physically an `Array(Tuple(K, V))`, and `CAST` converts between the two shapes at -/// the top level. It is also how a `Map` constant is written down when the analyzer serializes a -/// query for a remote server: `ConstantNode::toASTImpl` renders the field as an array of tuples and -/// wraps it into `_CAST(..., 'Array(Map(K, V))')`. Counting a `Map` as the array it stands for puts -/// both shapes at the same depth, so the nesting check below does not reject a cast the element -/// wrappers can perform. A genuine depth mismatch, such as `Array(String)` to `Array(Array(String))`, -/// is still rejected. A `Tuple` adds no depth, hence a `Map` counts as one dimension regardless of -/// its value type. -size_t getNumberOfDimensionsWithMapAsArray(const IDataType & type) +/// The number of `Array` dimensions of a type as `CAST` sees it. It is a range rather than a single +/// number, because `Map(K, V)` is physically an `Array(Tuple(K, V))` and `CAST` converts between the +/// two spellings: a `Map` can be matched either by another `Map`, contributing no dimension of its +/// own, or by the `Array(Tuple(K, V))` it stands for, contributing one. A `Tuple` adds no dimension, +/// hence a `Map` is at most one dimension whatever its value type is. +/// +/// The array spelling is how a `Map` constant is written down when the analyzer serializes a query +/// for a remote server: `ConstantNode::toASTImpl` renders the field as an array of tuples and wraps +/// it into `_CAST(..., 'Array(Map(K, V))')`. +struct DimensionsRange +{ + size_t min; + size_t max; +}; + +DimensionsRange getNumberOfDimensionsForCast(const IDataType & type) { if (const auto * type_array = typeid_cast(&type)) - return 1 + getNumberOfDimensionsWithMapAsArray(*type_array->getNestedType()); + { + const DimensionsRange nested = getNumberOfDimensionsForCast(*type_array->getNestedType()); + return {nested.min + 1, nested.max + 1}; + } if (typeid_cast(&type)) - return 1; - return 0; + return {0, 1}; + return {0, 0}; +} + +/// The nesting depths have to be reconcilable under some spelling of the `Map`s the two types +/// contain. This rejects a genuine depth mismatch, such as `Array(String)` to `Array(Array(String))`, +/// before an element wrapper does something unexpected with it (`CAST` from a `String` parses it). +/// Anything else is left to the element wrappers, which name the types that cannot be converted. +bool canHaveSameNumberOfDimensions(const IDataType & from, const IDataType & to) +{ + const DimensionsRange from_dimensions = getNumberOfDimensionsForCast(from); + const DimensionsRange to_dimensions = getNumberOfDimensionsForCast(to); + return from_dimensions.min <= to_dimensions.max && to_dimensions.min <= from_dimensions.max; } } @@ -914,7 +935,7 @@ FunctionCast::WrapperType FunctionCast::createArrayWrapper(const DataTypePtr & f /// In query SELECT CAST([] AS Array(Array(String))) from type is Array(Nothing) bool from_empty_array = isNothing(from_nested_type); - if (getNumberOfDimensionsWithMapAsArray(*from_type) != getNumberOfDimensionsWithMapAsArray(to_type) && !from_empty_array) + if (!canHaveSameNumberOfDimensions(*from_type, to_type) && !from_empty_array) throw Exception(ErrorCodes::TYPE_MISMATCH, "CAST AS Array can only be performed between same-dimensional array types"); diff --git a/src/IO/AzureBlobStorage/copyAzureBlobStorageFile.cpp b/src/IO/AzureBlobStorage/copyAzureBlobStorageFile.cpp index 7c036bf4041b..a43b91dc86e9 100644 --- a/src/IO/AzureBlobStorage/copyAzureBlobStorageFile.cpp +++ b/src/IO/AzureBlobStorage/copyAzureBlobStorageFile.cpp @@ -436,16 +436,22 @@ void copyAzureBlobStorageFile( auto copy_status = properties_model.CopyStatus; auto copy_status_description = properties_model.CopyStatusDescription; - + /// `CopySource` and `CopyStatusDescription` are optional in the properties of a blob: + /// the SDK models them as `Nullable`, and `Nullable::Value()` of an empty one aborts the + /// process in a release build (`AZURE_ASSERT_MSG` expands to a bare `std::abort` under + /// `NDEBUG`). The properties polled here come from the remote endpoint, which is under no + /// obligation to send either header, so nothing below dereferences them unchecked. if (copy_status.HasValue() && copy_status.Value() == Azure::Storage::Blobs::Models::CopyStatus::Success) { - LOG_TRACE(log, "Copy of {} to {} finished", properties_model.CopySource.Value(), dest_blob); + LOG_TRACE(log, "Copy of {} to {} finished", src_blob, dest_blob); } else { if (copy_status.HasValue()) throw Exception(ErrorCodes::AZURE_BLOB_STORAGE_ERROR, "Copy from {} to {} failed with status {} description {} (operation is done {})", - src_blob, dest_blob, copy_status.Value().ToString(), copy_status_description.Value(), operation.IsDone()); + src_blob, dest_blob, copy_status.Value().ToString(), + copy_status_description.HasValue() ? copy_status_description.Value() : String(""), + operation.IsDone()); throw Exception( ErrorCodes::AZURE_BLOB_STORAGE_ERROR, "Copy from {} to {} didn't complete with success status (operation is done {})", diff --git a/src/IO/LibdeflateInflatingReadBuffer.cpp b/src/IO/LibdeflateInflatingReadBuffer.cpp index b04e2a324bad..8f1c438e4824 100644 --- a/src/IO/LibdeflateInflatingReadBuffer.cpp +++ b/src/IO/LibdeflateInflatingReadBuffer.cpp @@ -91,7 +91,8 @@ bool LibdeflateInflatingReadBuffer::fillInput() in->nextIfAtEnd(); /// Copy a bounded amount per call so in_buf stays small even when the nested buffer exposes a /// lot at once (e.g. a memory-mapped file): the rest stays in the nested buffer for next time. - /// libdeflate consumes whole DEFLATE blocks, so the leftover we must keep is at most one block. + /// libdeflate consumes input up to the last decoded symbol, so the leftover we must keep is a + /// few bytes inside a Huffman block, and at most one stored block or block header otherwise. const size_t avail = std::min(in->buffer().end() - in->position(), INPUT_CHUNK); if (avail == 0) { @@ -290,11 +291,11 @@ bool LibdeflateInflatingReadBuffer::decompressImpl() window_nbytes = new_win; /// Decompress into the output region after the window, accumulating across as many - /// libdeflate calls as needed. Output produced at a non-final block boundary - /// (LIBDEFLATE_STREAM_NEED_INPUT) is deliberately NOT exposed yet: the stream may end at - /// the very next block, and its trailer must be validated before the final bytes reach the - /// caller. We expose the accumulated output only when the buffer fills at a block boundary - /// (the stream is then provably incomplete, so more output follows) or once the final block + /// libdeflate calls as needed. Output produced when the input runs out + /// (LIBDEFLATE_STREAM_NEED_INPUT) is deliberately NOT exposed yet: the stream may end + /// very soon after, and its trailer must be validated before the final bytes reach the + /// caller. We expose the accumulated output only when the buffer fills mid-stream (the + /// suspended item did not fit, so more output provably follows) or once the final block /// is decoded and its trailer verified. This mirrors ZlibInflatingReadBuffer, which has /// validated the trailer by the time it hands back the final bytes, so a reader that /// consumes an exact byte count with readStrict (and never calls nextImpl again) cannot @@ -338,11 +339,12 @@ bool LibdeflateInflatingReadBuffer::decompressImpl() if (r == LIBDEFLATE_STREAM_NEED_INPUT) { - /// At a non-final block boundary with the input exhausted. With end_of_input=true - /// the decoder never returns NEED_INPUT, so this means more input must exist; if the - /// nested stream is already at EOF the data is truncated. Otherwise pull more input - /// (or mark EOF, so the retry passes end_of_input=true and the final block finishes) - /// and keep accumulating into the same buffer. + /// Input exhausted; everything up to the last decoded symbol (or block boundary) was + /// consumed, so the retry never re-decodes more than a constant amount. With + /// end_of_input=true the decoder never returns NEED_INPUT, so this means more input + /// must exist; if the nested stream is already at EOF the data is truncated. + /// Otherwise pull more input (or mark EOF, so the retry passes end_of_input=true and + /// the final block finishes) and keep accumulating into the same buffer. if (input_eof) throw Exception(ErrorCodes::CANNOT_DECOMPRESS, "Unexpected end of {} stream", gzip ? "gzip" : "zlib"); fillInput(); @@ -353,23 +355,20 @@ bool LibdeflateInflatingReadBuffer::decompressImpl() { if (produced == 0) { - /* A single DEFLATE block's uncompressed size exceeds the whole output buffer. - * libdeflate's streaming decoder only suspends at block boundaries, so the whole - * block must be buffered before any of its output is exposed: grow the buffer (the - * 32 KiB window at the front is preserved) and retry. We grow geometrically and - * don't impose an artificial ceiling; the buffer is allocated through ClickHouse's - * tracked allocator, so a crafted single-block decompression bomb runs into the - * query/server memory limit and throws MEMORY_LIMIT_EXCEEDED, exactly like any - * other oversized allocation. Every mainstream gzip/zlib/deflate encoder bounds its - * blocks to well under a megabyte of uncompressed data (zlib flushes roughly every - * lit_bufsize symbols, libdeflate's SOFT_MAX_BLOCK_LENGTH is 300000 bytes, etc.), - * so a real-world stream never reaches this grow path; only a hand-crafted one does. */ + /* The streaming decoder suspends at symbol boundaries, so no progress at all + * means a single indivisible item - one match (<= 258 bytes) or one stored + * block (<= 64 KiB) - exceeds the whole free output region. That can only + * happen when the caller passed a tiny buf_size; growth is bounded by those + * item sizes, not by anything an attacker controls. Grow geometrically (the + * 32 KiB window at the front is preserved) and retry; the buffer is allocated + * through ClickHouse's tracked allocator, so even that bounded growth stays + * subject to the query/server memory limits. */ memory.resize(std::max(memory.size() + out_capacity, memory.size() * 2)); continue; } - /// Output buffer full at a block boundary: the stream is provably incomplete (the - /// final block was not reached), so more output follows and it is safe to expose - /// what we have without a trailer check. The window carries to the next call. + /// Output buffer full mid-stream: the suspended item itself did not fit, so more + /// output provably follows and it is safe to expose what we have without a + /// trailer check. The window carries to the next call. produced_end = window_nbytes + produced; break; } diff --git a/src/IO/LibdeflateInflatingReadBuffer.h b/src/IO/LibdeflateInflatingReadBuffer.h index e29bcae08d37..eac747fc3be9 100644 --- a/src/IO/LibdeflateInflatingReadBuffer.h +++ b/src/IO/LibdeflateInflatingReadBuffer.h @@ -16,9 +16,10 @@ struct libdeflate_decompressor; namespace DB { -/// Streaming gzip/zlib decompressor built on libdeflate's block-boundary-suspendable decoder +/// Streaming gzip/zlib decompressor built on libdeflate's symbol-boundary-suspendable decoder /// (libdeflate_deflate_decompress_stream, a ClickHouse addition). It is faster than the zlib -/// streaming path while keeping memory bounded. +/// streaming path while keeping memory bounded and time linear regardless of the stream's +/// DEFLATE block structure (a single block may span the whole stream, as with zlib-ng level 1). /// /// We parse the gzip/zlib header and trailer ourselves and feed the raw DEFLATE body to libdeflate, /// carrying the last 32 KiB of output as the back-reference window. Concatenated members are handled diff --git a/src/IO/ReadHelpers.h b/src/IO/ReadHelpers.h index cf24492e4bd0..2d216684a75b 100644 --- a/src/IO/ReadHelpers.h +++ b/src/IO/ReadHelpers.h @@ -187,16 +187,16 @@ inline void readIPv6Binary(IPv6 & ip, ReadBuffer & buf) } template -void readVectorBinary(V & v, ReadBuffer & buf) +void readVectorBinary(V & v, ReadBuffer & buf, size_t max_size = DEFAULT_MAX_STRING_SIZE) { using T = typename V::value_type; size_t size = 0; readVarUInt(size, buf); - if (size > DEFAULT_MAX_STRING_SIZE) + if (size > max_size) throw Exception(ErrorCodes::TOO_LARGE_ARRAY_SIZE, - "Too large array size (maximum: {})", DEFAULT_MAX_STRING_SIZE); + "Too large array size (maximum: {})", max_size); v.resize(size); diff --git a/src/IO/SharedThreadPools.h b/src/IO/SharedThreadPools.h index 7bb85c730c1d..07e76d8e12f7 100644 --- a/src/IO/SharedThreadPools.h +++ b/src/IO/SharedThreadPools.h @@ -70,7 +70,8 @@ class StaticThreadPool M(DatabaseCatalogDropTables, "DropTablesThreadPool", DatabaseCatalog) \ M(MergeTreePrefixesDeserialization, "MergeTreePrefixesDeserializationThreadPool", MergeTreeSubcolumnsReader) \ M(DropDistributedCache, "DropDistributedCacheThreadPool", DropDistributedCache) \ - M(FormatParsing, "FormatParsingThreadPool", FormatParsing) + M(FormatParsing, "FormatParsingThreadPool", FormatParsing) \ + M(IcebergManifestDecode, "IcebergManifestDecodeThreadPool", IcebergManifestDecode) #define DECLARE_STATIC_THREAD_POOL_GETTER(SUFFIX, NAME, METRIC) StaticThreadPool & get##SUFFIX##ThreadPool(); APPLY_FOR_STATIC_THREAD_POOLS(DECLARE_STATIC_THREAD_POOL_GETTER) diff --git a/src/IO/tests/gtest_azure_copy_blob_optional_properties.cpp b/src/IO/tests/gtest_azure_copy_blob_optional_properties.cpp new file mode 100644 index 000000000000..b0c8e706645e --- /dev/null +++ b/src/IO/tests/gtest_azure_copy_blob_optional_properties.cpp @@ -0,0 +1,155 @@ +#include "config.h" + +#if USE_AZURE_BLOB_STORAGE + +#include +#include +#include +#include + +#include +#include +#include +#include + +#include +#include +#include +#include + +#include + +namespace +{ + +constexpr size_t blob_size = 100; + +/// A body stream that owns what it serves. Every response needs one: the transport policy of the +/// SDK buffers the body by calling `ReadToEnd` on it unconditionally, so a response without a body +/// stream dereferences a null pointer - including the answer to a HEAD request, which has no body. +class OwningBodyStream : public Azure::Core::IO::BodyStream +{ +public: + explicit OwningBodyStream(std::string data_) : data(std::move(data_)) { } + + int64_t Length() const override { return static_cast(data.size()); } + + void Rewind() override { position = 0; } + +private: + size_t OnRead(uint8_t * buffer, size_t count, const Azure::Core::Context &) override + { + const size_t to_read = std::min(count, data.size() - position); + if (to_read != 0) + memcpy(buffer, data.data() + position, to_read); + position += to_read; + return to_read; + } + + std::string data; + size_t position = 0; +}; + +/// An endpoint that accepts an asynchronous `Copy Blob` (`StartCopyFromUri`) and reports it as +/// completed in the polled properties of the destination, without the optional `x-ms-copy-source` +/// header. The SDK models `BlobProperties::CopySource` as `Nullable`, and `Nullable::Value()` of an +/// empty one aborts the process in a release build (`AZURE_ASSERT_MSG` expands to a bare +/// `std::abort` under `NDEBUG`), so an endpoint behaving this way must not be able to take the +/// server down. +class CopyWithoutCopySourceTransport : public Azure::Core::Http::HttpTransport +{ +public: + std::unique_ptr Send(Azure::Core::Http::Request & request, const Azure::Core::Context &) override + { + /// `Copy Blob`: a `PUT` of the destination with `x-ms-copy-source`. Accepted, still pending. + if (request.GetMethod() == Azure::Core::Http::HttpMethod::Put) + { + ++copies_started; + auto response = std::make_unique(1, 1, Azure::Core::Http::HttpStatusCode::Accepted, "Accepted"); + response->SetHeader("Content-Length", "0"); + response->SetHeader("ETag", "\"0x8DA000000000001\""); + response->SetHeader("Last-Modified", "Wed, 21 Oct 2015 07:28:00 GMT"); + response->SetHeader("x-ms-copy-id", "copy-id"); + response->SetHeader("x-ms-copy-status", "pending"); + response->SetBodyStream(std::make_unique("")); + return response; + } + + /// The properties of the destination, polled until the copy completes. These are the headers + /// the SDK reads from such a response; `x-ms-copy-source` is left out on purpose. + if (request.GetMethod() == Azure::Core::Http::HttpMethod::Head) + { + ++properties_polled; + auto response = std::make_unique(1, 1, Azure::Core::Http::HttpStatusCode::Ok, "OK"); + response->SetHeader("Content-Length", std::to_string(blob_size)); + response->SetHeader("ETag", "\"0x8DA000000000001\""); + response->SetHeader("Last-Modified", "Wed, 21 Oct 2015 07:28:00 GMT"); + response->SetHeader("x-ms-creation-time", "Wed, 21 Oct 2015 07:28:00 GMT"); + response->SetHeader("x-ms-blob-type", "BlockBlob"); + response->SetHeader("x-ms-lease-state", "available"); + response->SetHeader("x-ms-lease-status", "unlocked"); + response->SetHeader("x-ms-server-encrypted", "true"); + response->SetHeader("x-ms-copy-id", "copy-id"); + response->SetHeader("x-ms-copy-status", "success"); + response->SetHeader("x-ms-copy-progress", std::to_string(blob_size) + "/" + std::to_string(blob_size)); + response->SetHeader("x-ms-copy-completion-time", "Wed, 21 Oct 2015 07:28:00 GMT"); + response->SetBodyStream(std::make_unique("")); + return response; + } + + /// Anything else would be the read-and-write fallback, which must not be reached: the native + /// copy completed. + ++unexpected_requests; + auto response = std::make_unique(1, 1, Azure::Core::Http::HttpStatusCode::NotFound, "Not Found"); + response->SetHeader("Content-Length", "0"); + response->SetBodyStream(std::make_unique("")); + return response; + } + + size_t copies_started = 0; + size_t properties_polled = 0; + size_t unexpected_requests = 0; +}; + +} + +TEST(AzureNativeCopy, CompletionWithoutTheCopySourceInTheProperties) +{ + /// The copy source is logged at the trace level, and a `LOG_TRACE` evaluates its arguments only + /// when the logger is at that level - which is the default level of the server, so this is the + /// configuration in which the property is dereferenced. + getLogger("copyAzureBlobStorageFile")->setLevel("trace"); + + auto transport = std::make_shared(); + + Azure::Storage::Blobs::BlobClientOptions client_options; + client_options.Retry.MaxRetries = 0; + client_options.Transport.Transport = transport; + + auto container_client = std::make_shared( + Azure::Storage::Blobs::BlobContainerClient("http://azure.invalid/container", client_options), /* blob_prefix */ ""); + + auto settings = std::make_shared(); + settings->use_native_copy = true; + /// A blob of at least `max_single_part_copy_size` bytes is copied with the asynchronous `Copy Blob`, + /// whose completion is read from the polled properties. + settings->max_single_part_copy_size = blob_size; + + ASSERT_NO_THROW(DB::copyAzureBlobStorageFile( + container_client, + container_client, + /* src_container_for_logging */ "container", + /* src_blob */ "blob", + /* src_size */ blob_size, + /* dest_container_for_logging */ "container", + /* dest_blob */ "copy", + settings, + DB::ReadSettings{}, + /* object_to_attributes */ std::nullopt)); + + ASSERT_EQ(transport->copies_started, 1u); + ASSERT_GE(transport->properties_polled, 1u); + ASSERT_EQ(transport->unexpected_requests, 0u); +} + +#endif diff --git a/src/IO/tests/gtest_libdeflate_inflate.cpp b/src/IO/tests/gtest_libdeflate_inflate.cpp index 912867a82e59..4ec771d36d71 100644 --- a/src/IO/tests/gtest_libdeflate_inflate.cpp +++ b/src/IO/tests/gtest_libdeflate_inflate.cpp @@ -11,9 +11,15 @@ #include +#include + +#include +#include +#include #include #include #include +#include #include using namespace DB; @@ -97,6 +103,19 @@ TEST_P(LibdeflateInflateTest, RoundTripFromZlibNg) } } +/// zlib-ng's deflate_quick path (compression level 1) opens a single static-Huffman block and +/// never closes it until the end of the stream, so a multi-megabyte level-1 stream is the +/// real-encoder version of the single-block shape of issue #114045; keep it large and the input +/// chunks small so a decoder that re-decodes from the block start on every refill cannot hide. +TEST_P(LibdeflateInflateTest, RoundTripFromZlibNgLevelOneLarge) +{ + const CompressionMethod method = GetParam(); + const std::string data = makeData(8 << 20, 3); + const std::string compressed = zlibCompress(data, method, 1); + for (size_t chunk : {size_t(16 * 1024), size_t(256 * 1024)}) + EXPECT_EQ(decompressViaBuffer(compressed, method, chunk, 1 << 20), data) << "chunk=" << chunk; +} + /// Two concatenated members must decode to the concatenation of their contents. TEST_P(LibdeflateInflateTest, MultiMember) { @@ -178,4 +197,216 @@ TEST(LibdeflateInflateHeaderValidation, RejectsBadGzipHeaderCrc) EXPECT_ANY_THROW(decompressViaBuffer(bad, CompressionMethod::Gzip, 4, 4096)); } +/// Regression tests for issue #114045: a single DEFLATE block spanning the whole stream - the shape +/// zlib-ng's deflate_quick path (compression level 1, the default of the official .NET SDK) emits - +/// must decompress in linear time with bounded memory. No real encoder is available here, so the +/// stream is crafted directly: a static-Huffman DEFLATE writer producing one block of arbitrary size. + +namespace +{ + +/// Emits a raw DEFLATE stream as a single static-Huffman block (RFC 1951 section 3.2.6). +class SingleBlockDeflateWriter +{ +public: + void putLiteral(uint8_t b) + { + if (b < 144) + putCode(0x30 + b, 8); + else + putCode(0x190 + (b - 144), 9); + reference.push_back(static_cast(b)); + } + + void putMatch(unsigned len /* 3..258 */, unsigned dist /* 1..32768, <= bytes emitted */) + { + static constexpr struct { unsigned base; int extra; } lengths[29] = { + {3,0},{4,0},{5,0},{6,0},{7,0},{8,0},{9,0},{10,0}, + {11,1},{13,1},{15,1},{17,1},{19,2},{23,2},{27,2},{31,2}, + {35,3},{43,3},{51,3},{59,3},{67,4},{83,4},{99,4},{115,4}, + {131,5},{163,5},{195,5},{227,5},{258,0}}; + static constexpr struct { unsigned base; int extra; } distances[30] = { + {1,0},{2,0},{3,0},{4,0},{5,1},{7,1},{9,2},{13,2},{17,3},{25,3}, + {33,4},{49,4},{65,5},{97,5},{129,6},{193,6},{257,7},{385,7}, + {513,8},{769,8},{1025,9},{1537,9},{2049,10},{3073,10}, + {4097,11},{6145,11},{8193,12},{12289,12},{16385,13},{24577,13}}; + + int li = len == 258 ? 28 : 27; + while (lengths[li].base > len) + --li; + unsigned length_sym = 257 + li; + if (length_sym < 280) + putCode(length_sym - 256, 7); + else + putCode(0xC0 + (length_sym - 280), 8); + if (lengths[li].extra) + putBits(len - lengths[li].base, lengths[li].extra); + + int di = 29; + while (distances[di].base > dist) + --di; + putCode(di, 5); + if (distances[di].extra) + putBits(dist - distances[di].base, distances[di].extra); + + for (unsigned i = 0; i < len; ++i) + reference.push_back(reference[reference.size() - dist]); + } + + /// Returns the raw DEFLATE stream; the object must not be used afterwards. + std::string finish() + { + putCode(0, 7); /* end-of-block */ + if (nbits) + deflate.push_back(static_cast(bitbuf)); + return std::move(deflate); + } + + std::string reference; /* the uncompressed bytes described so far */ + + SingleBlockDeflateWriter() + { + putBits(1, 1); /* BFINAL */ + putBits(1, 2); /* BTYPE: static Huffman */ + } + +private: + /// Raw bits, LSB-first (DEFLATE bit order for headers and extra bits). + void putBits(uint32_t v, int n) + { + bitbuf |= static_cast(v) << nbits; + nbits += n; + while (nbits >= 8) + { + deflate.push_back(static_cast(bitbuf)); + bitbuf >>= 8; + nbits -= 8; + } + } + + /// A Huffman codeword is written starting from its most significant bit. + void putCode(uint32_t c, int n) + { + uint32_t reversed = 0; + for (int i = 0; i < n; ++i) + reversed |= ((c >> i) & 1) << (n - 1 - i); + putBits(reversed, n); + } + + std::string deflate; + uint64_t bitbuf = 0; + int nbits = 0; +}; + +/// One gzip/zlib member whose DEFLATE payload is a single static-Huffman block: 'literal_bytes' of +/// pseudo-random literals followed by matches (with distances reaching the full 32 KiB window) up to +/// 'total_bytes' of output. Returns {member, reference}. +std::pair craftSingleBlockMember(CompressionMethod method, size_t literal_bytes, size_t total_bytes) +{ + SingleBlockDeflateWriter writer; + std::mt19937 rng(42); /// NOLINT(bugprone-random-generator-seed,cert-msc32-c,cert-msc51-cpp) deterministic test data on purpose + for (size_t i = 0; i < literal_bytes; ++i) + writer.putLiteral(static_cast(rng())); + while (writer.reference.size() < total_bytes) + writer.putMatch( + 3 + rng() % 256, + static_cast(1 + rng() % std::min(writer.reference.size(), 32768))); + std::string reference = writer.reference; + const std::string deflate = writer.finish(); + + std::string member; + if (method == CompressionMethod::Gzip) + { + member = std::string("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff", 10); + member += deflate; + const uint32_t crc = libdeflate_crc32(0, reference.data(), reference.size()); + const auto size32 = static_cast(reference.size()); + for (int i = 0; i < 4; ++i) + member.push_back(static_cast(crc >> (8 * i))); + for (int i = 0; i < 4; ++i) + member.push_back(static_cast(size32 >> (8 * i))); + } + else + { + member = std::string("\x78\x9c", 2); + member += deflate; + const uint32_t adler = libdeflate_adler32(1, reference.data(), reference.size()); + for (int i = 3; i >= 0; --i) + member.push_back(static_cast(adler >> (8 * i))); + } + return {member, reference}; +} + +} + +/// The whole member is one DEFLATE block: decompression must be correct through the read buffer, +/// including matches that cross the suspension points where the nested input chunks end. +TEST_P(LibdeflateInflateTest, SingleDeflateBlockSpanningWholeMember) +{ + const CompressionMethod method = GetParam(); + const auto [member, reference] = craftSingleBlockMember(method, 4 << 20, 8 << 20); + for (size_t chunk : {size_t(16 * 1024), size_t(256 * 1024)}) + EXPECT_EQ(decompressViaBuffer(member, method, chunk, 1 << 20), reference) << "chunk=" << chunk; +} + +/// The linearity contract underlying #114045: when the input runs out inside a Huffman block, the +/// streaming decoder must have consumed everything up to the last decoded symbol, so that the caller +/// never re-feeds (and the decoder never re-decodes) more than a constant amount. Before the fix, a +/// suspension rolled back to the block start: every call consumed nothing and re-decoded the whole +/// fed prefix, making ingestion quadratic in the block's compressed size. +TEST(LibdeflateSingleBlock, StreamingConsumesInputWithinBlock) +{ + SingleBlockDeflateWriter writer; + std::mt19937 rng(7); /// NOLINT(bugprone-random-generator-seed,cert-msc32-c,cert-msc51-cpp) deterministic test data on purpose + for (size_t i = 0; i < (4 << 20); ++i) + writer.putLiteral(static_cast(rng())); + const std::string reference = writer.reference; + const std::string deflate = writer.finish(); + + libdeflate_decompressor * decompressor = libdeflate_alloc_decompressor(); + ASSERT_NE(decompressor, nullptr); + libdeflate_deflate_decompress_stream_reset(decompressor); + + constexpr size_t window_max = 32768; + constexpr size_t chunk = 64 * 1024; + std::vector out(window_max + (1 << 20)); + std::string result; + size_t window = 0; + size_t consumed = 0; + size_t fed = std::min(deflate.size(), chunk); + + while (true) + { + const bool end_of_input = fed == deflate.size(); + size_t in_used = 0; + size_t out_used = 0; + const libdeflate_result r = libdeflate_deflate_decompress_stream( + decompressor, end_of_input ? 1 : 0, + deflate.data() + consumed, fed - consumed, + out.data() + window, out.size() - window, window, &in_used, &out_used); + result.append(out.data() + window, out_used); + consumed += in_used; + + const size_t total = window + out_used; + const size_t new_window = std::min(total, window_max); + memmove(out.data(), out.data() + (total - new_window), new_window); + window = new_window; + + if (r == LIBDEFLATE_SUCCESS) + break; + if (r == LIBDEFLATE_STREAM_NEED_INPUT) + { + ASSERT_FALSE(end_of_input); + /// The heart of the regression test: all but at most a suspended symbol plus the + /// refill lookahead must have been consumed. + ASSERT_LE(fed - consumed, 64u); + fed = std::min(deflate.size(), fed + chunk); + continue; + } + ASSERT_EQ(r, LIBDEFLATE_STREAM_NEED_OUTPUT); + } + libdeflate_free_decompressor(decompressor); + EXPECT_EQ(result, reference); +} + #endif diff --git a/src/Interpreters/Access/InterpreterExecuteAsQuery.cpp b/src/Interpreters/Access/InterpreterExecuteAsQuery.cpp index eeb5faae0d1b..feb284043a18 100644 --- a/src/Interpreters/Access/InterpreterExecuteAsQuery.cpp +++ b/src/Interpreters/Access/InterpreterExecuteAsQuery.cpp @@ -100,7 +100,13 @@ BlockIO InterpreterExecuteAsQuery::execute() { /// EXECUTE AS auto subquery_context = impersonateQueryContext(getContext(), target_user_name); - return executeQuery(query.subquery->formatWithSecretsOneLine(), subquery_context, QueryFlags{ .internal = true }).second; + /// The subquery is nested, hence `internal`, but its text comes from the user, hence `user_initiated`: + /// without it the access checks of `CREATE` subqueries would be skipped, so the impersonated statement + /// would not be limited to the privileges of the target user. + return executeQuery( + query.subquery->formatWithSecretsOneLine(), subquery_context, + QueryFlags{ .internal = true, .user_initiated = true }) + .second; } else { diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index 03884fccbad1..c719b7b231ec 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -323,6 +323,7 @@ namespace Setting extern const SettingsBool enable_filesystem_read_prefetches_log; extern const SettingsBool enable_blob_storage_log; extern const SettingsBool enable_blob_storage_log_for_read_operations; + extern const SettingsUInt64 filesystem_cache_boundary_alignment; extern const SettingsUInt64 filesystem_cache_max_download_size; extern const SettingsUInt64 filesystem_cache_reserve_space_wait_lock_timeout_milliseconds; extern const SettingsUInt64 filesystem_cache_wait_for_concurrent_download_timeout_milliseconds; @@ -3093,7 +3094,9 @@ StoragePtr Context::executeTableFunction(const ASTPtr & table_expression, const create.set(create.sql_security, sql_security); auto view_context = view_metadata->getSQLSecurityOverriddenContext(shared_from_this()); - auto sample_block = InterpreterSelectWithUnionQuery::getSampleBlock(query, view_context); + auto sample_block = getSettingsRef()[Setting::allow_experimental_analyzer] + ? InterpreterSelectQueryAnalyzer::getSampleBlock(query, view_context) + : InterpreterSelectWithUnionQuery::getSampleBlock(query, view_context); auto res = std::make_shared(StorageID(database_name, table_name), create, ColumnsDescription(sample_block->getNamesAndTypesList()), @@ -8822,6 +8825,9 @@ ReadSettings Context::getReadSettings() const = settings_ref[Setting::filesystem_cache_enable_background_download_during_fetch]; res.filesystem_cache_settings.prefer_bigger_buffer_size = settings_ref[Setting::filesystem_cache_prefer_bigger_buffer_size]; + if (settings_ref[Setting::filesystem_cache_boundary_alignment]) + res.filesystem_cache_settings.boundary_alignment = settings_ref[Setting::filesystem_cache_boundary_alignment]; + res.filesystem_cache_settings.max_download_size_per_query = settings_ref[Setting::filesystem_cache_max_download_size]; res.filesystem_cache_settings.skip_download_if_exceeds_per_query_cache_write_limit = settings_ref[Setting::filesystem_cache_skip_download_if_exceeds_per_query_cache_write_limit]; diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index f955487b1dd7..c54513e64c1f 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -3646,6 +3646,11 @@ void InterpreterCreateQuery::convertMergeTreeTableIfPossible(ASTCreateQuery & cr else if (!to_replicated) throw Exception(ErrorCodes::INCORRECT_QUERY, "Can not attach table as not replicated, table is already not replicated"); + /// Must precede every side effect below: neither the transaction metadata removal nor the + /// metadata rewrite can be rolled back. The other direction takes no Keeper path at all. + if (to_replicated) + DatabaseOrdinary::checkReplicaPathIsSafe(create, getContext()); + /// Ensure the old detached table instance is destroyed before we remove /// transaction metadata files. Otherwise the old table's parts still hold /// in-memory version metadata referencing those files, and the debug diff --git a/src/Interpreters/InterpreterParallelWithQuery.cpp b/src/Interpreters/InterpreterParallelWithQuery.cpp index eb6f14dcc54d..ad7466083bc3 100644 --- a/src/Interpreters/InterpreterParallelWithQuery.cpp +++ b/src/Interpreters/InterpreterParallelWithQuery.cpp @@ -122,7 +122,11 @@ void InterpreterParallelWithQuery::executeSubqueries(const ASTs & subqueries) void InterpreterParallelWithQuery::executeSubquery(ASTPtr subquery, ContextMutablePtr subquery_context) { - auto query_io = executeQuery(subquery->formatWithSecretsOneLine(), subquery_context, QueryFlags{ .internal = true }).second; + /// The subqueries are nested, hence `internal`, but their text comes from the user, hence `user_initiated`: + /// without it the access checks of `CREATE` subqueries would be skipped. + auto query_io = executeQuery( + subquery->formatWithSecretsOneLine(), subquery_context, QueryFlags{ .internal = true, .user_initiated = true }) + .second; auto & pipeline = query_io.pipeline; diff --git a/src/Interpreters/PreparedSets.cpp b/src/Interpreters/PreparedSets.cpp index 1d332973319e..587797c873e1 100644 --- a/src/Interpreters/PreparedSets.cpp +++ b/src/Interpreters/PreparedSets.cpp @@ -521,6 +521,9 @@ SetPtr FutureSetFromSubquery::buildOrderedSetInplace(const ContextPtr & context) if (!context->getSettingsRef()[Setting::use_index_for_in_with_subqueries]) return nullptr; + /// Concurrent index analyses may share this set through cloned filter DAGs, and the build mutates + /// `set_and_key->set` and `source`. A mutex and not `callOnce` because this build may stop without + /// creating the set (e.g. a subquery timeout with `overflow_mode = 'break'`) and then be retried. std::lock_guard lock(mutex); if (auto set = get_unsafe()) diff --git a/src/Interpreters/PreparedSets.h b/src/Interpreters/PreparedSets.h index e1df40752fdf..7983a76b14fa 100644 --- a/src/Interpreters/PreparedSets.h +++ b/src/Interpreters/PreparedSets.h @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Interpreters/QueryFlags.h b/src/Interpreters/QueryFlags.h index 24273e5ca031..3e8eb3707f71 100644 --- a/src/Interpreters/QueryFlags.h +++ b/src/Interpreters/QueryFlags.h @@ -6,6 +6,11 @@ namespace DB struct QueryFlags { bool internal = false; /// If true, this query is caused by another query and thus needn't be registered in the ProcessList. + /// If true, the query was written by the user even though it is executed as an `internal` query, i.e. it is not + /// initiated by the server itself. Subqueries of `PARALLEL WITH` are like that: they are re-executed as nested + /// queries, but their text comes from the user. Such queries must be subject to all the restrictions of a regular + /// user query, in particular to access checks - `internal` alone must never be treated as a permission to skip them. + bool user_initiated = false; bool distributed_backup_restore = false; /// If true, this query is a part of backup restore. bool parse_query_from_initial_buffer = false; /// If true, do not read more data while parsing the query. The remaining input can be streaming insert data. bool background = false; /// If true, this query is the background run scheduled by executeQueryInBackground. diff --git a/src/Interpreters/executeQuery.cpp b/src/Interpreters/executeQuery.cpp index 3f3ce1dc2661..8615932970d4 100644 --- a/src/Interpreters/executeQuery.cpp +++ b/src/Interpreters/executeQuery.cpp @@ -3107,7 +3107,10 @@ static BlockIO executeQueryImpl( if (auto * create_interpreter = typeid_cast(interpreter.get())) { create_interpreter->setIsRestoreFromBackup(flags.distributed_backup_restore); - create_interpreter->setInternal(internal); + /// `InterpreterCreateQuery` uses `internal` to mean "initiated by the server itself, so all + /// the restrictions for user queries (access checks among them) can be skipped". A query + /// written by the user is never that, even when it is executed as a nested `internal` query. + create_interpreter->setInternal(internal && !flags.user_initiated); } std::unique_ptr span; diff --git a/src/Parsers/ExpressionElementParsers.cpp b/src/Parsers/ExpressionElementParsers.cpp index f9ab6405fb00..18850116a6d1 100644 --- a/src/Parsers/ExpressionElementParsers.cpp +++ b/src/Parsers/ExpressionElementParsers.cpp @@ -220,11 +220,23 @@ bool ParserSubquery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) if (pos->type != TokenType::OpeningRoundBracket) return false; + const auto opening_bracket_pos = pos; ++pos; /// Lookahead for inner subquery const bool possible_inner_subquery = pos->type == TokenType::OpeningRoundBracket; + /// A subquery in the FROM-first form, where the SELECT clause is omitted (`(FROM t)` means + /// `(SELECT * FROM t)`), starts with a word that is also a valid unquoted column name: `from`. + /// Unlike a leading `SELECT`, `EXPLAIN` or `VALUES`, that word is therefore not evidence that the + /// parentheses hold a subquery at all. Where the contents read as an expression over a column + /// named `from`, that older reading wins and these are not a subquery. + const bool starts_with_from_clause + = pos->type == TokenType::BareWord && equalsCaseInsensitive(std::string_view(pos->begin, pos->size()), "from"); + + if (starts_with_from_clause && parenthesesHoldExpressionOverColumnNamedFrom(opening_bracket_pos)) + return false; + ASTPtr result_node = nullptr; if (ASTPtr select_node; select.parse(pos, select_node, expected)) @@ -314,7 +326,7 @@ bool ParserSubquery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) } /// Inner subquery should be handled separately - starts_with_valid_select_or_explain = !possible_inner_subquery && result_node != nullptr; + starts_with_valid_select_or_explain = !possible_inner_subquery && !starts_with_from_clause && result_node != nullptr; if (pos->type != TokenType::ClosingRoundBracket) return false; diff --git a/src/Parsers/ExpressionListParsers.cpp b/src/Parsers/ExpressionListParsers.cpp index 7fa127a1873a..10695c824f5f 100644 --- a/src/Parsers/ExpressionListParsers.cpp +++ b/src/Parsers/ExpressionListParsers.cpp @@ -3506,6 +3506,44 @@ static bool isArrayQuantifierPredicate(std::string_view function_name) return predicates.contains(function_name); } +/// See the declaration for the ambiguity this resolves. The word is read as a column only when an +/// operator follows it, because otherwise it is the clause keyword in front of a table expression and +/// the subquery reading is the only one: `(FROM t)`, `(FROM numbers(10) |> LIMIT 1)`, +/// `(FROM (SELECT 1))`. Even then the reading is taken only if the parentheses really do hold an +/// expression, which keeps a relation named after an operator readable as one: `1 IN (FROM in)`. +bool parenthesesHoldExpressionOverColumnNamedFrom(IParser::Pos pos) +{ + if (pos->type != TokenType::OpeningRoundBracket) + return false; + ++pos; + + /// A clause keyword is always a BareWord token, so a quoted `` `from` `` never starts a FROM clause. + auto contents_pos = pos; + if (pos->type != TokenType::BareWord || !equalsCaseInsensitive(std::string_view(pos->begin, pos->size()), "from")) + return false; + ++pos; + + Expected expected; + bool operator_follows = false; + for (const auto & [lexeme, unused_operator] : ParserExpressionImpl::operators_table) + { + auto operator_pos = pos; + if (parseOperator(operator_pos, lexeme, expected)) + { + operator_follows = true; + break; + } + } + + if (!operator_follows) + return false; + + /// The contents can also be a tuple or carry an alias, so parse them the way RoundBracketsLayer does. + ASTPtr contents; + ParserExpressionList contents_parser(/*allow_alias_without_as_keyword*/ false); + return contents_parser.parse(contents_pos, contents, expected) && contents_pos->type == TokenType::ClosingRoundBracket; +} + Action ParserExpressionImpl::tryParseOperand(Layers & layers, IParser::Pos & pos, Expected & expected) { ASTPtr tmp; diff --git a/src/Parsers/ExpressionListParsers.h b/src/Parsers/ExpressionListParsers.h index 069700dd4d4a..8cc2465ea6b8 100644 --- a/src/Parsers/ExpressionListParsers.h +++ b/src/Parsers/ExpressionListParsers.h @@ -334,6 +334,16 @@ class ParserTTLExpressionList : public IParserBase bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; }; +/// Whether the parentheses that `pos` points at hold an expression over a column named `from`, rather +/// than a subquery. Does not move `pos`. +/// +/// `from` is a keyword that ClickHouse has always accepted as an unquoted identifier, and a subquery +/// may omit its SELECT clause and start with the FROM clause instead (`(FROM t)` means +/// `(SELECT * FROM t)`), so both readings parse for `WHERE (from IN ('a'))`: the expression +/// `from IN ('a')`, and the subquery `SELECT * FROM IN('a')` over a table function named `IN`. The +/// column reading is older than the FROM-first form, so it wins the ambiguity. +bool parenthesesHoldExpressionOverColumnNamedFrom(IParser::Pos pos); + } #pragma clang diagnostic pop diff --git a/src/Parsers/FunctionSecretArgumentsFinder.cpp b/src/Parsers/FunctionSecretArgumentsFinder.cpp index 980626c879ea..5fd8fc32d992 100644 --- a/src/Parsers/FunctionSecretArgumentsFinder.cpp +++ b/src/Parsers/FunctionSecretArgumentsFinder.cpp @@ -1199,6 +1199,11 @@ void FunctionSecretArgumentsFinder::findDatabaseEngineSecretArguments() { findBackupDatabaseSecretArguments(); } + else if (engine_name == "URL") + { + /// URL('base_url') + findURLSecretArguments(); + } } void FunctionSecretArgumentsFinder::findMySQLDatabaseSecretArguments() diff --git a/src/Planner/PlannerJoinTree.cpp b/src/Planner/PlannerJoinTree.cpp index 3ddeaacd9305..dfb388f06ce1 100644 --- a/src/Planner/PlannerJoinTree.cpp +++ b/src/Planner/PlannerJoinTree.cpp @@ -838,7 +838,7 @@ void prepareBuildQueryPlanForTableExpression(const QueryTreeNodePtr & table_expr /// would let the query read the view without any `SELECT` grant. Enforce the same column-aware `SELECT` /// check the underlying view would receive as a `TableNode`. const auto & storage = table_function_node->getStorage(); - if (const auto * storage_view = storage ? storage->as() : nullptr; storage_view && storage_view->isParameterizedView()) + if (table_function_node->isParameterizedView()) { const auto & column_names_with_aliases = table_expression_data.getSelectedColumnsNames(); columns_names_allowed_to_select = checkAccessRights( @@ -1128,16 +1128,6 @@ UInt64 mainQueryNodeBlockSizeByLimit(const SelectQueryInfo & select_query_info) limit_offset = offset_uint->getUInt(0); } - /// `arrayJoin` in the projection expands one input row into several output rows after the - /// source has run. Capping the source to `limit + offset` rows would truncate input BEFORE - /// expansion, so hard consumers of `trivial_limit` (StorageLoop, system.zeros, generateRandom) - /// could drop output rows that the LIMIT should keep. See issue #82279 and the sibling guard - /// in `numbersLikeUtils::shouldPushdownLimit`. (The `ARRAY JOIN` clause is lowered to a - /// separate table expression in the analyzer, so it is not a single-table read and never - /// reaches this optimization.) - if (hasFunctionNode(main_query_node.getProjectionNode(), "arrayJoin")) - return 0; - /** If not specified DISTINCT, WHERE, GROUP BY, HAVING, ORDER BY, JOIN, LIMIT BY, LIMIT WITH TIES * but LIMIT is specified with UInt64 value, and limit + offset < max_block_size, * then as the block size we will use limit + offset (not to read more from the table than requested), @@ -1366,7 +1356,7 @@ void pushOrderByIntoView( /// source rows before the expansion runs, so if the top ordered rows have /// empty arrays the rewritten query would return too few rows instead of /// continuing to lower ordered rows to fill the `LIMIT`. Mirror the existing - /// guard in `mainQueryNodeBlockSizeByLimit`. + /// `trivial_limit` guard in `buildQueryPlanForTableExpression`. if (hasFunctionNode(outer->getProjectionNode(), "arrayJoin")) return; @@ -1809,6 +1799,8 @@ JoinTreeQueryPlan buildQueryPlanForTableExpression(TableExpressionNodePtr table_ UInt64 max_block_size = settings[Setting::max_block_size]; UInt64 max_block_size_limited = 0; + /// LIMIT + OFFSET as the most rows the source has to produce, when that holds. + UInt64 max_source_rows = 0; if (is_single_table_expression && !select_query_options.only_analyze) { /** If not specified DISTINCT, WHERE, GROUP BY, HAVING, ORDER BY, JOIN, LIMIT BY, LIMIT WITH TIES @@ -1825,24 +1817,46 @@ JoinTreeQueryPlan buildQueryPlanForTableExpression(TableExpressionNodePtr table_ max_block_size_limited = mainQueryNodeBlockSizeByLimit(select_query_info); if (max_block_size_limited) { - if (max_block_size_limited < max_block_size) + const bool has_array_join = hasFunctionNode(select_query_info.query_tree->as().getProjectionNode(), "arrayJoin"); + const bool shrink_block = max_block_size_limited < max_block_size; + if (shrink_block) { - max_block_size = std::max(1, max_block_size_limited); - max_streams = 1; - max_threads_execute_query = 1; + /// With `arrayJoin` the source cannot stop at the LIMIT, so over a long run of empty arrays it streams + /// every row anyway, and one-row blocks make that hundreds of times slower than the default block. + /// Keep a few hundred rows per block: still a small read, and the empty prefix stays cheap. + constexpr UInt64 min_block_size_above_array_join = 256; + if (has_array_join) + max_block_size = std::min(max_block_size, std::max(max_block_size_limited, min_block_size_above_array_join)); + else + max_block_size = std::max(1, max_block_size_limited); } - if (select_query_info.local_storage_limits.local_limits.size_limits.max_rows != 0) + /// With `arrayJoin` the LIMIT does not bound the source rows, so only the block size shrinks (#82279). + if (!has_array_join) { - if (max_block_size_limited < select_query_info.local_storage_limits.local_limits.size_limits.max_rows) + max_source_rows = max_block_size_limited; + if (shrink_block) + { + max_streams = 1; + max_threads_execute_query = 1; + } + + if (select_query_info.local_storage_limits.local_limits.size_limits.max_rows != 0) + { + if (max_block_size_limited < select_query_info.local_storage_limits.local_limits.size_limits.max_rows) + table_expression_query_info.trivial_limit = max_block_size_limited; + /// Ask to read just enough rows to make the max_rows limit effective (so it has a chance to be triggered). + else if (select_query_info.local_storage_limits.local_limits.size_limits.max_rows < std::numeric_limits::max()) + table_expression_query_info.trivial_limit = 1 + select_query_info.local_storage_limits.local_limits.size_limits.max_rows; + } + else + { table_expression_query_info.trivial_limit = max_block_size_limited; - /// Ask to read just enough rows to make the max_rows limit effective (so it has a chance to be triggered). - else if (select_query_info.local_storage_limits.local_limits.size_limits.max_rows < std::numeric_limits::max()) - table_expression_query_info.trivial_limit = 1 + select_query_info.local_storage_limits.local_limits.size_limits.max_rows; + } } else { - table_expression_query_info.trivial_limit = max_block_size_limited; + table_expression_query_info.small_limit_above_array_join = shrink_block; } } @@ -2724,8 +2738,8 @@ JoinTreeQueryPlan buildQueryPlanForTableExpression(TableExpressionNodePtr table_ if (table_expression_query_info.trivial_limit > 0 && table_expression_query_info.trivial_limit < rows_to_read) rows_to_read = table_expression_query_info.trivial_limit; - if (max_block_size_limited && (max_block_size_limited < rows_to_read)) - rows_to_read = max_block_size_limited; + if (max_source_rows && (max_source_rows < rows_to_read)) + rows_to_read = max_source_rows; const size_t number_of_replicas_to_use = rows_to_read / settings[Setting::parallel_replicas_min_number_of_rows_per_replica]; diff --git a/src/Planner/Utils.cpp b/src/Planner/Utils.cpp index 67311e1635a3..a54f859f69fb 100644 --- a/src/Planner/Utils.cpp +++ b/src/Planner/Utils.cpp @@ -26,7 +26,6 @@ #include #include -#include #include #include @@ -639,12 +638,10 @@ static void checkAccessRightsForFilter(const QueryTreeNodePtr & filter_query_tre { /// A parameterized view is resolved as a `TableFunctionNode` wrapping a real `StorageView`, see /// `prepareBuildQueryPlanForTableExpression`. Regular table functions are checked in `ITableFunction::execute`. - const auto & table_function_storage = table_function_node->getStorage(); - const auto * storage_view = table_function_storage ? table_function_storage->as() : nullptr; - if (!storage_view || !storage_view->isParameterizedView()) + if (!table_function_node->isParameterizedView()) return; - storage = table_function_storage; + storage = table_function_node->getStorage(); storage_id = table_function_node->getStorageID(); storage_snapshot = table_function_node->getStorageSnapshot(); } diff --git a/src/Processors/Formats/Impl/ArrowIPC/ArrowIPCBlockInputFormat.cpp b/src/Processors/Formats/Impl/ArrowIPC/ArrowIPCBlockInputFormat.cpp index f4eca795c114..031004f4dfef 100644 --- a/src/Processors/Formats/Impl/ArrowIPC/ArrowIPCBlockInputFormat.cpp +++ b/src/Processors/Formats/Impl/ArrowIPC/ArrowIPCBlockInputFormat.cpp @@ -1235,6 +1235,8 @@ cat forex_eurusd.arrow | clickhouse-client --query="INSERT INTO some_table FORMA | `output_format_arrow_compression_method` | Compression method for Arrow output format. Supported codecs: lz4_frame, zstd, none (uncompressed) | `lz4_frame` | | `output_format_arrow_fixed_string_as_fixed_byte_array` | Use Arrow FIXED_SIZE_BINARY type instead of Binary for FixedString columns. | `1` | | `output_format_arrow_low_cardinality_as_dictionary` | Enable output LowCardinality type as Dictionary Arrow type | `0` | +| `output_format_arrow_record_batch_size` | Target rows per record batch when combining small blocks. Buffering can increase memory use and delay the first batch until the query finishes. `0` disables the row target. | `0` | +| `output_format_arrow_record_batch_size_bytes` | Target bytes of accumulated block data per record batch. Buffering can increase memory use and delay the first batch until the query finishes. `0` disables the byte target. | `0` | | `output_format_arrow_string_as_string` | Use Arrow String type instead of Binary for String columns | `1` | | `output_format_arrow_unsupported_types_as_binary` | Output a type that has no Arrow equivalent (e.g. `BFloat16`, `AggregateFunction`) as raw binary data. If false, such a type raises an exception. | `1` | | `output_format_arrow_use_64_bit_indexes_for_dictionary` | Always use 64 bit integers for dictionary indexes in Arrow format | `0` | @@ -1369,6 +1371,8 @@ the blog post | `output_format_arrow_date_as_uint16` | Write Date values as plain 16-bit numbers (read back as UInt16), instead of converting to a 32-bit Arrow DATE32 type (read back as Date32). | `0` | | `output_format_arrow_fixed_string_as_fixed_byte_array` | Use Arrow FIXED_SIZE_BINARY type instead of Binary for FixedString columns. | `1` | | `output_format_arrow_low_cardinality_as_dictionary` | Enable output LowCardinality type as Dictionary Arrow type | `0` | +| `output_format_arrow_record_batch_size` | Target rows per record batch when combining small blocks. Buffering can increase memory use and delay the first batch until the query finishes. `0` disables the row target. | `0` | +| `output_format_arrow_record_batch_size_bytes` | Target bytes of accumulated block data per record batch. Buffering can increase memory use and delay the first batch until the query finishes. `0` disables the byte target. | `0` | | `output_format_arrow_string_as_string` | Use Arrow String type instead of Binary for String columns | `1` | | `output_format_arrow_unsupported_types_as_binary` | Output a type that has no Arrow equivalent (e.g. `BFloat16`, `AggregateFunction`) as raw binary data. If false, such a type raises an exception. | `1` | | `output_format_arrow_use_64_bit_indexes_for_dictionary` | Always use 64 bit integers for dictionary indexes in Arrow format | `0` | diff --git a/src/Processors/Formats/Impl/ArrowIPC/ArrowIPCBlockOutputFormat.cpp b/src/Processors/Formats/Impl/ArrowIPC/ArrowIPCBlockOutputFormat.cpp index f448d28199ae..6253eb086fd8 100644 --- a/src/Processors/Formats/Impl/ArrowIPC/ArrowIPCBlockOutputFormat.cpp +++ b/src/Processors/Formats/Impl/ArrowIPC/ArrowIPCBlockOutputFormat.cpp @@ -353,8 +353,49 @@ std::pair ArrowIPCBlockOutputFormat::substituteDictionar void ArrowIPCBlockOutputFormat::consume(Chunk chunk) { + /// Not in `writeChunk`: a reader gets the schema on the first chunk even when batches are combined. writeSchemaIfNeeded(); + const size_t target_rows = format_settings.arrow.output_record_batch_rows; + const size_t target_bytes = format_settings.arrow.output_record_batch_bytes; + if (!target_rows && !target_bytes) + { + writeChunk(std::move(chunk)); + return; + } + + if (chunk.getNumRows() == 0) + return; + + /// `ColumnConst::insertRangeFrom` advances the row count without copying the value, and a lazily sparse + /// or replicated column cannot be appended to a materialized one. It also has to precede the size + /// measured below, which `ColumnConst::byteSize` reports as a single stored value. + materializeChunk(chunk); + + auto reached = [&](size_t rows, size_t bytes) + { return (target_rows && rows >= target_rows) || (target_bytes && bytes >= target_bytes); }; + + /// A chunk that already fills a batch is written as it stands, so a large block is neither copied nor + /// combined with its neighbours. `Chunk::append` also requires at least one column. + if (chunk.getNumColumns() == 0 || reached(chunk.getNumRows(), chunk.bytes())) + { + if (staged.getNumRows()) + writeChunk(std::move(staged)); + writeChunk(std::move(chunk)); + return; + } + + if (staged.getNumRows()) + staged.append(chunk); + else + staged = std::move(chunk); + + if (reached(staged.getNumRows(), staged.bytes())) + writeChunk(std::move(staged)); +} + +void ArrowIPCBlockOutputFormat::writeChunk(Chunk chunk) +{ const size_t num_rows = chunk.getNumRows(); const Columns & columns = chunk.getColumns(); @@ -378,6 +419,10 @@ void ArrowIPCBlockOutputFormat::consume(Chunk chunk) void ArrowIPCBlockOutputFormat::finalizeImpl() { + /// The whole output of a result that never reached a target leaves through here. + if (staged.getNumRows()) + writeChunk(std::move(staged)); + /// Make sure even an empty result produces a valid stream/file (schema, then EOS or footer). writeSchemaIfNeeded(); @@ -399,6 +444,8 @@ void ArrowIPCBlockOutputFormat::finalizeImpl() void ArrowIPCBlockOutputFormat::resetFormatterImpl() { + /// `finalizeImpl` drains it on every normal path; this also covers a reset after an interrupted one. + staged.clear(); message_writer.emplace(out); schema_written = false; dictionary_blocks.clear(); diff --git a/src/Processors/Formats/Impl/ArrowIPC/ArrowIPCBlockOutputFormat.h b/src/Processors/Formats/Impl/ArrowIPC/ArrowIPCBlockOutputFormat.h index 13cc336dac92..2be27e359c1f 100644 --- a/src/Processors/Formats/Impl/ArrowIPC/ArrowIPCBlockOutputFormat.h +++ b/src/Processors/Formats/Impl/ArrowIPC/ArrowIPCBlockOutputFormat.h @@ -43,6 +43,9 @@ class ArrowIPCBlockOutputFormat final : public IOutputFormat void finalizeImpl() override; void resetFormatterImpl() override; + /// Encodes one chunk as exactly one record batch, preceded by the dictionary batches it references. + void writeChunk(Chunk chunk); + void writeSchemaIfNeeded(); /// Writes one encapsulated message for an encoded batch (a record batch, or a dictionary batch /// when `dictionary_id` is set), returning its location for recording an Arrow file `Block`. @@ -94,6 +97,11 @@ class ArrowIPCBlockOutputFormat final : public IOutputFormat }; ArrowIPC::DictPlans column_dict_plans; VectorWithMemoryTracking dictionary_states; + + /// Rows of consecutive chunks that are individually smaller than the configured record batch target, + /// to be written as one record batch. The first chunk becomes the accumulator itself, so a staged batch + /// can hold one source block's allocations until it is written. + Chunk staged; }; } diff --git a/src/Processors/QueryPlan/AggregatingStep.cpp b/src/Processors/QueryPlan/AggregatingStep.cpp index 11d17539b520..272d0f4f0e35 100644 --- a/src/Processors/QueryPlan/AggregatingStep.cpp +++ b/src/Processors/QueryPlan/AggregatingStep.cpp @@ -690,6 +690,7 @@ void AggregatingStep::transformPipeline(QueryPipelineBuilder & pipeline, const B many_data, counter++, limit_hint, + limit_hint_prefix_columns, nullptr // `dataflow_cache_updater` will be passed to `MergingAggregatedBucketTransform` below ); }); @@ -741,6 +742,7 @@ void AggregatingStep::transformPipeline(QueryPipelineBuilder & pipeline, const B sort_description_for_merging, group_by_sort_description, max_block_size, aggregation_in_order_max_block_bytes, limit_hint, + limit_hint_prefix_columns, dataflow_cache_updater); }); diff --git a/src/Processors/QueryPlan/AggregatingStep.h b/src/Processors/QueryPlan/AggregatingStep.h index 3140a1d30995..57ebee67f2e5 100644 --- a/src/Processors/QueryPlan/AggregatingStep.h +++ b/src/Processors/QueryPlan/AggregatingStep.h @@ -94,7 +94,13 @@ class AggregatingStep : public ITransformingStep void applyTopKOptimization(Aggregator::Params::TopKParams top_k); bool memoryBoundMergingWillBeUsed() const; void skipMerging() { skip_merging = true; } - void setLimitHint(size_t limit) { limit_hint = limit; } + /// `prefix_columns` is the number of leading columns of the group-by sort description + /// the query is ordered by; the in-order streams may stop only at a boundary of them. + void setLimitHint(size_t limit, size_t prefix_columns) + { + limit_hint = limit; + limit_hint_prefix_columns = prefix_columns; + } size_t getLimitHint() const { return limit_hint; } const SortDescription & getGroupBySortDescription() const { return group_by_sort_description; } @@ -186,6 +192,7 @@ class AggregatingStep : public ITransformingStep bool enable_sharding_aggregator; size_t limit_hint = 0; + size_t limit_hint_prefix_columns = 0; Processors aggregating_in_order; Processors aggregating_sorted; diff --git a/src/Processors/QueryPlan/JoinStepLogical.cpp b/src/Processors/QueryPlan/JoinStepLogical.cpp index 9ca69c92f9ac..88dc3f137ea7 100644 --- a/src/Processors/QueryPlan/JoinStepLogical.cpp +++ b/src/Processors/QueryPlan/JoinStepLogical.cpp @@ -809,6 +809,45 @@ static void predicateOperandsToCommonType( } } +/// Under `join_use_nulls`, a right column selected from a LEFT or FULL JOIN is output through `toNullable(x)` +/// (see `addToNullableIfNeeded`). When that column is also a join key, joining on the `Nullable` node makes +/// it the single right column that is both the key and the output, which the join restores from the left +/// key. Joining on the plain input instead leaves the `Nullable` wrapper as a payload column next to the +/// key: a whole extra column where the join keeps keys only in its arena, and a second count of the same +/// bytes toward the spill threshold where it saves the key columns too. +static void preferNullableRightKey( + JoinActionRef & right_node, + const JoinPlanningContext & planning_context, + std::vector> & shared_runtime_filter_descriptors) +{ + /// The `Join` engine and a dictionary are looked up by the key they declare. + if (planning_context.is_storage_join) + return; + + const auto * input = right_node.getNode(); + if (input->type != ActionsDAG::ActionType::INPUT) + return; + + auto it = planning_context.actions_after_join_map.find(input->result_name); + if (it == planning_context.actions_after_join_map.end()) + return; + + const auto * to_nullable = it->second; + if (to_nullable->type != ActionsDAG::ActionType::FUNCTION || to_nullable->children.size() != 1 + || to_nullable->children.front() != input || to_nullable->function_base->getName() != "toNullable") + return; + + /// The build-side key name is the rendezvous with the shared runtime filter descriptors, as in + /// `predicateOperandsToCommonType`. + String name_before = right_node.getColumnName(); + right_node = JoinActionRef::transform({right_node}, [to_nullable](auto &, auto &&) { return to_nullable; }); + for (auto & descriptor : shared_runtime_filter_descriptors) + { + if (descriptor.second == name_before) + descriptor.second = right_node.getColumnName(); + } +} + static bool addJoinPredicatesToTableJoin(std::vector & predicates, TableJoin::JoinOnClause & table_join_clause, std::vector & used_expressions, const JoinSettings & join_settings, const JoinPlanningContext & planning_context, std::vector> & shared_runtime_filter_descriptors) @@ -830,6 +869,8 @@ static bool addJoinPredicatesToTableJoin(std::vector & predicates predicateOperandsToCommonType(lhs, rhs, join_settings, planning_context, shared_runtime_filter_descriptors); bool null_safe_comparison = JoinConditionOperator::NullSafeEquals == predicate_op; + if (!null_safe_comparison) + preferNullableRightKey(rhs, planning_context, shared_runtime_filter_descriptors); if (null_safe_comparison && isNullableOrLowCardinalityNullable(lhs.getType()) && isNullableOrLowCardinalityNullable(rhs.getType())) { /** @@ -1656,6 +1697,34 @@ static QueryPlanNode buildPhysicalJoinImpl( for (const auto * node : dag_inputs) name_to_nodes[node->result_name].push_back(node); + /// An input that only feeds a used expression, such as the `toNullable(x)` key under `join_use_nulls` + /// or a key cast to a common type, is not passed to the join as a column of its own: the join would + /// keep it as payload for nothing. `ActionsDAG::updateHeader` drops such consumed inputs anyway. + std::unordered_set consumed_inputs; + { + std::unordered_set used_nodes; + for (const auto & expression : used_expressions) + used_nodes.insert(expression.getNode()); + + std::stack stack; + for (const auto * node : used_nodes) + for (const auto * child : node->children) + stack.push(child); + while (!stack.empty()) + { + const auto * node = stack.top(); + stack.pop(); + if (node->type == ActionsDAG::ActionType::INPUT) + { + if (!used_nodes.contains(node)) + consumed_inputs.insert(node); + continue; + } + for (const auto * child : node->children) + stack.push(child); + } + } + for (const auto * child : children) { for (const auto & column : *child->step->getOutputHeader()) @@ -1669,8 +1738,10 @@ static QueryPlanNode buildPhysicalJoinImpl( fmt::join(children | std::views::transform([](const auto & c) { return fmt::format("[{}]", c->step->getOutputHeader()->dumpNames()); }), ", "), expression_actions.getActionsDAG()->dumpDAG()); - used_expressions.emplace_back(input_it->second.front(), expression_actions); + const auto * input = input_it->second.front(); input_it->second.pop_front(); + if (!consumed_inputs.contains(input)) + used_expressions.emplace_back(input, expression_actions); } } diff --git a/src/Processors/QueryPlan/Optimizations/filterPushDown.cpp b/src/Processors/QueryPlan/Optimizations/filterPushDown.cpp index 475176db0221..8e7a93b2a58a 100644 --- a/src/Processors/QueryPlan/Optimizations/filterPushDown.cpp +++ b/src/Processors/QueryPlan/Optimizations/filterPushDown.cpp @@ -455,11 +455,6 @@ static void projectDagInputs(ActionsDAG & actions_dag) } } -static bool isAnyInnerJoin(JoinKind kind, JoinStrictness strictness) -{ - return kind == JoinKind::Inner && (strictness == JoinStrictness::Any || strictness == JoinStrictness::RightAny); -} - std::optional tryToExtractPartialPredicate( const ActionsDAG & original_dag, const std::string & filter_name, @@ -676,23 +671,6 @@ static size_t tryPushDownOverJoinStep(QueryPlan::Node * parent_node, QueryPlan:: right_stream_filter_push_down_input_columns_available = canPrefilterJoinSide(kind, strictness, JoinTableSide::Right); } - /** `ANY INNER` join emits at most one row per key, deduplicating both sides. - * Both sides are blocked: filtering the right stream can change which match is taken for the left row. - * If the left side has multiple rows with the same value, only one survives - * and pushing the filter down to the left may affect which one survives. - * Also the optimizer is allowed to swap the sides of the join after that pass, - * thus a filter on the left may end up on the right. - * Predicates over the equi-join keys are still pushed to both sides through the equivalent-columns path below. - */ - const bool is_any_inner_join = (table_join_ptr && isAnyInnerJoin(table_join_ptr->kind(), table_join_ptr->strictness())) - || (logical_join && isAnyInnerJoin(logical_join->getJoinOperator().kind, logical_join->getJoinOperator().strictness)); - - if (is_any_inner_join) - { - right_stream_filter_push_down_input_columns_available = false; - left_stream_filter_push_down_input_columns_available = false; - } - /** `canPrefilterJoinSide` only decides whether this side's own columns may be * used as ordinary filter inputs (false on the null-producing outer-JOIN side). * Equivalent-key filters can still be attached to that child. That attach is @@ -870,14 +848,9 @@ static size_t tryPushDownOverJoinStep(QueryPlan::Node * parent_node, QueryPlan:: for (const auto & [name, _] : equivalent_left_stream_column_to_right_stream_column) equivalent_columns_to_push_down.push_back(name); } - else if (is_any_inner_join - || (logical_join && logical_join->getJoinOperator().kind == JoinKind::Right && logical_join->getJoinOperator().strictness == JoinStrictness::Semi)) + else if (logical_join && logical_join->getJoinOperator().kind == JoinKind::Right && logical_join->getJoinOperator().strictness == JoinStrictness::Semi) { - /// The `typeChangingSides` check is not needed in case of physical join, because - /// 1. `INNER` joins never widen the columns' types. - /// 2. On this path an `INNER` join cannot have been produced from an `OUTER` one by - /// `tryConvertOuterJoinToInnerJoin` and thus it's not carrying the preserved nullability. - if (!logical_join || !logical_join->typeChangingSides().contains(JoinTableSide::Left)) + if (!logical_join->typeChangingSides().contains(JoinTableSide::Left)) { /// In this case we can also push down to left side of JOIN using equivalent sets. for (const auto & [name, _] : equivalent_left_stream_column_to_right_stream_column) @@ -890,14 +863,9 @@ static size_t tryPushDownOverJoinStep(QueryPlan::Node * parent_node, QueryPlan:: for (const auto & [name, _] : equivalent_right_stream_column_to_left_stream_column) equivalent_columns_to_push_down.push_back(name); } - else if (is_any_inner_join - || (logical_join && logical_join->getJoinOperator().kind == JoinKind::Left && logical_join->getJoinOperator().strictness == JoinStrictness::Semi)) + else if (logical_join && logical_join->getJoinOperator().kind == JoinKind::Left && logical_join->getJoinOperator().strictness == JoinStrictness::Semi) { - /// The `typeChangingSides` check is not needed in case of physical join, because - /// 1. `INNER` joins never widen the columns' types. - /// 2. On this path an `INNER` join cannot have been produced from an `OUTER` one by - /// `tryConvertOuterJoinToInnerJoin` and thus it's not carrying the preserved nullability. - if (!logical_join || !logical_join->typeChangingSides().contains(JoinTableSide::Right)) + if (!logical_join->typeChangingSides().contains(JoinTableSide::Right)) { /// In this case we can also push down to right side of JOIN using equivalent sets. for (const auto & [name, _] : equivalent_right_stream_column_to_left_stream_column) diff --git a/src/Processors/QueryPlan/Optimizations/makeDistributed.cpp b/src/Processors/QueryPlan/Optimizations/makeDistributed.cpp index 3353325b9594..52ce73ca539c 100644 --- a/src/Processors/QueryPlan/Optimizations/makeDistributed.cpp +++ b/src/Processors/QueryPlan/Optimizations/makeDistributed.cpp @@ -587,6 +587,19 @@ void tryMakeDistributedAggregation(QueryPlan::Node & node, QueryPlan::Nodes & no if (optimization_settings.distributed_plan_force_shuffle_aggregation && !aggregation_keys.empty()) strategy = Shuffle; + /// Shuffle moves the aggregation step unchanged, so each of the `bucket_count` instances keeps the + /// promise of bucket order while ordering only its own share, and the gather cannot restore a global + /// order: it merges by a sort description, and the bucket number is chunk metadata, not a column. + /// Shuffle is therefore impossible here, so `distributed_plan_force_shuffle_aggregation` cannot + /// apply either, as with `GROUPING SETS` below. + if (aggregating_step->shouldProduceResultsInBucketOrder()) + { + if (!can_use_partial_aggregation) + throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, + "make_distributed_plan does not support aggregation in order which must produce results in bucket order"); + strategy = PartialAggregation; + } + /// Shuffle scatters by the full key set, so GROUPING SETS subtotals (over key subsets) would be /// produced in several buckets and duplicated. Partial aggregation has no such problem: every /// worker produces partial states for every grouping set over its share of the data, tagged with diff --git a/src/Processors/QueryPlan/Optimizations/optimizeDirectReadFromTextIndex.cpp b/src/Processors/QueryPlan/Optimizations/optimizeDirectReadFromTextIndex.cpp index ad24c8857abf..a1f99567b656 100644 --- a/src/Processors/QueryPlan/Optimizations/optimizeDirectReadFromTextIndex.cpp +++ b/src/Processors/QueryPlan/Optimizations/optimizeDirectReadFromTextIndex.cpp @@ -64,6 +64,7 @@ struct TextIndexReadInfo MergeTreeIndexPtr index_helper = nullptr; bool is_materialized = false; bool is_fully_materialized = false; + bool has_patched_parts = false; }; using TextIndexReadInfos = absl::flat_hash_map; @@ -213,6 +214,7 @@ void collectTextIndexReadInfos(const ReadFromMergeTree * read_from_merge_tree_st /// other partitions/parts not in `parts_with_ranges`, disabling direct text index reads even when /// the queried parts have no on-the-fly updates for the index columns. NameSet all_updated_columns; + bool has_patched_parts = false; for (const auto & part : unique_parts) { auto alter_conversions = MergeTreeData::getAlterConversionsForPart(part, mutations_snapshot, context @@ -222,8 +224,12 @@ void collectTextIndexReadInfos(const ReadFromMergeTree * read_from_merge_tree_st ); const auto & part_updated_columns = alter_conversions->getAllUpdatedColumns(); all_updated_columns.insert(part_updated_columns.begin(), part_updated_columns.end()); + has_patched_parts |= alter_conversions->hasPatches(); } + if (has_patched_parts) + LOG_TRACE(logger, "Cannot use direct reading from text index. Reason: a part has a pending patch"); + for (const auto & index : indexes->skip_indexes.useful_indices) { if (!index.index->isTextIndex()) @@ -246,7 +252,8 @@ void collectTextIndexReadInfos(const ReadFromMergeTree * read_from_merge_tree_st .condition = index.condition_template->generateUnsubstituted(), .index = &index, .is_materialized = num_materialized_parts > 0, - .is_fully_materialized = num_materialized_parts == unique_parts.size() + .is_fully_materialized = num_materialized_parts == unique_parts.size(), + .has_patched_parts = has_patched_parts }; } } @@ -586,9 +593,11 @@ class TextIndexDAGReplacer const bool is_index_analyzed = !require_index_analyzed_predicate || isIndexAnalyzedPredicate(index_name, info, canonical_node); - /// Use direct read only when enabled and the entry is direct-read-eligible (has `index`). Otherwise - /// just inject the tokenizer/preprocessor/postprocessor (no virtual column), same as None mode. - if (!direct_read_from_text_index || !info.index || search_query->getDirectReadMode() == TextIndexDirectReadMode::None) + /// Use direct read only when enabled and the entry is direct-read-eligible (has `index`) and has no + /// patched parts. Otherwise just inject the tokenizer/preprocessor/postprocessor (no virtual column), + /// same as None mode. + if (!direct_read_from_text_index || !info.index || info.has_patched_parts + || search_query->getDirectReadMode() == TextIndexDirectReadMode::None) { selected_conditions.emplace_back(search_query, index_name, String{}, &info, is_index_analyzed); used_index_columns.insert(index_header.begin()->name); diff --git a/src/Processors/QueryPlan/Optimizations/optimizeLimitForAggregationInOrder.cpp b/src/Processors/QueryPlan/Optimizations/optimizeLimitForAggregationInOrder.cpp index ffec0be141a2..9481e1cee3da 100644 --- a/src/Processors/QueryPlan/Optimizations/optimizeLimitForAggregationInOrder.cpp +++ b/src/Processors/QueryPlan/Optimizations/optimizeLimitForAggregationInOrder.cpp @@ -122,6 +122,11 @@ void optimizeLimitForAggregationInOrder(QueryPlan::Node & root) || aggregating_step->isGroupingSets()) break; + /// The sort description may be a strict prefix of the group-by sort description. + /// Each in-order stream then stops only at a boundary of that prefix (see + /// `AggregatingInOrderTransform`), so the groups it drops sort strictly after + /// at least `limit` complete groups and the final sort can never pick a group + /// with a partial aggregate value (issue #116849). const auto & sort_desc = sorting_step->getSortDescription(); const auto & agg_sort_desc = aggregating_step->getGroupBySortDescription(); if (sort_desc.empty() || !agg_sort_desc.hasPrefix(sort_desc)) @@ -130,7 +135,7 @@ void optimizeLimitForAggregationInOrder(QueryPlan::Node & root) /// Use the smallest limit if multiple LimitSteps point to the same AggregatingStep. size_t current_hint = aggregating_step->getLimitHint(); if (!current_hint || limit < current_hint) - aggregating_step->setLimitHint(limit); + aggregating_step->setLimitHint(limit, sort_desc.size()); break; } diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.cpp b/src/Processors/QueryPlan/ReadFromMergeTree.cpp index 81a5ea1c95e5..db565127c152 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.cpp +++ b/src/Processors/QueryPlan/ReadFromMergeTree.cpp @@ -707,7 +707,8 @@ Pipe ReadFromMergeTree::readFromPool( * Because time spend during filling per thread tasks can be greater than whole query * execution for big tables with small limit. */ - bool use_prefetched_read_pool = query_info.trivial_limit == 0 && (allow_prefetched_remote || allow_prefetched_local); + bool use_prefetched_read_pool = query_info.trivial_limit == 0 && !query_info.small_limit_above_array_join + && (allow_prefetched_remote || allow_prefetched_local); if (use_prefetched_read_pool) { diff --git a/src/Processors/Transforms/AggregatingInOrderTransform.cpp b/src/Processors/Transforms/AggregatingInOrderTransform.cpp index b4d0d8a5fbb7..f4e048fae60e 100644 --- a/src/Processors/Transforms/AggregatingInOrderTransform.cpp +++ b/src/Processors/Transforms/AggregatingInOrderTransform.cpp @@ -27,12 +27,14 @@ AggregatingInOrderTransform::AggregatingInOrderTransform( const SortDescription & group_by_description_, size_t max_block_size_, size_t max_block_bytes_, size_t limit_hint_, + size_t limit_prefix_columns_, RuntimeDataflowStatisticsCacheUpdaterPtr dataflow_cache_updater_) : AggregatingInOrderTransform(std::move(header), std::move(params_), sort_description_for_merging, group_by_description_, max_block_size_, max_block_bytes_, std::make_unique(1), 0, limit_hint_, + limit_prefix_columns_, std::move(dataflow_cache_updater_)) { } @@ -44,6 +46,7 @@ AggregatingInOrderTransform::AggregatingInOrderTransform( size_t max_block_size_, size_t max_block_bytes_, ManyAggregatedDataPtr many_data_, size_t current_variant, size_t limit_hint_, + size_t limit_prefix_columns_, RuntimeDataflowStatisticsCacheUpdaterPtr dataflow_cache_updater_) : IProcessor({std::move(header)}, {params_->getCustomHeader(false)}) , max_block_size(max_block_size_) @@ -72,6 +75,22 @@ AggregatingInOrderTransform::AggregatingInOrderTransform( /// group_by_description may contains duplicates, so we use keys_size from Aggregator::params key_columns_raw.resize(params->params.keys_size); } + + /// The `ORDER BY` may extend past the columns the input is sorted by; a boundary of the + /// sorted columns alone is then a boundary of the `ORDER BY` prefix as well. + limit_prefix_columns = std::min(limit_prefix_columns_, group_by_description.size()); +} + +bool AggregatingInOrderTransform::isLimitPrefixBoundary(const Columns & key_columns, size_t row) const +{ + for (size_t i = 0; i < limit_prefix_columns; ++i) + { + const auto & elem = group_by_description[i]; + size_t ind = elem.column_number; + if (res_key_columns[ind]->compareAt(cur_block_size - 1, row, *key_columns[ind], elem.base.nulls_direction) != 0) + return true; + } + return false; } AggregatingInOrderTransform::~AggregatingInOrderTransform() = default; @@ -214,9 +233,16 @@ void AggregatingInOrderTransform::consume(Chunk chunk) if (!group_by_key) params->aggregator.addSingleKeyToAggregateColumns(variants, res_aggregate_columns); + /// Enough groups have been emitted and the new key starts a new value of the + /// `ORDER BY` prefix, so the rest of the input cannot be needed (see `limit_prefix_columns`). + /// With `group_by_key`, `cur_block_size` counts runs of the sorted key columns rather than + /// groups; the groups accumulated for the current block are the entries of the hash table. + size_t cur_block_groups = group_by_key ? variants.size() : cur_block_size; + if (limit_hint && cur_block_groups + res_rows >= limit_hint && isLimitPrefixBoundary(key_columns, key_end)) + limit_reached = true; + /// If max_block_size or limit_hint is reached we have to stop consuming and generate the block. Save the extra rows into new chunk. - if (cur_block_size >= max_block_size || cur_block_bytes + current_memory_usage >= max_block_bytes - || (limit_hint && cur_block_size + res_rows >= limit_hint)) + if (cur_block_size >= max_block_size || cur_block_bytes + current_memory_usage >= max_block_bytes || limit_reached) { if (group_by_key) group_by_chunk @@ -225,9 +251,8 @@ void AggregatingInOrderTransform::consume(Chunk chunk) /// When limit is reached, don't save leftover rows — we're done. /// Set block_end_reached + need_generate to trigger generate(), - /// which will produce the output and then set is_consume_finished - /// via the limit check after res_rows is updated. - if (limit_hint && cur_block_size + res_rows >= limit_hint) + /// which will produce the output and then set is_consume_finished. + if (limit_reached) { block_end_reached = true; need_generate = true; @@ -416,8 +441,8 @@ void AggregatingInOrderTransform::generate() res_rows += to_push_chunk.getNumRows(); need_generate = false; - /// If we have emitted enough groups, stop consuming more input. - if (limit_hint && res_rows >= limit_hint) + /// If we have emitted enough groups up to a boundary of the `ORDER BY` prefix, stop consuming more input. + if (limit_reached) is_consume_finished = true; } diff --git a/src/Processors/Transforms/AggregatingInOrderTransform.h b/src/Processors/Transforms/AggregatingInOrderTransform.h index 28b312c607d4..e308f30187a0 100644 --- a/src/Processors/Transforms/AggregatingInOrderTransform.h +++ b/src/Processors/Transforms/AggregatingInOrderTransform.h @@ -35,6 +35,7 @@ class AggregatingInOrderTransform final : public IProcessor ManyAggregatedDataPtr many_data, size_t current_variant, size_t limit_hint_, + size_t limit_prefix_columns_, RuntimeDataflowStatisticsCacheUpdaterPtr dataflow_cache_updater_); AggregatingInOrderTransform( @@ -45,6 +46,7 @@ class AggregatingInOrderTransform final : public IProcessor size_t max_block_size_, size_t max_block_bytes_, size_t limit_hint_, + size_t limit_prefix_columns_, RuntimeDataflowStatisticsCacheUpdaterPtr dataflow_cache_updater_); ~AggregatingInOrderTransform() override; @@ -61,6 +63,9 @@ class AggregatingInOrderTransform final : public IProcessor private: void generate(); void finalizeCurrentChunk(Chunk chunk, size_t key_end); + /// Whether the key at `row` of `key_columns` differs from the current key in the first + /// `limit_prefix_columns` columns, i.e. whether a boundary of the `ORDER BY` prefix is reached. + bool isLimitPrefixBoundary(const Columns & key_columns, size_t row) const; size_t max_block_size; size_t max_block_bytes; @@ -89,7 +94,14 @@ class AggregatingInOrderTransform final : public IProcessor UInt64 src_bytes = 0; UInt64 res_rows = 0; + /// The stream may stop early once it has emitted `limit_hint` groups, but only at a + /// boundary of the first `limit_prefix_columns` columns of `group_by_description`, which + /// are the columns the query is ordered by. Otherwise a group tied on those columns with + /// the emitted ones could be cut off here while another stream emits its partial state, + /// and the final sort could pick that partial group. size_t limit_hint = 0; + size_t limit_prefix_columns = 0; + bool limit_reached = false; bool need_generate = false; bool block_end_reached = false; diff --git a/src/Storages/MergeTree/MergeTask.cpp b/src/Storages/MergeTree/MergeTask.cpp index 74f4d50043e7..7138a2685bb3 100644 --- a/src/Storages/MergeTree/MergeTask.cpp +++ b/src/Storages/MergeTree/MergeTask.cpp @@ -758,7 +758,31 @@ bool MergeTask::ExecuteAndFinalizeHorizontalPart::prepare() const hasLightweightDelete(global_ctx->future_part) || global_ctx->merging_params.mode != MergeTreeData::MergingParams::Ordinary; - prepareProjectionsToMergeAndRebuild(); + /// For TTLDrop merges, all source parts are fully expired. + /// Skip creating the read pipeline to avoid opening source parts + /// and allocating read/prefetch buffers. + /// + /// We restrict this to tables that have only an unconditional rows TTL + /// (no column TTL, moves, recompression, GROUP BY, or WHERE-clause TTL). + /// When other TTL families are present, TTLTransform::finalize rebuilds + /// their maps from scratch, and replicating that logic here would be + /// fragile. hasOnlyRowsTTL already excludes WHERE-clause TTLs. + /// + /// A merge cancelled after selection has `need_remove_expired_values` cleared above and must + /// not drop rows, so it falls through to the normal pipeline, which builds no TTLTransform. + const bool can_short_circuit_ttl_drop = + global_ctx->future_part->merge_type == MergeType::TTLDrop + && global_ctx->metadata_snapshot->hasOnlyRowsTTL() + && ctx->need_remove_expired_values; + + /// The short-circuit below commits a 0-row part without ever running a pipeline, so nothing + /// would retire these projections. Decide before the bookkeeping rather than undoing it + /// after: `prepareProjectionsToMergeAndRebuild` increments `MergedProjections` and + /// `RebuiltProjections` and pushes the names into `MergeListElement::projections_pending`, + /// which only the rebuild and merge paths erase from. Clearing the worklists afterwards left + /// those names in `system.merges.projections_remaining` for the lifetime of the merge entry. + if (!can_short_circuit_ttl_drop) + prepareProjectionsToMergeAndRebuild(); const auto & merge_tree_settings = global_ctx->data_settings; @@ -1036,19 +1060,6 @@ bool MergeTask::ExecuteAndFinalizeHorizontalPart::prepare() const && !use_const_adaptive_granularity && global_ctx->chosen_merge_algorithm == MergeAlgorithm::Vertical; - /// For TTLDrop merges, all source parts are fully expired. - /// Skip creating the read pipeline to avoid opening source parts - /// and allocating read/prefetch buffers. - /// - /// We restrict this to tables that have only an unconditional rows TTL - /// (no column TTL, moves, recompression, GROUP BY, or WHERE-clause TTL). - /// When other TTL families are present, TTLTransform::finalize rebuilds - /// their maps from scratch, and replicating that logic here would be - /// fragile. hasOnlyRowsTTL already excludes WHERE-clause TTLs. - const bool can_short_circuit_ttl_drop = - global_ctx->future_part->merge_type == MergeType::TTLDrop - && global_ctx->metadata_snapshot->hasOnlyRowsTTL(); - if (can_short_circuit_ttl_drop) { LOG_DEBUG(ctx->log, "TTLDrop merge: skipping data pipeline, " @@ -1063,10 +1074,11 @@ bool MergeTask::ExecuteAndFinalizeHorizontalPart::prepare() const global_ctx->new_data_part->ttl_infos = {}; global_ctx->new_data_part->ttl_infos.table_ttl = {0, 0, true}; - /// Clear projections — no rows means no projection data to merge or rebuild. - global_ctx->projections_to_rebuild.clear(); - global_ctx->projections_to_merge.clear(); - global_ctx->projections_to_merge_parts.clear(); + /// No rows means no projection data to merge or rebuild, and the bookkeeping that would + /// have announced some was skipped above, so there is nothing to undo here. + chassert(global_ctx->projections_to_rebuild.empty()); + chassert(global_ctx->projections_to_merge.empty()); + chassert(global_ctx->projections_to_merge_parts.empty()); /// Force Horizontal algorithm. This prevents the Vertical stage from trying /// to finalize an empty rows_sources file, and ensures finalizePart takes @@ -1090,15 +1102,27 @@ bool MergeTask::ExecuteAndFinalizeHorizontalPart::prepare() const global_ctx->skip_indexes_by_column.clear(); global_ctx->text_indexes_to_merge.clear(); - auto all_skip_indexes = global_ctx->metadata_snapshot->getSecondaryIndices(); - for (const auto & index : all_skip_indexes) + /// Repopulate only when the setting asks for it: the clear above this branch already + /// honoured `materialize_skip_indexes_on_merge = 0`, and putting the indexes back would + /// override the user's choice on a TTLDrop merge only. + if ((*merge_tree_settings)[MergeTreeSetting::materialize_skip_indexes_on_merge]) { - if (!exclude_index_names.contains(index.name)) + auto all_skip_indexes = global_ctx->metadata_snapshot->getSecondaryIndices(); + for (const auto & index : all_skip_indexes) { - if (index.type == "text") - global_ctx->text_indexes_to_merge.push_back(index); - else - global_ctx->merging_skip_indexes.push_back(index); + if (!exclude_index_names.contains(index.name)) + { + /// Inert indices (a removed index type kept only for attach compatibility) hold + /// no data and cannot be recomputed. Skip them so the merge does not wedge + /// trying to aggregate them. + if (MergeTreeIndexFactory::instance().get(global_ctx->metadata_snapshot, index, *global_ctx->data_settings)->isInert()) + continue; + + if (index.type == "text") + global_ctx->text_indexes_to_merge.push_back(index); + else + global_ctx->merging_skip_indexes.push_back(index); + } } } } @@ -2478,7 +2502,12 @@ bool MergeTask::MergeTextIndexStage::prepare() const auto index_ptr = MergeTreeIndexFactory::instance().get(global_ctx->metadata_snapshot, index, *global_ctx->data_settings); std::vector segments; - if (global_ctx->merge_may_reduce_rows) + if (global_ctx->ttl_drop_short_circuit) + { + /// No read pipeline ran, so no transform built segments. The resulting part has no + /// rows, which is exactly what a 0-row pipeline would have produced anyway. + } + else if (global_ctx->merge_may_reduce_rows) { /// Text index was built for the resulting part. segments = getTextIndexSegments(global_ctx->new_data_part->name, index.name, 0); diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index df436b80aa27..bd6cfcd0c1ed 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -462,6 +462,17 @@ namespace FailPoints /// transient error (e.g. temporary disk unavailability). Used to test that the refresh task /// reschedules itself after such an error instead of stopping permanently. extern const char merge_tree_refresh_parts_throw_once[]; + /// Throws a retryable error (`MEMORY_LIMIT_EXCEEDED`) while loading every outdated part in the + /// background. Used to test that the loading is retried later instead of terminating the server. + extern const char merge_tree_load_outdated_parts_retryable_error[]; + /// Pauses every worker that loads an outdated part in the background until the failpoint is disabled. + /// Used to cancel the loading (e.g. with `DETACH TABLE`) while the workers are in flight. + extern const char merge_tree_load_outdated_parts_pause[]; +} + +namespace ErrorCodes +{ + extern const int MEMORY_LIMIT_EXCEEDED; } static String getPartNameFromAST(const ASTPtr & partition) @@ -3530,10 +3541,42 @@ try /// Acquire shared lock because 'relative_data_path' is used while loading parts. TableLockHolder shared_lock; if (is_async) - shared_lock = lockForShare(RWLockImpl::NO_QUERY, (*getSettings())[MergeTreeSetting::lock_acquire_timeout_for_background_operations]); + { + shared_lock = tryLockForShare(RWLockImpl::NO_QUERY, (*getSettings())[MergeTreeSetting::lock_acquire_timeout_for_background_operations]); + if (!shared_lock) + { + /// The table is being dropped, detached or restarted (e.g. by `SYSTEM RESTART REPLICA`). + /// It is not an inconsistency of the set of parts, so retry later instead of terminating: + /// if the table is going away, the task is deactivated in `shutdown` anyway. + LOG_DEBUG(log, "Cannot lock the table to load outdated data parts because it is being dropped or restarted, will retry later"); + outdated_data_parts_loading_task->scheduleAfter(loading_parts_max_backoff_ms); + return; + } + } std::atomic_size_t num_loaded_parts = 0; + /// A retryable error (e.g. not enough memory or a network error) is not a sign of an inconsistent + /// set of parts, so the parts that failed with it are put back to the queue and loaded later. + /// + /// The workers collect them under a dedicated mutex and must not touch `outdated_data_parts_mutex`: + /// the cancellation branch below waits for all the workers while holding that mutex, + /// so a worker that takes it before its future becomes ready would deadlock + /// `DETACH`, `DROP`, shutdown and `SYSTEM RESTART REPLICA`. The failed parts are returned + /// to `outdated_unloaded_data_parts` only after all the workers have finished. + std::mutex failed_parts_mutex; + PartLoadingTreeNodes failed_parts; + std::exception_ptr retryable_exception; + std::atomic_bool has_retryable_exception = false; + + /// Must be called after `runner.waitForAllToFinishAndRethrowFirstError()` and under `outdated_data_parts_mutex`. + auto requeue_failed_parts = [&]() TSA_REQUIRES(outdated_data_parts_mutex) + { + std::lock_guard lock(failed_parts_mutex); + outdated_unloaded_data_parts.insert(outdated_unloaded_data_parts.end(), failed_parts.begin(), failed_parts.end()); + failed_parts.clear(); + }; + auto blocker = CannotAllocateThreadFaultInjector::blockFaultInjections(); ThreadPoolCallbackRunnerLocal runner(getOutdatedPartsLoadingThreadPool().get(), ThreadName::MERGETREE_LOAD_OUTDATED_PARTS); @@ -3552,6 +3595,7 @@ try /// Wait for every scheduled task /// In case of any exception it will be re-thrown and server will be terminated. runner.waitForAllToFinishAndRethrowFirstError(); + requeue_failed_parts(); LOG_DEBUG(log, "Stopped loading outdated data parts because task was canceled. " @@ -3559,22 +3603,47 @@ try return; } - if (outdated_unloaded_data_parts.empty()) + /// Do not start loading the remaining parts if the loading is going to be retried later anyway. + if (outdated_unloaded_data_parts.empty() || has_retryable_exception) break; part = outdated_unloaded_data_parts.back(); outdated_unloaded_data_parts.pop_back(); } - /// num_loaded_parts will outlive runner, so capturing by reference is ok - runner.enqueueAndKeepTrack([this, my_part = part, &num_loaded_parts, replicated]() + /// The captured locals will outlive runner, so capturing by reference is ok + runner.enqueueAndKeepTrack([this, my_part = part, &num_loaded_parts, &failed_parts_mutex, &failed_parts, &retryable_exception, &has_retryable_exception, replicated]() { auto blocker_for_runner_thread = CannotAllocateThreadFaultInjector::blockFaultInjections(); - auto res = loadDataPartWithRetries( - my_part->info, my_part->name, my_part->disk, - DataPartState::Outdated, data_parts_mutex, loading_parts_initial_backoff_ms, - loading_parts_max_backoff_ms, loading_parts_max_tries); + LoadPartResult res; + try + { + FailPointInjection::pauseFailPoint(FailPoints::merge_tree_load_outdated_parts_pause); + + fiu_do_on(FailPoints::merge_tree_load_outdated_parts_retryable_error, + { + throw Exception(ErrorCodes::MEMORY_LIMIT_EXCEEDED, "Injected retryable error while loading outdated part {}", my_part->name); + }); + + res = loadDataPartWithRetries( + my_part->info, my_part->name, my_part->disk, + DataPartState::Outdated, data_parts_mutex, loading_parts_initial_backoff_ms, + loading_parts_max_backoff_ms, loading_parts_max_tries); + } + catch (...) + { + if (!isRetryableException(std::current_exception())) + throw; + + /// The part is not added to the set of parts if the loading failed, so it can be loaded again from scratch. + std::lock_guard lock(failed_parts_mutex); + failed_parts.push_back(my_part); + if (!retryable_exception) + retryable_exception = std::current_exception(); + has_retryable_exception = true; + return; + } ++num_loaded_parts; if (res.is_broken) @@ -3591,6 +3660,28 @@ try runner.waitForAllToFinishAndRethrowFirstError(); + /// All the workers have finished, so no synchronization is needed to read `retryable_exception`. + if (has_retryable_exception) + { + size_t num_unloaded_parts = 0; + { + std::lock_guard lock(outdated_data_parts_mutex); + requeue_failed_parts(); + num_unloaded_parts = outdated_unloaded_data_parts.size(); + } + + /// Synchronous loading (on table drop) has no task to retry with, so it fails fast as before. + if (!is_async) + std::rethrow_exception(retryable_exception); + + LOG_WARNING(log, "Loading of outdated data parts was interrupted by a retryable error, will retry later. " + "Loaded {} parts, {} left unloaded. Error: {}", + num_loaded_parts.load(), num_unloaded_parts, getExceptionMessage(retryable_exception, /*with_stacktrace=*/ false)); + + outdated_data_parts_loading_task->scheduleAfter(loading_parts_max_backoff_ms); + return; + } + LOG_DEBUG(log, "Loaded {} outdated data parts {}", num_loaded_parts.load(), is_async ? "asynchronously" : "synchronously"); @@ -7318,12 +7409,25 @@ MergeTreeData::getColumnDefaultnessStats(const String & column_name, ContextPtr return std::nullopt; } + auto metadata_snapshot = getInMemoryMetadataPtr(query_context, /*bypass_metadata_cache=*/ false); + auto column_in_metadata = metadata_snapshot->getColumns().tryGetPhysical(column_name); + if (!column_in_metadata) + return std::nullopt; + ColumnDefaultnessStats aggregate; for (const auto & part : getActivePartsForColumnDefaultnessStats(query_context)) { if (part->isEmpty()) continue; + /// A metadata-only `MODIFY COLUMN` (e.g. `UInt64` -> `Nullable(UInt64)`) does not rewrite the part, + /// so its `num_defaults` counts defaults of the old type while reads return the new type. + if (part->getColumnsDescription().tryGetPhysical(column_name) != column_in_metadata) + { + LOG_DEBUG(log, "No defaultness stats for column {}: type in part {} differs from the type in metadata", column_name, part->name); + return std::nullopt; + } + const auto & infos = part->getSerializationInfos(); auto it = infos.find(column_name); if (it == infos.end()) diff --git a/src/Storages/MergeTree/MergeTreeDataPartChecksum.cpp b/src/Storages/MergeTree/MergeTreeDataPartChecksum.cpp index 33b16fa8d1a1..e8f417f8f9dd 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartChecksum.cpp +++ b/src/Storages/MergeTree/MergeTreeDataPartChecksum.cpp @@ -112,6 +112,34 @@ UInt64 MergeTreeDataPartChecksums::getTotalSizeUncompressedOnDisk() const return res; } +namespace +{ + +/// File names in checksums.txt are used verbatim to build paths inside the part directory +/// (e.g. when checking sizes or removing the part), so they must not escape it. +void assertFileNameIsRelativeAndContained(const String & name) +{ + if (name.empty()) + throw Exception(ErrorCodes::UNEXPECTED_FILE_IN_DATA_PART, "Empty file name in checksums of data part"); + + if (name.starts_with('/')) + throw Exception(ErrorCodes::UNEXPECTED_FILE_IN_DATA_PART, "Absolute file name '{}' in checksums of data part", name); + + /// A NUL byte would be kept inside a single path component here, but the local disk layer + /// passes the joined path to C APIs which truncate at the first NUL, so "..\0/x" would act as "..". + if (name.contains('\0')) + throw Exception(ErrorCodes::UNEXPECTED_FILE_IN_DATA_PART, "File name '{}' in checksums of data part contains a NUL byte", name); + + for (const auto & component : std::filesystem::path(name)) + { + if (component == "." || component == "..") + throw Exception(ErrorCodes::UNEXPECTED_FILE_IN_DATA_PART, + "File name '{}' in checksums of data part contains '{}' path component", name, component.string()); + } +} + +} + bool MergeTreeDataPartChecksums::read(ReadBuffer & in, size_t format_version) { switch (format_version) @@ -155,6 +183,7 @@ bool MergeTreeDataPartChecksums::readV2(ReadBuffer & in) Checksum sum; readString(name, in); + assertFileNameIsRelativeAndContained(name); assertString("\n\tsize: ", in); readText(sum.file_size, in); assertString("\n\thash: ", in); @@ -192,6 +221,7 @@ bool MergeTreeDataPartChecksums::readV3(ReadBuffer & in) Checksum sum; readStringBinary(name, in); + assertFileNameIsRelativeAndContained(name); readVarUInt(sum.file_size, in); readBinaryLittleEndian(sum.file_hash, in); readBinaryLittleEndian(sum.is_compressed, in); diff --git a/src/Storages/MergeTree/MergeTreeIndexConditionText.cpp b/src/Storages/MergeTree/MergeTreeIndexConditionText.cpp index 8f341ba083f5..548148d12bd7 100644 --- a/src/Storages/MergeTree/MergeTreeIndexConditionText.cpp +++ b/src/Storages/MergeTree/MergeTreeIndexConditionText.cpp @@ -152,31 +152,35 @@ MergeTreeIndexConditionText::MergeTreeIndexConditionText( return; } + /// Plain LRU: the caches live for one query and readers sweep the segments in task order. + /// Not using SLRU, because it would only pin stale entries. + static constexpr auto cache_policy = "LRU"; + /// Local caches: ~10% of the query memory budget, capped at 200 MiB. + static constexpr size_t local_cache_size_cap = 200ULL * 1024 * 1024; + const auto & settings = context_->getSettingsRef(); - static constexpr auto cache_policy = "SLRU"; - /// Local caches: ~10% of the query memory budget, capped at 100 MiB; max_memory_usage == 0 (unlimited) uses the cap, not a 0-size cache. - static constexpr size_t local_cache_size_cap = 100ULL * 1024 * 1024; const size_t query_memory_limit = settings[Setting::max_memory_usage]; - const size_t local_cache_max_size - = query_memory_limit == 0 ? local_cache_size_cap : std::min(query_memory_limit / 10, local_cache_size_cap); + const size_t local_cache_max_size = query_memory_limit == 0 + ? local_cache_size_cap + : std::min(query_memory_limit / 10, local_cache_size_cap); /// If usage of global text index caches is disabled, create local /// one to share them between threads that read the same data parts. if (settings[Setting::use_text_index_tokens_cache]) tokens_cache = context_->getTextIndexTokensCache(); else - tokens_cache = std::make_shared(cache_policy, local_cache_max_size, 0, 1.0); + tokens_cache = std::make_shared(cache_policy, local_cache_max_size, 0, /*size_ratio=*/ 0.0); use_global_header_cache = settings[Setting::use_text_index_header_cache]; if (use_global_header_cache) header_cache = context_->getTextIndexHeaderCache(); else - header_cache = std::make_shared(cache_policy, local_cache_max_size, 0, 1.0); + header_cache = std::make_shared(cache_policy, local_cache_max_size, 0, /*size_ratio=*/ 0.0); if (settings[Setting::use_text_index_postings_cache]) postings_cache = context_->getTextIndexPostingsCache(); else - postings_cache = std::make_shared(cache_policy, local_cache_max_size, 0, 1.0); + postings_cache = std::make_shared(cache_policy, local_cache_max_size, 0, /*size_ratio=*/ 0.0); rpn = std::move(RPNBuilder( predicate, diff --git a/src/Storages/MergeTree/MergeTreeReaderWide.cpp b/src/Storages/MergeTree/MergeTreeReaderWide.cpp index 6785042a07c5..71bd9c2aeaa1 100644 --- a/src/Storages/MergeTree/MergeTreeReaderWide.cpp +++ b/src/Storages/MergeTree/MergeTreeReaderWide.cpp @@ -332,7 +332,7 @@ ReadBuffer * MergeTreeReaderWide::getStream( ISerialization::SubstreamsCache & cache) { /// If substream have already been read. - if (cache.contains(ISerialization::getSubcolumnNameForStream(substream_path))) + if (cache.contains(ISerialization::getSubstreamsCacheKeyForStream(substream_path))) return nullptr; auto stream_name = IMergeTreeDataPart::getStreamNameForColumn(name_and_type, substream_path, ".bin", checksums, storage_settings); diff --git a/src/Storages/NATS/StorageNATS.cpp b/src/Storages/NATS/StorageNATS.cpp index 43913a8fc5f4..5e2f139a9c76 100644 --- a/src/Storages/NATS/StorageNATS.cpp +++ b/src/Storages/NATS/StorageNATS.cpp @@ -1196,7 +1196,8 @@ void registerStorageNATS(StorageFactory & factory) else if (!args.storage_def->settings) throw Exception(ErrorCodes::BAD_ARGUMENTS, "NATS engine must have settings"); - nats_settings->loadFromQuery(*args.storage_def); + if (args.storage_def->settings) + nats_settings->loadFromQuery(*args.storage_def); /// A credential source assigned in the `SETTINGS` clause is query-level even when the named /// collection provides the same key: the clause is applied on top of the collection values, diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergIterator.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergIterator.cpp index 3ccc4248d9c8..8f9d6e285ad3 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergIterator.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergIterator.cpp @@ -36,6 +36,7 @@ #include #include #include +#include #include #include @@ -54,6 +55,7 @@ #include #include #include +#include #include #include @@ -81,7 +83,8 @@ extern const int ICEBERG_SPECIFICATION_VIOLATION; namespace Setting { extern const SettingsBool use_iceberg_partition_pruning; -extern const SettingsNonZeroUInt64 iceberg_delete_manifest_decode_concurrency; +extern const SettingsNonZeroUInt64 iceberg_file_entries_queue_size; +extern const SettingsNonZeroUInt64 iceberg_manifest_decode_concurrency; }; @@ -90,8 +93,8 @@ using namespace Iceberg; namespace { -/// All entries of one delete manifest file, produced by a single decode task. -using DeleteManifestBatch = std::vector; +/// All entries of one manifest file, produced by a single decode task. +using ManifestEntryBatch = std::vector; /// decode path agrees on whether pruning applies. std::shared_ptr makeManifestFilterDag(const ActionsDAG * filter_dag, const ContextPtr & context) @@ -180,114 +183,151 @@ std::span defineDeletesSpan( } -std::optional SingleThreadIcebergKeysIterator::next() +namespace Iceberg { - if (!data_snapshot) + +DataFileEntriesStream::DataFileEntriesStream( + size_t queue_size_, + size_t decode_concurrency_, + IcebergDataSnapshotPtr data_snapshot_, + std::function prepare_, + CreateManifestIterator create_manifest_iterator_) + : chunk_size(queue_size_) + , decode_concurrency(decode_concurrency_) + , data_snapshot(std::move(data_snapshot_)) + , prepare(std::move(prepare_)) + , create_manifest_iterator(std::move(create_manifest_iterator_)) + , queue(queue_size_) +{ + producer = std::make_unique( + [this, thread_group = CurrentThread::getGroup()]() + { + DB::ThreadGroupSwitcher switcher(thread_group, DB::ThreadName::ICEBERG_ITERATOR); + try + { + run(); + } + catch (...) + { + std::lock_guard lock(exception_mutex); + if (!exception) + { + exception = std::current_exception(); + } + } + stop(); + }); +} + +DataFileEntriesStream::~DataFileEntriesStream() +{ + stop(); + if (producer) { - return std::nullopt; + producer->join(); } +} - while (true) - { - /// Try to get the next entry from the current manifest file iterator. - if (current_manifest_file_iterator) - { - auto entry = current_manifest_file_iterator->next(); - if (entry) - return entry; +bool DataFileEntriesStream::pop(ProcessedManifestFileEntryPtr & entry) +{ + return queue.pop(entry); +} - /// next() returned nullptr, meaning the manifest file is fully exhausted. - current_manifest_file_iterator = nullptr; - } +void DataFileEntriesStream::clearAndFinish() +{ + stopped.store(true, std::memory_order_relaxed); + queue.clearAndFinish(); +} - /// Make sure a prefetch of the next matching manifest is in flight. - schedulePrefetchIfPossible(); - if (!prefetched_manifest.has_value()) - return std::nullopt; - - /// Take ownership of the in-flight prefetch: move it out of the member and reset the - /// optional (a moved-from optional is still engaged, so schedulePrefetchIfPossible would - /// otherwise see an occupied slot and never start the next fetch). - auto curr_prefetched = std::move(prefetched_manifest); - prefetched_manifest.reset(); - /// While the future lives only in this local, the destructor's wait no longer covers it, - /// so the task capturing `this` could outlive the iterator if anything below throws - /// (for example scheduling the next prefetch while the pool is shutting down). - SCOPE_EXIT({ - if (curr_prefetched.has_value() && curr_prefetched->future.valid()) - curr_prefetched->future.wait(); - }); +void DataFileEntriesStream::stop() +{ + stopped.store(true, std::memory_order_relaxed); + queue.finish(); +} - /// Start scheduling the next prefetch before we block and parse. - schedulePrefetchIfPossible(); - - auto manifest_file_cacheable_part = curr_prefetched->future.get(); - const auto & manifest_list_entry = data_snapshot->manifest_list_entries[curr_prefetched->manifest_list_index]; - - current_manifest_file_iterator = Iceberg::ManifestFileIterator::create( - manifest_file_cacheable_part.deserializer, - manifest_list_entry.manifest_file_path, - persistent_components.path_resolver, - *persistent_components.schema_processor, - manifest_list_entry.added_sequence_number, - manifest_list_entry.added_snapshot_id, - local_context, - filter_dag, - table_snapshot->schema_id); - } +std::exception_ptr DataFileEntriesStream::getException() const +{ + std::lock_guard lock(exception_mutex); + return exception; } -void SingleThreadIcebergKeysIterator::schedulePrefetchIfPossible() +void DataFileEntriesStream::run() { - if (!data_snapshot || prefetched_manifest.has_value()) + if (!data_snapshot) return; - while (manifest_file_index < data_snapshot->manifest_list_entries.size()) + if (prepare) + prepare(); + + auto stream_runner = threadPoolCallbackRunnerUnsafe(getIcebergManifestDecodeThreadPool().get(), DB::ThreadName::ICEBERG_ITERATOR); + + std::deque> in_flight; + SCOPE_EXIT({ + for (auto & manifest : in_flight) + { + if (manifest->future.valid()) + manifest->future.wait(); + } + }); + + const auto & manifest_list_entries = data_snapshot->manifest_list_entries; + size_t next_index = 0; + while (!stopped.load(std::memory_order_relaxed)) { - const size_t index = manifest_file_index++; - const auto & manifest_list_entry = data_snapshot->manifest_list_entries[index]; - if (manifest_list_entry.content_type != manifest_file_content_type) - continue; - - auto fetch = [this, - path = manifest_list_entry.manifest_file_path, - bytes = manifest_list_entry.manifest_file_byte_size]() + while (in_flight.size() < decode_concurrency && next_index < manifest_list_entries.size()) { - return Iceberg::getManifestFile(object_storage, persistent_components, local_context, log, path, bytes); - }; - prefetched_manifest = PrefetchedManifest{index, prefetch_runner(std::move(fetch), Priority{})}; - return; + const size_t index = next_index++; + if (manifest_list_entries[index].content_type != ManifestFileContentType::DATA) + continue; + auto manifest = std::make_unique(manifest_list_entries[index]); + auto * scheduled = manifest.get(); + manifest->future = stream_runner([this, scheduled] { decodeChunk(*scheduled); }, Priority{}); + in_flight.push_back(std::move(manifest)); + } + + if (in_flight.empty()) + return; + + auto & manifest = *in_flight.front(); + manifest.future.get(); + + for (auto & entry : manifest.chunk) + { + if (!queue.push(std::move(entry))) + return; + } + manifest.chunk.clear(); + + if (manifest.exhausted) + in_flight.pop_front(); + else + manifest.future = stream_runner([this, scheduled = &manifest] { decodeChunk(*scheduled); }, Priority{}); } } -SingleThreadIcebergKeysIterator::~SingleThreadIcebergKeysIterator() +void DataFileEntriesStream::decodeChunk(InFlightManifest & manifest) { - /// The scheduled task captures `this`, so it must not outlive the iterator. - if (prefetched_manifest.has_value() && prefetched_manifest->future.valid()) - prefetched_manifest->future.wait(); + if (stopped.load(std::memory_order_relaxed)) + { + manifest.exhausted = true; + return; + } + + if (!manifest.iterator) + manifest.iterator = create_manifest_iterator(manifest.key, &stopped); + + while (manifest.chunk.size() < chunk_size) + { + auto entry = manifest.iterator->next(); + if (!entry) + { + manifest.exhausted = true; + return; + } + manifest.chunk.push_back(std::move(entry)); + } } -SingleThreadIcebergKeysIterator::SingleThreadIcebergKeysIterator( - ObjectStoragePtr object_storage_, - ContextPtr local_context_, - Iceberg::ManifestFileContentType manifest_file_content_type_, - const ActionsDAG * filter_dag_, - Iceberg::TableStateSnapshotPtr table_snapshot_, - Iceberg::IcebergDataSnapshotPtr data_snapshot_, - PersistentTableComponents persistent_components_) - : object_storage(object_storage_) - , filter_dag(makeManifestFilterDag(filter_dag_, local_context_)) - , local_context(local_context_) - , table_snapshot(table_snapshot_) - , data_snapshot(data_snapshot_) - , persistent_components(persistent_components_) - , log(getLogger("IcebergIterator")) - , manifest_file_content_type(manifest_file_content_type_) - , prefetch_runner(threadPoolCallbackRunnerUnsafe( - getIOThreadPool().get(), DB::ThreadName::ICEBERG_ITERATOR)) -{ - /// Warm the first manifest fetch. - schedulePrefetchIfPossible(); } IcebergIterator::IcebergIterator( @@ -304,65 +344,30 @@ IcebergIterator::IcebergIterator( , table_state_snapshot(table_snapshot_) , data_snapshot(data_snapshot_) , persistent_components(persistent_components_) - , deletes_filter_dag(makeManifestFilterDag(filter_dag_, local_context_)) - , data_files_iterator( - object_storage, - local_context_, - Iceberg::ManifestFileContentType::DATA, - filter_dag_, - table_snapshot_, - data_snapshot_, - persistent_components_) - , blocking_queue(100) + , manifest_filter_dag(makeManifestFilterDag(filter_dag_, local_context_)) , callback(std::move(callback_)) , table_schema_id(table_snapshot_->schema_id) { - /// Decoding any manifest reads settings from the context, so a missing one is fatal either way. - if (!local_context) - throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "Context is required to construct IcebergIterator"); + chassert(local_context); - producer_task = std::make_unique( - [this, thread_group = CurrentThread::getGroup()]() + data_files_stream = std::make_unique( + local_context->getSettingsRef()[Setting::iceberg_file_entries_queue_size], + local_context->getSettingsRef()[Setting::iceberg_manifest_decode_concurrency], + data_snapshot, + [this] { - DB::ThreadGroupSwitcher switcher(thread_group, DB::ThreadName::ICEBERG_ITERATOR); - while (!blocking_queue.isFinished()) - { - std::optional entry; - try - { - entry = data_files_iterator.next(); - } - catch (...) - { - std::lock_guard lock(exception_mutex); - if (!exception) - { - exception = std::current_exception(); - } - blocking_queue.finish(); - break; - } - if (!entry.has_value()) - break; - while (!blocking_queue.push(std::move(entry.value()))) - { - if (blocking_queue.isFinished()) - { - break; - } - } - } - blocking_queue.finish(); - }); + if (manifest_filter_dag) + VirtualColumnUtils::buildOrderedSetsForDAG(*manifest_filter_dag, local_context); + }, + [this](const ManifestFileCacheKey & manifest_list_entry, const std::atomic * stop_flag) + { return createManifestIterator(manifest_list_entry, stop_flag); }); } -// ensureDeletesReady is called lazily so that delete and data manifest downloads overlap. void IcebergIterator::ensureDeletesReady() { std::lock_guard lock(deletes_mutex); if (!deletes_ready) { - /// Deferred part of the iterator initialization, charged to the same event as the constructor. ProfileEventTimeIncrement watch(ProfileEvents::IcebergIteratorInitializationMicroseconds); try { @@ -371,8 +376,7 @@ void IcebergIterator::ensureDeletesReady() catch (...) { deletes_exception = std::current_exception(); - /// Every `next` rethrows this exception, so the buffered entries are dead and the producer must stop. - blocking_queue.clearAndFinish(); + data_files_stream->clearAndFinish(); } deletes_ready = true; } @@ -381,6 +385,43 @@ void IcebergIterator::ensureDeletesReady() std::rethrow_exception(deletes_exception); } +Iceberg::ManifestIteratorPtr IcebergIterator::createManifestIterator(const ManifestFileCacheKey & manifest_list_entry, const std::atomic * stop_flag) const +{ + auto manifest_file_cacheable_part = Iceberg::getManifestFile( + object_storage, + persistent_components, + local_context, + logger, + manifest_list_entry.manifest_file_path, + manifest_list_entry.manifest_file_byte_size); + + return Iceberg::ManifestFileIterator::create( + manifest_file_cacheable_part.deserializer, + manifest_list_entry.manifest_file_path, + persistent_components.path_resolver, + *persistent_components.schema_processor, + manifest_list_entry.added_sequence_number, + manifest_list_entry.added_snapshot_id, + local_context, + manifest_filter_dag, + table_state_snapshot->schema_id, + stop_flag); +} + +std::vector IcebergIterator::decodeManifest(const ManifestFileCacheKey & manifest_list_entry, const std::atomic * stop_flag) const +{ + if (stop_flag && stop_flag->load(std::memory_order_relaxed)) + return {}; + + auto manifest_file_iterator = createManifestIterator(manifest_list_entry, stop_flag); + + ManifestEntryBatch batch; + while (auto entry = manifest_file_iterator->next()) + batch.push_back(entry); + /// Iterator and deserializer die here, before the batch is handed over. + return batch; +} + void IcebergIterator::decodeDeleteManifests() { std::vector delete_manifests; @@ -394,12 +435,12 @@ void IcebergIterator::decodeDeleteManifests() } /// Cap concurrency: each in-flight manifest holds its decoded contents. - const size_t max_in_flight = local_context->getSettingsRef()[Setting::iceberg_delete_manifest_decode_concurrency]; + const size_t max_in_flight = local_context->getSettingsRef()[Setting::iceberg_manifest_decode_concurrency]; auto decode_runner - = threadPoolCallbackRunnerUnsafe(getIOThreadPool().get(), DB::ThreadName::ICEBERG_ITERATOR); + = threadPoolCallbackRunnerUnsafe(getIOThreadPool().get(), DB::ThreadName::ICEBERG_DELETE_DECODE); - std::deque> in_flight; + std::deque> in_flight; /// The tasks capture `this`, so none of them may still be running when this function is left. SCOPE_EXIT({ for (auto & future : in_flight) @@ -415,32 +456,7 @@ void IcebergIterator::decodeDeleteManifests() while (in_flight.size() < max_in_flight && next_to_decode < delete_manifests.size()) { auto decode = [this, manifest_list_entry = delete_manifests[next_to_decode++]]() - { - auto manifest_file_cacheable_part = Iceberg::getManifestFile( - object_storage, - persistent_components, - local_context, - logger, - manifest_list_entry.manifest_file_path, - manifest_list_entry.manifest_file_byte_size); - - auto manifest_file_iterator = Iceberg::ManifestFileIterator::create( - manifest_file_cacheable_part.deserializer, - manifest_list_entry.manifest_file_path, - persistent_components.path_resolver, - *persistent_components.schema_processor, - manifest_list_entry.added_sequence_number, - manifest_list_entry.added_snapshot_id, - local_context, - deletes_filter_dag, - table_state_snapshot->schema_id); - - DeleteManifestBatch batch; - while (auto entry = manifest_file_iterator->next()) - batch.push_back(entry); - /// Iterator and deserializer die here, before the batch is handed over. - return batch; - }; + { return decodeManifest(manifest_list_entry, /* stop_flag */ nullptr); }; in_flight.push_back(decode_runner(std::move(decode), Priority{})); } @@ -478,7 +494,7 @@ ObjectInfoPtr IcebergIterator::next(size_t) ProfileEventTimeIncrement watch(ProfileEvents::IcebergMetadataReadWaitTimeMicroseconds); ensureDeletesReady(); Iceberg::ProcessedManifestFileEntryPtr manifest_file_entry; - if (blocking_queue.pop(manifest_file_entry)) + if (data_files_stream->pop(manifest_file_entry)) { IcebergDataObjectInfoPtr object_info = std::make_shared( @@ -623,14 +639,11 @@ ObjectInfoPtr IcebergIterator::next(size_t) return object_info; } + if (auto exception = data_files_stream->getException()) { - std::lock_guard lock(exception_mutex); - if (exception) - { - auto exception_message = getExceptionMessage(exception, true, true); - auto exception_code = getExceptionErrorCode(exception); - throw DB::Exception(exception_code, "Iceberg iterator is failed with exception: {}", exception_message); - } + auto exception_message = getExceptionMessage(exception, true, true); + auto exception_code = getExceptionErrorCode(exception); + throw DB::Exception(exception_code, "Iceberg iterator is failed with exception: {}", exception_message); } return nullptr; @@ -641,14 +654,7 @@ size_t IcebergIterator::estimatedKeysCount() return std::numeric_limits::max(); } -IcebergIterator::~IcebergIterator() -{ - blocking_queue.finish(); - if (producer_task) - { - producer_task->join(); - } -} +IcebergIterator::~IcebergIterator() = default; } #endif diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergIterator.h b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergIterator.h index 6b27c8a37c81..15d85487e786 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergIterator.h +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergIterator.h @@ -17,10 +17,11 @@ #include #include -#include +#include +#include #include -#include +#include #include #include @@ -35,45 +36,54 @@ namespace DB namespace Iceberg { -class SingleThreadIcebergKeysIterator +class DataFileEntriesStream { public: - SingleThreadIcebergKeysIterator( - ObjectStoragePtr object_storage_, - ContextPtr local_context_, - Iceberg::ManifestFileContentType manifest_file_content_type_, - const ActionsDAG * filter_dag_, - TableStateSnapshotPtr table_snapshot_, + using CreateManifestIterator = std::function *)>; + + DataFileEntriesStream( + size_t queue_size_, + size_t decode_concurrency_, IcebergDataSnapshotPtr data_snapshot_, - PersistentTableComponents persistent_components); + std::function prepare_, + CreateManifestIterator create_manifest_iterator_); - ~SingleThreadIcebergKeysIterator(); + ~DataFileEntriesStream(); - std::optional next(); + bool pop(ProcessedManifestFileEntryPtr & entry); + void clearAndFinish(); + std::exception_ptr getException() const; private: - void schedulePrefetchIfPossible(); - - ObjectStoragePtr object_storage; - std::shared_ptr filter_dag; - ContextPtr local_context; - Iceberg::TableStateSnapshotPtr table_snapshot; - Iceberg::IcebergDataSnapshotPtr data_snapshot; - PersistentTableComponents persistent_components; - LoggerPtr log; - - size_t manifest_file_index = 0; - Iceberg::ManifestIteratorPtr current_manifest_file_iterator; - - const Iceberg::ManifestFileContentType manifest_file_content_type; - - struct PrefetchedManifest + struct InFlightManifest { - size_t manifest_list_index; - std::future future; + explicit InFlightManifest(ManifestFileCacheKey key_) + : key(std::move(key_)) + { + } + + ManifestFileCacheKey key; + ManifestIteratorPtr iterator; + std::vector chunk; + bool exhausted = false; + std::future future; }; - std::optional prefetched_manifest; - ThreadPoolCallbackRunnerUnsafe prefetch_runner; + + void run(); + void decodeChunk(InFlightManifest & manifest); + void stop(); + + const size_t chunk_size; + const size_t decode_concurrency; + const IcebergDataSnapshotPtr data_snapshot; + + const std::function prepare; + const CreateManifestIterator create_manifest_iterator; + ConcurrentBoundedQueue queue; + std::atomic stopped{false}; + mutable std::mutex exception_mutex; + std::exception_ptr exception TSA_GUARDED_BY(exception_mutex); + std::unique_ptr producer; }; } @@ -98,6 +108,8 @@ class IcebergIterator : public IObjectIterator private: void ensureDeletesReady(); void decodeDeleteManifests(); + Iceberg::ManifestIteratorPtr createManifestIterator(const ManifestFileCacheKey & manifest_list_entry, const std::atomic * stop_flag) const; + std::vector decodeManifest(const ManifestFileCacheKey & manifest_list_entry, const std::atomic * stop_flag) const; LoggerPtr logger; ObjectStoragePtr object_storage; @@ -105,10 +117,8 @@ class IcebergIterator : public IObjectIterator const Iceberg::TableStateSnapshotPtr table_state_snapshot; Iceberg::IcebergDataSnapshotPtr data_snapshot; Iceberg::PersistentTableComponents persistent_components; - std::shared_ptr deletes_filter_dag; - Iceberg::SingleThreadIcebergKeysIterator data_files_iterator; - ConcurrentBoundedQueue blocking_queue; - std::unique_ptr producer_task; + /// Shared read-only by the concurrent data- and delete-manifest decode tasks. + std::shared_ptr manifest_filter_dag; IDataLakeMetadata::FileProgressCallback callback; /// Filled once under `deletes_mutex` and never mutated afterwards, so `next` may read them /// unguarded once it has gone through `ensureDeletesReady`. @@ -118,9 +128,10 @@ class IcebergIterator : public IObjectIterator std::mutex deletes_mutex; bool deletes_ready TSA_GUARDED_BY(deletes_mutex) = false; std::exception_ptr deletes_exception TSA_GUARDED_BY(deletes_mutex); - std::exception_ptr exception; - std::mutex exception_mutex; Int32 table_schema_id; + /// Declared last: its tasks call back into `createManifestIterator`, so it must be destroyed + /// (producer joined, tasks drained) before any other member. + std::unique_ptr data_files_stream; }; } diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFileIterator.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFileIterator.cpp index cecfc51c3c4f..9f9b153c5e0e 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFileIterator.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFileIterator.cpp @@ -151,7 +151,8 @@ std::shared_ptr ManifestFileIterator::create( Int64 inherited_snapshot_id_, DB::ContextPtr context_, std::shared_ptr filter_dag_, - Int32 table_snapshot_schema_id_) + Int32 table_snapshot_schema_id_, + const std::atomic * stop_flag_) { auto dump_metadata = [&]()->String { return manifest_file_deserializer_->getMetadataContent(); }; insertRowToLogTable( @@ -255,7 +256,8 @@ std::shared_ptr ManifestFileIterator::create( std::move(partition_key_description), total_rows, std::move(filter_dag_), - table_snapshot_schema_id_)); + table_snapshot_schema_id_, + stop_flag_)); } ManifestFileIterator::ManifestFileIterator( @@ -272,7 +274,8 @@ ManifestFileIterator::ManifestFileIterator( std::optional partition_key_description_, size_t total_rows_, std::shared_ptr filter_dag_, - Int32 table_snapshot_schema_id_) + Int32 table_snapshot_schema_id_, + const std::atomic * stop_flag_) : manifest_file_deserializer(std::move(manifest_file_deserializer_)) , path_to_manifest_file(path_to_manifest_file_) , format_version(format_version_) @@ -285,6 +288,7 @@ ManifestFileIterator::ManifestFileIterator( , partition_key_description(std::move(partition_key_description_)) , table_snapshot_schema_id(table_snapshot_schema_id_) , total_rows(total_rows_) + , stop_flag(stop_flag_) , data_files_without_deleted(std::make_shared>()) , position_deletes_files_without_deleted(std::make_shared>()) , equality_deletes_files_without_deleted(std::make_shared>()) @@ -536,6 +540,11 @@ ProcessedManifestFileEntryPtr ManifestFileIterator::next() fully_initialized.store(true); return nullptr; } + /// The data manifest decode tasks pass the stream's stopped flag here, so a cancelled + /// query stops decoding mid-manifest. Checked between rows rather than by the caller, + /// because a long stretch of pruned rows yields nothing the caller could check on. + if (stop_flag && stop_flag->load(std::memory_order_relaxed)) + return nullptr; auto entry = processRow(row_index); if (entry) return entry; diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFileIterator.h b/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFileIterator.h index 229ef01cc888..b65b0db66582 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFileIterator.h +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFileIterator.h @@ -11,6 +11,8 @@ #include #include +#include +#include #include #include @@ -92,7 +94,8 @@ class ManifestFileIterator : public boost::noncopyable Int64 inherited_snapshot_id, DB::ContextPtr context, std::shared_ptr filter_dag_, - Int32 table_snapshot_schema_id_); + Int32 table_snapshot_schema_id_, + const std::atomic * stop_flag = nullptr); ManifestFileEntriesHandle getFilesWithoutDeletedHandle() const; @@ -131,7 +134,8 @@ class ManifestFileIterator : public boost::noncopyable std::optional partition_key_description, size_t total_rows, std::shared_ptr filter_dag, - Int32 table_snapshot_schema_id); + Int32 table_snapshot_schema_id, + const std::atomic * stop_flag); ProcessedManifestFileEntryPtr processRow(size_t row_index); @@ -151,6 +155,8 @@ class ManifestFileIterator : public boost::noncopyable /// Iteration state const size_t total_rows; + /// When set and observed true, `next` gives up between rows and returns nullptr as on EOF. + const std::atomic * const stop_flag; std::atomic current_row_index{0}; std::atomic fully_initialized{false}; std::atomic active_fetchers{0}; diff --git a/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp b/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp index 1bed63dba006..77046d76f4ec 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp +++ b/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp @@ -98,6 +98,11 @@ namespace CurrentMetrics namespace DB { +namespace FailPoints +{ +extern const char object_storage_source_pause_before_virtual_columns[]; +} + namespace ErrorCodes { extern const int CANNOT_COMPILE_REGEXP; @@ -699,6 +704,8 @@ Chunk StorageObjectStorageSource::generate() else if (object_metadata->is_size_known) object_size = object_metadata->size_bytes; + FailPointInjection::pauseFailPoint(FailPoints::object_storage_source_pause_before_virtual_columns); + VirtualColumnUtils::addRequestedFileLikeStorageVirtualsToChunk( chunk, read_from_format_info.requested_virtual_columns, diff --git a/src/Storages/SelectQueryInfo.h b/src/Storages/SelectQueryInfo.h index 150e34dee225..a86ca8b23ce9 100644 --- a/src/Storages/SelectQueryInfo.h +++ b/src/Storages/SelectQueryInfo.h @@ -212,6 +212,8 @@ struct SelectQueryInfo // If not 0, that means it's a trivial limit query. UInt64 trivial_limit = 0; + /// A trivial limit query whose rows `arrayJoin` expands: the source must not stop at the limit, but should read small. + bool small_limit_above_array_join = false; /// For IStorageSystemOneBlock std::vector columns_mask; diff --git a/src/Storages/StorageFile.cpp b/src/Storages/StorageFile.cpp index 61cea1c2f7e5..955dfbc8b60a 100644 --- a/src/Storages/StorageFile.cpp +++ b/src/Storages/StorageFile.cpp @@ -345,6 +345,16 @@ std::string getTablePath(const std::string & table_dir_path, const std::string & return table_dir_path + "/data." + escapeForFileName(format_name); } +/// Every syscall a path is passed to (`stat`, `open`, `opendir`) stops at the first NUL byte, while the +/// containment checks see the whole value. A path with an embedded NUL therefore addresses a location the +/// checks never look at, so it is rejected before the first filesystem probe. The path is not echoed in the +/// message: it would put the NUL byte into the logs. +void throwIfPathContainsEmbeddedNul(const std::string & path) +{ + if (path.contains('\0')) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "File path contains an embedded NUL byte"); +} + /// Both db_dir_path and table_path must be converted to absolute paths (in particular, path cannot contain '..'). void checkCreationIsAllowed( ContextPtr context_global, @@ -352,6 +362,8 @@ void checkCreationIsAllowed( const std::string & table_path, bool can_be_directory) { + throwIfPathContainsEmbeddedNul(table_path); + if (context_global->getApplicationType() != Context::ApplicationType::SERVER) return; @@ -399,6 +411,10 @@ std::pair splitToArchivePathAndPathInArchive(const String & sour /// Finds files matching a specified pattern with globs. Strings getPathsList(const String & path_with_globs, const String & user_files_path, const ContextPtr & context, size_t & total_bytes_to_read) { + /// The listing below stats the path and expands its globs on the filesystem before the matched paths + /// are checked for containment in `user_files_path`, so the path must be rejected here, not there. + throwIfPathContainsEmbeddedNul(path_with_globs); + fs::path user_files_absolute_path = fs::weakly_canonical(user_files_path); fs::path fs_pattern(path_with_globs); if (fs_pattern.is_relative()) diff --git a/src/Storages/StorageLog.cpp b/src/Storages/StorageLog.cpp index dd5ceb02d120..9fc213141e70 100644 --- a/src/Storages/StorageLog.cpp +++ b/src/Storages/StorageLog.cpp @@ -304,7 +304,7 @@ void LogSource::readPrefix(const NameAndTypePair & name_and_type, ISerialization ISerialization::DeserializeBinaryBulkSettings settings; settings.getter = [&](const ISerialization::SubstreamPath & path) -> ReadBuffer * { - if (cache.contains(ISerialization::getSubcolumnNameForStream(path))) + if (cache.contains(ISerialization::getSubstreamsCacheKeyForStream(path))) return nullptr; String data_file_name = ISerialization::getFileNameForStream(name_and_type, path, {}); @@ -333,7 +333,7 @@ void LogSource::readData(const NameAndTypePair & name_and_type, MutableColumnPtr settings.getter = [&] (const ISerialization::SubstreamPath & path) -> ReadBuffer * { - if (cache.contains(ISerialization::getSubcolumnNameForStream(path))) + if (cache.contains(ISerialization::getSubstreamsCacheKeyForStream(path))) return nullptr; String data_file_name = ISerialization::getFileNameForStream(name_and_type, path, {}); diff --git a/src/Storages/StorageMergeTreeCodecBlockCounts.cpp b/src/Storages/StorageMergeTreeCodecBlockCounts.cpp index b15f13cf0a24..a02ea2d0473f 100644 --- a/src/Storages/StorageMergeTreeCodecBlockCounts.cpp +++ b/src/Storages/StorageMergeTreeCodecBlockCounts.cpp @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -19,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -296,25 +298,39 @@ void ReadFromMergeTreeCodecBlockCounts::initializePipeline(QueryPipelineBuilder } StorageMergeTreeCodecBlockCounts::StorageMergeTreeCodecBlockCounts( - const StorageID & table_id_, StoragePtr source_table_, const ColumnsDescription & columns_) + const StorageID & table_id_, StorageID source_table_id_, const ColumnsDescription & columns_) : IStorage(table_id_) - , source_table(std::move(source_table_)) + , source_table_id(std::move(source_table_id_)) { - const auto * merge_tree = dynamic_cast(source_table.get()); - if (!merge_tree) - throw Exception( - ErrorCodes::BAD_ARGUMENTS, "Storage MergeTreeCodecBlockCounts expected MergeTree table, got: {}", source_table->getName()); - - /// `system.parts_columns` lists patch parts, so this function does too. - data_parts = merge_tree->getDataPartsVectorForInternalUsage( - {MergeTreeData::DataPartState::Active}, {MergeTreeData::DataPartKind::Regular, MergeTreeData::DataPartKind::Patch}); - std::erase_if(data_parts, [](const MergeTreeData::DataPartPtr & part) { return part->isEmpty(); }); - StorageInMemoryMetadata storage_metadata; storage_metadata.setColumns(columns_); setInMemoryMetadata(storage_metadata); } +void StorageMergeTreeCodecBlockCounts::checkSourceTableAccess(const StoragePtr & source_table, const ContextPtr & context) +{ + const auto source_metadata = source_table->getInMemoryMetadataPtr(context, false); + context->checkAccess(AccessType::SELECT, source_table->getStorageID(), source_metadata->getColumns().getNamesOfPhysical()); +} + +StoragePtr StorageMergeTreeCodecBlockCounts::resolveSourceTable(const StorageID & source_table_id, const ContextPtr & context) +{ + /// `SHOW TABLES` is the privilege that governs whether the table's existence may be learned, and it is implied + /// by a grant on any single column of it, so this only adds a tier below the `SELECT` check on every column + /// that follows the resolution. `SHOW COLUMNS`, which `DESCRIBE` of the source table requires, is not implied + /// by column-level grants, so it would reject a user who holds `SELECT` on every column separately. + context->checkAccess(AccessType::SHOW_TABLES, source_table_id); + + auto source_table = DatabaseCatalog::instance().getTable(source_table_id, context); + checkSourceTableAccess(source_table, context); + + if (!dynamic_cast(source_table.get())) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, "Table function mergeTreeCodecBlockCounts expected MergeTree table, got: {}", source_table->getName()); + + return source_table; +} + void StorageMergeTreeCodecBlockCounts::read( QueryPlan & query_plan, const Names & column_names, @@ -327,9 +343,18 @@ void StorageMergeTreeCodecBlockCounts::read( { storage_snapshot->check(column_names); - const auto source_metadata = source_table->getInMemoryMetadataPtr(context, false); + /// Under the reader's context, see the constructor. + auto source_table = resolveSourceTable(source_table_id, context); const auto source_storage_id = source_table->getStorageID(); - context->checkAccess(AccessType::SELECT, source_storage_id, source_metadata->getColumns().getNamesOfPhysical()); + + /// A cast to a base class, so not `assert_cast`, which asserts the exact type and would reject every + /// `MergeTree` table. `resolveSourceTable` has already rejected a source table that is not a `MergeTree`. + const auto & merge_tree = dynamic_cast(*source_table); + + /// `system.parts_columns` lists patch parts, so this function does too. + auto data_parts = merge_tree.getDataPartsVectorForInternalUsage( + {MergeTreeData::DataPartState::Active}, {MergeTreeData::DataPartKind::Regular, MergeTreeData::DataPartKind::Patch}); + std::erase_if(data_parts, [](const MergeTreeData::DataPartPtr & part) { return part->isEmpty(); }); auto sample_block = std::make_shared(storage_snapshot->getSampleBlockForColumns(column_names)); @@ -353,7 +378,7 @@ void StorageMergeTreeCodecBlockCounts::read( query_plan.addStep( std::make_unique( - column_names, query_info, storage_snapshot, std::move(context), std::move(sample_block), data_parts, source_storage_id)); + column_names, query_info, storage_snapshot, std::move(context), std::move(sample_block), std::move(data_parts), source_storage_id)); } } diff --git a/src/Storages/StorageMergeTreeCodecBlockCounts.h b/src/Storages/StorageMergeTreeCodecBlockCounts.h index 0a07928f9cc2..e8852112a0ae 100644 --- a/src/Storages/StorageMergeTreeCodecBlockCounts.h +++ b/src/Storages/StorageMergeTreeCodecBlockCounts.h @@ -1,7 +1,7 @@ #pragma once +#include #include -#include namespace DB { @@ -11,10 +11,24 @@ namespace DB class StorageMergeTreeCodecBlockCounts final : public IStorage { public: - StorageMergeTreeCodecBlockCounts(const StorageID & table_id_, StoragePtr source_table_, const ColumnsDescription & columns_); + /// Holds only the name of the source table. It is resolved and checked on every read, under the context of the + /// user who reads, so that the checks run where the data is disclosed rather than where the storage is built, + /// and so that the storage keeps no reference to the source table between reads. + StorageMergeTreeCodecBlockCounts(const StorageID & table_id_, StorageID source_table_id_, const ColumnsDescription & columns_); std::string getName() const override { return "MergeTreeCodecBlockCounts"; } + /// Every column of this function is derived from the source table's data, so reading any of them requires + /// `SELECT` on all of the source table's columns. Called both when the function's structure is resolved and + /// when it is read, so that resolving the structure cannot reveal anything about a table the user cannot select from. + static void checkSourceTableAccess(const StoragePtr & source_table, const ContextPtr & context); + + /// Resolves the source table for `context`'s user, in the order that discloses nothing the user may not learn: + /// `SHOW TABLES` on the name, before the catalog is consulted, so that an inaccessible table and a missing one + /// answer alike; then `SELECT` on every column, before the engine is examined, so that a user without it cannot + /// learn the engine from the `BAD_ARGUMENTS` that rejects a table that is not a `MergeTree`. + static StoragePtr resolveSourceTable(const StorageID & source_table_id, const ContextPtr & context); + void read( QueryPlan & query_plan, const Names & column_names, @@ -26,8 +40,7 @@ class StorageMergeTreeCodecBlockCounts final : public IStorage size_t num_streams) override; private: - StoragePtr source_table; - MergeTreeData::DataPartsVector data_parts; + StorageID source_table_id; }; } diff --git a/src/Storages/StorageMergeTreeTextIndex.cpp b/src/Storages/StorageMergeTreeTextIndex.cpp index 8c16dd595e1d..19db88c09619 100644 --- a/src/Storages/StorageMergeTreeTextIndex.cpp +++ b/src/Storages/StorageMergeTreeTextIndex.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -25,7 +26,6 @@ #include #include #include -#include namespace DB { @@ -361,7 +361,10 @@ void ReadFromMergeTreeTextIndex::applyFilters(ActionDAGNodes added_filter_nodes) void ReadFromMergeTreeTextIndex::initializePipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings &) { - auto filtered_parts = VirtualColumnUtils::filterDataPartsWithExpression(storage->data_parts, virtual_columns_filter); + /// Taken at read time: the storage outlives the query in a table created from the function before that was forbidden. + auto data_parts = dynamic_cast(*storage->source_table).getDataPartsVectorForInternalUsage(); + std::erase_if(data_parts, [](const MergeTreeData::DataPartPtr & part) { return part->isEmpty(); }); + auto filtered_parts = VirtualColumnUtils::filterDataPartsWithExpression(data_parts, virtual_columns_filter); if (filtered_parts.empty()) { @@ -407,13 +410,9 @@ StorageMergeTreeTextIndex::StorageMergeTreeTextIndex( , source_table(source_table_) , text_index(std::move(text_index_)) { - const auto * merge_tree = dynamic_cast(source_table.get()); - if (!merge_tree) + if (!dynamic_cast(source_table.get())) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Storage MergeTreeTextIndex expected MergeTree table, got: {}", source_table->getName()); - data_parts = merge_tree->getDataPartsVectorForInternalUsage(); - std::erase_if(data_parts, [](const MergeTreeData::DataPartPtr & part) { return part->isEmpty(); }); - StorageInMemoryMetadata storage_metadata; storage_metadata.setColumns(columns); storage_metadata.setVirtuals(createVirtuals()); @@ -428,6 +427,35 @@ VirtualColumnsDescription StorageMergeTreeTextIndex::createVirtuals() return desc; } +void StorageMergeTreeTextIndex::checkAccess(const ContextPtr & context, const StorageID & source_storage_id, const IMergeTreeIndex & index) +{ + /// The checks below are for the user who runs the query, so a shard of a distributed query may run it only as the + /// initiating user: authenticated by the interserver secret, or reached by `remote(...)` as the same user, which the + /// initiator confirms by pushing its roles (it does not when it rewrote the initial user to the connection user). + const auto & client_info = context->getClientInfo(); + const bool same_user = client_info.initial_user == client_info.current_user && client_info.current_roles.has_value(); + if ((client_info.query_kind == ClientInfo::QueryKind::SECONDARY_QUERY + || (client_info.distributed_depth > 0 && client_info.client_name == "ClickHouse server")) + && client_info.interface != ClientInfo::Interface::TCP_INTERSERVER && !same_user) + throw Exception(ErrorCodes::ACCESS_DENIED, + "Table function `mergeTreeTextIndex` checks the access of the user who runs the query, so a shard of a " + "distributed query can execute it only as the initiating user: through a cluster with an interserver secret, " + "or through `remote` with the credentials of that user"); + + context->checkAccess(AccessType::SELECT, source_storage_id, index.getColumnsRequiredForIndexCalc()); + + /// The index is built over all rows of a part, so it contains tokens of the rows a row policy hides, + /// regardless of which columns the policy filters on. The policy cannot be applied to the dictionary. + auto row_policy_filter = context->getRowPolicyFilter( + source_storage_id.getDatabaseName(), source_storage_id.getTableName(), RowPolicyFilterType::SELECT_FILTER); + + if (row_policy_filter && !row_policy_filter->isAlwaysTrue()) + throw Exception(ErrorCodes::ACCESS_DENIED, + "Cannot read from `mergeTreeTextIndex` because a row policy is applied on table {}. " + "The text index covers all rows of the table, so reading its tokens would violate the row policy", + source_storage_id.getNameForLogs()); +} + void StorageMergeTreeTextIndex::readImpl( QueryPlan & query_plan, const Names & column_names, @@ -438,30 +466,7 @@ void StorageMergeTreeTextIndex::readImpl( size_t max_block_size, size_t num_streams) { - auto source_storage_id = source_table->getStorageID(); - auto required_columns = text_index->getColumnsRequiredForIndexCalc(); - context->checkAccess(AccessType::SELECT, source_storage_id, required_columns); - /// If the row policy filter references any column required for building the index, - /// reading from the text index would expose tokens derived from those columnsand violate the row policy. - auto row_policy_filter = context->getRowPolicyFilter(source_storage_id.getDatabaseName(), source_storage_id.getTableName(), RowPolicyFilterType::SELECT_FILTER); - - if (row_policy_filter && !row_policy_filter->isAlwaysTrue()) - { - RequiredSourceColumnsVisitor::Data columns_context; - RequiredSourceColumnsVisitor(columns_context).visit(row_policy_filter->expression); - NameSet row_policy_columns = columns_context.requiredColumns(); - - for (const auto & column_name : required_columns) - { - if (row_policy_columns.contains(column_name)) - { - throw Exception(ErrorCodes::ACCESS_DENIED, - "Cannot read from `mergeTreeTextIndex` because a row policy on column `{}` " - "is applied on table {}. Reading text index tokens could violate the row policy", - column_name, source_storage_id.getNameForLogs()); - } - } - } + checkAccess(context, source_table->getStorageID(), *text_index); auto sample_block = std::make_shared(storage_snapshot->getSampleBlockForColumns(column_names)); auto this_ptr = std::static_pointer_cast(shared_from_this()); diff --git a/src/Storages/StorageMergeTreeTextIndex.h b/src/Storages/StorageMergeTreeTextIndex.h index 92245724d26e..f39db5f33dfa 100644 --- a/src/Storages/StorageMergeTreeTextIndex.h +++ b/src/Storages/StorageMergeTreeTextIndex.h @@ -34,12 +34,14 @@ class StorageMergeTreeTextIndex final : public StorageWithCommonVirtualColumns static VirtualColumnsDescription createVirtuals(); + /// Throws if the user may not read the tokens of the index of the table `source_storage_id`. + static void checkAccess(const ContextPtr & context, const StorageID & source_storage_id, const IMergeTreeIndex & index); + private: friend class ReadFromMergeTreeTextIndex; StoragePtr source_table; MergeTreeIndexPtr text_index; - MergeTreeData::DataPartsVector data_parts; }; } diff --git a/src/Storages/StorageQueryRunner.cpp b/src/Storages/StorageQueryRunner.cpp index 845b2bd814bb..e10d6f5d681d 100644 --- a/src/Storages/StorageQueryRunner.cpp +++ b/src/Storages/StorageQueryRunner.cpp @@ -442,7 +442,12 @@ class QueryRunnerDispatcher : WithContext void executeLocally(const QueryRunnerJob & job, ContextMutablePtr job_context) const { - auto io = executeQuery(job.query, job_context, QueryFlags{ .internal = true }).second; + /// The job is nested, hence `internal` - which is also what marks these queries with + /// `is_internal = 1` in `system.query_log`. Its text comes from the user who inserted it, + /// hence `user_initiated`: without it the access checks of `CREATE` jobs would be skipped, so + /// a job would not be limited to the privileges of the principal it runs as. + auto io + = executeQuery(job.query, job_context, QueryFlags{ .internal = true, .user_initiated = true }).second; try { if (io.pipeline.initialized()) diff --git a/src/Storages/TableZnodeInfo.h b/src/Storages/TableZnodeInfo.h index b75cfbf65cae..ff0c69ec681f 100644 --- a/src/Storages/TableZnodeInfo.h +++ b/src/Storages/TableZnodeInfo.h @@ -58,7 +58,9 @@ struct TableZnodeInfo /// `validate_substitutions` rejects a {database}/{table} value that would not stay a single safe /// ZooKeeper path component. It must be requested only for a freshly supplied definition: enabling /// it while merely re-deriving the path of an existing table (a short ATTACH, a Replicated-database - /// recovery replay, a RESTORE) would break that table. + /// recovery replay, a RESTORE) would break that table. Converting a MergeTree table to a + /// replicated engine counts as fresh: the path is minted from the server's template at + /// conversion time rather than read back from the table's own metadata. static TableZnodeInfo resolve( const String & requested_path, const String & requested_replica_name, const StorageID & table_id, const ASTCreateQuery & query, LoadingStrictnessLevel mode, diff --git a/src/TableFunctions/TableFunctionMergeTreeCodecBlockCounts.cpp b/src/TableFunctions/TableFunctionMergeTreeCodecBlockCounts.cpp index 854115b42309..16dc445c4fa2 100644 --- a/src/TableFunctions/TableFunctionMergeTreeCodecBlockCounts.cpp +++ b/src/TableFunctions/TableFunctionMergeTreeCodecBlockCounts.cpp @@ -4,9 +4,8 @@ #include #include #include -#include +#include #include -#include #include #include #include @@ -18,8 +17,8 @@ namespace DB namespace ErrorCodes { extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; -extern const int BAD_ARGUMENTS; extern const int LOGICAL_ERROR; +extern const int BAD_ARGUMENTS; } class TableFunctionMergeTreeCodecBlockCounts : public ITableFunction @@ -28,10 +27,22 @@ class TableFunctionMergeTreeCodecBlockCounts : public ITableFunction static constexpr auto name = "mergeTreeCodecBlockCounts"; std::string getName() const override { return name; } + /// Refused in line with the other `MergeTree` introspection table functions. This storage holds only the source + /// table's name and resolves it on every read, so it pins nothing, but the persisted form has no uses and is + /// refused for all of them alike, so that none of them has to stay correct while outliving its query. + void validateUseToCreateTable() const override + { + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Table function '{}' cannot be used to create a table", getName()); + } + void parseArguments(const ASTPtr & ast_function, ContextPtr context) override; ColumnsDescription getActualTableStructure(ContextPtr context, bool is_insert_query) const override; private: + /// The fixed structure. It is not derived from the source table, but the table function does not declare it as + /// static: resolving it is a read of the source table and has to check access to it, see `getActualTableStructure`. + static ColumnsDescription getColumns(); + StoragePtr executeImpl( const ASTPtr & ast_function, ContextPtr context, @@ -69,13 +80,15 @@ void TableFunctionMergeTreeCodecBlockCounts::parseArguments(const ASTPtr & ast_f ColumnsDescription TableFunctionMergeTreeCodecBlockCounts::getActualTableStructure(ContextPtr context, bool /*is_insert_query*/) const { - auto source_table = DatabaseCatalog::instance().getTable(source_table_id, context); - - const auto * merge_tree = dynamic_cast(source_table.get()); - if (!merge_tree) - throw Exception( - ErrorCodes::BAD_ARGUMENTS, "Table function {} expected MergeTree table, got: {}", getName(), source_table->getName()); + /// The structure is fixed, so nothing in it depends on the source table. The table is still resolved here, because + /// resolving the structure is a read of it and needs the same access as reading it: this is what `DESCRIBE` + /// goes through, under the context of the user who asks. + StorageMergeTreeCodecBlockCounts::resolveSourceTable(source_table_id, context); + return getColumns(); +} +ColumnsDescription TableFunctionMergeTreeCodecBlockCounts::getColumns() +{ auto codec_map = std::make_shared(std::make_shared(), std::make_shared()); ColumnsDescription columns; @@ -99,16 +112,17 @@ ColumnsDescription TableFunctionMergeTreeCodecBlockCounts::getActualTableStructu StoragePtr TableFunctionMergeTreeCodecBlockCounts::executeImpl( const ASTPtr & /*ast_function*/, - ContextPtr context, + ContextPtr /*context*/, const std::string & table_name, ColumnsDescription /*cached_columns*/, - bool is_insert_query) const + bool /*is_insert_query*/) const { - auto source_table = DatabaseCatalog::instance().getTable(source_table_id, context); - auto columns = getActualTableStructure(context, is_insert_query); - + /// Deliberately does not resolve the source table. Building the storage discloses nothing, because its structure + /// is fixed, and the storage may be built and read under different contexts (`EXPLAIN` builds it without reading). + /// The source is resolved and checked by `StorageMergeTreeCodecBlockCounts::read`, on the path that reads its data, + /// under the context of the user who reads. StorageID storage_id(getDatabaseName(), table_name); - auto res = std::make_shared(std::move(storage_id), std::move(source_table), std::move(columns)); + auto res = std::make_shared(std::move(storage_id), source_table_id, getColumns()); res->startup(); return res; @@ -125,6 +139,8 @@ Selecting `codec_block_counts` reads `.bin` data files, not just metadata. The o Parts that do not record their substreams in `columns_substreams.txt` are not listed. +Every reported value is derived from the table's data, so reading any column of the result requires the `SELECT` privilege on all columns of the table. A grant that covers only some of the columns is not enough. The privilege is also required to resolve the structure of the function, e.g. by `DESCRIBE`. A user who is not allowed to see the table at all, that is, one without the `SHOW TABLES` privilege on it, gets `ACCESS_DENIED` whether or not it exists, so the function does not tell such a user which tables exist. + If a row policy applies to the table for the current user, reading `codec_block_counts` throws `ACCESS_DENIED`, because the counts would cover rows the policy hides. The other columns stay readable, `system.parts_columns` reports them regardless of row policies. ## Syntax {#syntax} diff --git a/src/TableFunctions/TableFunctionMergeTreeTextIndex.cpp b/src/TableFunctions/TableFunctionMergeTreeTextIndex.cpp index 60a024e9a02f..35075b28af60 100644 --- a/src/TableFunctions/TableFunctionMergeTreeTextIndex.cpp +++ b/src/TableFunctions/TableFunctionMergeTreeTextIndex.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include #include @@ -10,13 +11,16 @@ #include #include #include +#include #include +#include namespace DB { namespace ErrorCodes { + extern const int ACCESS_DENIED; extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; extern const int BAD_ARGUMENTS; extern const int LOGICAL_ERROR; @@ -28,6 +32,13 @@ class TableFunctionMergeTreeTextIndex : public ITableFunction static constexpr auto name = "mergeTreeTextIndex"; std::string getName() const override { return name; } + /// The returned storage holds its source table's storage object, so a persisted table would keep the source undroppable. + /// A persisted definition would also resolve the source table under the global context or the engine credentials. + void validateUseToCreateTable() const override + { + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Table function '{}' cannot be used to create a table", getName()); + } + void parseArguments(const ASTPtr & ast_function, ContextPtr context) override; ColumnsDescription getActualTableStructure(ContextPtr context, bool is_insert_query) const override; @@ -39,6 +50,10 @@ class TableFunctionMergeTreeTextIndex : public ITableFunction ColumnsDescription cached_columns, bool is_insert_query) const override; + /// Resolves the index of the source table and checks that the user may read it. + std::pair resolveIndex(ContextPtr context) const; + static ColumnsDescription getColumns(); + const char * getStorageEngineName() const override { return ""; @@ -78,7 +93,7 @@ static std::shared_ptr getDictionaryCompressionType() return std::make_shared(std::move(values)); } -ColumnsDescription TableFunctionMergeTreeTextIndex::getActualTableStructure(ContextPtr, bool /*is_insert_query*/) const +ColumnsDescription TableFunctionMergeTreeTextIndex::getColumns() { return ColumnsDescription{{ {"part_name", std::make_shared()}, @@ -92,13 +107,19 @@ ColumnsDescription TableFunctionMergeTreeTextIndex::getActualTableStructure(Cont }}; } -StoragePtr TableFunctionMergeTreeTextIndex::executeImpl( - const ASTPtr & /*ast_function*/, - ContextPtr context, - const std::string & table_name, - ColumnsDescription /*cached_columns*/, - bool is_insert_query) const +std::pair TableFunctionMergeTreeTextIndex::resolveIndex(ContextPtr context) const { + /// A table persisted before that was forbidden resolves the function under the load context, which has no user. + if (!context->getUserID()) + { + context = CurrentThread::tryGetQueryContext(); + if (!context) + throw Exception(ErrorCodes::ACCESS_DENIED, "Table function 'mergeTreeTextIndex' cannot check the access of the user outside of a query"); + } + + /// Otherwise the errors below would reveal the engine and the indexes of a table the user cannot see. + context->checkAccess(AccessType::SHOW_TABLES, source_database, source_table); + auto source_table_ptr = DatabaseCatalog::instance().getTable(StorageID{source_database, source_table}, context); auto metadata_snapshot = source_table_ptr->getInMemoryMetadataPtr(context, false); const auto & index_desc = metadata_snapshot->getSecondaryIndices().getByName(source_index_name); @@ -114,14 +135,32 @@ StoragePtr TableFunctionMergeTreeTextIndex::executeImpl( throw Exception(ErrorCodes::BAD_ARGUMENTS, "Storage MergeTreeTextIndex expected MergeTree table, got: {}", source_table_ptr->getName()); auto text_index = MergeTreeIndexFactory::instance().get(metadata_snapshot, index_desc, *merge_tree->getSettings()); - auto columns = getActualTableStructure(context, is_insert_query); + StorageMergeTreeTextIndex::checkAccess(context, source_table_ptr->getStorageID(), *text_index); + return {std::move(source_table_ptr), std::move(text_index)}; +} + +ColumnsDescription TableFunctionMergeTreeTextIndex::getActualTableStructure(ContextPtr context, bool /*is_insert_query*/) const +{ + /// The structure is static, but this is where e.g. `remote` over a local shard checks the access of the user. + resolveIndex(context); + return getColumns(); +} + +StoragePtr TableFunctionMergeTreeTextIndex::executeImpl( + const ASTPtr & /*ast_function*/, + ContextPtr context, + const std::string & table_name, + ColumnsDescription /*cached_columns*/, + bool /*is_insert_query*/) const +{ + auto [source_table_ptr, text_index] = resolveIndex(context); StorageID storage_id(getDatabaseName(), table_name); auto res = std::make_shared( std::move(storage_id), std::move(source_table_ptr), std::move(text_index), - std::move(columns)); + getColumns()); res->startup(); return res; diff --git a/tests/docker_scripts/stress_runner.sh b/tests/docker_scripts/stress_runner.sh index 1fdaee27310e..0d013452f68a 100755 --- a/tests/docker_scripts/stress_runner.sh +++ b/tests/docker_scripts/stress_runner.sh @@ -97,7 +97,9 @@ if [ "$cache_policy" = "SLRU" ]; then sed -i.tmp "s|LRU|SLRU|" /etc/clickhouse-server/config.d/storage_conf*.xml fi -start_server || { echo "Failed to start server"; exit 1; } +# Preload writes system logs the restart must load before port 9000 opens. +# The default wait (~70s) expires first on some ARM runners. +start_server 10 || { echo "Failed to start server"; exit 1; } clickhouse-client --query "SYSTEM STOP THREAD FUZZER" @@ -302,7 +304,9 @@ fi # hang the server under sanitizers and trip the hung check. cp -av --dereference /repo/ci/jobs/scripts/fuzzer/limit-recursion-settings.xml /etc/clickhouse-server/users.d/ -start_server || { echo "Failed to start server"; exit 1; } +# Same wait as the other restarts: ARM sanitizer + S3 + async_load_databases=false +# can miss the default ~70s window before port 9000 opens. +start_server 10 || { echo "Failed to start server"; exit 1; } # clickhouse-test must know which storage backend the server actually uses, or its storage skip # tags are inert and incompatible tests run on an unsupported backend. Both variables are already diff --git a/tests/integration/compose/docker_compose_iceberg_rest_catalog_with_trino.yml b/tests/integration/compose/docker_compose_iceberg_rest_catalog_with_trino.yml index 8a4729847812..49f4dea0800b 100644 --- a/tests/integration/compose/docker_compose_iceberg_rest_catalog_with_trino.yml +++ b/tests/integration/compose/docker_compose_iceberg_rest_catalog_with_trino.yml @@ -20,7 +20,9 @@ services: stop_grace_period: 5s cpus: 3 minio: - image: quay.io/minio/minio:RELEASE.2024-07-31T05-46-26Z + # Mirror of quay.io/minio/minio:RELEASE.2024-07-31T05-46-26Z (see compose/mirror-images.sh). + # The minio/minio Docker Hub repository was deleted upstream; quay.io is not behind the proxy. + image: clickhouse/minio-minio:RELEASE.2024-07-31T05-46-26Z environment: - MINIO_ROOT_USER=minio - MINIO_ROOT_PASSWORD=ClickHouse_Minio_P@ssw0rd @@ -38,7 +40,9 @@ services: mc: depends_on: - minio - image: quay.io/minio/mc:RELEASE.2025-04-16T18-13-26Z + # Mirror of quay.io/minio/mc:RELEASE.2025-04-16T18-13-26Z (see compose/mirror-images.sh). + # The minio/mc Docker Hub repository was deleted upstream; quay.io is not behind the proxy. + image: clickhouse/minio-mc:RELEASE.2025-04-16T18-13-26Z environment: - AWS_ACCESS_KEY_ID=minio - AWS_SECRET_ACCESS_KEY=ClickHouse_Minio_P@ssw0rd diff --git a/tests/integration/compose/docker_compose_minio.yml b/tests/integration/compose/docker_compose_minio.yml index f37c461b549b..6142f47552b6 100644 --- a/tests/integration/compose/docker_compose_minio.yml +++ b/tests/integration/compose/docker_compose_minio.yml @@ -1,6 +1,8 @@ services: minio1: - image: quay.io/minio/minio:RELEASE.2024-09-13T20-26-02Z + # Mirror of quay.io/minio/minio:RELEASE.2024-09-13T20-26-02Z (see compose/mirror-images.sh). + # The minio/minio Docker Hub repository was deleted upstream; quay.io is not behind the proxy. + image: clickhouse/minio-minio:RELEASE.2024-09-13T20-26-02Z volumes: - data1-1:/data1 - ${MINIO_CERTS_DIR:-}:/certs diff --git a/tests/integration/compose/mirror-images.sh b/tests/integration/compose/mirror-images.sh index 839fe27ac603..cfe83ab43a54 100755 --- a/tests/integration/compose/mirror-images.sh +++ b/tests/integration/compose/mirror-images.sh @@ -4,10 +4,12 @@ # Why: CI runners pull Docker Hub images through the dockerhub-proxy cache # (registry:2 + nginx, backed by S3 — see tests/ci/terraform/dockerhub-proxy.md). # That proxy only fronts Docker Hub. Images hosted on other registries -# (mcr.microsoft.com, ghcr.io) bypass the proxy and are pulled directly, so CI is -# exposed to those registries' anonymous rate limits (e.g. mcr.microsoft.com +# (mcr.microsoft.com, ghcr.io, quay.io) bypass the proxy and are pulled directly, so +# CI is exposed to those registries' anonymous rate limits (e.g. mcr.microsoft.com # returns HTTP 429 "toomanyrequests" under load). Re-hosting them under # clickhouse/ routes the pulls back through the proxy + S3 cache. +# A mirror also outlives the upstream repository: minio/minio and minio/mc were +# deleted from Docker Hub, and quay.io is the only remaining source of those tags. # # Usage: log in to Docker Hub with an account that can push to the clickhouse org, # then run this script. It is idempotent — re-run it to add images or bump versions. @@ -22,6 +24,9 @@ IMAGES=( "ghcr.io/ytsaurus/local:stable-24.2 clickhouse/ytsaurus-local:stable-24.2" "ghcr.io/letsencrypt/pebble:2.9.0 clickhouse/letsencrypt-pebble:2.9.0" "ghcr.io/letsencrypt/pebble-challtestsrv:2.9.0 clickhouse/letsencrypt-pebble-challtestsrv:2.9.0" + "quay.io/minio/minio:RELEASE.2024-09-13T20-26-02Z clickhouse/minio-minio:RELEASE.2024-09-13T20-26-02Z" + "quay.io/minio/minio:RELEASE.2024-07-31T05-46-26Z clickhouse/minio-minio:RELEASE.2024-07-31T05-46-26Z" + "quay.io/minio/mc:RELEASE.2025-04-16T18-13-26Z clickhouse/minio-mc:RELEASE.2025-04-16T18-13-26Z" ) for entry in "${IMAGES[@]}"; do diff --git a/tests/integration/test_accept_invalid_certificate/configs/ssl_config_ca_signed.xml b/tests/integration/test_accept_invalid_certificate/configs/ssl_config_ca_signed.xml new file mode 100644 index 000000000000..b1dccbb15c66 --- /dev/null +++ b/tests/integration/test_accept_invalid_certificate/configs/ssl_config_ca_signed.xml @@ -0,0 +1,15 @@ + + + 9440 + + + + /etc/clickhouse-server/config.d/client-cert.pem + /etc/clickhouse-server/config.d/client-key.pem + /etc/clickhouse-server/config.d/ca-cert.pem + none + + + diff --git a/tests/integration/test_accept_invalid_certificate/test.py b/tests/integration/test_accept_invalid_certificate/test.py index afea8011c4b9..b66f311b9235 100644 --- a/tests/integration/test_accept_invalid_certificate/test.py +++ b/tests/integration/test_accept_invalid_certificate/test.py @@ -9,6 +9,7 @@ SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) MAX_RETRY = 5 +CA_CERT = f"{SCRIPT_DIR}/certs/ca-cert.pem" cluster = ClickHouseCluster(__file__) instance = cluster.add_instance( @@ -35,6 +36,18 @@ ) +node2 = cluster.add_instance( + "node2", + main_configs=[ + "configs/ssl_config_ca_signed.xml", + "certs/client-key.pem", + "certs/client-cert.pem", + "certs/ca-cert.pem", + ], + with_zookeeper=False, +) + + @pytest.fixture(scope="module", autouse=True) def started_cluster(): try: @@ -61,6 +74,36 @@ def started_cluster(): """ +# node2 presents a certificate this CA signs, so the chain is valid and only the name can fail. +config_ca_signed = """ + + + {caConfig} + false + + +""" + +config_ca_signed_sni_override = """ + client + + + {caConfig} + false + + +""" + +config_ca_signed_no_extended_verification = """ + + + {caConfig} + false + false + + +""" + def execute_query_native(node, query, config): fd, config_path = tempfile.mkstemp( @@ -132,3 +175,29 @@ def test_strict_connection_reject(): config_connection_accept.format(ip_address=f"{instance.ip_address}"), ) assert "certificate verify failed" in str(err.value) + + +def test_hostname_mismatch_rejected_by_default(): + with pytest.raises(Exception) as err: + execute_query_native(node2, "SELECT 1", config_ca_signed.format(caConfig=CA_CERT)) + assert "Unacceptable certificate" in str(err.value) + + +def test_hostname_match_accepted(): + assert ( + execute_query_native( + node2, "SELECT 1", config_ca_signed_sni_override.format(caConfig=CA_CERT) + ) + == "1\n" + ) + + +def test_extended_verification_disabled(): + assert ( + execute_query_native( + node2, + "SELECT 1", + config_ca_signed_no_extended_verification.format(caConfig=CA_CERT), + ) + == "1\n" + ) diff --git a/tests/integration/test_acme_tls/configs/config.xml b/tests/integration/test_acme_tls/configs/config.xml index 4594c72e885e..0977ed12d711 100644 --- a/tests/integration/test_acme_tls/configs/config.xml +++ b/tests/integration/test_acme_tls/configs/config.xml @@ -14,6 +14,9 @@ + + false AcceptCertificateHandler diff --git a/tests/integration/test_acme_tls/configs/config_multi.xml b/tests/integration/test_acme_tls/configs/config_multi.xml index 56c9d415886c..9cda6eb0ba87 100644 --- a/tests/integration/test_acme_tls/configs/config_multi.xml +++ b/tests/integration/test_acme_tls/configs/config_multi.xml @@ -14,6 +14,9 @@ + + false AcceptCertificateHandler diff --git a/tests/integration/test_bind_host/configs/config.d/ssl_conf.xml b/tests/integration/test_bind_host/configs/config.d/ssl_conf.xml index c11c5736d96f..e51749f35438 100644 --- a/tests/integration/test_bind_host/configs/config.d/ssl_conf.xml +++ b/tests/integration/test_bind_host/configs/config.d/ssl_conf.xml @@ -7,6 +7,9 @@ /etc/clickhouse-server/config.d/server.key + + false AcceptCertificateHandler diff --git a/tests/integration/test_database_hdfs_read_grant/__init__.py b/tests/integration/test_database_hdfs_read_grant/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tests/integration/test_database_hdfs_read_grant/configs/allowlist.xml b/tests/integration/test_database_hdfs_read_grant/configs/allowlist.xml new file mode 100644 index 000000000000..b6ed5de3a79a --- /dev/null +++ b/tests/integration/test_database_hdfs_read_grant/configs/allowlist.xml @@ -0,0 +1,5 @@ + + + allowed.example.com:9000 + + diff --git a/tests/integration/test_database_hdfs_read_grant/test.py b/tests/integration/test_database_hdfs_read_grant/test.py new file mode 100644 index 000000000000..2a6f1b0d531e --- /dev/null +++ b/tests/integration/test_database_hdfs_read_grant/test.py @@ -0,0 +1,101 @@ +"""A `HDFS` database must not serve a table from its cache unchecked. + +The cache is keyed on the table name alone, so an entry one user resolved is handed to every later +caller. Both the read source grant and the remote host filter therefore have to be checked above it. +The same shape was fixed for the `Filesystem` database in +https://github.com/ClickHouse/ClickHouse/issues/118042. +""" + +import pytest + +from helpers.cluster import ClickHouseCluster, is_arm +from helpers.config_manager import ConfigManager + +if is_arm(): + pytestmark = pytest.mark.skip + +cluster = ClickHouseCluster(__file__) +node = cluster.add_instance("node", with_hdfs=True, stay_alive=True) + + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster.start() + yield cluster + finally: + cluster.shutdown() + + +def test_hdfs_database_cache_is_checked(started_cluster): + hdfs_api = started_cluster.hdfs_api + for name in ("warm.tsv", "cold.tsv", "cold2.tsv"): + hdfs_api.write_data("/" + name, "7\n") + + # The engine rejects a source that carries a path, so table names here are relative, which is the + # form the `EXISTS TABLE` arms below need. + node.query("CREATE DATABASE hdfs_db ENGINE = HDFS('hdfs://hdfs1:9000')") + # The source grant is the variable under test, so the table grant is deliberately broad and cannot + # confound the result. It also shows that it does not confer the source grant by itself. + node.query("CREATE USER u") + node.query("GRANT SELECT ON *.* TO u") + + # A user holding the grant resolves the name, which is what caches the storage for it. + assert node.query("SELECT * FROM hdfs_db.`warm.tsv`") == "7\n" + + # `tryGetTable` reports a denied resolution as `UNKNOWN_TABLE` (code 60). What this asserts is that + # the cached name is refused exactly like the name that was never resolved. + error = node.query_and_get_error("SELECT * FROM hdfs_db.`warm.tsv`", user="u") + assert "Code: 60." in error, error + error = node.query_and_get_error("SELECT * FROM hdfs_db.`cold.tsv`", user="u") + assert "Code: 60." in error, error + + # `EXISTS TABLE` needs only `SHOW TABLES`, so it must answer alike for a cached name and for one + # that was never resolved, or it reports which names other users have read. + assert node.query("EXISTS TABLE hdfs_db.`warm.tsv`", user="u") == "1\n" + assert node.query("EXISTS TABLE hdfs_db.`never.tsv`", user="u") == "1\n" + # A name that forms no usable URL still answers 0, and answers it to everyone: the two lines above + # are a measurement, and this probe does not depend on the grant. + assert node.query("EXISTS TABLE hdfs_db.`hdfs://hdfs1:9000`", user="u") == "0\n" + assert node.query("EXISTS TABLE hdfs_db.`hdfs://hdfs1:9000`") == "0\n" + + node.query("GRANT READ ON HDFS TO u") + + # With the grant the cached name is served. `TableFunctionExecute` counts calls of the table + # function, which serving from the cache does not make, so a counter that does not move is what + # shows the read was answered from the cache. The resolve below is the control that moves it. + calls = "SELECT sum(value) FROM system.events WHERE event = 'TableFunctionExecute'" + before = int(node.query(calls)) + assert node.query("SELECT * FROM hdfs_db.`warm.tsv`", user="u") == "7\n" + assert int(node.query(calls)) == before + assert node.query("SELECT * FROM hdfs_db.`cold2.tsv`") == "7\n" + assert int(node.query(calls)) > before + + # A grant restricted by URL must keep working: the filter is matched against the URI the table + # function reports, which for HDFS is the host of the table and not its path. + node.query("CREATE USER f") + node.query("GRANT SELECT ON *.* TO f") + node.query("GRANT READ ON HDFS('hdfs://hdfs1:9000') TO f") + assert node.query("SELECT * FROM hdfs_db.`warm.tsv`", user="f") == "7\n" + + # The host filter is re-read on `SYSTEM RELOAD CONFIG`, which keeps the cache, so a cached table + # must stop being served once its host is no longer allowed. These run as the fully granted user, + # so the grant cannot be the cause. + with ConfigManager() as cm: + cm.add_main_config(node, "configs/allowlist.xml") + error = node.query_and_get_error("SELECT * FROM hdfs_db.`warm.tsv`") + assert "Code: 60." in error, error + assert node.query("EXISTS TABLE hdfs_db.`warm.tsv`") == "0\n" + # A path that reports the refusal instead of masking it names the URL that was rejected. + error = node.query_and_get_error("INSERT INTO hdfs_db.`warm.tsv` VALUES (1)") + assert "UNACCEPTABLE_URL" in error, error + # The name that is not in the cache is refused the same way, as it already was. + error = node.query_and_get_error("SELECT * FROM hdfs_db.`cold.tsv`") + assert "Code: 60." in error, error + + # Removing the file and reloading again restores the answer, so the refusals above came from the + # filter and not from an evicted or poisoned cache entry. + assert node.query("SELECT * FROM hdfs_db.`warm.tsv`") == "7\n" + + node.query("DROP DATABASE hdfs_db SYNC") + node.query("DROP USER u, f") diff --git a/tests/integration/test_iceberg_manifest_decode_pool/__init__.py b/tests/integration/test_iceberg_manifest_decode_pool/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tests/integration/test_iceberg_manifest_decode_pool/configs/decode_pool.xml b/tests/integration/test_iceberg_manifest_decode_pool/configs/decode_pool.xml new file mode 100644 index 000000000000..e0ad5b6cbbf8 --- /dev/null +++ b/tests/integration/test_iceberg_manifest_decode_pool/configs/decode_pool.xml @@ -0,0 +1,3 @@ + + 1 + diff --git a/tests/integration/test_iceberg_manifest_decode_pool/test.py b/tests/integration/test_iceberg_manifest_decode_pool/test.py new file mode 100644 index 000000000000..3492ffa904c4 --- /dev/null +++ b/tests/integration/test_iceberg_manifest_decode_pool/test.py @@ -0,0 +1,98 @@ +import pytest + +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) +node = cluster.add_instance("node", main_configs=["configs/decode_pool.xml"]) + + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster.start() + yield cluster + finally: + cluster.shutdown() + + +def test_nested_iceberg_read_with_single_pool_thread(started_cluster): + """A pruning subquery reading another Iceberg table must not deadlock the decode + pool: the sets are built on the producer thread before any pool task runs, so the + single pool thread is never held by a task that waits for a nested read.""" + node.query( + """ + CREATE TABLE ice_main (id Int64, part Int64) + ENGINE = IcebergLocal(concat(getServerSetting('user_files_path'), '/ice_main/')) + """ + ) + node.query( + """ + CREATE TABLE ice_parts (part Int64) + ENGINE = IcebergLocal(concat(getServerSetting('user_files_path'), '/ice_parts/')) + """ + ) + # One commit per insert, so both tables have several manifests to decode. + for part in range(8): + node.query( + f"INSERT INTO ice_main SELECT number + {part} * 10, {part} FROM numbers(10) " + "SETTINGS allow_insert_into_iceberg = 1" + ) + for part in [1, 3, 6]: + node.query( + f"INSERT INTO ice_parts SELECT {part} " + "SETTINGS allow_insert_into_iceberg = 1" + ) + + result = node.query( + "SELECT count() FROM ice_main WHERE part IN (SELECT part FROM ice_parts)", + settings={ + "iceberg_manifest_decode_concurrency": 4, + "use_iceberg_metadata_files_cache": 0, + }, + timeout=120, + ) + assert int(result.strip()) == 30 + + node.query("DROP TABLE ice_main") + node.query("DROP TABLE ice_parts") + + +def test_join_of_two_iceberg_tables_with_single_pool_thread(started_cluster): + node.query( + """ + CREATE TABLE ice_left (id Int64) + ENGINE = IcebergLocal(concat(getServerSetting('user_files_path'), '/ice_left/')) + """ + ) + node.query( + """ + CREATE TABLE ice_right (id Int64) + ENGINE = IcebergLocal(concat(getServerSetting('user_files_path'), '/ice_right/')) + """ + ) + # One commit per insert, so each side has several data files to hand over. + for i in range(8): + node.query( + f"INSERT INTO ice_left SELECT number + {i} * 10 FROM numbers(10) " + "SETTINGS allow_insert_into_iceberg = 1" + ) + node.query( + f"INSERT INTO ice_right SELECT number + {i} * 10 FROM numbers(10) " + "SETTINGS allow_insert_into_iceberg = 1" + ) + + result = node.query( + "SELECT count() FROM ice_left AS l INNER JOIN ice_right AS r ON l.id = r.id", + settings={ + "iceberg_manifest_decode_concurrency": 4, + # The smallest queue: the second data file of a side already has to wait for the + # query to consume the first one, which is what used to park a pool thread. + "iceberg_file_entries_queue_size": 1, + "use_iceberg_metadata_files_cache": 0, + }, + timeout=120, + ) + assert int(result.strip()) == 80 + + node.query("DROP TABLE ice_left") + node.query("DROP TABLE ice_right") diff --git a/tests/integration/test_keeper_empty_multi/__init__.py b/tests/integration/test_keeper_empty_multi/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tests/integration/test_keeper_empty_multi/configs/keeper.xml b/tests/integration/test_keeper_empty_multi/configs/keeper.xml new file mode 100644 index 000000000000..33d1f784a67c --- /dev/null +++ b/tests/integration/test_keeper_empty_multi/configs/keeper.xml @@ -0,0 +1,28 @@ + + + + node + 9181 + + + + + 9181 + 1 + /var/lib/clickhouse/coordination/log + /var/lib/clickhouse/coordination/snapshots + + + 5000 + 10000 + + + + + 1 + node + 9234 + + + + diff --git a/tests/integration/test_keeper_empty_multi/test.py b/tests/integration/test_keeper_empty_multi/test.py new file mode 100644 index 000000000000..fe73c9f635cf --- /dev/null +++ b/tests/integration/test_keeper_empty_multi/test.py @@ -0,0 +1,55 @@ +import pytest + +import helpers.keeper_utils as keeper_utils +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) +node = cluster.add_instance( + "node", + main_configs=["configs/keeper.xml"], + stay_alive=True, + with_zookeeper=False, +) + + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster.start() + keeper_utils.wait_nodes(cluster, [node]) + yield cluster + finally: + cluster.shutdown() + + +def test_empty_multi_request(started_cluster): + # A `Multi` request whose subrequest list holds nothing but the terminator record - what a client + # sends when it builds a transaction from a list that turns out to be empty; kazoo serializes it + # as exactly that for a transaction without operations. It parses, and it is preprocessed into + # no deltas at all, so it used to be written to the changelog and then read `deltas.front()` on + # an empty range on the raft commit thread, terminating the process - and terminating it again + # on every restart that replayed the entry. + zk = keeper_utils.get_fake_zk(cluster, node.name) + try: + # ZooKeeper answers with an empty successful multi response, which kazoo turns into an + # empty list of results. + assert zk.transaction().commit() == [] + + assert keeper_utils.send_4lw_cmd(cluster, node, "ruok") == "imok" + + # The session and the server keep working. + zk.create("/after_empty_multi", b"1") + assert zk.get("/after_empty_multi")[0] == b"1" + # A multi transaction with subrequests keeps working. + transaction = zk.transaction() + transaction.create("/after_empty_multi/child", b"2") + assert transaction.commit() == ["/after_empty_multi/child"] + assert zk.get("/after_empty_multi/child")[0] == b"2" + finally: + zk.stop() + zk.close() + + # The entry of the empty multi is in the changelog, and the restart replays it. + node.restart_clickhouse() + keeper_utils.wait_nodes(cluster, [node]) + assert keeper_utils.send_4lw_cmd(cluster, node, "ruok") == "imok" diff --git a/tests/integration/test_keeper_secure_client/configs/generate_certs.sh b/tests/integration/test_keeper_secure_client/configs/generate_certs.sh new file mode 100755 index 000000000000..e398289b8cc9 --- /dev/null +++ b/tests/integration/test_keeper_secure_client/configs/generate_certs.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +# Self-signed certificate that Keeper presents on its secure port. It is also its own trust +# anchor (CA:TRUE), so a client can point caConfig at it directly. +# +# The subjectAltName has to list every name a client may connect to: `node1` is the cluster +# member name used in use_secure_keeper.xml, and `localhost` is kept because the raft +# configuration and the existing arms use it. Once a dNSName SAN is present OpenSSL ignores +# the common name, so localhost must be listed explicitly rather than left to CN. +openssl req -newkey rsa:2048 -x509 -days 36500 -nodes -batch \ + -keyout server.key -out server.crt \ + -subj "/CN=localhost" \ + -addext "subjectAltName=DNS:localhost,DNS:node1" \ + -addext "basicConstraints=critical,CA:TRUE" diff --git a/tests/integration/test_keeper_secure_client/configs/server.crt b/tests/integration/test_keeper_secure_client/configs/server.crt index 7ade2d962733..d9ad8941d615 100644 --- a/tests/integration/test_keeper_secure_client/configs/server.crt +++ b/tests/integration/test_keeper_secure_client/configs/server.crt @@ -1,19 +1,19 @@ -----BEGIN CERTIFICATE----- -MIIC/TCCAeWgAwIBAgIJANjx1QSR77HBMA0GCSqGSIb3DQEBCwUAMBQxEjAQBgNV -BAMMCWxvY2FsaG9zdDAgFw0xODA3MzAxODE2MDhaGA8yMjkyMDUxNDE4MTYwOFow -FDESMBAGA1UEAwwJbG9jYWxob3N0MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB -CgKCAQEAs9uSo6lJG8o8pw0fbVGVu0tPOljSWcVSXH9uiJBwlZLQnhN4SFSFohfI -4K8U1tBDTnxPLUo/V1K9yzoLiRDGMkwVj6+4+hE2udS2ePTQv5oaMeJ9wrs+5c9T -4pOtlq3pLAdm04ZMB1nbrEysceVudHRkQbGHzHp6VG29Fw7Ga6YpqyHQihRmEkTU -7UCYNA+Vk7aDPdMS/khweyTpXYZimaK9f0ECU3/VOeG3fH6Sp2X6FN4tUj/aFXEj -sRmU5G2TlYiSIUMF2JPdhSihfk1hJVALrHPTU38SOL+GyyBRWdNcrIwVwbpvsvPg -pryMSNxnpr0AK0dFhjwnupIv5hJIOQIDAQABo1AwTjAdBgNVHQ4EFgQUjPLb3uYC -kcamyZHK4/EV8jAP0wQwHwYDVR0jBBgwFoAUjPLb3uYCkcamyZHK4/EV8jAP0wQw -DAYDVR0TBAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEAM/ocuDvfPus/KpMVD51j -4IdlU8R0vmnYLQ+ygzOAo7+hUWP5j0yvq4ILWNmQX6HNvUggCgFv9bjwDFhb/5Vr -85ieWfTd9+LTjrOzTw4avdGwpX9G+6jJJSSq15tw5ElOIFb/qNA9O4dBiu8vn03C -L/zRSXrARhSqTW5w/tZkUcSTT+M5h28+Lgn9ysx4Ff5vi44LJ1NnrbJbEAIYsAAD -+UA+4MBFKx1r6hHINULev8+lCfkpwIaeS8RL+op4fr6kQPxnULw8wT8gkuc8I4+L -P9gg/xDHB44T3ADGZ5Ib6O0DJaNiToO6rnoaaxs0KkotbvDWvRoxEytSbXKoYjYp -0g== +MIIDKDCCAhCgAwIBAgIUBTZCvmMS+9yaX8IRKGkGqbcMBOIwDQYJKoZIhvcNAQEL +BQAwFDESMBAGA1UEAwwJbG9jYWxob3N0MCAXDTI2MDgyNzExMzMzOFoYDzIxMjYw +ODAzMTEzMzM4WjAUMRIwEAYDVQQDDAlsb2NhbGhvc3QwggEiMA0GCSqGSIb3DQEB +AQUAA4IBDwAwggEKAoIBAQCyOozYThV2lJBPteVCm7YMVJ/w4hralGidnbkpGZxQ +OnILomL9mym+e2+xdtztN3fbqZhiX3Cpgns6EBM0RYv7SQtcoUG2YErKHuASHSSc +DdCZv18EW1BuJSwA9Qq1fRiySjW+wkyOobxhUFROGgguoIdVOKUtDwib/mq27048 ++uz8i8bYcvlJqMRmiJy2LcwsQMdgYopGtwUGle4rMlAR8oTDBmHlrowoEOjN/mwd +WnLni5I7/4TIrsaARaGSivKLgcMlgs5JTUItvXreLuvdt1wQoCC1bH6h6K3ymsu8 +1xiC0zL9QTbGvhZItybzez/RzXcq3nsGEKCFXtunioRbAgMBAAGjcDBuMB0GA1Ud +DgQWBBQP0fRsGOqD4ZxQ1oTe7v9HIzc3RTAfBgNVHSMEGDAWgBQP0fRsGOqD4ZxQ +1oTe7v9HIzc3RTAbBgNVHREEFDASgglsb2NhbGhvc3SCBW5vZGUxMA8GA1UdEwEB +/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAGw73J/dlsYphCnXZzVfOWiwFtRX +ij9M3/GjtCfmfh2MriYcNlCGqw+7GvzUUJ44rKjRxWyF+FfVM7DC4O0a6oimvtwE +yhb9mFshGDUpMX6WtpjhFGDtNjbgXrz6FXkt8U5SQayVoVLQjIT6XZSmXEccMJpM +U3lGAqPaeMVYVZqQPcvdO+t+Jfrjjznpa66biD87BvJ4CCWNynWi/ftpK76bk1fj +SbL0RgG1SUBAJikJirL2OW+ADgCyPxP86IgTKdOV92e+00spcfvSlms7UzrChXA7 +Kv0Z/9WUgL3tL87kiP0eG3XaZp83w+WS0OOcaU8uQ0AhofzWXMnQT0Bgq/8= -----END CERTIFICATE----- diff --git a/tests/integration/test_keeper_secure_client/configs/server.key b/tests/integration/test_keeper_secure_client/configs/server.key index f0fb61ac443f..390ae5e9995f 100644 --- a/tests/integration/test_keeper_secure_client/configs/server.key +++ b/tests/integration/test_keeper_secure_client/configs/server.key @@ -1,28 +1,28 @@ -----BEGIN PRIVATE KEY----- -MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCz25KjqUkbyjyn -DR9tUZW7S086WNJZxVJcf26IkHCVktCeE3hIVIWiF8jgrxTW0ENOfE8tSj9XUr3L -OguJEMYyTBWPr7j6ETa51LZ49NC/mhox4n3Cuz7lz1Pik62WreksB2bThkwHWdus -TKxx5W50dGRBsYfMenpUbb0XDsZrpimrIdCKFGYSRNTtQJg0D5WTtoM90xL+SHB7 -JOldhmKZor1/QQJTf9U54bd8fpKnZfoU3i1SP9oVcSOxGZTkbZOViJIhQwXYk92F -KKF+TWElUAusc9NTfxI4v4bLIFFZ01ysjBXBum+y8+CmvIxI3GemvQArR0WGPCe6 -ki/mEkg5AgMBAAECggEATrbIBIxwDJOD2/BoUqWkDCY3dGevF8697vFuZKIiQ7PP -TX9j4vPq0DfsmDjHvAPFkTHiTQXzlroFik3LAp+uvhCCVzImmHq0IrwvZ9xtB43f -7Pkc5P6h1l3Ybo8HJ6zRIY3TuLtLxuPSuiOMTQSGRL0zq3SQ5DKuGwkz+kVjHXUN -MR2TECFwMHKQ5VLrC+7PMpsJYyOMlDAWhRfUalxC55xOXTpaN8TxNnwQ8K2ISVY5 -212Jz/a4hn4LdwxSz3Tiu95PN072K87HLWx3EdT6vW4Ge5P/A3y+smIuNAlanMnu -plHBRtpATLiTxZt/n6npyrfQVbYjSH7KWhB8hBHtaQKBgQDh9Cq1c/KtqDtE0Ccr -/r9tZNTUwBE6VP+3OJeKdEdtsfuxjOCkS1oAjgBJiSDOiWPh1DdoDeVZjPKq6pIu -Mq12OE3Doa8znfCXGbkSzEKOb2unKZMJxzrz99kXt40W5DtrqKPNb24CNqTiY8Aa -CjtcX+3weat82VRXvph6U8ltMwKBgQDLxjiQQzNoY7qvg7CwJCjf9qq8jmLK766g -1FHXopqS+dTxDLM8eJSRrpmxGWJvNeNc1uPhsKsKgotqAMdBUQTf7rSTbt4MyoH5 -bUcRLtr+0QTK9hDWMOOvleqNXha68vATkohWYfCueNsC60qD44o8RZAS6UNy3ENq -cM1cxqe84wKBgQDKkHutWnooJtajlTxY27O/nZKT/HA1bDgniMuKaz4R4Gr1PIez -on3YW3V0d0P7BP6PWRIm7bY79vkiMtLEKdiKUGWeyZdo3eHvhDb/3DCawtau8L2K -GZsHVp2//mS1Lfz7Qh8/L/NedqCQ+L4iWiPnZ3THjjwn3CoZ05ucpvrAMwKBgB54 -nay039MUVq44Owub3KDg+dcIU62U+cAC/9oG7qZbxYPmKkc4oL7IJSNecGHA5SbU -2268RFdl/gLz6tfRjbEOuOHzCjFPdvAdbysanpTMHLNc6FefJ+zxtgk9sJh0C4Jh -vxFrw9nTKKzfEl12gQ1SOaEaUIO0fEBGbe8ZpauRAoGAMAlGV+2/K4ebvAJKOVTa -dKAzQ+TD2SJmeR1HZmKDYddNqwtZlzg3v4ZhCk4eaUmGeC1Bdh8MDuB3QQvXz4Dr -vOIP4UVaOr+uM+7TgAgVnP4/K6IeJGzUDhX93pmpWhODfdu/oojEKVcpCojmEmS1 -KCBtmIrQLqzMpnBpLNuSY+Q= +MIIEugIBADANBgkqhkiG9w0BAQEFAASCBKQwggSgAgEAAoIBAQCyOozYThV2lJBP +teVCm7YMVJ/w4hralGidnbkpGZxQOnILomL9mym+e2+xdtztN3fbqZhiX3Cpgns6 +EBM0RYv7SQtcoUG2YErKHuASHSScDdCZv18EW1BuJSwA9Qq1fRiySjW+wkyOobxh +UFROGgguoIdVOKUtDwib/mq27048+uz8i8bYcvlJqMRmiJy2LcwsQMdgYopGtwUG +le4rMlAR8oTDBmHlrowoEOjN/mwdWnLni5I7/4TIrsaARaGSivKLgcMlgs5JTUIt +vXreLuvdt1wQoCC1bH6h6K3ymsu81xiC0zL9QTbGvhZItybzez/RzXcq3nsGEKCF +XtunioRbAgMBAAECgf9xetYy44TG+9Dm7+oJfxtqvncX6N8wTU5PDR3YDXZ5nrz+ +q2ZcE2+A2RdE52nDoI2wT0u3Vw1k4K+VSKbxoVZ/qQKS+BnfBXCZQQ9qeSPWOPWY +70CQhZOwjfp2oY8v/3a6DoYp74zPRCutynfjQq/zAEMq/+Yrymaw7E0GLVijehE7 +eLhAFL/1IAjzvr5ikaN2MgSGHwcZ7XNpTN0Lvpz1PH2LkpDmh7PWUKcvu7nV3cQc +x8BkEFFaqvlXq0NfgPNCZ9qL9/Y+VrJjv0Bta7KCO/6nbmJubUGVZbzv0qwOJBVB +/3iki0BoEG2YJzJLuKnJJ0Ugf6usNai1NY9PKtkCgYEA3V+QJzcL0fWrvw6DBP0m +WWv1afZGr2ONjHmz65IvegFpt9qmm2CnRVzFPXet1VJkxK+MNFiPs+h29s1d2G7/ +VdwU5+b2KdQkgG9QKWsRv9bkNUqK48LWvLYkUCNhYW7oxCH4SxBNbSdVDUpXnZJ/ +u6T71YRgY9yYRI6HnHrJr40CgYEAzhtZ7phK4inHokcXWLJBhYy6HwiENzNUlEm9 +dLRTeExzrdb/zA8WCPc+Vb1dSoNa0l/5TFfyoRpZ2RsfX9iGZIR2GiQhC8ftK2XH +ZynQBN+rGNBoarWhPfpyMU8g5pHCIxChOJaMTA5sb52oHE5D03oQXa0jUQDS7ie8 +IIUh9YcCgYB2I8zLkM3nFAO/J6B2Zh80M7A6B2t7qFZfNIf3XQGnS9++dZraav95 +veOpRRhpMtqCOOlK+kRej94FIl5F5h6wFI63oAOZGRUN0dnm8LP9PFJ3ujtbB50Q +AuhoOCl9FPJ10WcvsBiR+x6hxB30Ar3WR1r6fPXv9Ubxt1raAJFn9QKBgCCNg1vC +4/jqnmRMFCznIqAkRPVH1BIf4lj1eJ3pKVHTyWWIkqg5LcLGwiqqmQR+0KZvkCF1 +tpWpYs1+kisCON/BaCHi2PFSZ2w2TBNIMFnEtfJaYFOSUmBGoSiWldy04tPE+aHF +bW1JzeinHvkxh1bsbY9OHEjb93S0A+ui+2UbAoGADWmYVq4ksRSrHMa/9gwqGC5k +bWx791nrTf2UKlpsBA2b7kQ76DLMzgPGzCAGo5condd65y/4oP/Mr2+mEZkDu+Dh +W8jCsFN9RW2XP4ekgJj41hKAyKedP7JkbBb7pznvtqq1oLndIKN00jjrlfIbsKD0 +Reoz9LQnDowqs/fGhh8= -----END PRIVATE KEY----- diff --git a/tests/integration/test_keeper_secure_client/configs/ssl_conf_verify.xml b/tests/integration/test_keeper_secure_client/configs/ssl_conf_verify.xml new file mode 100644 index 000000000000..55a744c632f5 --- /dev/null +++ b/tests/integration/test_keeper_secure_client/configs/ssl_conf_verify.xml @@ -0,0 +1,23 @@ + + + + /etc/clickhouse-server/config.d/server.crt + /etc/clickhouse-server/config.d/server.key + none + true + sslv2,sslv3 + + + + /etc/clickhouse-server/config.d/server.crt + false + relaxed + true + sslv2,sslv3 + + RejectCertificateHandler + + + + diff --git a/tests/integration/test_keeper_secure_client/test.py b/tests/integration/test_keeper_secure_client/test.py index 1f00e6881028..fe8f65aa7bde 100644 --- a/tests/integration/test_keeper_secure_client/test.py +++ b/tests/integration/test_keeper_secure_client/test.py @@ -24,6 +24,16 @@ "configs/server.key", ], ) +# Same secure Keeper, but this client actually verifies the certificate it is presented. +node3 = cluster.add_instance( + "node3", + main_configs=[ + "configs/use_secure_keeper.xml", + "configs/ssl_conf_verify.xml", + "configs/server.crt", + "configs/server.key", + ], +) @pytest.fixture(scope="module") @@ -40,3 +50,9 @@ def started_cluster(): def test_connection(started_cluster): # just nothrow node2.query_with_retry("SELECT * FROM system.zookeeper WHERE path = '/'") + + +def test_connection_verifying_certificate(started_cluster): + # The socket connects to the address node1 resolves to, so this only succeeds if + # the certificate is matched against `node1` rather than against that address. + node3.query_with_retry("SELECT * FROM system.zookeeper WHERE path = '/'") diff --git a/tests/integration/test_mask_sensitive_info/test.py b/tests/integration/test_mask_sensitive_info/test.py index 49d9e69e0ea5..b3d06747b358 100644 --- a/tests/integration/test_mask_sensitive_info/test.py +++ b/tests/integration/test_mask_sensitive_info/test.py @@ -473,6 +473,7 @@ def test_create_database(): f"Backup('', S3('http://minio1:9001/root/data/backup', 'minio', '{password}'))", "DNS_ERROR", ), + f"URL('https://username:{password}@localhost:11111/x/')", ] def make_test_case(i): @@ -501,6 +502,7 @@ def make_test_case(i): "CREATE DATABASE database3 ENGINE = S3(named_collection_2, secret_access_key = '[HIDDEN]', access_key_id = 'minio')", # "CREATE DATABASE database4 ENGINE = PostgreSQL('localhost:5432', 'postgres_db', 'postgres_user', '[HIDDEN]')", "CREATE DATABASE database4 ENGINE = Backup('', S3('http://minio1:9001/root/data/backup', 'minio', '[HIDDEN]'))", + "CREATE DATABASE database5 ENGINE = URL('https://username:[HIDDEN]@localhost:11111/x/')", ], must_not_contain=[password], ) diff --git a/tests/integration/test_modify_engine_on_restart/configs/config.d/clusters_name_path.xml b/tests/integration/test_modify_engine_on_restart/configs/config.d/clusters_name_path.xml new file mode 100644 index 000000000000..7f5c93e20a71 --- /dev/null +++ b/tests/integration/test_modify_engine_on_restart/configs/config.d/clusters_name_path.xml @@ -0,0 +1,23 @@ + + + + + true + + ch1 + 9000 + + + + + + + 01 + + + +/clickhouse/tables/{database}/{table} + + diff --git a/tests/integration/test_modify_engine_on_restart/test_unsafe_name.py b/tests/integration/test_modify_engine_on_restart/test_unsafe_name.py new file mode 100644 index 000000000000..d6619fa56640 --- /dev/null +++ b/tests/integration/test_modify_engine_on_restart/test_unsafe_name.py @@ -0,0 +1,222 @@ +import pytest + +from helpers.cluster import ClickHouseCluster +from test_modify_engine_on_restart.common import get_table_path, set_convert_flags + +cluster = ClickHouseCluster(__file__) +ch1 = cluster.add_instance( + "ch1", + main_configs=[ + "configs/config.d/clusters_name_path.xml", + "configs/config.d/distributed_ddl.xml", + ], + with_zookeeper=True, + macros={"replica": "node1"}, + stay_alive=True, +) + +database_name = "modify_engine_unsafe_name" + +# A name-based `default_replica_path` splices the table's own name into the Keeper path, so a name +# carrying '/' resolves inside another table's subtree. `victim` is the co-tenant that gets damaged; +# the ghost's name is exactly the replica path underneath it. +VICTIM = "victim" +GHOST = "victim/replicas/ghost" + + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster.start() + yield cluster + + finally: + cluster.shutdown() + + +def q(query): + return ch1.query(database=database_name, sql=query) + + +def victim_total_replicas(): + return q( + f"SELECT total_replicas FROM system.replicas WHERE database = '{database_name}' AND table = '{VICTIM}'" + ).strip() + + +def active_part_paths(table): + """Absolute in-container paths of the table's active parts, each with a trailing '/'. + + `system.parts` rather than a directory listing: a MergeTree table's data directory always holds + a top-level `detached` directory too, and Outdated parts keep their own, so enumerating + directories counts more than the parts. + """ + return q( + "SELECT path FROM system.parts" + f" WHERE database = '{database_name}' AND table = '{table}' AND active" + ).split() + + +def count_txn_version_files(table): + path = get_table_path(ch1, table, database_name) + return int( + ch1.exec_in_container( + ["bash", "-c", f"find {path} -name txn_version.txt | wc -l"] + ).strip() + ) + + +def plant_txn_version_files(table): + """Write a valid non-transactional `txn_version.txt` onto every active part of `table`. + + Transactions are off by default, so parts carry no such file and a bare count of them cannot + tell whether the conversion removed anything. The content is the form `VersionInfo` itself + emits, so a surviving file still loads. + """ + nil = "00000000-0000-0000-0000-000000000000" + content = ( + "version: 1\\n" + "storing_version: 1\\n" + f"creation_tid: (1, 1, {nil})\\n" + "creation_csn: 1\\n" + f"removal_tid: (0, 0, {nil})\\n" + "removal_csn: 0" + ) + for part_path in active_part_paths(table): + ch1.exec_in_container( + ["bash", "-c", f'printf "{content}" > "{part_path}txn_version.txt"'] + ) + + +def test_attach_as_replicated_rejects_unsafe_name(started_cluster): + ch1.query(f"DROP DATABASE IF EXISTS {database_name} SYNC") + ch1.query(f"CREATE DATABASE {database_name}") + + q(f"CREATE TABLE `{VICTIM}` ( A Int64 ) ENGINE = ReplicatedMergeTree ORDER BY A") + q(f"INSERT INTO `{VICTIM}` VALUES (1)") + # The whole scenario needs the path to be derived from the name, so assert that rather than + # assuming the config took effect. + assert ( + q( + f"SELECT zookeeper_path FROM system.replicas WHERE database = '{database_name}' AND table = '{VICTIM}'" + ).strip() + == f"/clickhouse/tables/{database_name}/{VICTIM}" + ) + assert victim_total_replicas() == "1" + + # Merges are pinned off so the set of active parts, and therefore the planted file count, is the + # same before the conversion and after the re-attach below. + q( + f"CREATE TABLE `{GHOST}` ( A Int64 ) ENGINE = MergeTree ORDER BY A" + " SETTINGS max_bytes_to_merge_at_max_space_in_pool = 1" + ) + q(f"INSERT INTO `{GHOST}` VALUES (7)") + q(f"INSERT INTO `{GHOST}` VALUES (8)") + planted = len(active_part_paths(GHOST)) + # Arming assertion: with no part to plant onto, the file count reads 0 in every arm and the + # transaction-metadata assertion below discriminates nothing. + assert planted > 0 + plant_txn_version_files(GHOST) + assert count_txn_version_files(GHOST) == planted + q(f"DETACH TABLE `{GHOST}`") + + # The conversion must be refused. Without the check it succeeds and the table takes a path + # under the victim's own subtree. + assert "BAD_ARGUMENTS" in ch1.query_and_get_error( + f"ATTACH TABLE `{GHOST}` AS REPLICATED", database=database_name + ) + + # The rejection ran before the metadata rewrite, so a plain ATTACH brings the table back as + # the MergeTree it always was. A rejection sited after the rewrite reports ReplicatedMergeTree. + q(f"ATTACH TABLE `{GHOST}`") + assert ( + q( + f"SELECT engine FROM system.tables WHERE database = '{database_name}' AND name = '{GHOST}'" + ).strip() + == "MergeTree" + ) + + # It also ran before the table's transaction metadata was removed. That removal is + # irreversible, so the file count is what pins the check ahead of `clearTransactionMetadata`; the + # row count below is a plain no-regression line. + assert count_txn_version_files(GHOST) == planted + assert q(f"SELECT count() FROM `{GHOST}`").strip() == "2" + + # The victim is untouched, and still accepts a metadata ALTER. A planted ghost replica + # leaves this failing with a Keeper error over the ghost's missing log_pointer. + assert victim_total_replicas() == "1" + assert ( + q( + f"SELECT groupArray(name) FROM system.zookeeper WHERE path = '/clickhouse/tables/{database_name}/{VICTIM}/replicas'" + ).strip() + == "['node1']" + ) + q(f"ALTER TABLE `{VICTIM}` ADD COLUMN B UInt64 SETTINGS alter_sync = 2") + + # Control: a path-safe name converts fine under the very same configuration, so the + # rejection above is about the name and not about the config or the conversion route. + q("CREATE TABLE safe_name ( A Int64 ) ENGINE = MergeTree ORDER BY A") + q("INSERT INTO safe_name VALUES (1), (2)") + q("DETACH TABLE safe_name") + q("ATTACH TABLE safe_name AS REPLICATED") + assert ( + q( + f"SELECT engine FROM system.tables WHERE database = '{database_name}' AND name = 'safe_name'" + ).strip() + == "ReplicatedMergeTree" + ) + assert q("SELECT count() FROM safe_name").strip() == "2" + + # Control: the reverse direction mints no Keeper path, so an unsafe name must not block it. + # The table has to be replicated already AND unsafely named, which conversion cannot produce, so + # it is created directly with an explicit path outside the victim's subtree. + q( + f"CREATE TABLE `{GHOST}2` ( A Int64 )" + f" ENGINE = ReplicatedMergeTree('/clickhouse/unrelated/{database_name}', 'node1') ORDER BY A" + ) + q(f"INSERT INTO `{GHOST}2` VALUES (1), (2)") + q(f"DETACH TABLE `{GHOST}2`") + q(f"ATTACH TABLE `{GHOST}2` AS NOT REPLICATED") + assert ( + q( + f"SELECT engine FROM system.tables WHERE database = '{database_name}' AND name = '{GHOST}2'" + ).strip() + == "MergeTree" + ) + assert q(f"SELECT count() FROM `{GHOST}2`").strip() == "2" + + ch1.query(f"DROP DATABASE IF EXISTS {database_name} SYNC") + + +def test_convert_flag_rejects_unsafe_name(started_cluster): + ch1.query(f"DROP DATABASE IF EXISTS {database_name} SYNC") + ch1.query(f"CREATE DATABASE {database_name}") + + q(f"CREATE TABLE `{GHOST}` ( A Int64 ) ENGINE = MergeTree ORDER BY A") + q(f"INSERT INTO `{GHOST}` VALUES (1), (2)") + set_convert_flags(ch1, database_name, [GHOST]) + # Read while the server is up: every helper here answers from a query, and between the stop and + # the recovery start below there is provably no server to answer one. + table_data_path = get_table_path(ch1, GHOST, database_name) + + # The flag-file route refuses the same conversion, and because it runs during startup the + # server does not come up. That is the behaviour the sibling `checkReplicaPathExists` already + # has on this route (see test_zk_path_exists.py), and the recovery is the same: delete the flag. + ch1.stop_clickhouse() + ch1.start_clickhouse(start_wait_sec=120, expected_to_fail=True) + + ch1.exec_in_container( + ["bash", "-c", f"rm {table_data_path}convert_to_replicated"] + ) + ch1.start_clickhouse() + + # The table is intact: still a MergeTree, still holding its rows. + assert ( + q( + f"SELECT engine FROM system.tables WHERE database = '{database_name}' AND name = '{GHOST}'" + ).strip() + == "MergeTree" + ) + assert q(f"SELECT count() FROM `{GHOST}`").strip() == "2" + + ch1.query(f"DROP DATABASE IF EXISTS {database_name} SYNC") diff --git a/tests/integration/test_s3_client_refresh/__init__.py b/tests/integration/test_s3_client_refresh/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tests/integration/test_s3_client_refresh/configs/s3.xml b/tests/integration/test_s3_client_refresh/configs/s3.xml new file mode 100644 index 000000000000..dec0558850b1 --- /dev/null +++ b/tests/integration/test_s3_client_refresh/configs/s3.xml @@ -0,0 +1,21 @@ + + + 0 + + http://minio1:9001/root/refresh/endpoint/ + 0 + + + http://minio1:9001/root/refresh/keys/ + invalid + invalid + + + + + http://minio1:9001/root/refresh/ + 1 + TSV + + + diff --git a/tests/integration/test_s3_client_refresh/test.py b/tests/integration/test_s3_client_refresh/test.py new file mode 100644 index 000000000000..b9b6a4dabe53 --- /dev/null +++ b/tests/integration/test_s3_client_refresh/test.py @@ -0,0 +1,95 @@ +import pytest + +from helpers.cluster import ClickHouseCluster +from helpers.config_cluster import minio_access_key, minio_secret_key + +cluster = ClickHouseCluster(__file__) +node = cluster.add_instance( + "node", + with_minio=True, + main_configs=["configs/s3.xml"], + env_variables={ + "AWS_ACCESS_KEY_ID": minio_access_key, + "AWS_SECRET_ACCESS_KEY": minio_secret_key, + "AWS_EC2_METADATA_DISABLED": "true", + }, +) + +TRUSTED = {"s3_allow_server_credentials_in_user_queries": 1} +RESTRICTED = {"s3_allow_server_credentials_in_user_queries": 0} + + +@pytest.fixture(scope="module", autouse=True) +def started_cluster(): + try: + cluster.start() + yield + finally: + cluster.shutdown() + + +@pytest.mark.parametrize("scope", ["global", "endpoint"]) +@pytest.mark.parametrize("partitioned", [False, True]) +def test_named_collection_credentials_survive_restricted_read(scope, partitioned): + # Refresh must preserve the collection's ambient-credential opt-in over server defaults, + # while still refusing restricted reads. Partitioned tables exercise the system-log write path. + table = f"refresh_{scope}_{int(partitioned)}" + filename = f"{scope}/{table}/data" + if partitioned: + filename += "_{_partition_id}" + filename += ".tsv" + partition_by = "PARTITION BY x" if partitioned else "" + node.query( + f"CREATE TABLE {table} (x UInt8) " + f"ENGINE = S3(refresh_env, filename = '{filename}') {partition_by}", + settings=TRUSTED, + ) + try: + for value in (1, 2): + node.query( + f"INSERT INTO {table} VALUES ({value})", + settings={**TRUSTED, "s3_truncate_on_insert": 1}, + ) + # Read back with explicit keys so verification cannot refresh the table's client. + object_name = filename.replace("{_partition_id}", str(value)) + assert ( + node.query( + f"SELECT * FROM s3('http://minio1:9001/root/refresh/{object_name}', " + f"'{minio_access_key}', '{minio_secret_key}', 'TSV', 'x UInt8')", + settings=RESTRICTED, + ) + == f"{value}\n" + ) + + error = node.query_and_get_error( + f"SELECT * FROM {table}", settings=RESTRICTED + ) + if partitioned: + assert "ACCESS_DENIED" in error or "NOT_IMPLEMENTED" in error, error + else: + assert "ACCESS_DENIED" in error, error + finally: + node.query(f"DROP TABLE {table} SYNC") + + +@pytest.mark.parametrize("named_collection", [False, True]) +def test_explicit_keys_survive_client_refresh(named_collection): + # Static keys must also survive a session change, even with conflicting endpoint credentials. + table = f"refresh_keys_{int(named_collection)}" + url = f"http://minio1:9001/root/refresh/keys/{table}.tsv" + if named_collection: + arguments = ( + f"refresh_env, url = '{url}', access_key_id = '{minio_access_key}', " + f"secret_access_key = '{minio_secret_key}'" + ) + else: + arguments = f"'{url}', '{minio_access_key}', '{minio_secret_key}', 'TSV'" + node.query( + f"CREATE TABLE {table} (x UInt8) ENGINE = S3({arguments})", settings=TRUSTED + ) + try: + node.query(f"INSERT INTO {table} VALUES (7)", settings=TRUSTED) + for settings in (RESTRICTED, TRUSTED, RESTRICTED): + assert node.query(f"SELECT * FROM {table}", settings=settings) == "7\n" + finally: + node.query(f"DROP TABLE {table} SYNC") diff --git a/tests/integration/test_ssl_cert_authentication/test.py b/tests/integration/test_ssl_cert_authentication/test.py index 4edb7865241b..977bc82ea276 100644 --- a/tests/integration/test_ssl_cert_authentication/test.py +++ b/tests/integration/test_ssl_cert_authentication/test.py @@ -44,6 +44,7 @@ def started_cluster(): config = """ + {sslHost} strict @@ -62,6 +63,7 @@ def execute_query_native(node, query, user, cert_name, password=None): certificateFile=f"{SCRIPT_DIR}/certs/{cert_name}-cert.pem", privateKeyFile=f"{SCRIPT_DIR}/certs/{cert_name}-key.pem", caConfig=f"{SCRIPT_DIR}/certs/ca-cert.pem", + sslHost=SSL_HOST, ) file = open(config_path, "w") diff --git a/tests/integration/test_storage_iceberg_with_spark/test_data_manifest_decode_concurrency.py b/tests/integration/test_storage_iceberg_with_spark/test_data_manifest_decode_concurrency.py new file mode 100644 index 000000000000..e027ec542382 --- /dev/null +++ b/tests/integration/test_storage_iceberg_with_spark/test_data_manifest_decode_concurrency.py @@ -0,0 +1,349 @@ +from helpers.iceberg_utils import ( + create_iceberg_table, + default_upload_directory, + get_uuid_str, +) + +# One data manifest per `INSERT`, see `write_table`. +DATA_MANIFEST_COUNT = 8 +ROWS_PER_INSERT = 4 +ROW_COUNT = DATA_MANIFEST_COUNT * ROWS_PER_INSERT + +# Must match the delay the `iceberg_slow_manifest_read` failpoint injects. +SLEEP_PER_MANIFEST_SECONDS = 0.40 + +STORAGE_TYPE = "local" + + +def get_array(query_result: str): + return sorted([int(x) for x in query_result.strip().split("\n")]) + + +def elapsed(node, query, **kwargs): + query_id = get_uuid_str() + node.query(query, query_id=query_id, **kwargs) + node.query("SYSTEM FLUSH LOGS query_log") + duration_result = node.query( + f"""SELECT query_duration_ms / 1000.0 as duration FROM system.query_log + WHERE type = 'QueryFinish' AND query_id = '{query_id}' LIMIT 1""" + ) + return float(duration_result.strip()) + + +def write_table( + started_cluster, table_name: str, merge_on_read: bool = False, format_version: int = 2 +): + spark = started_cluster.spark_session + + extra_properties = "" + if merge_on_read: + extra_properties = """, + 'write.update.mode' = 'merge-on-read', + 'write.delete.mode' = 'merge-on-read', + 'write.merge.mode' = 'merge-on-read'""" + spark.sql( + f""" + CREATE TABLE {table_name} (id bigint, part int, data string) USING iceberg + PARTITIONED BY (part) + TBLPROPERTIES ( + 'format-version' = '{format_version}', + 'commit.manifest-merge.enabled' = 'false'{extra_properties} + ) + """ + ) + # One commit per partition so that every `INSERT` produces its own data manifest. + for part in range(DATA_MANIFEST_COUNT): + spark.sql( + f""" + INSERT INTO {table_name} + SELECT id, {part}, char(id % 26 + ascii('a')) + FROM range({part * ROWS_PER_INSERT}, {(part + 1) * ROWS_PER_INSERT}) + """ + ) + + default_upload_directory( + started_cluster, + STORAGE_TYPE, + f"/iceberg_data/default/{table_name}/", + f"/iceberg_data/default/{table_name}/", + ) + + +def check_data_manifests(instance, table_expression: str) -> None: + settings = {"iceberg_metadata_log_level": "manifest_list_entry"} + + query_id = "check_data_manifests_" + get_uuid_str() + instance.query( + f"SELECT id FROM {table_expression} FORMAT Null", query_id=query_id, settings=settings + ) + instance.query("SYSTEM FLUSH LOGS iceberg_metadata_log") + # Data manifests have content = 0. (Delete manifests have content = 1). + count = instance.query( + f""" + SELECT uniqExact(JSONExtractString(content, 'manifest_path')) + FROM system.iceberg_metadata_log + WHERE query_id = '{query_id}' + AND content_type = 'ManifestListEntry' + AND JSONExtractInt(content, 'content') = 0 + """ + ) + count = int(count.strip()) + assert count == DATA_MANIFEST_COUNT + + +def test_data_manifest_decode_concurrency(started_cluster_iceberg_with_spark): + """The result must not depend on `iceberg_manifest_decode_concurrency`.""" + instance = started_cluster_iceberg_with_spark.instances["node1"] + TABLE_NAME = "test_data_manifest_decode_concurrency_" + get_uuid_str() + + write_table(started_cluster_iceberg_with_spark, TABLE_NAME) + create_iceberg_table( + STORAGE_TYPE, instance, TABLE_NAME, started_cluster_iceberg_with_spark + ) + check_data_manifests(instance, TABLE_NAME) + + expected = list(range(ROW_COUNT)) + for concurrency in [1, 2, 4, 16]: + assert ( + get_array( + instance.query( + f"SELECT id FROM {TABLE_NAME} ORDER BY id", + settings={ + "iceberg_manifest_decode_concurrency": concurrency + }, + ) + ) + == expected + ), f"wrong result with iceberg_manifest_decode_concurrency={concurrency}" + + # A filter on the partition column makes every concurrent decode task evaluate the same + # shared filter DAG (with its lazily materialized IN set) while pruning manifest entries. + filtered_parts = [1, 3, 6] + expected_filtered = sorted( + row_id + for row_id in range(ROW_COUNT) + if row_id // ROWS_PER_INSERT in filtered_parts + ) + for concurrency in [1, 2, 4, 16]: + result = get_array( + instance.query( + f"SELECT id FROM {TABLE_NAME} WHERE part IN (1, 3, 6) ORDER BY id", + settings={ + "iceberg_manifest_decode_concurrency": concurrency + }, + ) + ) + assert ( + result == expected_filtered + ), f"wrong filtered result with iceberg_manifest_decode_concurrency={concurrency}" + + instance.query(f"DROP TABLE {TABLE_NAME}") + + +def test_data_and_delete_manifest_decode_concurrency( + started_cluster_iceberg_with_spark, +): + """Data- and delete-manifest decode run concurrently and share one filter DAG; + the result must not depend on the concurrency setting.""" + instance = started_cluster_iceberg_with_spark.instances["node1"] + TABLE_NAME = "test_data_and_delete_manifest_decode_concurrency_" + get_uuid_str() + + write_table(started_cluster_iceberg_with_spark, TABLE_NAME, merge_on_read=True) + spark = started_cluster_iceberg_with_spark.spark_session + # One delete manifest per `DELETE`; ids 4, 13 and 25 fall inside the filtered + # partitions below, id 0 outside them. + deleted_ids = [0, 4, 13, 25] + for row_id in deleted_ids: + spark.sql(f"DELETE FROM {TABLE_NAME} WHERE id = {row_id}") + default_upload_directory( + started_cluster_iceberg_with_spark, + STORAGE_TYPE, + f"/iceberg_data/default/{TABLE_NAME}/", + f"/iceberg_data/default/{TABLE_NAME}/", + ) + create_iceberg_table( + STORAGE_TYPE, instance, TABLE_NAME, started_cluster_iceberg_with_spark + ) + + filtered_parts = [1, 3, 6] + expected = sorted( + row_id + for row_id in range(ROW_COUNT) + if row_id // ROWS_PER_INSERT in filtered_parts and row_id not in deleted_ids + ) + for concurrency in [1, 4, 16]: + result = get_array( + instance.query( + f"SELECT id FROM {TABLE_NAME} WHERE part IN (1, 3, 6) ORDER BY id", + settings={ + "iceberg_manifest_decode_concurrency": concurrency, + }, + ) + ) + assert result == expected, ( + f"wrong result with iceberg_manifest_decode_concurrency={concurrency}" + ) + + instance.query(f"DROP TABLE {TABLE_NAME}") + + +def test_data_manifest_decode_large_manifest(started_cluster_iceberg_with_spark): + """A single manifest holding three times more entries than the producer queue's + capacity (100), so pushes block in the middle of the manifest at every concurrency; + the result must not depend on the concurrency.""" + instance = started_cluster_iceberg_with_spark.instances["node1"] + spark = started_cluster_iceberg_with_spark.spark_session + TABLE_NAME = "test_data_manifest_decode_large_manifest_" + get_uuid_str() + + entry_count = 300 + spark.sql( + f""" + CREATE TABLE {TABLE_NAME} (id bigint) USING iceberg + PARTITIONED BY (id) + TBLPROPERTIES ('commit.manifest-merge.enabled' = 'false') + """ + ) + # One commit writes one data file per partition value, producing a single + # manifest with `entry_count` entries. + spark.sql(f"INSERT INTO {TABLE_NAME} SELECT id FROM range({entry_count})") + default_upload_directory( + started_cluster_iceberg_with_spark, + STORAGE_TYPE, + f"/iceberg_data/default/{TABLE_NAME}/", + f"/iceberg_data/default/{TABLE_NAME}/", + ) + create_iceberg_table( + STORAGE_TYPE, instance, TABLE_NAME, started_cluster_iceberg_with_spark + ) + + expected = list(range(entry_count)) + for concurrency in [1, 4]: + result = get_array( + instance.query( + f"SELECT id FROM {TABLE_NAME} ORDER BY id", + settings={ + "iceberg_manifest_decode_concurrency": concurrency, + "use_iceberg_metadata_files_cache": 0, + }, + ) + ) + assert ( + result == expected + ), f"wrong result with iceberg_manifest_decode_concurrency={concurrency}" + + # A filter pruning every entry keeps the row loop inside `ManifestFileIterator::next` + # busy for the whole manifest without yielding. + for concurrency in [1, 16]: + count = instance.query( + f"SELECT count() FROM {TABLE_NAME} WHERE id < 0", + settings={ + "iceberg_manifest_decode_concurrency": concurrency, + "use_iceberg_metadata_files_cache": 0, + }, + ) + assert ( + int(count.strip()) == 0 + ), f"fully pruned read returned rows with iceberg_manifest_decode_concurrency={concurrency}" + + instance.query(f"DROP TABLE {TABLE_NAME}") + + +def test_data_manifest_decode_concurrency_subquery_filter( + started_cluster_iceberg_with_spark, +): + """`part IN (SELECT ...)` is backed by `FutureSetFromSubquery`, which the concurrent + decode tasks build lazily through the shared filter DAG; the result must not depend + on the concurrency. (Literal tuple `IN` goes through `FutureSetFromTuple` instead + and is covered above.)""" + instance = started_cluster_iceberg_with_spark.instances["node1"] + TABLE_NAME = "test_data_manifest_decode_concurrency_subquery_" + get_uuid_str() + + write_table(started_cluster_iceberg_with_spark, TABLE_NAME) + create_iceberg_table( + STORAGE_TYPE, instance, TABLE_NAME, started_cluster_iceberg_with_spark + ) + + # The subquery yields parts {1, 3, 5}. + filtered_parts = [1, 3, 5] + expected = sorted( + row_id + for row_id in range(ROW_COUNT) + if row_id // ROWS_PER_INSERT in filtered_parts + ) + for concurrency in [1, 4, 16]: + for _ in range(2): + result = get_array( + instance.query( + f"SELECT id FROM {TABLE_NAME} " + "WHERE part IN (SELECT toInt32(number * 2 + 1) FROM numbers(3)) " + "ORDER BY id", + settings={ + "iceberg_manifest_decode_concurrency": concurrency, + "use_iceberg_metadata_files_cache": 0, + }, + ) + ) + assert result == expected, ( + f"wrong subquery-filtered result with " + f"iceberg_manifest_decode_concurrency={concurrency}" + ) + + instance.query(f"DROP TABLE {TABLE_NAME}") + + +def test_data_manifest_decode_concurrency_bounds_reads( + started_cluster_iceberg_with_spark, +): + """`iceberg_manifest_decode_concurrency = 1` reads the manifests one at a + time, so with the failpoint delaying every manifest read the query cannot run + faster than one delay per manifest; a higher value overlaps the reads.""" + instance = started_cluster_iceberg_with_spark.instances["node1"] + + # Skip timing asserts on slow builds. + slow_build = ( + instance.is_built_with_sanitizer() or instance.is_built_with_llvm_coverage() + ) + + TABLE_NAME = "test_data_manifest_decode_concurrency_bounds_" + get_uuid_str() + + write_table(started_cluster_iceberg_with_spark, TABLE_NAME) + create_iceberg_table( + STORAGE_TYPE, instance, TABLE_NAME, started_cluster_iceberg_with_spark + ) + + instance.query("SYSTEM ENABLE FAILPOINT iceberg_slow_manifest_read") + try: + serial_duration = elapsed( + instance, + f"SELECT * FROM {TABLE_NAME} FORMAT Null", + settings={ + "iceberg_manifest_decode_concurrency": 1, + "use_iceberg_metadata_files_cache": 0, + }, + ) + parallel_duration = elapsed( + instance, + f"SELECT * FROM {TABLE_NAME} FORMAT Null", + settings={ + "iceberg_manifest_decode_concurrency": 16, + "use_iceberg_metadata_files_cache": 0, + }, + ) + finally: + instance.query("SYSTEM DISABLE FAILPOINT iceberg_slow_manifest_read") + instance.query(f"DROP TABLE {TABLE_NAME}") + + if slow_build: + return + + serial_floor = 0.9 * DATA_MANIFEST_COUNT * SLEEP_PER_MANIFEST_SECONDS + assert serial_duration >= serial_floor, ( + f"the serial read of {DATA_MANIFEST_COUNT} manifests took {serial_duration:.3f}s, " + f"below the {serial_floor:.3f}s floor of one failpoint delay per manifest, so " + f"iceberg_manifest_decode_concurrency = 1 did not decode them one at a time" + ) + assert parallel_duration < 0.6 * serial_duration, ( + f"the read with iceberg_manifest_decode_concurrency = 16 took " + f"{parallel_duration:.3f}s against {serial_duration:.3f}s serially, so the " + f"manifest reads were likely not overlapped" + ) diff --git a/tests/integration/test_storage_iceberg_with_spark/test_lifetime_sources_bug.py b/tests/integration/test_storage_iceberg_with_spark/test_lifetime_sources_bug.py new file mode 100644 index 000000000000..cd538da005a6 --- /dev/null +++ b/tests/integration/test_storage_iceberg_with_spark/test_lifetime_sources_bug.py @@ -0,0 +1,156 @@ +import pytest + +from helpers.iceberg_utils import ( + default_upload_directory, + write_iceberg_from_df, + generate_data, + create_iceberg_table, + get_uuid_str, +) + +import threading +import time + + +FAILPOINT = "object_storage_source_pause_before_virtual_columns" + + +def uptime(instance): + # Uptime only ever grows while the server keeps running and drops back to nearly zero when it + # restarts, so comparing two samples tells a survived run from a crashed one. Deriving the start + # time as `now() - uptime()` instead would be off by a second whenever the two functions, both of + # them second-granular, are evaluated on different sides of a second boundary. `system.crash_log` + # is no good either: the table only appears once something has already crashed. + return int(instance.query("SELECT uptime()")) + + +@pytest.mark.parametrize("format_version", ["2"]) +@pytest.mark.parametrize("storage_type", ["s3"]) +def test_source_must_not_outlive_storage(started_cluster_iceberg_with_spark, format_version, storage_type): + """Dropping a table must not pull the storage out from under a reading pipeline. + + `StorageObjectStorageSource::generate` resolves `storage_snapshot->storage.getStorageID()` for every + chunk, and `StorageSnapshot::storage` is a bare `const IStorage &`: the source keeps the snapshot + alive, not the storage behind it. If `DROP TABLE` destroys the storage while a pipeline thread sits + in `generate`, that call locks the mutex of a destroyed `IStorage` and glibc aborts the whole server. + Staging shows 597 crashes sharing exactly this stack, all of them signal 6 through + `IStorage::getStorageID` from `StorageObjectStorageSource::generate`, reported for single node + execution with parallel replicas. + + So the test parks a source inside `generate`, drops the table underneath it, and then lets it wake + up. Whether `DROP` waits for the reading pipeline is the crux: if it goes through while the source + is parked, the table lock is not being held for that path, and the wake-up touches freed memory. + """ + instance = started_cluster_iceberg_with_spark.instances["node1"] + spark = started_cluster_iceberg_with_spark.spark_session + TABLE_NAME = ( + "test_source_must_not_outlive_storage_" + + format_version + + "_" + + storage_type + + "_" + + get_uuid_str() + ) + + NUM_FILES = 8 + ROWS_PER_FILE = 100 + + write_iceberg_from_df( + spark, generate_data(spark, 0, ROWS_PER_FILE), TABLE_NAME, mode="overwrite", format_version=format_version + ) + for i in range(1, NUM_FILES): + write_iceberg_from_df( + spark, + generate_data(spark, i * 1000, i * 1000 + ROWS_PER_FILE), + TABLE_NAME, + mode="append", + format_version=format_version, + ) + + default_upload_directory( + started_cluster_iceberg_with_spark, + storage_type, + f"/iceberg_data/default/{TABLE_NAME}/", + f"/iceberg_data/default/{TABLE_NAME}/", + ) + + create_iceberg_table(storage_type, instance, TABLE_NAME, started_cluster_iceberg_with_spark) + + uptime_before = uptime(instance) + + # `_path` is what drags the storage into `generate`, and parallel replicas keep the reading pipeline + # on the initiator, which is the shape the crashes were reported for. + select_id = get_uuid_str() + select = ( + f"SELECT _path, count() FROM {TABLE_NAME} GROUP BY _path " + "SETTINGS enable_parallel_replicas = 1, max_parallel_replicas = 3, " + "parallel_replicas_local_plan = 1, cluster_for_parallel_replicas = 'cluster_simple'" + ) + + select_error = [] + drop_error = [] + drop_finished = threading.Event() + + def run_select(): + try: + instance.query(select, query_id=select_id) + except Exception as e: # the query may legitimately die with the table, the server may not + select_error.append(str(e)) + + def run_drop(): + try: + instance.query(f"DROP TABLE {TABLE_NAME} SYNC") + except Exception as e: + drop_error.append(str(e)) + finally: + drop_finished.set() + + select_thread = threading.Thread(target=run_select) + drop_thread = threading.Thread(target=run_drop) + + instance.query(f"SYSTEM ENABLE FAILPOINT {FAILPOINT}") + try: + select_thread.start() + + deadline = time.time() + 60 + running = 0 + while time.time() < deadline: + running = int(instance.query( + f"SELECT count() FROM system.processes WHERE query_id = '{select_id}'" + )) + if running: + break + time.sleep(0.2) + + assert running, "the query never started, so no source is parked in generate()" + + # `system.processes` reports the query from its start, before the pipeline has reached the + # failpoint, so give the source a moment to actually park there. + time.sleep(3) + + drop_thread.start() + # A DROP that returns while the source is parked means nothing held the table for the reading + # pipeline. Recorded rather than asserted on: the outcome that matters is whether the server + # survives the wake-up below. + dropped_under_reader = drop_finished.wait(timeout=10) + finally: + # Releases the parked source, which then reaches `storage_snapshot->storage.getStorageID()`. + instance.query(f"SYSTEM DISABLE FAILPOINT {FAILPOINT}") + + select_thread.join(timeout=180) + drop_thread.join(timeout=180) + assert not select_thread.is_alive(), "the source never woke up after the failpoint was released" + assert not drop_thread.is_alive(), "DROP TABLE never returned" + + assert instance.query("SELECT 1").strip() == "1" + assert uptime(instance) >= uptime_before, ( + "the server restarted during the test: DROP TABLE destroyed the storage while a source was " + "parked in StorageObjectStorageSource::generate, and the wake-up locked the mutex of a " + "destroyed IStorage" + + ( + " (DROP TABLE returned while the source was still parked, so the reading pipeline was not" + " holding the table)" + if dropped_under_reader + else "" + ) + ) diff --git a/tests/integration/test_storage_iceberg_with_spark/test_variant_type.py b/tests/integration/test_storage_iceberg_with_spark/test_variant_type.py new file mode 100644 index 000000000000..9b85b2cb9dcb --- /dev/null +++ b/tests/integration/test_storage_iceberg_with_spark/test_variant_type.py @@ -0,0 +1,75 @@ +import pytest + +from helpers.iceberg_utils import ( + create_iceberg_table, + default_upload_directory, + get_creation_expression, + get_uuid_str, +) + +from helpers.test_tools import TSV + + +@pytest.mark.parametrize("storage_type", ["s3", "azure", "local"]) +def test_variant_type(started_cluster_iceberg_with_spark, storage_type): + instance = started_cluster_iceberg_with_spark.instances["node1"] + spark = started_cluster_iceberg_with_spark.spark_session + if int(spark.version.split(".")[0]) < 4: + pytest.skip(f"the VARIANT type needs Spark 4.0, the runner has {spark.version}") + TABLE_NAME = "test_variant_type_" + storage_type + "_" + get_uuid_str() + + # `variant` was introduced in Iceberg format version 3. Iceberg writes it as an unshredded + # group of two plain BYTE_ARRAY leaves annotated with the `VARIANT` logical type: + # + # optional group v (Variant(1)) { + # required binary metadata; + # required binary value; + # } + spark.sql( + f""" + CREATE TABLE {TABLE_NAME} (id INT, v VARIANT) + USING iceberg + TBLPROPERTIES ('format-version'='3') + """ + ) + spark.sql( + f""" + INSERT INTO {TABLE_NAME} VALUES + (1, cast(100000 as variant)), + (2, cast('hello' as variant)) + """ + ) + + default_upload_directory( + started_cluster_iceberg_with_spark, + storage_type, + f"/iceberg_data/default/{TABLE_NAME}/", + f"/iceberg_data/default/{TABLE_NAME}/", + ) + + create_iceberg_table( + storage_type, + instance, + TABLE_NAME, + started_cluster_iceberg_with_spark, + format_version=3, + ) + + table_function_expr = get_creation_expression( + storage_type, + TABLE_NAME, + started_cluster_iceberg_with_spark, + table_function=True, + format_version=3, + ) + + assert instance.query(f"DESCRIBE {table_function_expr} FORMAT TSV") == TSV( + [ + ["id", "Nullable(Int32)"], + ["v", "Dynamic"], + ] + ) + + assert instance.query( + f"SELECT id, dynamicType(v), v FROM {table_function_expr} ORDER BY id FORMAT TSV" + ).strip() == ("1\tInt32\t100000\n" "2\tString\thello") diff --git a/tests/performance/iceberg_data_manifests.xml b/tests/performance/iceberg_data_manifests.xml new file mode 100644 index 000000000000..2cc4e4c081d0 --- /dev/null +++ b/tests/performance/iceberg_data_manifests.xml @@ -0,0 +1,220 @@ + + + + + n + + 0 + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8 + 9 + 10 + 11 + 12 + 13 + 14 + 15 + 16 + 17 + 18 + 19 + 20 + 21 + 22 + 23 + 24 + 25 + 26 + 27 + 28 + 29 + 30 + 31 + 32 + 33 + 34 + 35 + 36 + 37 + 38 + 39 + 40 + 41 + 42 + 43 + 44 + 45 + 46 + 47 + 48 + 49 + 50 + 51 + 52 + 53 + 54 + 55 + 56 + 57 + 58 + 59 + 60 + 61 + 62 + 63 + 64 + 65 + 66 + 67 + 68 + 69 + 70 + 71 + 72 + 73 + 74 + 75 + 76 + 77 + 78 + 79 + 80 + 81 + 82 + 83 + 84 + 85 + 86 + 87 + 88 + 89 + 90 + 91 + 92 + 93 + 94 + 95 + 96 + 97 + 98 + 99 + 100 + 101 + 102 + 103 + 104 + 105 + 106 + 107 + 108 + 109 + 110 + 111 + 112 + 113 + 114 + 115 + 116 + 117 + 118 + 119 + 120 + 121 + 122 + 123 + 124 + 125 + 126 + 127 + 128 + 129 + 130 + 131 + 132 + 133 + 134 + 135 + 136 + 137 + 138 + 139 + 140 + 141 + 142 + 143 + 144 + 145 + 146 + 147 + 148 + 149 + 150 + 151 + 152 + 153 + 154 + 155 + 156 + 157 + 158 + 159 + 160 + 161 + 162 + 163 + 164 + 165 + 166 + 167 + 168 + 169 + 170 + 171 + 172 + 173 + 174 + 175 + 176 + 177 + 178 + 179 + 180 + 181 + 182 + 183 + 184 + 185 + 186 + 187 + 188 + 189 + 190 + 191 + 192 + 193 + 194 + 195 + 196 + 197 + 198 + 199 + + + + + DROP TABLE IF EXISTS perf_iceberg_data_manifests + CREATE TABLE perf_iceberg_data_manifests (id Int64, data String) ENGINE = IcebergLocal(concat(getServerSetting('user_files_path'), '/perf_iceberg_data_manifests/')) + INSERT INTO perf_iceberg_data_manifests SELECT number + {n} * 500, toString(number) FROM numbers(500) SETTINGS allow_insert_into_iceberg = 1 + + SELECT max(id) FROM perf_iceberg_data_manifests SETTINGS use_iceberg_metadata_files_cache = 0 + + DROP TABLE IF EXISTS perf_iceberg_data_manifests + diff --git a/tests/queries/0_stateless/01076_parallel_alter_replicated_zookeeper.reference b/tests/queries/0_stateless/01076_parallel_alter_replicated_zookeeper.reference index ff9c6824f00a..c7200be9f317 100644 --- a/tests/queries/0_stateless/01076_parallel_alter_replicated_zookeeper.reference +++ b/tests/queries/0_stateless/01076_parallel_alter_replicated_zookeeper.reference @@ -15,3 +15,4 @@ Finishing alters 0 1 0 +1 diff --git a/tests/queries/0_stateless/01076_parallel_alter_replicated_zookeeper.sh b/tests/queries/0_stateless/01076_parallel_alter_replicated_zookeeper.sh index d995dbf731e7..9e2b81423db3 100755 --- a/tests/queries/0_stateless/01076_parallel_alter_replicated_zookeeper.sh +++ b/tests/queries/0_stateless/01076_parallel_alter_replicated_zookeeper.sh @@ -48,14 +48,13 @@ done INITIAL_SUM=$($CLICKHOUSE_CLIENT --query "SELECT SUM(value1) FROM concurrent_mutate_mt_1") -# Run mutation on random replica +# Run mutation on replica 1, which detach_attach_thread never takes down function correct_alter_thread() { local TIMELIMIT=$((SECONDS+TIMEOUT)) while [ $SECONDS -lt "$TIMELIMIT" ] do - REPLICA=$(($RANDOM % 5 + 1)) - $CLICKHOUSE_CLIENT --query "ALTER TABLE concurrent_mutate_mt_$REPLICA UPDATE value1 = value1 + 1 WHERE 1"; + $CLICKHOUSE_CLIENT --query "ALTER TABLE concurrent_mutate_mt_1 UPDATE value1 = value1 + 1 WHERE 1"; sleep 1 done } @@ -79,7 +78,9 @@ function detach_attach_thread() local TIMELIMIT=$((SECONDS+TIMEOUT)) while [ $SECONDS -lt "$TIMELIMIT" ] do - REPLICA=$(($RANDOM % 5 + 1)) + # Replicas 2.. only: correct_alter_thread needs replica 1 attached, or an ALTER that draws a + # DETACHed replica fails with UNKNOWN_TABLE and the run assigns no mutation to check. + REPLICA=$(($RANDOM % (REPLICAS - 1) + 2)) $CLICKHOUSE_CLIENT --query "DETACH TABLE concurrent_mutate_mt_$REPLICA" sleep 0.$RANDOM sleep 0.$RANDOM @@ -142,10 +143,15 @@ for i in $(seq $REPLICAS); do $CLICKHOUSE_CLIENT --query "SYSTEM SYNC REPLICA concurrent_mutate_mt_$i" $CLICKHOUSE_CLIENT --query "CHECK TABLE concurrent_mutate_mt_$i" &> /dev/null # if we will remove something the output of select will be wrong $CLICKHOUSE_CLIENT --query "SELECT SUM(toUInt64(value1)) > $INITIAL_SUM FROM concurrent_mutate_mt_$i" - $CLICKHOUSE_CLIENT --query "SELECT COUNT() FROM system.mutations WHERE table='concurrent_mutate_mt_$i' and is_done=0" # all mutations have to be done - $CLICKHOUSE_CLIENT --query "SELECT * FROM system.mutations WHERE table='concurrent_mutate_mt_$i' and is_done=0" # for verbose output + $CLICKHOUSE_CLIENT --query "SELECT COUNT() FROM system.mutations WHERE database='${CLICKHOUSE_DATABASE}' and table='concurrent_mutate_mt_$i' and is_done=0" # all mutations have to be done + $CLICKHOUSE_CLIENT --query "SELECT * FROM system.mutations WHERE database='${CLICKHOUSE_DATABASE}' and table='concurrent_mutate_mt_$i' and is_done=0" # for verbose output done +# The stress phase must have assigned at least one mutation: every check above is also satisfied by a +# run that assigned none, because the concurrent inserts alone raise the sum and a run without +# mutations has none unfinished. The SYSTEM SYNC REPLICA calls above make the entries visible here. +$CLICKHOUSE_CLIENT --query "SELECT count() > 0 FROM system.mutations WHERE database='${CLICKHOUSE_DATABASE}' AND table LIKE 'concurrent_mutate_mt_%'" + for i in $(seq $REPLICAS); do $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS concurrent_mutate_mt_$i" done diff --git a/tests/queries/0_stateless/01655_plan_optimizations.reference b/tests/queries/0_stateless/01655_plan_optimizations.reference index 9543d5aebc82..fe7cc6d1eb85 100644 --- a/tests/queries/0_stateless/01655_plan_optimizations.reference +++ b/tests/queries/0_stateless/01655_plan_optimizations.reference @@ -130,7 +130,7 @@ Filter column: notEquals(__table1.number, 1_UInt8) > (analyzer) one condition of filter is pushed down before INNER JOIN Join Filter column: and(notEquals(__table1.number, 1_UInt8), notEquals(__table1.number, 2_UInt8)) -Filter column: and(notEquals(__table2.b, 1_UInt8), notEquals(__table2.b, 2_UInt8)) +Filter column: and(notEquals(__table2.b, 2_UInt8), notEquals(__table2.b, 1_UInt8)) 3 3 > filter is pushed down before UNION Union diff --git a/tests/queries/0_stateless/01655_plan_optimizations.sh b/tests/queries/0_stateless/01655_plan_optimizations.sh index 0890be12d6e2..96760de57b9f 100755 --- a/tests/queries/0_stateless/01655_plan_optimizations.sh +++ b/tests/queries/0_stateless/01655_plan_optimizations.sh @@ -182,7 +182,7 @@ $CLICKHOUSE_CLIENT --enable_analyzer=1 -q " select number as a, r.b from numbers(4) as l any inner join ( select number + 2 as b from numbers(3) ) as r on a = r.b where a != 1 and b != 2 settings enable_optimize_predicate_expression = 0, query_plan_join_swap_table = 0, enable_join_runtime_filters = 0" | - grep -o " Join\|Filter column: and(notEquals(__table1.number, 1_UInt8), notEquals(__table1.number, 2_UInt8))\|Filter column: and(notEquals(__table2.b, 1_UInt8), notEquals(__table2.b, 2_UInt8))" + grep -o " Join\|Filter column: and(notEquals(__table1.number, 1_UInt8), notEquals(__table1.number, 2_UInt8))\|Filter column: and(notEquals(__table2.b, 2_UInt8), notEquals(__table2.b, 1_UInt8))" $CLICKHOUSE_CLIENT -q " select number as a, r.b from numbers(4) as l any inner join ( select number + 2 as b from numbers(3) diff --git a/tests/queries/0_stateless/02346_text_index_bug106460.reference b/tests/queries/0_stateless/02346_text_index_bug106460.reference new file mode 100644 index 000000000000..d31e718aeb3c --- /dev/null +++ b/tests/queries/0_stateless/02346_text_index_bug106460.reference @@ -0,0 +1,21 @@ +100 1 +100 1 +100 1 +1 1 +11 0 +21 0 +-- materialized +100 1 +1 +-- With preprocessor +0 +10 +0 +10 +patched indexed column +99 +99 +99 +1 +1 +1 diff --git a/tests/queries/0_stateless/02346_text_index_bug106460.sql b/tests/queries/0_stateless/02346_text_index_bug106460.sql new file mode 100644 index 000000000000..04459c383bfe --- /dev/null +++ b/tests/queries/0_stateless/02346_text_index_bug106460.sql @@ -0,0 +1,109 @@ +-- Tags: no-parallel-replicas +-- no-parallel-replicas: the distributed plan forces `use_skip_indexes_on_data_read = 0`, which turns +-- the direct read off, so those runs would only ever exercise the fallback path. + +-- Checks that a text index search returns the same rows while a lightweight update is pending on the +-- table as it does without, and as it does with direct read turned off. + +SET query_plan_direct_read_from_text_index = 1; +SET enable_lightweight_update = 1; + +DROP TABLE IF EXISTS tab; + +CREATE TABLE tab ( + id UInt64, + c UInt64, + s String, + INDEX idx s TYPE text(tokenizer = splitByNonAlpha)) +ENGINE = MergeTree ORDER BY id +SETTINGS + enable_block_number_column = 1, + enable_block_offset_column = 1, + apply_patches_on_merge = 0; -- keep the patch pending + +INSERT INTO tab SELECT number, 0, concat('tok', toString(number % 10), ' word') FROM numbers(1000); + +SYSTEM STOP MERGES tab; +UPDATE tab SET c = 1 WHERE id < 10; +-- the update is now pending + +-- 100 rows match `tok1`, of which only `id = 1` was updated. +SELECT count(), sum(c) FROM tab WHERE hasToken(s, 'tok1') SETTINGS query_plan_direct_read_from_text_index = 1; +SELECT count(), sum(c) FROM tab WHERE hasToken(s, 'tok1') SETTINGS query_plan_direct_read_from_text_index = 0; +SELECT count(), sum(c) FROM tab WHERE hasToken(s, 'tok1') SETTINGS use_skip_indexes = 0; + +-- A second predicate alongside the search: the rows disappeared here too. +SELECT id, c FROM tab WHERE hasToken(s, 'tok1') AND id < 25 ORDER BY id; + +SELECT '-- materialized'; +SYSTEM START MERGES tab; +ALTER TABLE tab MODIFY SETTING apply_patches_on_merge = 1; +OPTIMIZE TABLE tab FINAL; + +SELECT count(), sum(c) FROM tab WHERE hasToken(s, 'tok1'); + +-- Direct read must be used +SELECT count() > 0 FROM +( + EXPLAIN actions = 1 SELECT count() FROM tab WHERE hasToken(s, 'tok1') +) WHERE explain LIKE '%\_\_text_index%'; + +DROP TABLE tab; + +SELECT '-- With preprocessor'; + +CREATE TABLE tab +( + id UInt64, + c UInt64, + s String, + INDEX idx_s s TYPE text(tokenizer = splitByNonAlpha, preprocessor = lower(s)) +) +ENGINE = MergeTree ORDER BY id +SETTINGS + enable_block_number_column = 1, + enable_block_offset_column = 1, + apply_patches_on_merge = 0; + +INSERT INTO tab SELECT number, 0, if(number < 10, 'Hello World', 'foo bar') FROM numbers(1000); + +-- The preprocessor applies on the index path only, so these two legitimately differ. +SELECT count() FROM tab WHERE hasAnyTokens(s, 'hello') SETTINGS use_skip_indexes = 0; +SELECT count() FROM tab WHERE hasAnyTokens(s, 'hello') SETTINGS use_skip_indexes = 1; + +SYSTEM STOP MERGES tab; +UPDATE tab SET c = 1 WHERE id = 500; + +-- Same two answers with an update pending on `c`, which the index does not cover. +SELECT count() FROM tab WHERE hasAnyTokens(s, 'hello') SETTINGS use_skip_indexes = 0; +SELECT count() FROM tab WHERE hasAnyTokens(s, 'hello') SETTINGS use_skip_indexes = 1; + +DROP TABLE tab; + +-- Updating the indexed column itself. The search must see the new value. +SELECT 'patched indexed column'; + +CREATE TABLE tab ( + id UInt64, + s String, + INDEX idx s TYPE text(tokenizer = splitByNonAlpha)) +ENGINE = MergeTree ORDER BY id +SETTINGS + enable_block_number_column = 1, + enable_block_offset_column = 1, + apply_patches_on_merge = 0; + +INSERT INTO tab SELECT number, concat('tok', toString(number % 10), ' word') FROM numbers(1000); + +SYSTEM STOP MERGES tab; +UPDATE tab SET s = 'zebra word' WHERE id = 1; + +SELECT count() FROM tab WHERE hasToken(s, 'tok1') SETTINGS query_plan_direct_read_from_text_index = 1; +SELECT count() FROM tab WHERE hasToken(s, 'tok1') SETTINGS query_plan_direct_read_from_text_index = 0; +SELECT count() FROM tab WHERE hasToken(s, 'tok1') SETTINGS use_skip_indexes = 0; + +SELECT count() FROM tab WHERE hasToken(s, 'zebra') SETTINGS query_plan_direct_read_from_text_index = 1; +SELECT count() FROM tab WHERE hasToken(s, 'zebra') SETTINGS query_plan_direct_read_from_text_index = 0; +SELECT count() FROM tab WHERE hasToken(s, 'zebra') SETTINGS use_skip_indexes = 0; + +DROP TABLE tab; diff --git a/tests/queries/0_stateless/02346_text_index_mergeTreeTextIndex_access.reference b/tests/queries/0_stateless/02346_text_index_mergeTreeTextIndex_access.reference index 14efafd0c46c..d219eb8e6a85 100644 --- a/tests/queries/0_stateless/02346_text_index_mergeTreeTextIndex_access.reference +++ b/tests/queries/0_stateless/02346_text_index_mergeTreeTextIndex_access.reference @@ -9,5 +9,5 @@ ACCESS_DENIED OK ACCESS_DENIED ACCESS_DENIED -OK +ACCESS_DENIED ACCESS_DENIED diff --git a/tests/queries/0_stateless/02346_text_index_mergeTreeTextIndex_access.sh b/tests/queries/0_stateless/02346_text_index_mergeTreeTextIndex_access.sh index 87e86c4edc6c..ae07f79cd846 100755 --- a/tests/queries/0_stateless/02346_text_index_mergeTreeTextIndex_access.sh +++ b/tests/queries/0_stateless/02346_text_index_mergeTreeTextIndex_access.sh @@ -66,7 +66,7 @@ check_access "SELECT * FROM mergeTreeTextIndex(currentDatabase(), tab, idx_ab)" ## Row policy tests $CLICKHOUSE_CLIENT -q "GRANT SELECT ON $CLICKHOUSE_DATABASE.tab TO $user_name;" -# Row policy on column `a`: idx_a and idx_ab denied, idx_b allowed +# Row policy on column `a`: every index is denied, because each dictionary contains tokens of the hidden rows $CLICKHOUSE_CLIENT -q "CREATE ROW POLICY p1_03917 ON $CLICKHOUSE_DATABASE.tab FOR SELECT USING a = 'hello' TO $user_name;" check_access "SELECT * FROM mergeTreeTextIndex(currentDatabase(), tab, idx_a)" diff --git a/tests/queries/0_stateless/02354_vector_search_rescoring_distance_in_select_list.reference b/tests/queries/0_stateless/02354_vector_search_rescoring_distance_in_select_list.reference index e307b0ef6ae9..0bf9c660e01a 100644 --- a/tests/queries/0_stateless/02354_vector_search_rescoring_distance_in_select_list.reference +++ b/tests/queries/0_stateless/02354_vector_search_rescoring_distance_in_select_list.reference @@ -2,32 +2,32 @@ Create tables with Array(Float32) and Array(BFloat16) column Column: Array(Float32) -- Search vector: Array(Float64) 5 0 -6 0.09375 -7 0.203125 -8 0.296875 +6 0.1 +7 0.2 +8 0.3 -- Search vector: Array(Float32) 5 0 -6 0.09375 -7 0.203125 -8 0.296875 +6 0.1 +7 0.2 +8 0.3 -- Search vector: Array(BFloat16) 5 0 -6 0.09375 -7 0.203125 -8 0.296875 +6 0.1 +7 0.2 +8 0.3 Column: Array(BFloat16) -- Search vector: Array(Float64) 5 0 -6 0.09375 -7 0.1875 -8 0.296875 +6 0.1 +7 0.2 +8 0.3 -- Search vector: Array(Float32) 5 0 -6 0.09375 -7 0.1875 -8 0.296875 +6 0.1 +7 0.2 +8 0.3 -- Search vector: Array(BFloat16) 5 0 -6 0.09375 -7 0.1875 -8 0.296875 +6 0.1 +7 0.2 +8 0.3 diff --git a/tests/queries/0_stateless/02354_vector_search_rescoring_distance_in_select_list.sql b/tests/queries/0_stateless/02354_vector_search_rescoring_distance_in_select_list.sql index c4aa1debd865..e0e49b060904 100644 --- a/tests/queries/0_stateless/02354_vector_search_rescoring_distance_in_select_list.sql +++ b/tests/queries/0_stateless/02354_vector_search_rescoring_distance_in_select_list.sql @@ -1,6 +1,8 @@ -- Tags: no-fasttest, no-ordinary-database, no-parallel-replicas --- no-parallel-replicas: Because the test records and verifies --- _distance values returned from the rescoring optimization. +--- Distances are rounded to 1 decimal: USearch's SimSIMD kernels and its scalar +--- fallback disagree in the last bits (0.203125 vs 0.1875 for id 7). -- Issue #85514 -- @@ -65,21 +67,21 @@ SELECT 'Column: Array(Float32)'; SELECT '-- Search vector: Array(Float64)'; WITH CAST([0.0, 2.0] AS Array(Float64)) AS reference_vec -SELECT id, L2Distance(vec, reference_vec) +SELECT id, round(L2Distance(vec, reference_vec), 1) FROM tab_f32 ORDER BY L2Distance(vec, reference_vec) LIMIT 4; SELECT '-- Search vector: Array(Float32)'; WITH CAST([0.0, 2.0] AS Array(BFloat16)) AS reference_vec -SELECT id, L2Distance(vec, reference_vec) +SELECT id, round(L2Distance(vec, reference_vec), 1) FROM tab_f32 ORDER BY L2Distance(vec, reference_vec) LIMIT 4; SELECT '-- Search vector: Array(BFloat16)'; WITH CAST([0.0, 2.0] AS Array(BFloat16)) AS reference_vec -SELECT id, L2Distance(vec, reference_vec) +SELECT id, round(L2Distance(vec, reference_vec), 1) FROM tab_f32 ORDER BY L2Distance(vec, reference_vec) LIMIT 4; @@ -88,21 +90,21 @@ SELECT 'Column: Array(BFloat16)'; SELECT '-- Search vector: Array(Float64)'; WITH [0.0, 2.0] AS reference_vec -SELECT id, L2Distance(vec, reference_vec) +SELECT id, round(L2Distance(vec, reference_vec), 1) FROM tab_bf16 ORDER BY L2Distance(vec, reference_vec) LIMIT 4; SELECT '-- Search vector: Array(Float32)'; WITH CAST([0.0, 2.0] AS Array(BFloat16)) AS reference_vec -SELECT id, L2Distance(vec, reference_vec) +SELECT id, round(L2Distance(vec, reference_vec), 1) FROM tab_bf16 ORDER BY L2Distance(vec, reference_vec) LIMIT 4; SELECT '-- Search vector: Array(BFloat16)'; WITH CAST([0.0, 2.0] AS Array(Float32)) AS reference_vec -SELECT id, L2Distance(vec, reference_vec) +SELECT id, round(L2Distance(vec, reference_vec), 1) FROM tab_bf16 ORDER BY L2Distance(vec, reference_vec) LIMIT 4; diff --git a/tests/queries/0_stateless/03357_join_pk_sharding.reference b/tests/queries/0_stateless/03357_join_pk_sharding.reference index 110b13f33c48..a411a626b93d 100644 --- a/tests/queries/0_stateless/03357_join_pk_sharding.reference +++ b/tests/queries/0_stateless/03357_join_pk_sharding.reference @@ -210,13 +210,13 @@ Expression ((Project names + Projection)) Expression (Post Join Actions) Join (JOIN FillRightFirst) Algorithm: ConcurrentHashJoin - Clauses: [(multiply(__table1.a, 2_UInt8), plus(__table1.b, __table1.c), __table1.d) = (multiply(__table3.a, 2_UInt8), multiply(__table3.c, 2_UInt8), __table3.d)] + Clauses: [(multiply(__table1.a, 2_UInt8), plus(__table1.b, __table1.c), __table1.d) = (multiply(__table3.a, 2_UInt8), multiply(__table3.c, 2_UInt8), toNullable(__table3.d))] Sharding: [(multiply(a, 2) = multiply(a, 2)), (plus(b, c) = multiply(c, 2))] Expression (Left Pre Join Actions) Expression (Post Join Actions) Join (JOIN FillRightFirst) Algorithm: ConcurrentHashJoin - Clauses: [(multiply(__table1.a, 2_UInt8), __table1.d, plus(__table1.b, __table1.c)) = (plus(__table2.c, __table2.d), __table2.a, multiply(__table2.b, 2_UInt8))] + Clauses: [(multiply(__table1.a, 2_UInt8), __table1.d, plus(__table1.b, __table1.c)) = (plus(__table2.c, __table2.d), toNullable(__table2.a), multiply(__table2.b, 2_UInt8))] Sharding: [(multiply(a, 2) = plus(c, d)), (plus(b, c) = multiply(b, 2))] Expression (Left Pre Join Actions) Expression (Change column names to column identifiers) diff --git a/tests/queries/0_stateless/03402_concurrent_right_full_join.reference b/tests/queries/0_stateless/03402_concurrent_right_full_join.reference index 302682474cfc..866ff4078b73 100644 --- a/tests/queries/0_stateless/03402_concurrent_right_full_join.reference +++ b/tests/queries/0_stateless/03402_concurrent_right_full_join.reference @@ -38,8 +38,8 @@ Positions: 4 0 1 Actions: INPUT : 0 -> __table1.id UInt32 : 0 INPUT : 1 -> __table1.value String : 1 FUNCTION toNullable(__table1.id : 0) -> toNullable(__table1.id) Nullable(UInt32) : 2 - FUNCTION toNullable(__table1.value : 1) -> toNullable(__table1.value) Nullable(String) : 3 - Positions: 0 2 3 1 + FUNCTION toNullable(__table1.value :: 1) -> toNullable(__table1.value) Nullable(String) : 3 + Positions: 0 2 3 Expression (Change column names to column identifiers) Actions: INPUT : 0 -> id UInt32 : 0 INPUT : 1 -> value String : 1 diff --git a/tests/queries/0_stateless/04357_analysisOfVariance_deserialize_size.reference b/tests/queries/0_stateless/04357_analysisOfVariance_deserialize_size.reference new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tests/queries/0_stateless/04357_analysisOfVariance_deserialize_size.sql b/tests/queries/0_stateless/04357_analysisOfVariance_deserialize_size.sql new file mode 100644 index 000000000000..5474dc68c519 --- /dev/null +++ b/tests/queries/0_stateless/04357_analysisOfVariance_deserialize_size.sql @@ -0,0 +1,10 @@ +-- analysisOfVariance keeps three parallel arrays in its aggregate state (one entry per group). +-- The finalize path iterates one array and indexes the others at the same position, so a state +-- deserialized from raw bytes with mismatched array lengths reads out of bounds. Such a state is +-- reachable from any user via CAST of a String to AggregateFunction. + +-- Normal aggregation builds equal-length arrays and must keep working. +SELECT analysisOfVariance(number, number % 3) FROM numbers(30) FORMAT Null; + +-- xs1 and xs2 have two groups, ns has one -> ns is read past its end. +SELECT finalizeAggregation(CAST(unhex('02000000000000f03f000000000000f03f02000000000000f03f000000000000f03f010500000000000000') AS AggregateFunction(analysisOfVariance, Float64, UInt8))); -- { serverError INCORRECT_DATA } diff --git a/tests/queries/0_stateless/04367_distributed_plan_merge_scatter_multishard.reference b/tests/queries/0_stateless/04367_distributed_plan_merge_scatter_multishard.reference index 1e2b514dd4ce..6007533519d5 100644 --- a/tests/queries/0_stateless/04367_distributed_plan_merge_scatter_multishard.reference +++ b/tests/queries/0_stateless/04367_distributed_plan_merge_scatter_multishard.reference @@ -1,17 +1,17 @@ Expression ((Project names + Projection)) MergingAggregated ReadFromMerge - GatherExchange - Expression - Aggregating - ShuffleExchange + Expression + MergingAggregated (merge) + GatherExchange + Aggregating (partial) Expression (Before GROUP BY) Filter ReadFromMergeTree (default.base107946_1) - GatherExchange - Expression - Aggregating - ShuffleExchange + Expression + MergingAggregated (merge) + GatherExchange + Aggregating (partial) Expression (Before GROUP BY) Filter ReadFromMergeTree (default.base107946_4) diff --git a/tests/queries/0_stateless/04367_distributed_plan_merge_scatter_multishard.sql b/tests/queries/0_stateless/04367_distributed_plan_merge_scatter_multishard.sql index d756bd43797b..ae1d43a893cc 100644 --- a/tests/queries/0_stateless/04367_distributed_plan_merge_scatter_multishard.sql +++ b/tests/queries/0_stateless/04367_distributed_plan_merge_scatter_multishard.sql @@ -26,18 +26,23 @@ CREATE TABLE m107946 ENGINE = Merge(currentDatabase(), '^d107946_(1|4)$'); -- aggregation with a non-zero limit (Code 344). -- prefer_localhost_replica must be 1: with 0 the Distributed child ships its plan to the localhost -- replica over the classic protocol, which cannot deserialize distributed-plan steps (Code 47). --- distributed_plan_max_rows_to_broadcast = 0 forces shuffle aggregation and bucketed reads, so the --- child plan deterministically contains exchanges. +-- distributed_plan_max_rows_to_broadcast = 0 makes the size-based strategy choice pick shuffle +-- aggregation, and forces bucketed reads, so the child plan deterministically contains exchanges. -- distributed_plan_default_shuffle_join_bucket_count is pinned > 1: with 1 bucket the broken path -- is not exercised. +-- distributed_aggregation_memory_efficient must be 1: the plan below depends on the bucket-order +-- promise, which holds when either it or enable_memory_bound_merging_of_aggregation_results is on, and +-- the runner randomizes both. SET make_distributed_plan = 1, enable_parallel_replicas = 0, distributed_plan_execute_locally = 1, use_statistics = 1, distributed_plan_optimize_exchanges = 1, enable_join_runtime_filters = 0, max_rows_to_group_by = 0, prefer_localhost_replica = 1, distributed_plan_max_rows_to_broadcast = 0, - distributed_plan_default_shuffle_join_bucket_count = 8, explain_query_plan_default = 'legacy'; + distributed_plan_default_shuffle_join_bucket_count = 8, explain_query_plan_default = 'legacy', + distributed_aggregation_memory_efficient = 1; -- The outer plan stays single-stage: the children aggregate up to the mergeable state themselves, --- so there are no exchanges above ReadFromMerge. Each child plan under it carries a single layer --- of exchanges: a gather over the aggregation over the shuffle. +-- so there are no exchanges above ReadFromMerge. Aggregating to the mergeable state promises results +-- in bucket order, and the shuffle strategy cannot keep that promise, so each child plan carries a +-- partial aggregation and its merge around a gather rather than an aggregation over a shuffle. EXPLAIN SELECT count(_table) FROM m107946 WHERE _table = 'base107946_1' GROUP BY _table; -- The reproducer from the issue. It must not throw; no rows match because _table exposes the diff --git a/tests/queries/0_stateless/04602_group_by_all_suspicious_types.reference b/tests/queries/0_stateless/04602_group_by_all_suspicious_types.reference index e0a24a12d854..cb962acd614c 100644 --- a/tests/queries/0_stateless/04602_group_by_all_suspicious_types.reference +++ b/tests/queries/0_stateless/04602_group_by_all_suspicious_types.reference @@ -1,2 +1,8 @@ 1 ('1',1) +1 +(1,2) +1 +\N +1 +v0 diff --git a/tests/queries/0_stateless/04602_group_by_all_suspicious_types.sql b/tests/queries/0_stateless/04602_group_by_all_suspicious_types.sql index f2e23cb156ae..d622b03c603c 100644 --- a/tests/queries/0_stateless/04602_group_by_all_suspicious_types.sql +++ b/tests/queries/0_stateless/04602_group_by_all_suspicious_types.sql @@ -16,3 +16,31 @@ SELECT tuple(d, 1) FROM (SELECT 1::Dynamic AS d) GROUP BY tuple(d, 1) SETTINGS a -- Allowed once the setting permits suspicious types. SELECT d FROM (SELECT 1::Dynamic AS d) GROUP BY ALL SETTINGS allow_suspicious_types_in_group_by = 1; SELECT tuple(d, 1) FROM (SELECT 1::Dynamic AS d) GROUP BY ALL SETTINGS allow_suspicious_types_in_group_by = 1; + +-- `validate_group_by_all_key_types` gates the key types that `GROUP BY ALL` expands into, and only +-- those: with it off `GROUP BY ALL` accepts a suspicious key type again while an explicit `GROUP BY` +-- keeps rejecting it, which is what distinguishes this setting from `allow_suspicious_types_in_group_by`. +SELECT d FROM (SELECT 1::Dynamic AS d) GROUP BY ALL SETTINGS allow_suspicious_types_in_group_by = 0, validate_group_by_all_key_types = 0; +SELECT d FROM (SELECT 1::Dynamic AS d) GROUP BY d SETTINGS allow_suspicious_types_in_group_by = 0, validate_group_by_all_key_types = 0; -- { serverError ILLEGAL_COLUMN } + +-- The tuple expansion is not gated, so a tuple grouping key is still unwrapped into its elements with +-- the validation off and an `ORDER BY` of the same tuple still finds them in the aggregated block. +SELECT tuple(c0, c1) AS t FROM (SELECT 1 c0, 2 c1) v0 GROUP BY ALL ORDER BY t + SETTINGS validate_group_by_all_key_types = 0, optimize_injective_functions_in_group_by = 0; + +-- Under `group_by_use_nulls` with a `WITH ROLLUP` modifier the `GROUP BY ALL` keys are expanded before +-- they are resolved, so they are validated on the same path as an explicit `GROUP BY`. The setting +-- reaches that path too, while an explicit `GROUP BY` there stays unconditional. The second row of the +-- accepted arm is the rollup total, whose key is NULL because `group_by_use_nulls` promoted it. +SELECT d FROM (SELECT 1::Dynamic AS d) GROUP BY ALL WITH ROLLUP SETTINGS group_by_use_nulls = 1, allow_suspicious_types_in_group_by = 0; -- { serverError ILLEGAL_COLUMN } +SELECT d FROM (SELECT 1::Dynamic AS d) GROUP BY ALL WITH ROLLUP SETTINGS group_by_use_nulls = 1, allow_suspicious_types_in_group_by = 0, validate_group_by_all_key_types = 0; +SELECT d FROM (SELECT 1::Dynamic AS d) GROUP BY d WITH ROLLUP SETTINGS group_by_use_nulls = 1, allow_suspicious_types_in_group_by = 0, validate_group_by_all_key_types = 0; -- { serverError ILLEGAL_COLUMN } + +-- `compatibility` with a version before 26.7 restores the earlier acceptance; with 26.7 itself it does +-- not, because 26.7 is the version that started rejecting such a key. +SELECT d FROM (SELECT 1::Dynamic AS d) GROUP BY ALL SETTINGS allow_suspicious_types_in_group_by = 0, compatibility = '26.6'; +SELECT d FROM (SELECT 1::Dynamic AS d) GROUP BY ALL SETTINGS allow_suspicious_types_in_group_by = 0, compatibility = '26.7'; -- { serverError ILLEGAL_COLUMN } + +-- The reported shape: an untyped JSON subpath is a `Dynamic` grouping key, so it is gated the same way. +SELECT c1.p1 FROM (SELECT '{"p1":"v0"}'::JSON AS c1) GROUP BY ALL SETTINGS allow_suspicious_types_in_group_by = 0, validate_group_by_all_key_types = 0; +SELECT c1.p1 FROM (SELECT '{"p1":"v0"}'::JSON AS c1) GROUP BY c1.p1 SETTINGS allow_suspicious_types_in_group_by = 0, validate_group_by_all_key_types = 0; -- { serverError ILLEGAL_COLUMN } diff --git a/tests/queries/0_stateless/04616_replicated_serialization_oob_index_native_protocol.python b/tests/queries/0_stateless/04616_replicated_serialization_oob_index_native_protocol.python index 4ecd4becd513..048ef838a2e0 100644 --- a/tests/queries/0_stateless/04616_replicated_serialization_oob_index_native_protocol.python +++ b/tests/queries/0_stateless/04616_replicated_serialization_oob_index_native_protocol.python @@ -436,10 +436,8 @@ def indexRowCountMismatchScenario(): def sendSparseOverReplicatedBlock(s): - # A Sparse-over-Replicated kind stack makes SerializationSparse hand its values column - # (a fresh ColumnSparse always seeds one default value) to the nested Replicated - # deserialization, which then hits the "non-empty column" check. That check is decided - # by the wire-supplied kind stack and must be INCORRECT_DATA, not LOGICAL_ERROR. + # Sparse can sit inside Replicated but not the other way round, so this kind stack is rejected as + # out of order. A wire-supplied kind stack must produce INCORRECT_DATA, not LOGICAL_ERROR. END_OF_GRANULE_FLAG = 1 << 62 limit = 4 @@ -474,7 +472,7 @@ def sparseOverReplicatedScenario(): s.connect((CLICKHOUSE_HOST, CLICKHOUSE_PORT)) connectAndStartInsert(s) sendSparseOverReplicatedBlock(s) - expectIncorrectData(s, "Reading into non-empty column ColumnReplicated") + expectIncorrectData(s, "is out of order") print("sparse over replicated: caught expected exception INCORRECT_DATA") diff --git a/tests/queries/0_stateless/04627_url_database_engine.reference b/tests/queries/0_stateless/04627_url_database_engine.reference index 5be03016e806..9b1a3484096a 100644 --- a/tests/queries/0_stateless/04627_url_database_engine.reference +++ b/tests/queries/0_stateless/04627_url_database_engine.reference @@ -23,5 +23,12 @@ ENGINE = URL('http://server/') --- URL database engine on the server Ok. Ok. ---- the base URL must contain a scheme +--- the base URL must contain a scheme, and the rejected value is not echoed back (last line counts the leaks) must contain a scheme +must contain a scheme +0 +--- the base-url password is masked in the display surfaces (the last line counts the leaks) +URL(\'https://user:[HIDDEN]@localhost:11111/x/\') +CREATE DATABASE secret_db +ENGINE = URL('https://user:[HIDDEN]@localhost:11111/x/') +0 diff --git a/tests/queries/0_stateless/04627_url_database_engine.sh b/tests/queries/0_stateless/04627_url_database_engine.sh index baaf4615ed99..6c2a3cb40351 100755 --- a/tests/queries/0_stateless/04627_url_database_engine.sh +++ b/tests/queries/0_stateless/04627_url_database_engine.sh @@ -54,5 +54,16 @@ ${CLICKHOUSE_CLIENT} -q "SELECT * FROM ${WEB_DB}.\`ping\`" ${CLICKHOUSE_CLIENT} -q "SELECT * FROM ${WEB_DB}.\`${SERVER_URL}/ping\`" ${CLICKHOUSE_CLIENT} -q "DROP DATABASE ${WEB_DB}" -echo '--- the base URL must contain a scheme' +echo '--- the base URL must contain a scheme, and the rejected value is not echoed back (last line counts the leaks)' ${CLICKHOUSE_LOCAL} -q "CREATE DATABASE bad ENGINE = URL('localhost/dir/')" 2>&1 | grep -oF 'must contain a scheme' | head -1 +${CLICKHOUSE_LOCAL} -q "CREATE DATABASE bad ENGINE = URL('user:SEKRIT_PW@localhost/dir/')" 2>&1 | grep -oF 'must contain a scheme' | head -1 +${CLICKHOUSE_LOCAL} -q "CREATE DATABASE bad ENGINE = URL('user:SEKRIT_PW@localhost/dir/')" 2>&1 | grep -c SEKRIT_PW + +echo '--- the base-url password is masked in the display surfaces (the last line counts the leaks)' +SECRET_DB="${CLICKHOUSE_DATABASE}_04627_secret" +${CLICKHOUSE_CLIENT} -q "DROP DATABASE IF EXISTS ${SECRET_DB}" +${CLICKHOUSE_CLIENT} -q "CREATE DATABASE ${SECRET_DB} ENGINE = URL('https://user:SEKRIT_PW@localhost:11111/x/')" +${CLICKHOUSE_CLIENT} -q "SELECT engine_full FROM system.databases WHERE name = '${SECRET_DB}'" +${CLICKHOUSE_CLIENT} -q "SHOW CREATE DATABASE ${SECRET_DB} FORMAT TabSeparatedRaw" | sed "s|${SECRET_DB}|secret_db|" +${CLICKHOUSE_CLIENT} -q "SHOW CREATE DATABASE ${SECRET_DB}" | grep -c SEKRIT_PW +${CLICKHOUSE_CLIENT} -q "DROP DATABASE ${SECRET_DB}" diff --git a/tests/queries/0_stateless/04653_parameterized_view_own_name.reference b/tests/queries/0_stateless/04653_parameterized_view_own_name.reference new file mode 100644 index 000000000000..00065a0b9fce --- /dev/null +++ b/tests/queries/0_stateless/04653_parameterized_view_own_name.reference @@ -0,0 +1,37 @@ +-- a parameterized view may be joined without an alias +1 +1 +1 +1 +-- its columns may be qualified with the view name +1 +1 +t1 +1 +1 +1 +-- matcher-expanded columns are qualified with the view name +tenant_id String +host_id UInt64 +pv.tenant_id String +pv.host_id UInt64 +tenant_id String +host_id UInt64 +pv.host_id UInt64 +-- control: a real table function contributes no qualifier +tenant_id String +host_id UInt64 +number UInt64 +-- the parameter type is irrelevant +1 +-- controls: an alias or no qualifier always worked +1 +1 +-- controls: a regular table function and an ordinary view must not gain a bindable name +tenant_id String +host_id UInt64 +tenant_id String +-- a view in another database binds by the view name too +1 +1 +-- control: qualifying with a database that does not hold the view does not bind diff --git a/tests/queries/0_stateless/04653_parameterized_view_own_name.sql b/tests/queries/0_stateless/04653_parameterized_view_own_name.sql new file mode 100644 index 000000000000..99a2904a47be --- /dev/null +++ b/tests/queries/0_stateless/04653_parameterized_view_own_name.sql @@ -0,0 +1,78 @@ +-- Tags: need-query-parameters + +-- The fix is in the query-tree analyzer, so pin it: the `old analyzer` CI jobs link +-- `users.d/analyzer.xml` and the randomized `compatibility='<24.3'` setting also reverts +-- `allow_experimental_analyzer`, and on the legacy path a JOINed parameterized view is not +-- expanded at all (only the left table storage is), so these rows would assert a different +-- code path's behaviour. +SET enable_analyzer = 1; + +DROP TABLE IF EXISTS local_data; +DROP VIEW IF EXISTS pv; +DROP VIEW IF EXISTS pv2; + +CREATE TABLE local_data (tenant_id String, host_id UInt64) ENGINE = MergeTree ORDER BY tenant_id; +INSERT INTO local_data SELECT 't1', 1; + +CREATE VIEW pv AS SELECT tenant_id, host_id FROM local_data WHERE tenant_id IN ({tenants:Array(String)}); +CREATE VIEW pv2 AS SELECT tenant_id, host_id FROM local_data WHERE tenant_id = {tenant:String}; + +SELECT '-- a parameterized view may be joined without an alias'; +SELECT count() FROM local_data JOIN pv(tenants = ['t1']) USING (tenant_id); +SELECT count() FROM local_data AS t JOIN pv(tenants = ['t1']) ON t.tenant_id = pv.tenant_id; +SELECT count() FROM local_data, pv(tenants = ['t1']); +SELECT count() FROM pv(tenants = ['t1']) JOIN local_data AS t USING (tenant_id); + +SELECT '-- its columns may be qualified with the view name'; +SELECT pv.host_id FROM pv(tenants = ['t1']); +SELECT host_id FROM pv(tenants = ['t1']) WHERE pv.host_id = 1; +SELECT pv.tenant_id FROM pv(tenants = ['t1']) GROUP BY pv.tenant_id; +SELECT pv.host_id FROM pv(tenants = ['t1']) ORDER BY pv.host_id; +SELECT {CLICKHOUSE_DATABASE:Identifier}.pv.host_id FROM pv(tenants = ['t1']); +SELECT count() FROM local_data AS t JOIN pv(tenants = ['t1']) ON t.tenant_id = pv.tenant_id SETTINGS joined_subquery_requires_alias = 0; + +SELECT '-- matcher-expanded columns are qualified with the view name'; +DESCRIBE (SELECT * FROM local_data, pv(tenants = ['t1'])); +DESCRIBE (SELECT * FROM local_data JOIN pv(tenants = ['t1']) USING (tenant_id)); + +SELECT '-- control: a real table function contributes no qualifier'; +DESCRIBE (SELECT * FROM local_data AS t JOIN numbers(3) AS n ON 1 = 1); + +SELECT '-- the parameter type is irrelevant'; +SELECT pv2.host_id FROM pv2(tenant = 't1'); + +SELECT '-- controls: an alias or no qualifier always worked'; +SELECT p.host_id FROM pv(tenants = ['t1']) AS p; +SELECT host_id FROM pv(tenants = ['t1']); + +SELECT '-- controls: a regular table function and an ordinary view must not gain a bindable name'; +SELECT numbers.number FROM numbers(3); -- { serverError UNKNOWN_IDENTIFIER } +SELECT count() FROM local_data JOIN numbers(3) ON 1 = 1; -- { serverError ALIAS_REQUIRED } +SELECT view.dummy FROM view(SELECT 1 AS dummy); -- { serverError UNKNOWN_IDENTIFIER } +SELECT count() FROM local_data JOIN view(SELECT 1 AS dummy) ON 1 = 1; -- { serverError ALIAS_REQUIRED } +-- `tenant_id` collides with `local_data`'s, so the matcher must decide whether to qualify it; +-- an ordinary view contributes no qualification parts, so the second one stays bare. +DESCRIBE (SELECT * FROM local_data, view(SELECT 't1' AS tenant_id)) SETTINGS joined_subquery_requires_alias = 0; + +DROP VIEW pv2; +DROP VIEW pv; +DROP TABLE local_data; + +-- The view also binds by its own name when it lives outside the session's current database. +-- The call itself stays unqualified: a query parameter is not accepted in the database +-- position of a table function, and a literal database name would not be parallel-safe. +DROP DATABASE IF EXISTS {CLICKHOUSE_DATABASE_1:Identifier}; +CREATE DATABASE {CLICKHOUSE_DATABASE_1:Identifier}; +USE {CLICKHOUSE_DATABASE_1:Identifier}; +CREATE TABLE local_data (tenant_id String, host_id UInt64) ENGINE = MergeTree ORDER BY tenant_id; +INSERT INTO local_data SELECT 't1', 1; +CREATE VIEW pv AS SELECT tenant_id, host_id FROM local_data WHERE tenant_id IN ({tenants:Array(String)}); + +SELECT '-- a view in another database binds by the view name too'; +SELECT pv.host_id FROM pv(tenants = ['t1']); +SELECT {CLICKHOUSE_DATABASE_1:Identifier}.pv.host_id FROM pv(tenants = ['t1']); + +SELECT '-- control: qualifying with a database that does not hold the view does not bind'; +SELECT {CLICKHOUSE_DATABASE:Identifier}.pv.host_id FROM pv(tenants = ['t1']); -- { serverError UNKNOWN_IDENTIFIER } + +DROP DATABASE {CLICKHOUSE_DATABASE_1:Identifier}; diff --git a/tests/queries/0_stateless/04790_ttl_drop_merge_text_index.reference b/tests/queries/0_stateless/04790_ttl_drop_merge_text_index.reference new file mode 100644 index 000000000000..d2e67b3bbdc3 --- /dev/null +++ b/tests/queries/0_stateless/04790_ttl_drop_merge_text_index.reference @@ -0,0 +1,26 @@ +-- Case 1: text index +0 +idx_txt text 1 +1 +-- Case 2: materialize_skip_indexes_on_merge = 0 is respected +0 +idx_set set 0 +idx_txt text 0 +1 +-- Case 3: inert index +1 +0 +i0 hypothesis 0 +idx_txt text 1 +1 +-- Case 4: default TTL settings +0 +1 +1 +-- Case 5: projections are not announced as pending work +0 +1 +1 +0 +1 +1 diff --git a/tests/queries/0_stateless/04790_ttl_drop_merge_text_index.sh b/tests/queries/0_stateless/04790_ttl_drop_merge_text_index.sh new file mode 100755 index 000000000000..f1c57f1260d9 --- /dev/null +++ b/tests/queries/0_stateless/04790_ttl_drop_merge_text_index.sh @@ -0,0 +1,280 @@ +#!/usr/bin/env bash +# Tags: long + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# A TTLDrop merge takes the short-circuit that skips the read pipeline. Every index the +# resulting empty part must still carry has to be handled there, because the builders that +# normally produce them live inside the skipped pipeline. +# +# Only the background selector assigns MergeType::TTLDrop; OPTIMIZE TABLE FINAL assigns +# MergeType::Regular. Every case below therefore drives a background TTL merge and waits for +# it, bounded by wall-clock time rather than a fixed iteration count. + +function wait_for_ttl_drop() +{ + local table=$1 + local deadline=$((SECONDS + 90)) + while [ "$SECONDS" -lt "$deadline" ]; do + local rows + rows=$(${CLICKHOUSE_CLIENT} -q "SELECT count() FROM $table") + if [ "$rows" = "0" ]; then + return + fi + sleep 0.5 + done + echo "timed out waiting for the TTL drop merge on $table" +} + +echo "-- Case 1: text index" + +# Before the fix the merge threw LOGICAL_ERROR 'Text index transform for index ... not found' +# and retried forever, so the expired rows were never dropped. +${CLICKHOUSE_CLIENT} -q " + SET allow_experimental_full_text_index = 1; + + CREATE TABLE t_ttl_drop_text + ( + id UInt64, + value String, + event_time DateTime DEFAULT now() - INTERVAL 2 DAY, + INDEX idx_txt value TYPE text(tokenizer = 'splitByNonAlpha') GRANULARITY 1 + ) + ENGINE = MergeTree() + ORDER BY id + TTL event_time + INTERVAL 1 DAY + SETTINGS + ttl_only_drop_parts = 1, + merge_with_ttl_timeout = 0, + min_bytes_for_wide_part = 1, + -- keep the 0-row part so the index assertions below have something to read + remove_empty_parts = 0; + + SYSTEM STOP MERGES t_ttl_drop_text; + + INSERT INTO t_ttl_drop_text (id, value) SELECT number, 'w' || toString(number) FROM numbers(100); + INSERT INTO t_ttl_drop_text (id, value) SELECT number, 'w' || toString(number) FROM numbers(100); + + SYSTEM START MERGES t_ttl_drop_text; +" + +wait_for_ttl_drop "t_ttl_drop_text" + +${CLICKHOUSE_CLIENT} -q "SELECT count() FROM t_ttl_drop_text;" + +# The empty part must still carry the text index files, byte-for-byte as the normal 0-row +# path writes them: a replica that fetches the part compares checksums, so an index dropped +# here would diverge. data_compressed_bytes counts the serialized index header, which is +# non-empty even with no tokens -- it reads 0 when the index files are missing entirely. +${CLICKHOUSE_CLIENT} -q " + SELECT name, type, data_compressed_bytes > 0 + FROM system.data_skipping_indices + WHERE database = currentDatabase() AND table = 't_ttl_drop_text'; +" + +${CLICKHOUSE_CLIENT} -q "CHECK TABLE t_ttl_drop_text SETTINGS check_query_single_value_result = 1;" +${CLICKHOUSE_CLIENT} -q "DROP TABLE t_ttl_drop_text;" + +echo "-- Case 2: materialize_skip_indexes_on_merge = 0 is respected" + +# The 'forget about skip indexes' clear runs before the short-circuit, which used to put +# every index straight back -- so the setting was honoured for a normal merge and silently +# ignored for a TTLDrop merge. With a text index that also reintroduced case 1's failure. +${CLICKHOUSE_CLIENT} -q " + SET allow_experimental_full_text_index = 1; + + CREATE TABLE t_ttl_drop_no_materialize + ( + id UInt64, + value String, + event_time DateTime DEFAULT now() - INTERVAL 2 DAY, + INDEX idx_txt value TYPE text(tokenizer = 'splitByNonAlpha') GRANULARITY 1, + INDEX idx_set id TYPE set(100) GRANULARITY 1 + ) + ENGINE = MergeTree() + ORDER BY id + TTL event_time + INTERVAL 1 DAY + SETTINGS + ttl_only_drop_parts = 1, + merge_with_ttl_timeout = 0, + min_bytes_for_wide_part = 1, + materialize_skip_indexes_on_merge = 0, + remove_empty_parts = 0; + + SYSTEM STOP MERGES t_ttl_drop_no_materialize; + + INSERT INTO t_ttl_drop_no_materialize (id, value) SELECT number, 'w' || toString(number) FROM numbers(100); + INSERT INTO t_ttl_drop_no_materialize (id, value) SELECT number, 'w' || toString(number) FROM numbers(100); + + SYSTEM START MERGES t_ttl_drop_no_materialize; +" + +wait_for_ttl_drop "t_ttl_drop_no_materialize" + +${CLICKHOUSE_CLIENT} -q "SELECT count() FROM t_ttl_drop_no_materialize;" + +# The setting suppresses both index families, so neither writes any files. +${CLICKHOUSE_CLIENT} -q " + SELECT name, type, data_compressed_bytes > 0 + FROM system.data_skipping_indices + WHERE database = currentDatabase() AND table = 't_ttl_drop_no_materialize' + ORDER BY name; +" + +${CLICKHOUSE_CLIENT} -q "CHECK TABLE t_ttl_drop_no_materialize SETTINGS check_query_single_value_result = 1;" +${CLICKHOUSE_CLIENT} -q "DROP TABLE t_ttl_drop_no_materialize;" + +echo "-- Case 3: inert index" + +# A legacy 'hypothesis' index is inert: it holds no data and cannot be recomputed, so +# createIndexAggregator rejects it with ILLEGAL_INDEX. The short-circuit used to omit the +# isInert filter the normal path applies, which wedged the merge and left the rows in place. +# The non-inert sibling pins that the filter is per index rather than per table: it must still +# materialize while the inert one is skipped. +# Full-definition ATTACH is the only way to get such a table, and it needs an explicit UUID. +uuid=$(${CLICKHOUSE_CLIENT} -q "SELECT generateUUIDv4()") +${CLICKHOUSE_CLIENT} -q " + SET send_logs_level = 'fatal'; + SET allow_experimental_full_text_index = 1; + + ATTACH TABLE t_ttl_drop_inert UUID '$uuid' + ( + id UInt64, + value String, + event_time DateTime DEFAULT now() - INTERVAL 2 DAY, + INDEX i0 90 % id TYPE hypothesis, + INDEX idx_txt value TYPE text(tokenizer = 'splitByNonAlpha') GRANULARITY 1 + ) + ENGINE = MergeTree() + PRIMARY KEY tuple() + TTL event_time + INTERVAL 1 DAY + SETTINGS + ttl_only_drop_parts = 1, + merge_with_ttl_timeout = 0, + min_bytes_for_wide_part = 1, + remove_empty_parts = 0; +" + +${CLICKHOUSE_CLIENT} -q " + SELECT count() FROM system.data_skipping_indices + WHERE database = currentDatabase() AND table = 't_ttl_drop_inert' AND type = 'hypothesis'; +" + +${CLICKHOUSE_CLIENT} -q " + SYSTEM STOP MERGES t_ttl_drop_inert; + + INSERT INTO t_ttl_drop_inert (id, value) SELECT number, 'w' || toString(number) FROM numbers(100); + INSERT INTO t_ttl_drop_inert (id, value) SELECT number + 100, 'w' || toString(number) FROM numbers(100); + + SYSTEM START MERGES t_ttl_drop_inert; +" + +wait_for_ttl_drop "t_ttl_drop_inert" + +${CLICKHOUSE_CLIENT} -q "SELECT count() FROM t_ttl_drop_inert;" + +# The inert index is skipped individually; its non-inert sibling still gets materialized. +${CLICKHOUSE_CLIENT} -q " + SELECT name, type, data_compressed_bytes > 0 + FROM system.data_skipping_indices + WHERE database = currentDatabase() AND table = 't_ttl_drop_inert' + ORDER BY name; +" + +${CLICKHOUSE_CLIENT} -q "CHECK TABLE t_ttl_drop_inert SETTINGS check_query_single_value_result = 1;" +${CLICKHOUSE_CLIENT} -q "DROP TABLE t_ttl_drop_inert;" + +echo "-- Case 4: default TTL settings" + +# TTLDrop selection ignores merge_with_ttl_timeout: TTLPartDropMergeSelector passes a null +# merge_due_times, and selectPartsToMerge schedules no next time for MergeType::TTLDrop. So +# the short-circuit is reachable with every TTL setting left at its default, including +# ttl_only_drop_parts = 0, which the cases above do not cover. +# +# No OPTIMIZE here: with one level-0 part it would select a MergeType::Regular merge of its +# own and could empty the table through the normal pipeline before the background TTLDrop is +# picked up. Only the background selector produces the merge type this case is about. +${CLICKHOUSE_CLIENT} -q " + SET allow_experimental_full_text_index = 1; + + CREATE TABLE t_ttl_drop_default + ( + c0 UInt64, + c1 Date, + c2 String, + INDEX idx0 c2 TYPE text(tokenizer = 'splitByNonAlpha') GRANULARITY 1 + ) + ENGINE = MergeTree() + ORDER BY c0 + TTL c1 + INTERVAL 1 SECOND DELETE; + + INSERT INTO t_ttl_drop_default SELECT number, toDate('2020-01-01'), 'w' || toString(number) FROM numbers(100); +" + +wait_for_ttl_drop "t_ttl_drop_default" + +${CLICKHOUSE_CLIENT} -q "SELECT count() FROM t_ttl_drop_default;" + +# Emptying the table is not enough on its own: assert the merge that did it was a TTLDrop +# that succeeded and read no rows. error = 0 excludes the failed-and-retried merge the +# unfixed code produced, which MergePlainMergeTreeTask also logs under this merge_reason; +# read_rows = 0 is what distinguishes the skipped pipeline from one that ran. +${CLICKHOUSE_CLIENT} -q " + SYSTEM FLUSH LOGS part_log; + SELECT countIf(merge_reason = 'TTLDropMerge' AND error = 0 AND read_rows = 0) > 0 + FROM system.part_log + WHERE database = currentDatabase() AND table = 't_ttl_drop_default' AND event_type = 'MergeParts'; +" + +${CLICKHOUSE_CLIENT} -q "CHECK TABLE t_ttl_drop_default SETTINGS check_query_single_value_result = 1;" +${CLICKHOUSE_CLIENT} -q "DROP TABLE t_ttl_drop_default;" + +echo "-- Case 5: projections are not announced as pending work" + +# prepareProjectionsToMergeAndRebuild both bumps the projection profile events and pushes the +# names into MergeListElement::projections_pending, which system.merges.projections_remaining +# reads and which only the rebuild and merge paths erase from. The short-circuit runs neither. +# +# projections_pending is fed from two lists, and deduplicate_merge_projection_mode decides which +# one a projection lands in, so both modes are covered: asserting RebuiltProjections alone passes +# on unfixed code under 'ignore', where the name arrives via projections_to_merge instead. +for mode in throw ignore +do + table="t_ttl_drop_projection_$mode" + + ${CLICKHOUSE_CLIENT} -q " + CREATE TABLE $table + ( + c0 UInt64, + c1 Date, + c2 String, + PROJECTION p0 (SELECT c2, count() GROUP BY c2) + ) + ENGINE = MergeTree() + ORDER BY c0 + TTL c1 + INTERVAL 1 SECOND DELETE + SETTINGS deduplicate_merge_projection_mode = '$mode'; + + INSERT INTO $table SELECT number, toDate('2020-01-01'), 'w' || toString(number) FROM numbers(100); + " + + wait_for_ttl_drop "$table" + + ${CLICKHOUSE_CLIENT} -q "SELECT count() FROM $table;" + + # read_rows = 0 pins the short-circuit; both counters at 0 are the assertion. Each mode gets + # its own table so this reads only its own merge. + ${CLICKHOUSE_CLIENT} -q " + SYSTEM FLUSH LOGS part_log; + SELECT countIf(merge_reason = 'TTLDropMerge' AND error = 0 AND read_rows = 0 + AND ProfileEvents['RebuiltProjections'] = 0 + AND ProfileEvents['MergedProjections'] = 0) > 0 + FROM system.part_log + WHERE database = currentDatabase() AND table = '$table' AND event_type = 'MergeParts'; + " + + ${CLICKHOUSE_CLIENT} -q "CHECK TABLE $table SETTINGS check_query_single_value_result = 1;" + ${CLICKHOUSE_CLIENT} -q "DROP TABLE $table;" +done diff --git a/tests/queries/0_stateless/04812_any_inner_join_filter_push_down.reference b/tests/queries/0_stateless/04812_any_inner_join_filter_push_down.reference deleted file mode 100644 index f6e2b4ce1d46..000000000000 --- a/tests/queries/0_stateless/04812_any_inner_join_filter_push_down.reference +++ /dev/null @@ -1,30 +0,0 @@ --- non-key filter must not be pushed to the right side - Filter column: equals(__table2.v, 1_UInt8) (removed) - Type: INNER - Strictness: ANY - ReadType: Default - ReadType: Default --- non-key filter must not be pushed to the left side - Filter column: equals(__table1.a, 1_UInt8) (removed) - Type: INNER - Strictness: ANY - ReadType: Default - ReadType: Default --- non-key filter may change the output set if pushed right -1 --- non-key filter may change the output set if pushed left -1 --- equi-key filter on the right column is still pushed to both sides - Type: INNER - Strictness: ANY - ReadType: Default - Prewhere filter column: equals(__table1.k, 1_UInt8) (removed) - ReadType: Default - Prewhere filter column: equals(__table2.k, 1_UInt8) (removed) --- equi-key filter on the left column is still pushed to both sides - Type: INNER - Strictness: ANY - ReadType: Default - Prewhere filter column: equals(__table1.k, 1_UInt8) (removed) - ReadType: Default - Prewhere filter column: equals(__table2.k, 1_UInt8) (removed) diff --git a/tests/queries/0_stateless/04812_any_inner_join_filter_push_down.sql b/tests/queries/0_stateless/04812_any_inner_join_filter_push_down.sql deleted file mode 100644 index 7f3c4abd1deb..000000000000 --- a/tests/queries/0_stateless/04812_any_inner_join_filter_push_down.sql +++ /dev/null @@ -1,49 +0,0 @@ -SET explain_query_plan_default = 'legacy'; -SET enable_analyzer = 1; -SET enable_parallel_replicas = 0; -SET query_plan_join_swap_table = 0; -SET enable_join_runtime_filters = 0; -SET query_plan_optimize_join_order_randomize = 0; -- the test asserts on the join plan -SET join_algorithm = 'hash'; -SET max_bytes_before_external_join = 0, max_bytes_ratio_before_external_join = 0; -- Disable automatic spilling for this test -SET optimize_move_to_prewhere = 1, query_plan_optimize_prewhere = 1; - -CREATE TABLE t1 (k UInt64, a UInt64) ENGINE = MergeTree ORDER BY k; -CREATE TABLE t2 (k UInt64, v UInt64) ENGINE = MergeTree ORDER BY k; -INSERT INTO t1 VALUES (1, 100), (2, 200), (1, 200); -INSERT INTO t2 VALUES (1, 10), (1, 20), (2, 30); - -SELECT '-- non-key filter must not be pushed to the right side'; -SELECT explain FROM ( - EXPLAIN actions = 1 - SELECT * FROM t1 ANY INNER JOIN t2 ON t1.k = t2.k WHERE t2.v = 1 -) WHERE explain ilike '%Filter column%' OR explain ilike '%Strictness%' OR explain ilike '%Type:%'; - -SELECT '-- non-key filter must not be pushed to the left side'; -SELECT explain FROM ( - EXPLAIN actions = 1 - SELECT * FROM t1 ANY INNER JOIN t2 ON t1.k = t2.k WHERE t1.a = 1 -) WHERE explain ilike '%Filter column%' OR explain ilike '%Strictness%' OR explain ilike '%Type:%'; - -SELECT '-- non-key filter may change the output set if pushed right'; -SELECT - (SELECT count() FROM t1 ANY JOIN t2 USING (k) WHERE t2.v > 10) - = (SELECT sum(t2.v > 10) FROM t1 ANY JOIN t2 USING (k)); - -SELECT '-- non-key filter may change the output set if pushed left'; -SELECT - (SELECT count() FROM t1 ANY JOIN t2 USING (k) WHERE t1.a = 200) - = (SELECT sum(t1.a = 200) FROM t1 ANY JOIN t2 USING (k)); - -SELECT '-- equi-key filter on the right column is still pushed to both sides'; -SELECT explain FROM ( - EXPLAIN actions = 1 - SELECT * FROM t1 ANY INNER JOIN t2 ON t1.k = t2.k WHERE t2.k = 1 -) WHERE explain ilike '%Filter column%' OR explain ilike '%Strictness%' OR explain ilike '%Type:%'; - -SELECT '-- equi-key filter on the left column is still pushed to both sides'; -SELECT explain FROM ( - EXPLAIN actions = 1 - SELECT * FROM t1 ANY INNER JOIN t2 ON t1.k = t2.k WHERE t1.k = 1 -) WHERE explain ilike '%Filter column%' OR explain ilike '%Strictness%' OR explain ilike '%Type:%'; - diff --git a/tests/queries/0_stateless/04812_filesystem_cache_boundary_alignment_for_disk_read.reference b/tests/queries/0_stateless/04812_filesystem_cache_boundary_alignment_for_disk_read.reference new file mode 100644 index 000000000000..0d66ea1aee95 --- /dev/null +++ b/tests/queries/0_stateless/04812_filesystem_cache_boundary_alignment_for_disk_read.reference @@ -0,0 +1,2 @@ +0 +1 diff --git a/tests/queries/0_stateless/04812_filesystem_cache_boundary_alignment_for_disk_read.sh b/tests/queries/0_stateless/04812_filesystem_cache_boundary_alignment_for_disk_read.sh new file mode 100755 index 000000000000..e31d6a228c90 --- /dev/null +++ b/tests/queries/0_stateless/04812_filesystem_cache_boundary_alignment_for_disk_read.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, no-random-settings, no-random-merge-tree-settings, no-distributed-cache + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# Query level setting `filesystem_cache_boundary_alignment` must override +# `boundary_alignment` of the cache configuration also for disk read. +ALIGNMENT=$((20 * 1024 * 1024)) + +$CLICKHOUSE_CLIENT -m -q " +DROP TABLE IF EXISTS test; +CREATE TABLE test (a Int32, b String) +ENGINE = MergeTree() +ORDER BY a +SETTINGS disk = disk(type = cache, + max_size = '1Gi', + max_file_segment_size = '40Mi', + boundary_alignment = '1Mi', + cache_on_write_operations = 0, + path = '$CLICKHOUSE_TEST_UNIQUE_NAME', + name = '$CLICKHOUSE_TEST_UNIQUE_NAME', + disk = 's3_disk'); + +INSERT INTO test SELECT number, randomString(100) FROM numbers(500000); + +SYSTEM DROP FILESYSTEM CACHE '$CLICKHOUSE_TEST_UNIQUE_NAME'; + +SET read_through_distributed_cache = 0; +SET filesystem_cache_boundary_alignment = $ALIGNMENT; + +-- Read a granule from the middle of the table, so that the file segments +-- of the big column are neither at the beginning nor at the end of the file. +SELECT * FROM test WHERE a = 250000 FORMAT Null; +" + +# File segments must start at a boundary of the requested alignment +# (and there must be a file segment which does not start at the beginning of the file, +# otherwise the check above is trivial). +$CLICKHOUSE_CLIENT -m -q " +SELECT count() FROM system.filesystem_cache +WHERE cache_name = '$CLICKHOUSE_TEST_UNIQUE_NAME' +AND file_segment_range_begin % $ALIGNMENT != 0; + +SELECT count() > 0 FROM system.filesystem_cache +WHERE cache_name = '$CLICKHOUSE_TEST_UNIQUE_NAME' +AND file_segment_range_begin > 0; +" + +$CLICKHOUSE_CLIENT -q "DROP TABLE test" diff --git a/tests/queries/0_stateless/04836_http_gzip_single_deflate_block.reference b/tests/queries/0_stateless/04836_http_gzip_single_deflate_block.reference new file mode 100644 index 000000000000..5213944ab0aa --- /dev/null +++ b/tests/queries/0_stateless/04836_http_gzip_single_deflate_block.reference @@ -0,0 +1,3 @@ +90 36000000 10606447025795110145 +90 36000000 10606447025795110145 +single-block ingest within 10x of multi-block: 1 diff --git a/tests/queries/0_stateless/04836_http_gzip_single_deflate_block.sh b/tests/queries/0_stateless/04836_http_gzip_single_deflate_block.sh new file mode 100755 index 000000000000..781746aec0af --- /dev/null +++ b/tests/queries/0_stateless/04836_http_gzip_single_deflate_block.sh @@ -0,0 +1,82 @@ +#!/usr/bin/env bash +# Regression test for https://github.com/ClickHouse/ClickHouse/issues/114045: ingesting a gzip +# HTTP body whose DEFLATE payload is a single block spanning the whole stream - the shape +# zlib-ng's deflate_quick path (compression level 1, the default of the official .NET SDK) +# emits. The streaming decompressor used to re-decode the block from its start on every socket +# refill, making the ingest quadratic in the compressed size; a correct decoder handles this +# shape in linear time. No available encoder produces the shape on demand, so the gzip file is +# crafted directly: with the static Huffman code, every literal byte below 144 is an 8-bit +# codeword, so the block body is a byte translation of the payload shifted by the 3 header bits. +# +# The assertion is on time, because the quadratic decoder still produces the right bytes, just +# far too slowly. An absolute budget is not robust: the linear ingest of a 36 MB body takes +# 0.2 s on a release build but well over 15 s under TSan when the flaky check runs 18 copies of +# the test at once. So the same payload is ingested twice - first as an ordinary multi-block +# gzip, which every decoder handles in linear time, then as the single block - and the test +# asserts that the single-block ingest is at most 10x slower than the multi-block one. Both +# sides scale together with machine speed and load, so the ratio does not depend on either: +# measured on a release build, the quadratic decoder needs about 22 s for the single block +# against about 0.3 s for the multi-block stream (over 70x), and the linear decoder about the +# same time for both. The quadratic growth was confirmed to hold over the whole range +# (10/20/40/60 MB took 1.8/6.9/27.3/61.1 s), so the pre-fix margin does not depend on the +# machine being as fast as the one measured. The payload is 90 lines of 400 KB rather than many +# short ones so that line parsing and the `MergeTree` write cost nothing next to the +# decompression under test (with 520000 short lines they dominated at 5.8 s). The multi-block +# baseline is ingested first so that any one-off warm-up cost lands on the baseline side. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +MULTI_BLOCK_FILE=${CLICKHOUSE_TMP}/04836_multi_block.gz +SINGLE_BLOCK_FILE=${CLICKHOUSE_TMP}/04836_single_block.gz + +python3 -c " +import hashlib, struct, sys, zlib +size, line = 36000000, 400000 +chunk = b''.join(hashlib.sha256(b'%d' % i).hexdigest().encode() for i in range(size // 64 + 1))[:size] +raw = b'\n'.join(chunk[i:i + line] for i in range(0, size, line)) + b'\n' + +# The baseline: an ordinary gzip stream of many DEFLATE blocks, as any regular encoder emits. +# Level 6 (not 1) so that the shape does not depend on whether Python links zlib or zlib-ng. +multi = zlib.compressobj(6, zlib.DEFLATED, 31) +open(sys.argv[1], 'wb').write(multi.compress(raw) + multi.flush()) + +# The shape under test: the whole payload as one static-Huffman DEFLATE block. +assert max(raw) < 144 +table = bytes(int(format(0x30 + b, '08b')[::-1], 2) for b in range(144)) + bytes(112) +body = raw.translate(table) +# 3 header bits (BFINAL=1, BTYPE=01 static), 8 bits per literal, 7 zero bits of end-of-block. +n = (int.from_bytes(body, 'little') << 3) | 0b011 +deflate = n.to_bytes(len(body) + 2, 'little') +blob = (b'\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff' + deflate + + struct.pack(' MULTI_BLOCK_NS * 10 )); then + echo "single-block ingest took ${SINGLE_BLOCK_NS} ns, multi-block ingest took ${MULTI_BLOCK_NS} ns" +fi +echo "single-block ingest within 10x of multi-block: $(( SINGLE_BLOCK_NS <= MULTI_BLOCK_NS * 10 ))" + +${CLICKHOUSE_CLIENT} --query "DROP TABLE t_04836" +rm -f "${MULTI_BLOCK_FILE}" "${SINGLE_BLOCK_FILE}" diff --git a/tests/queries/0_stateless/04853_attach_as_replicated_keeper_path_macro.reference b/tests/queries/0_stateless/04853_attach_as_replicated_keeper_path_macro.reference new file mode 100644 index 000000000000..d91901a6aeaf --- /dev/null +++ b/tests/queries/0_stateless/04853_attach_as_replicated_keeper_path_macro.reference @@ -0,0 +1,8 @@ +safe_name_converts ReplicatedMergeTree 2 +definition_not_rewritten 1 +as_not_replicated_allows_unsafe_name MergeTree +stored_macro_armed 1 +short_attach OK +short_attach_count 1 +restart_replica OK +restart_replica_count 1 diff --git a/tests/queries/0_stateless/04853_attach_as_replicated_keeper_path_macro.sh b/tests/queries/0_stateless/04853_attach_as_replicated_keeper_path_macro.sh new file mode 100755 index 000000000000..60ade65ffbc3 --- /dev/null +++ b/tests/queries/0_stateless/04853_attach_as_replicated_keeper_path_macro.sh @@ -0,0 +1,74 @@ +#!/usr/bin/env bash +# Tags: zookeeper, no-replicated-database, no-ordinary-database, no-shared-merge-tree +# Rows needing a per-copy unique database name, which a .sql file cannot interpolate. + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +# The rejection this test guards fires only on a name-based `default_replica_path`, which the test +# config does not set (it is {uuid}-based, and a {uuid} is never path-unsafe). So the treatment row +# lives in tests/integration/test_modify_engine_on_restart/test_unsafe_name.py, which configures +# that setting, and the rows here pin the exemptions the rejection must not disturb. +CLIENT="${CLICKHOUSE_CLIENT} --server_logs_file=/dev/null" + +# Converting a table with a path-safe name still works. Guards the check against rejecting the +# ordinary case: the shipped {uuid} template resolves through the same code path. +${CLIENT} -q "CREATE TABLE c1 (c0 Int) ENGINE = MergeTree ORDER BY c0" +${CLIENT} -q "INSERT INTO c1 VALUES (1), (2)" +${CLIENT} -q "DETACH TABLE c1" +${CLIENT} -q "ATTACH TABLE c1 AS REPLICATED" +${CLIENT} -q "SELECT 'safe_name_converts', engine, (SELECT count() FROM c1) FROM system.tables WHERE database = currentDatabase() AND name = 'c1'" + +# The check only reads the definition, it must not rewrite the engine arguments. The stored +# template stays unexpanded, which is what keeps a metadata file copyable between replicas. +${CLIENT} -q "SELECT 'definition_not_rewritten', create_table_query LIKE '%{uuid}%' FROM system.tables WHERE database = currentDatabase() AND name = 'c1'" +${CLIENT} -q "DROP TABLE c1" + +# The opposite direction strips the path arguments instead of minting one, so an unsafe name is +# irrelevant to it. The path is given explicitly here, so the CREATE itself is legal. +${CLIENT} -q "CREATE TABLE \`c3/unsafe\` (c0 Int) ENGINE = ReplicatedMergeTree('/clickhouse/04853/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/c3', 'r1') ORDER BY c0" +${CLIENT} -q "DETACH TABLE \`c3/unsafe\`" +${CLIENT} -q "ATTACH TABLE \`c3/unsafe\` AS NOT REPLICATED" +${CLIENT} -q "SELECT 'as_not_replicated_allows_unsafe_name', engine FROM system.tables WHERE database = currentDatabase() AND name = 'c3/unsafe'" +${CLIENT} -q "DROP TABLE \`c3/unsafe\`" + +# A table whose STORED path re-expands to a path-unsafe value keeps loading, both through a +# short ATTACH and through SYSTEM RESTART REPLICA. Two arming details: +# * only a CONFIGURED macro survives into metadata unexpanded; a direct {database} is unfolded at +# CREATE, leaving nothing to re-substitute. +# * the DATABASE is renamed rather than the table, because RenamingRestrictions refuses to rename +# a table whose path carries an implicit macro. +# Report the exemption from the client's exit status, not from a grep for one error code: the table +# is registered in the catalog before `startup()` runs on both routes, so a failure there leaves the +# table present and the count assertions below cannot see it. +run_exempt() { + local label=$1 query=$2 out rc + out=$(${CLIENT} -q "$query" 2>&1) + rc=$? + if [ "$rc" = "0" ]; then + echo "$label OK" + else + echo "$label FAILED rc=$rc: $out" + fi +} + +LEGACY_DB="${CLICKHOUSE_DATABASE}_legacy" +${CLIENT} -q "DROP DATABASE IF EXISTS \`${LEGACY_DB}/d\` SYNC" +${CLIENT} -q "DROP DATABASE IF EXISTS \`${LEGACY_DB}\` SYNC" +${CLIENT} -q "CREATE DATABASE \`${LEGACY_DB}\`" +${CLIENT} -q "CREATE TABLE \`${LEGACY_DB}\`.t (c0 Int) ENGINE = ReplicatedMergeTree('{default_path_test}04853legacy', 'r2') ORDER BY c0" +${CLIENT} -q "SELECT 'stored_macro_armed', create_table_query LIKE '%{default_path_test}%' FROM system.tables WHERE database = '${LEGACY_DB}' AND name = 't'" +${CLIENT} -q "RENAME DATABASE \`${LEGACY_DB}\` TO \`${LEGACY_DB}/d\`" +${CLIENT} -q "DETACH TABLE \`${LEGACY_DB}/d\`.t" +run_exempt short_attach "ATTACH TABLE \`${LEGACY_DB}/d\`.t" +${CLIENT} -q "SELECT 'short_attach_count', count() FROM system.tables WHERE database = '${LEGACY_DB}/d' AND name = 't'" +run_exempt restart_replica "SYSTEM RESTART REPLICA \`${LEGACY_DB}/d\`.t" +${CLIENT} -q "SELECT 'restart_replica_count', count() FROM system.tables WHERE database = '${LEGACY_DB}/d' AND name = 't'" +# Re-resolve the path under the original name before dropping: the stored path re-expands {database}, +# so under the new name the table points at a different znode tree than the CREATE made, and dropping +# it there would leave the original tree behind. +${CLIENT} -q "RENAME DATABASE \`${LEGACY_DB}/d\` TO \`${LEGACY_DB}\`" +${CLIENT} -q "DETACH TABLE \`${LEGACY_DB}\`.t" +${CLIENT} -q "ATTACH TABLE \`${LEGACY_DB}\`.t" +${CLIENT} -q "DROP DATABASE \`${LEGACY_DB}\` SYNC" diff --git a/tests/queries/0_stateless/04878_distributed_plan_shuffle_bucket_order_promise.reference b/tests/queries/0_stateless/04878_distributed_plan_shuffle_bucket_order_promise.reference new file mode 100644 index 000000000000..6523a0f5d10f --- /dev/null +++ b/tests/queries/0_stateless/04878_distributed_plan_shuffle_bucket_order_promise.reference @@ -0,0 +1,14 @@ +1 +1 +1 +1 +1 +1 +7 600056 +7 600056 +1 +1 +1 +1 +1 +1 diff --git a/tests/queries/0_stateless/04878_distributed_plan_shuffle_bucket_order_promise.sql b/tests/queries/0_stateless/04878_distributed_plan_shuffle_bucket_order_promise.sql new file mode 100644 index 000000000000..4a609c233414 --- /dev/null +++ b/tests/queries/0_stateless/04878_distributed_plan_shuffle_bucket_order_promise.sql @@ -0,0 +1,167 @@ +-- Tags: shard, no-old-analyzer + +SET enable_parallel_replicas = 0; +SET automatic_parallel_replicas_mode = 0; +SET explain_query_plan_default = 'legacy'; +-- Distributed aggregation cannot enforce a global `max_rows_to_group_by`, so pin it to 0. +SET max_rows_to_group_by = 0; + +DROP TABLE IF EXISTS t_shuffle_bucket_order; +CREATE TABLE t_shuffle_bucket_order (k UInt64, v UInt64) ENGINE = MergeTree ORDER BY tuple() + SETTINGS index_granularity = 256, auto_statistics_types = ''; +INSERT INTO t_shuffle_bucket_order SELECT number % 50000, number FROM numbers(200000); + +SET make_distributed_plan = 1; +SET distributed_plan_execute_locally = 1; +SET distributed_plan_default_shuffle_join_bucket_count = 2; +-- No statistics, so the strategy choice does not depend on an estimated group count. +SET use_statistics = 0; +SET distributed_plan_max_rows_to_broadcast = 0; +-- A shard plan otherwise carries `BlocksMarshallingStep`, which cannot run on a worker, and a plan +-- holding it is executed with its exchanges turned into no-ops instead of being distributed. +SET enable_parallel_blocks_marshalling = 0; +-- Two-level aggregation states in every producer, so the merge consumes several buckets per input. +SET group_by_two_level_threshold = 10000; +SET group_by_two_level_threshold_bytes = 1; +SET max_threads = 16; +SET distributed_aggregation_memory_efficient = 1; +-- Pinned because the runner randomizes it and the promise below is set from either setting, so an arm +-- that leaves this one open does not say which of the two it exercised. +SET enable_memory_bound_merging_of_aggregation_results = 1; +-- `EXPLAIN PLAN distributed = 1` prints a shipped shard plan as it was shipped, and a shard plan is +-- shipped before the distributed rewrite runs on it, so the rows below read the rewrite only when the +-- shard receives the query as text. +SET serialize_query_plan = 0; + +-- Arming, asserted separately from the results below, on the same rewritten plan the guard acts on: +-- `distributed = 1` shows the per-shard plans, and the settings sit on the inner query because the +-- wrapper's own `SETTINGS` clause, which keeps the wrapper itself out of the rewrite, would otherwise +-- apply to the plan being explained as well. The initiator merges bucket by bucket, it does so over +-- more than one producer, and both are required for a duplicated bucket to be observable. +SELECT count() > 0 FROM + (EXPLAIN PLAN actions = 1, distributed = 1 SELECT k, sum(v) FROM remote('127.0.0.{2,3}', currentDatabase(), t_shuffle_bucket_order) GROUP BY k + SETTINGS make_distributed_plan = 1, distributed_aggregation_memory_efficient = 1) + WHERE explain ILIKE '%memory-efficient%' + SETTINGS make_distributed_plan = 0; +SELECT count() > 1 FROM + (EXPLAIN PLAN actions = 1, distributed = 1 SELECT k, sum(v) FROM remote('127.0.0.{2,3}', currentDatabase(), t_shuffle_bucket_order) GROUP BY k + SETTINGS make_distributed_plan = 1, distributed_aggregation_memory_efficient = 1) + WHERE explain ILIKE '%ReadFromMergeTree%' + SETTINGS make_distributed_plan = 0; + +-- The guard itself, observed at the site it acts on: the shuffle scatter is gone from the shard plan, +-- so the strategy is partial aggregation plus merge. The second row pins that +-- `distributed_plan_force_shuffle_aggregation` loses to the guard, as it loses to GROUPING SETS. +SELECT count() = 0 FROM + (EXPLAIN PLAN actions = 1, distributed = 1 SELECT k, sum(v) FROM remote('127.0.0.{2,3}', currentDatabase(), t_shuffle_bucket_order) GROUP BY k + SETTINGS make_distributed_plan = 1, distributed_aggregation_memory_efficient = 1) + WHERE explain ILIKE '%by hash(%' + SETTINGS make_distributed_plan = 0; +SELECT count() = 0 FROM + (EXPLAIN PLAN actions = 1, distributed = 1 SELECT k, sum(v) FROM remote('127.0.0.{2,3}', currentDatabase(), t_shuffle_bucket_order) GROUP BY k + SETTINGS make_distributed_plan = 1, distributed_aggregation_memory_efficient = 1, + distributed_plan_force_shuffle_aggregation = 1) + WHERE explain ILIKE '%by hash(%' + SETTINGS make_distributed_plan = 0; + +-- And the strategy it demotes to is the partial aggregation plus its merge, each named at the site +-- that builds it, so a plan left undistributed rather than demoted does not satisfy these two rows. +SELECT count() > 0 FROM + (EXPLAIN PLAN actions = 1, distributed = 1 SELECT k, sum(v) FROM remote('127.0.0.{2,3}', currentDatabase(), t_shuffle_bucket_order) GROUP BY k + SETTINGS make_distributed_plan = 1, distributed_aggregation_memory_efficient = 1, + distributed_plan_force_shuffle_aggregation = 1) + WHERE explain ILIKE '%MergingAggregated (merge)%' + SETTINGS make_distributed_plan = 0; +SELECT count() > 0 FROM + (EXPLAIN PLAN actions = 1, distributed = 1 SELECT k, sum(v) FROM remote('127.0.0.{2,3}', currentDatabase(), t_shuffle_bucket_order) GROUP BY k + SETTINGS make_distributed_plan = 1, distributed_aggregation_memory_efficient = 1, + distributed_plan_force_shuffle_aggregation = 1) + WHERE explain ILIKE '%Aggregating (partial)%' + SETTINGS make_distributed_plan = 0; + +-- The aggregation must complete. The shuffle strategy keeps the promise to produce results in bucket +-- order while each of its instances orders only its own share, so the merge receives a bucket it has +-- already merged and rejects the whole query. Two shapes, because the merge reaches the duplicate +-- from both its ordered and its delayed-bucket push. The force setting pins the strategy, so neither +-- arm depends on the statistics-free default choice. +SELECT k, sum(v) FROM remote('127.0.0.{2,3}', currentDatabase(), t_shuffle_bucket_order) GROUP BY k FORMAT Null + SETTINGS distributed_plan_force_shuffle_aggregation = 1; +SELECT k FROM remote('127.0.0.{2,3}', currentDatabase(), t_shuffle_bucket_order) GROUP BY ALL FORMAT Null + SETTINGS distributed_plan_force_shuffle_aggregation = 1; +-- A shipped shard plan is rewritten by the shard that receives it rather than by the initiator. No +-- explain reaches a rewrite made there, so this row pins the outcome rather than the strategy: it does +-- not separate the demotion from an aggregation left unrewritten above the gather, which the rows above +-- separate whenever the plan is not shipped. +SELECT k, sum(v) FROM remote('127.0.0.{2,3}', currentDatabase(), t_shuffle_bucket_order) GROUP BY k FORMAT Null + SETTINGS distributed_plan_force_shuffle_aggregation = 1, serialize_query_plan = 1; + +-- The keys and the aggregate values must match the plain plan, not merely avoid the rejection. A +-- single group keeps the output deterministic without an ORDER BY, which this plan cannot distribute. +SELECT k, sum(v) FROM remote('127.0.0.{2,3}', currentDatabase(), t_shuffle_bucket_order) GROUP BY k HAVING k = 7; +SELECT k, sum(v) FROM remote('127.0.0.{2,3}', currentDatabase(), t_shuffle_bucket_order) GROUP BY k HAVING k = 7 + SETTINGS make_distributed_plan = 0; + +-- The shuffle strategy stays available, including under its force setting, when nothing downstream +-- requires bucket order. The first row is the shard plan of the query the guard acts on, with both +-- settings that create the promise turned off: the same plan loses the shuffle above when they are on, +-- so a guard demoting every non-final aggregation fails here. +SELECT count() > 0 FROM + (EXPLAIN PLAN actions = 1, distributed = 1 SELECT k, sum(v) FROM remote('127.0.0.{2,3}', currentDatabase(), t_shuffle_bucket_order) GROUP BY k + SETTINGS make_distributed_plan = 1, + distributed_aggregation_memory_efficient = 0, + enable_memory_bound_merging_of_aggregation_results = 0, + distributed_plan_force_shuffle_aggregation = 1) + WHERE explain ILIKE '%by hash(%' + SETTINGS make_distributed_plan = 0; + +-- The next two rows keep the `Complete` stage, with both settings pinned on, so what keeps the shuffle +-- there is the stage alone: a guard reading those settings instead of the promise fails them. That is +-- also the default configuration, since both settings default to 1. +SELECT count() > 0 FROM (EXPLAIN PLAN actions = 1 SELECT k, sum(v) FROM t_shuffle_bucket_order GROUP BY k + SETTINGS make_distributed_plan = 1, + distributed_aggregation_memory_efficient = 1, + enable_memory_bound_merging_of_aggregation_results = 1) + WHERE explain ILIKE '%by hash(%' + SETTINGS make_distributed_plan = 0; +SELECT count() > 0 FROM (EXPLAIN PLAN actions = 1 SELECT k, sum(v) FROM t_shuffle_bucket_order GROUP BY k + SETTINGS make_distributed_plan = 1, + distributed_aggregation_memory_efficient = 1, + enable_memory_bound_merging_of_aggregation_results = 1, + distributed_plan_force_shuffle_aggregation = 1) + WHERE explain ILIKE '%by hash(%' + SETTINGS make_distributed_plan = 0; + +-- The promise is set from either setting, so each of the two demotes on its own. Both rows pin both +-- settings, and each asserts the whole shape at once: no shuffle, and the partial aggregation plus its +-- merge, so a plan left undistributed does not satisfy them either. +SELECT countIf(explain ILIKE '%by hash(%') = 0 + AND countIf(explain ILIKE '%Aggregating (partial)%') > 0 + AND countIf(explain ILIKE '%MergingAggregated (merge)%') > 0 FROM + (EXPLAIN PLAN actions = 1, distributed = 1 SELECT k, sum(v) FROM remote('127.0.0.{2,3}', currentDatabase(), t_shuffle_bucket_order) GROUP BY k + SETTINGS make_distributed_plan = 1, + distributed_aggregation_memory_efficient = 1, + enable_memory_bound_merging_of_aggregation_results = 0, + distributed_plan_force_shuffle_aggregation = 1) + SETTINGS make_distributed_plan = 0; +SELECT countIf(explain ILIKE '%by hash(%') = 0 + AND countIf(explain ILIKE '%Aggregating (partial)%') > 0 + AND countIf(explain ILIKE '%MergingAggregated (merge)%') > 0 FROM + (EXPLAIN PLAN actions = 1, distributed = 1 SELECT k, sum(v) FROM remote('127.0.0.{2,3}', currentDatabase(), t_shuffle_bucket_order) GROUP BY k + SETTINGS make_distributed_plan = 1, + distributed_aggregation_memory_efficient = 0, + enable_memory_bound_merging_of_aggregation_results = 1, + distributed_plan_force_shuffle_aggregation = 1) + SETTINGS make_distributed_plan = 0; + +-- The boundary of the row above: with only memory-bound merging on, the merge over the shard output is +-- not the memory-efficient one, so nothing there reads the bucket order the demotion preserves. +SELECT count() = 0 FROM + (EXPLAIN PLAN actions = 1, distributed = 1 SELECT k, sum(v) FROM remote('127.0.0.{2,3}', currentDatabase(), t_shuffle_bucket_order) GROUP BY k + SETTINGS make_distributed_plan = 1, + distributed_aggregation_memory_efficient = 0, + enable_memory_bound_merging_of_aggregation_results = 1, + distributed_plan_force_shuffle_aggregation = 1) + WHERE explain ILIKE '%memory-efficient%' + SETTINGS make_distributed_plan = 0; + +DROP TABLE t_shuffle_bucket_order; diff --git a/tests/queries/0_stateless/04891_nats_credential_file_path_restriction.sql b/tests/queries/0_stateless/04891_nats_credential_file_path_restriction.sql index bebc4d7ac63d..6c6549b72bde 100644 --- a/tests/queries/0_stateless/04891_nats_credential_file_path_restriction.sql +++ b/tests/queries/0_stateless/04891_nats_credential_file_path_restriction.sql @@ -75,12 +75,10 @@ CREATE NAMED COLLECTION 04891_nats_existing_sql_collection AS nats_startup_connect_tries = 0, nats_reconnect_wait = 1; -- A full-definition `ATTACH` is validated exactly like `CREATE`, and unlike `CREATE` it tolerates -- a failing connection attempt, so the table exists afterwards and its metadata can be replayed. --- The `SETTINGS` clause carries an unrelated key on purpose: an engine definition whose settings --- are all inherited from the named collection is stored with an empty `SETTINGS` clause, which the --- metadata reload then fails to parse. +-- All of its settings come from the named collection, so the stored definition must carry no +-- `SETTINGS` clause at all. ATTACH TABLE nats_file_from_existing_sql_collection UUID 'c6d2423a-9ab2-4a37-8e56-10e479541002' (key UInt64) -ENGINE = NATS(04891_nats_existing_sql_collection) -SETTINGS nats_num_consumers = 1; +ENGINE = NATS(04891_nats_existing_sql_collection); DETACH TABLE nats_file_from_existing_sql_collection; ALTER NAMED COLLECTION 04891_nats_existing_sql_collection SET nats_credential_file = '/etc/passwd'; ATTACH TABLE nats_file_from_existing_sql_collection; -- { serverError BAD_ARGUMENTS } diff --git a/tests/queries/0_stateless/04927_substreams_cache_key_collision.reference b/tests/queries/0_stateless/04927_substreams_cache_key_collision.reference new file mode 100644 index 000000000000..ba91c81af2da --- /dev/null +++ b/tests/queries/0_stateless/04927_substreams_cache_key_collision.reference @@ -0,0 +1,184 @@ +[(100),(200)] ('abc',99) +[(300)] ('de',98) +[] ('',97) +4 +Row 1: +────── +c1: ([(100),(200)],1) +c2: ([('abc')],2) +c3: ([('abc')],3) +c4: ([([10,20])],4) +c5: ([(1),(NULL)],5) +c6: ([[(10),(20)]],6) +c7: ([[(10),(20)]],7) +c8: ([(1),NULL],8) +c9: ({'k':[(1),(2)]},9) +c10: (('abc',99),10) +c11: (([10,20],99),11) +c12: ((5,7),12) +Row 1: +────── +c1.x: [(100),(200)] +c2.x: [('abc')] +c3.x: [('abc')] +c4.x: [([10,20])] +c5.x: [(1),(NULL)] +c6.x: [[(10),(20)]] +c7.x: [[(10),(20)]] +c8.x: [(1),NULL] +c9.x: {'k':[(1),(2)]} +c10.x: ('abc',99) +c11.x: ([10,20],99) +c12.x: (5,7) +Row 1: +────── +c1: ([(100),(200)],1) +c2: ([('abc')],2) +c3: ([('abc')],3) +c4: ([([10,20])],4) +c5: ([(1),(NULL)],5) +c6: ([[(10),(20)]],6) +c7: ([[(10),(20)]],7) +c8: ([(1),NULL],8) +c9: ({'k':[(1),(2)]},9) +c10: (('abc',99),10) +c11: (([10,20],99),11) +c12: ((5,7),12) +Row 1: +────── +c1.x: [(100),(200)] +c2.x: [('abc')] +c3.x: [('abc')] +c4.x: [([10,20])] +c5.x: [(1),(NULL)] +c6.x: [[(10),(20)]] +c7.x: [[(10),(20)]] +c8.x: [(1),NULL] +c9.x: {'k':[(1),(2)]} +c10.x: ('abc',99) +c11.x: ([10,20],99) +c12.x: (5,7) +Row 1: +────── +c1: ('{"object_shared_data":{"0":{"size0":1}}}',1) +c2: ('{"a":[1,2],"a":{"size0":7}}',2) +c3: ('{"a":1,"a":{"null":7}}',3) +c4: ('{"a":"abc","a":{"size":7}}',4) +c5: ('{"a":1,"a":{"Int64":7}}',5) +c6: ('{"a":1,"a":{"Int64":7}}',6) +c7: ('{"a":{"b":0},"a":{"b":1}}',7) +Row 1: +────── +c1.x: { + "object_shared_data": { + "0": { + "size0": 1 + } + } +} +c2.x: { + "a": [1,2], + "a": { + "size0": 7 + } +} +c3.x: { + "a": 1, + "a": { + "null": 7 + } +} +c4.x: { + "a": "abc", + "a": { + "size": 7 + } +} +c5.x: { + "a": 1, + "a": { + "Int64": 7 + } +} +c6.x: { + "a": 1, + "a": { + "Int64": 7 + } +} +c7.x: { + "a": { + "b": 0 + }, + "a": { + "b": 1 + } +} +Row 1: +────── +c1: ('{"object_shared_data":{"0":{"size0":1}}}',1) +c2: ('{"a":[1,2],"a":{"size0":7}}',2) +c3: ('{"a":1,"a":{"null":7}}',3) +c4: ('{"a":"abc","a":{"size":7}}',4) +c5: ('{"a":1,"a":{"Int64":7}}',5) +c6: ('{"a":1,"a":{"Int64":7}}',6) +c7: ('{"a":{"b":0},"a":{"b":1}}',7) +Row 1: +────── +c1.x: { + "object_shared_data": { + "0": { + "size0": 1 + } + } +} +c2.x: { + "a": [1,2], + "a": { + "size0": 7 + } +} +c3.x: { + "a": 1, + "a": { + "null": 7 + } +} +c4.x: { + "a": "abc", + "a": { + "size": 7 + } +} +c5.x: { + "a": 1, + "a": { + "Int64": 7 + } +} +c6.x: { + "a": 1, + "a": { + "Int64": 7 + } +} +c7.x: { + "a": { + "b": 0 + }, + "a": { + "b": 1 + } +} +12495667661173940433 30 14365225858895003680 +5579088841599825588 20 3931234904921333272 +11007181315820846960 20 16997049742924259240 3194560342723893250 +9842334541689766390 20 3194560342723893250 +1328710051673613252 0 11053694533033933970 +9067169503920147426 190 +1 +9469320946673274046 2 13032090342589669733 2 2268159345449755554 10 +[(100),(200)] ('abc',99) +[(300)] ('de',98) +[(100),(200)] ('abc',99) +[(300)] ('de',98) diff --git a/tests/queries/0_stateless/04927_substreams_cache_key_collision.sql b/tests/queries/0_stateless/04927_substreams_cache_key_collision.sql new file mode 100644 index 000000000000..044b3a0d321b --- /dev/null +++ b/tests/queries/0_stateless/04927_substreams_cache_key_collision.sql @@ -0,0 +1,167 @@ +-- Two streams of one column whose files differ (`c.size0` and `c%2Esize0` for Array(Tuple(`size0` UInt64))) +-- used to share a substreams cache slot, so one of them got the other's column. The data on disk is correct. +-- The wide reader uses the cache for every read, the compact reader only for subcolumn reads. + +SET enable_nullable_tuple_type = 1; + +DROP TABLE IF EXISTS t_cache_key; + +CREATE TABLE t_cache_key (c Array(Tuple(`size0` UInt64)), d Tuple(`a` String, `a.size` UInt64)) +ENGINE = MergeTree ORDER BY tuple() SETTINGS min_bytes_for_wide_part = 0; +INSERT INTO t_cache_key VALUES ([(100), (200)], ('abc', 99)), ([(300)], ('de', 98)), ([], ('', 97)); +SELECT c, d FROM t_cache_key; + +-- A merge reads the parts the same way. +INSERT INTO t_cache_key VALUES ([(400)], ('f', 96)); +OPTIMIZE TABLE t_cache_key FINAL; +SELECT count() FROM t_cache_key; + +DROP TABLE t_cache_key; + +DROP TABLE IF EXISTS t_cache_wide; +DROP TABLE IF EXISTS t_cache_compact; + +-- Every column is wrapped in a Tuple: the compact reader allocates a substreams cache only for subcolumn +-- reads of a part that has substream marks, hence `write_marks_for_substreams_in_compact_parts` below. +CREATE TABLE t_cache_wide +( + -- A Tuple element named like the array sizes of an enclosing Array. + c1 Tuple(x Array(Tuple(`size0` UInt64)), y UInt8), + c2 Tuple(x Array(Tuple(`size0` String)), y UInt8), + c3 Tuple(x Array(Tuple(`size0` LowCardinality(String))), y UInt8), + c4 Tuple(x Array(Tuple(`size0` Array(UInt64))), y UInt8), + c5 Tuple(x Array(Tuple(`size0` Nullable(UInt64))), y UInt8), + c6 Tuple(x Array(Array(Tuple(`size0` UInt64))), y UInt8), + c7 Tuple(x Array(Array(Tuple(`size1` UInt64))), y UInt8), + c8 Tuple(x Array(Nullable(Tuple(`size0` Int64))), y UInt8), + c9 Tuple(x Map(String, Array(Tuple(`size1` UInt64))), y UInt8), + -- A Tuple element named like an automatic subcolumn of a sibling element. + c10 Tuple(x Tuple(`a` String, `a.size` UInt64), y UInt8), + c11 Tuple(x Tuple(`a` Array(UInt64), `a.size0` UInt64), y UInt8), + c12 Tuple(x Tuple(`a` Nullable(UInt64), `a.null` UInt8), y UInt8) +) +ENGINE = MergeTree ORDER BY tuple() SETTINGS min_bytes_for_wide_part = 0; + +CREATE TABLE t_cache_compact +( + c1 Tuple(x Array(Tuple(`size0` UInt64)), y UInt8), + c2 Tuple(x Array(Tuple(`size0` String)), y UInt8), + c3 Tuple(x Array(Tuple(`size0` LowCardinality(String))), y UInt8), + c4 Tuple(x Array(Tuple(`size0` Array(UInt64))), y UInt8), + c5 Tuple(x Array(Tuple(`size0` Nullable(UInt64))), y UInt8), + c6 Tuple(x Array(Array(Tuple(`size0` UInt64))), y UInt8), + c7 Tuple(x Array(Array(Tuple(`size1` UInt64))), y UInt8), + c8 Tuple(x Array(Nullable(Tuple(`size0` Int64))), y UInt8), + c9 Tuple(x Map(String, Array(Tuple(`size1` UInt64))), y UInt8), + c10 Tuple(x Tuple(`a` String, `a.size` UInt64), y UInt8), + c11 Tuple(x Tuple(`a` Array(UInt64), `a.size0` UInt64), y UInt8), + c12 Tuple(x Tuple(`a` Nullable(UInt64), `a.null` UInt8), y UInt8) +) +ENGINE = MergeTree ORDER BY tuple() SETTINGS min_bytes_for_wide_part = '10G', min_rows_for_wide_part = 1000000000, write_marks_for_substreams_in_compact_parts = 1; + +INSERT INTO t_cache_wide SELECT ([tuple(100), tuple(200)], 1), ([tuple('abc')], 2), ([tuple('abc')], 3), ([tuple([10, 20])], 4), ([tuple(1), tuple(NULL)], 5), ([[tuple(10), tuple(20)]], 6), ([[tuple(10), tuple(20)]], 7), ([tuple(1), NULL], 8), (map('k', [tuple(1), tuple(2)]), 9), (('abc', 99), 10), (([10, 20], 99), 11), ((5, 7), 12); +INSERT INTO t_cache_compact SELECT ([tuple(100), tuple(200)], 1), ([tuple('abc')], 2), ([tuple('abc')], 3), ([tuple([10, 20])], 4), ([tuple(1), tuple(NULL)], 5), ([[tuple(10), tuple(20)]], 6), ([[tuple(10), tuple(20)]], 7), ([tuple(1), NULL], 8), (map('k', [tuple(1), tuple(2)]), 9), (('abc', 99), 10), (([10, 20], 99), 11), ((5, 7), 12); + +SELECT * FROM t_cache_wide FORMAT Vertical; +SELECT c1.x, c2.x, c3.x, c4.x, c5.x, c6.x, c7.x, c8.x, c9.x, c10.x, c11.x, c12.x FROM t_cache_wide FORMAT Vertical; +SELECT * FROM t_cache_compact FORMAT Vertical; +SELECT c1.x, c2.x, c3.x, c4.x, c5.x, c6.x, c7.x, c8.x, c9.x, c10.x, c11.x, c12.x FROM t_cache_compact FORMAT Vertical; + +DROP TABLE t_cache_wide; +DROP TABLE t_cache_compact; + +-- The same with JSON typed paths named like a substream of a sibling path or like an internal stream. +DROP TABLE IF EXISTS t_cache_json_wide; +DROP TABLE IF EXISTS t_cache_json_compact; + +CREATE TABLE t_cache_json_wide +( + c1 Tuple(x JSON(`object_shared_data.0.size0` Int64), y UInt8), + c2 Tuple(x JSON(`a` Array(Int64), `a.size0` Int64), y UInt8), + c3 Tuple(x JSON(`a` Nullable(Int64), `a.null` Int64), y UInt8), + c4 Tuple(x JSON(`a` String, `a.size` Int64), y UInt8), + c5 Tuple(x JSON(`a` Dynamic, `a.Int64` Int64), y UInt8), + c6 Tuple(x JSON(`a` Variant(Int64, String), `a.Int64` Int64), y UInt8), + c7 Tuple(x JSON(`a` JSON(`b` Int64), `a.b` Int64), y UInt8) +) +ENGINE = MergeTree ORDER BY tuple() SETTINGS min_bytes_for_wide_part = 0; + +CREATE TABLE t_cache_json_compact +( + c1 Tuple(x JSON(`object_shared_data.0.size0` Int64), y UInt8), + c2 Tuple(x JSON(`a` Array(Int64), `a.size0` Int64), y UInt8), + c3 Tuple(x JSON(`a` Nullable(Int64), `a.null` Int64), y UInt8), + c4 Tuple(x JSON(`a` String, `a.size` Int64), y UInt8), + c5 Tuple(x JSON(`a` Dynamic, `a.Int64` Int64), y UInt8), + c6 Tuple(x JSON(`a` Variant(Int64, String), `a.Int64` Int64), y UInt8), + c7 Tuple(x JSON(`a` JSON(`b` Int64), `a.b` Int64), y UInt8) +) +ENGINE = MergeTree ORDER BY tuple() SETTINGS min_bytes_for_wide_part = '10G', min_rows_for_wide_part = 1000000000, write_marks_for_substreams_in_compact_parts = 1; + +INSERT INTO t_cache_json_wide VALUES (('{"object_shared_data.0.size0" : 1}', 1), ('{"a" : [1, 2], "a.size0" : 7}', 2), ('{"a" : 1, "a.null" : 7}', 3), ('{"a" : "abc", "a.size" : 7}', 4), ('{"a" : 1, "a.Int64" : 7}', 5), ('{"a" : 1, "a.Int64" : 7}', 6), ('{"a" : {"b" : 1}}', 7)); +INSERT INTO t_cache_json_compact VALUES (('{"object_shared_data.0.size0" : 1}', 1), ('{"a" : [1, 2], "a.size0" : 7}', 2), ('{"a" : 1, "a.null" : 7}', 3), ('{"a" : "abc", "a.size" : 7}', 4), ('{"a" : 1, "a.Int64" : 7}', 5), ('{"a" : 1, "a.Int64" : 7}', 6), ('{"a" : {"b" : 1}}', 7)); + +SELECT * FROM t_cache_json_wide FORMAT Vertical; +SELECT c1.x, c2.x, c3.x, c4.x, c5.x, c6.x, c7.x FROM t_cache_json_wide FORMAT Vertical; +SELECT * FROM t_cache_json_compact FORMAT Vertical; +SELECT c1.x, c2.x, c3.x, c4.x, c5.x, c6.x, c7.x FROM t_cache_json_compact FORMAT Vertical; + +DROP TABLE t_cache_json_wide; +DROP TABLE t_cache_json_compact; + +-- Whole column and subcolumn reads must keep the same key: the wide reader keys streams by file name, so +-- without sharing the second read continues from where the first stopped. Needs several granules. +DROP TABLE IF EXISTS t_cache_sharing; + +SET flatten_nested = 0; + +CREATE TABLE t_cache_sharing +( + i UInt64, + arr Array(Nullable(UInt64)), + arr2 Array(Array(String)), + m Map(String, UInt64), + n Nested(a UInt64, b String), + s Nullable(String), + j JSON(`p` Int64) +) +ENGINE = MergeTree ORDER BY i SETTINGS min_bytes_for_wide_part = 0, index_granularity = 3; + +INSERT INTO t_cache_sharing SELECT number, range(number % 4), [[toString(number)]], map('k', number), [(number, toString(number))], toString(number), '{"p" : ' || toString(number) || '}' FROM numbers(20); + +SELECT sum(cityHash64(arr)), sum(arr.size0), sum(cityHash64(arr.null)) FROM t_cache_sharing; +SELECT sum(cityHash64(arr2)), sum(arr2.size0), sum(cityHash64(arr2.size1)) FROM t_cache_sharing; +SELECT sum(cityHash64(m)), sum(m.size0), sum(cityHash64(m.keys)), sum(cityHash64(m.values)) FROM t_cache_sharing; +SELECT sum(cityHash64(n)), sum(n.size0), sum(cityHash64(n.a)) FROM t_cache_sharing; +SELECT sum(cityHash64(s)), sum(s.null), sum(cityHash64(s.size)) FROM t_cache_sharing; +SELECT sum(cityHash64(j)), sum(j.p) FROM t_cache_sharing; + +DROP TABLE t_cache_sharing; + +-- Sparse elements are a separate substream, which the key must keep distinct from the column itself. +DROP TABLE IF EXISTS t_cache_sparse; + +CREATE TABLE t_cache_sparse (i UInt64, s String, arr Array(UInt64), t Tuple(a UInt64, b String)) +ENGINE = MergeTree ORDER BY i +SETTINGS min_bytes_for_wide_part = 0, ratio_of_defaults_for_sparse_serialization = 0.1, index_granularity = 3; + +INSERT INTO t_cache_sparse SELECT number, if(number % 10 = 0, 'x', ''), if(number % 10 = 0, [number], []), (if(number % 10 = 0, number, 0), '') FROM numbers(20); + +SELECT sum(serialization_kind = 'Sparse') > 0 FROM system.parts_columns WHERE database = currentDatabase() AND table = 't_cache_sparse' AND active; +SELECT sum(cityHash64(s)), sum(s.size), sum(cityHash64(arr)), sum(arr.size0), sum(cityHash64(t)), sum(t.a) FROM t_cache_sparse; + +DROP TABLE t_cache_sparse; + +-- StorageLog and StorageTinyLog use the same cache. +DROP TABLE IF EXISTS t_cache_log; +DROP TABLE IF EXISTS t_cache_tiny_log; + +CREATE TABLE t_cache_log (c Array(Tuple(`size0` UInt64)), d Tuple(`a` String, `a.size` UInt64)) ENGINE = Log; +CREATE TABLE t_cache_tiny_log (c Array(Tuple(`size0` UInt64)), d Tuple(`a` String, `a.size` UInt64)) ENGINE = TinyLog; +INSERT INTO t_cache_log VALUES ([(100), (200)], ('abc', 99)), ([(300)], ('de', 98)); +INSERT INTO t_cache_tiny_log VALUES ([(100), (200)], ('abc', 99)), ([(300)], ('de', 98)); +SELECT c, d FROM t_cache_log ORDER BY c; +SELECT c, d FROM t_cache_tiny_log ORDER BY c; + +DROP TABLE t_cache_log; +DROP TABLE t_cache_tiny_log; diff --git a/tests/queries/0_stateless/04936_detached_serialization_from_client_native_protocol.python b/tests/queries/0_stateless/04936_detached_serialization_from_client_native_protocol.python new file mode 100755 index 000000000000..cd620dfafefa --- /dev/null +++ b/tests/queries/0_stateless/04936_detached_serialization_from_client_native_protocol.python @@ -0,0 +1,315 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +# Regression test for a post-auth type confusion over the native TCP protocol. +# +# The serialization kind of every column of a block is chosen by the peer that sends the +# block. `Kind::DETACHED` is an internal transport representation for parallel blocks +# marshalling: it produces a `ColumnBLOB` holding raw peer-supplied bytes while the block +# keeps the declared type of the column. A client that selects it for an external table +# therefore puts a column that is not a `ColumnMap` into a block declared as `Map`, and +# every later `assert_cast` (a plain `static_cast` in release builds) +# reinterprets attacker-controlled bytes. +# +# `NativeReader` must accept `DETACHED` only from a peer whose pipeline unmarshalls blobs, +# which never includes a client, and must reject kind stacks that no writer can produce. +# All scenarios below expect INCORRECT_DATA (code 117) and a server that stays alive. +# +# The client speaks the raw native protocol pinned to revision 54482 +# (DBMS_MIN_REVISION_WITH_REPLICATED_SERIALIZATION), so the set of exchanged, +# revision-gated fields is fixed and does not drift. + +import os +import socket +import struct +import uuid + +CLICKHOUSE_HOST = os.environ.get("CLICKHOUSE_HOST", "127.0.0.1") +CLICKHOUSE_PORT = int(os.environ.get("CLICKHOUSE_PORT_TCP", "9000")) +CLICKHOUSE_DATABASE = os.environ.get("CLICKHOUSE_DATABASE", "default") +CLIENT_NAME = "detached serialization poc" + +REVISION = 54482 + +# KindStackBinarySerializationType +KIND_STACK_DEFAULT = 0 +KIND_STACK_DETACHED = 2 +KIND_STACK_DETACHED_OVER_SPARSE = 3 +KIND_STACK_COMBINATION = 5 + +# ISerialization::Kind +KIND_DEFAULT = 0 +KIND_SPARSE = 1 +KIND_DETACHED = 2 +KIND_REPLICATED = 3 + + +def writeVarUInt(x, ba): + for _ in range(0, 9): + byte = x & 0x7F + if x > 0x7F: + byte |= 0x80 + ba.append(byte) + x >>= 7 + if x == 0: + return + + +def writeStringBinary(s, ba): + b = bytes(s, "utf-8") if isinstance(s, str) else s + writeVarUInt(len(b), ba) + ba.extend(b) + + +def readStrict(s, size=1): + res = bytearray() + while size: + cur = s.recv(size) + if not cur: + raise EOFError("Connection closed by server") + size -= len(cur) + res.extend(cur) + return res + + +def readUInt(s, size=1): + res = readStrict(s, size) + val = 0 + for i in range(len(res)): + val += res[i] << (i * 8) + return val + + +def readUInt32(s): + return readUInt(s, 4) + + +def readVarUInt(s): + x = 0 + for i in range(9): + byte = readStrict(s)[0] + x |= (byte & 0x7F) << (7 * i) + if not byte & 0x80: + return x + return x + + +def readStringBinary(s): + size = readVarUInt(s) + return readStrict(s, size).decode("utf-8") + + +def sendHello(s): + ba = bytearray() + writeVarUInt(0, ba) # Hello + writeStringBinary(CLIENT_NAME, ba) + writeVarUInt(24, ba) # major + writeVarUInt(9, ba) # minor + writeVarUInt(REVISION, ba) + writeStringBinary(CLICKHOUSE_DATABASE, ba) # database + writeStringBinary("default", ba) # user + writeStringBinary("", ba) # password + s.sendall(ba) + + +def receiveHello(s): + assert readVarUInt(s) == 0 # Hello + readStringBinary(s) # server name + readVarUInt(s) # major + readVarUInt(s) # minor + readVarUInt(s) # revision + readVarUInt(s) # parallel replicas protocol version (>= 54471) + readStringBinary(s) # timezone (>= 54058) + readStringBinary(s) # display name (>= 54372) + readVarUInt(s) # version patch (>= 54401) + readStringBinary(s) # proto_send chunked (>= 54470) + readStringBinary(s) # proto_recv chunked (>= 54470) + for _ in range(readVarUInt(s)): # password complexity rules (>= 54461) + readStringBinary(s) + readStringBinary(s) + readStrict(s, 8) # nonce, UInt64 (>= 54462) + # Server settings in STRINGS_WITH_FLAGS format: (name, flags, value)* terminated + # by an empty name (>= 54474). + while True: + if readStringBinary(s) == "": # setting name + break + readVarUInt(s) # flags + readStringBinary(s) # value + readVarUInt(s) # query plan serialization version (>= 54477) + readVarUInt(s) # cluster function protocol version (>= 54479) + + +def sendAddendum(s): + ba = bytearray() + writeStringBinary("", ba) # quota key (>= 54458) + writeStringBinary("notchunked", ba) # proto_send chunked (>= 54470) + writeStringBinary("notchunked", ba) # proto_recv chunked (>= 54470) + writeVarUInt(0, ba) # parallel replicas protocol version (>= 54471) + s.sendall(ba) + + +def serializeClientInfo(ba, query_id): + ba.append(1) # INITIAL_QUERY + writeStringBinary("default", ba) # initial_user + writeStringBinary(query_id, ba) # initial_query_id + writeStringBinary("127.0.0.1:9000", ba) # initial_address + ba.extend([0] * 8) # initial_query_start_time_microseconds (>= 54449) + ba.append(1) # interface = TCP + writeStringBinary("os_user", ba) + writeStringBinary("client_hostname", ba) + writeStringBinary(CLIENT_NAME, ba) + writeVarUInt(24, ba) # client major + writeVarUInt(9, ba) # client minor + writeVarUInt(REVISION, ba) # client tcp protocol version + writeStringBinary("", ba) # quota key (>= 54060) + writeVarUInt(0, ba) # distributed_depth (>= 54448) + writeVarUInt(1, ba) # client version patch (>= 54401) + ba.append(0) # opentelemetry: no trace id (>= 54442) + writeVarUInt(0, ba) # parallel replicas: collaborate_with_initiator (>= 54453) + writeVarUInt(0, ba) # parallel replicas: obsolete count + writeVarUInt(0, ba) # parallel replicas: number_of_current_replica + writeVarUInt(0, ba) # script query number (>= 54475) + writeVarUInt(0, ba) # script line number (>= 54475) + ba.append(0) # jwt: none (>= 54476) + + +def sendQuery(s, query): + ba = bytearray() + query_id = uuid.uuid4().hex + writeVarUInt(1, ba) # Query + writeStringBinary(query_id, ba) + serializeClientInfo(ba, query_id) + writeStringBinary("", ba) # empty per-query settings + writeStringBinary("", ba) # interserver externally granted roles (>= 54472) + writeStringBinary("", ba) # interserver secret (>= 54441) + writeVarUInt(2, ba) # stage = Complete + ba.append(0) # no compression + writeStringBinary(query, ba) + writeStringBinary("", ba) # query parameters terminator (>= 54459) + s.sendall(ba) + + +def serializeBlockInfo(ba): + writeVarUInt(1, ba) # field 1 + ba.append(0) # is_overflows = false + writeVarUInt(2, ba) # field 2 + ba.extend(struct.pack(" inside mapConcat. + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.settimeout(30) + s.connect((CLICKHOUSE_HOST, CLICKHOUSE_PORT)) + sendHello(s) + receiveHello(s) + sendAddendum(s) + sendQuery(s, "SELECT mapConcat(x) FROM e") + s.sendall(block) + expectIncorrectData(s, must_contain) + print("{}: caught expected exception INCORRECT_DATA".format(name)) + + +def main(): + # The reported vector: a Map column whose runtime representation becomes a ColumnBLOB. + runScenario( + "detached", + externalTableBlock("Map(UInt64, UInt64)", bytes([KIND_STACK_DETACHED])), + "Unexpected serialization kind Detached", + ) + + # The same, reached through the dedicated Detached-over-Sparse encoding. + runScenario( + "detached over sparse", + externalTableBlock("UInt64", bytes([KIND_STACK_DETACHED_OVER_SPARSE])), + "Unexpected serialization kind Detached", + ) + + # The same, reached through the generic kind stack encoding. + runScenario( + "detached in a combination", + externalTableBlock("Map(UInt64, UInt64)", combinationKindStack([KIND_DEFAULT, KIND_DETACHED])), + "Unexpected serialization kind Detached", + ) + + # Kind stacks are per tuple element, so a blob can also be requested for a nested column. + tuple_block = externalTableBlock("Tuple(UInt64, UInt64)", bytes([KIND_STACK_DEFAULT])) + tuple_block += bytes([KIND_STACK_DETACHED]) # element 0 + tuple_block += bytes([KIND_STACK_DEFAULT]) # element 1 + runScenario("detached tuple element", tuple_block, "Unexpected serialization kind Detached") + + # A repeated kind nests a column inside its own layout: a ColumnSparse whose values column is + # itself sparse never materializes back to a full column of the declared type. + runScenario( + "repeated kind", + externalTableBlock("UInt64", combinationKindStack([KIND_DEFAULT, KIND_SPARSE, KIND_SPARSE])), + "is out of order", + ) + + # Sparse can sit inside Replicated but not the other way round, and ColumnSparse over + # ColumnReplicated does not materialize back to a full column of the declared type either. + runScenario( + "kinds in the wrong order", + externalTableBlock("UInt64", combinationKindStack([KIND_DEFAULT, KIND_REPLICATED, KIND_SPARSE])), + "is out of order", + ) + + # Every kind stack starts with Default. + runScenario( + "kind stack without default", + externalTableBlock("UInt64", combinationKindStack([KIND_SPARSE])), + "must start with Default", + ) + + # A huge kind stack must be rejected before it is materialized into nested serializations. + huge = bytearray() + huge.append(KIND_STACK_COMBINATION) + writeVarUInt(1000000, huge) + runScenario("too many kinds", externalTableBlock("UInt64", bytes(huge)), "Too many serialization kinds") + + +if __name__ == "__main__": + main() diff --git a/tests/queries/0_stateless/04936_detached_serialization_from_client_native_protocol.reference b/tests/queries/0_stateless/04936_detached_serialization_from_client_native_protocol.reference new file mode 100644 index 000000000000..f362e10f2513 --- /dev/null +++ b/tests/queries/0_stateless/04936_detached_serialization_from_client_native_protocol.reference @@ -0,0 +1,9 @@ +detached: caught expected exception INCORRECT_DATA +detached over sparse: caught expected exception INCORRECT_DATA +detached in a combination: caught expected exception INCORRECT_DATA +detached tuple element: caught expected exception INCORRECT_DATA +repeated kind: caught expected exception INCORRECT_DATA +kinds in the wrong order: caught expected exception INCORRECT_DATA +kind stack without default: caught expected exception INCORRECT_DATA +too many kinds: caught expected exception INCORRECT_DATA +server alive 45 diff --git a/tests/queries/0_stateless/04936_detached_serialization_from_client_native_protocol.sh b/tests/queries/0_stateless/04936_detached_serialization_from_client_native_protocol.sh new file mode 100755 index 000000000000..55eae4a88645 --- /dev/null +++ b/tests/queries/0_stateless/04936_detached_serialization_from_client_native_protocol.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# We should have correct env vars from shell_config.sh to run this test +python3 "$CUR_DIR"/04936_detached_serialization_from_client_native_protocol.python + +# The kinds a client may legitimately select still work, and the server is still alive. +$CLICKHOUSE_CLIENT -q "SELECT 'server alive', sum(number) FROM numbers(10)" diff --git a/tests/queries/0_stateless/05054_postgresql_protocol_message_size.reference b/tests/queries/0_stateless/05054_postgresql_protocol_message_size.reference new file mode 100644 index 000000000000..ec35dc29b3b4 --- /dev/null +++ b/tests/queries/0_stateless/05054_postgresql_protocol_message_size.reference @@ -0,0 +1,6 @@ +SASL mechanism longer than the message: error response +password without a terminator: error response +well-formed exchange: query result received +query with a smuggled tail: error response +query shorter than declared: error response +oversized sync: error response diff --git a/tests/queries/0_stateless/05054_postgresql_protocol_message_size.sh b/tests/queries/0_stateless/05054_postgresql_protocol_message_size.sh new file mode 100755 index 000000000000..e5fd34c2db3f --- /dev/null +++ b/tests/queries/0_stateless/05054_postgresql_protocol_message_size.sh @@ -0,0 +1,158 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# Tag no-fasttest: the PostgreSQL compatibility port is not enabled in fasttest. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# Authentication messages are parsed before the client is authenticated, so a field of a message +# must not be allocated at the size the client declares for it, and parsing a message must not +# continue past the size the client declared for the message. + +USER_SCRAM="user_scram_${CLICKHOUSE_DATABASE}" +USER_PLAIN="user_plain_${CLICKHOUSE_DATABASE}" + +$CLICKHOUSE_CLIENT --query "CREATE USER ${USER_SCRAM} IDENTIFIED WITH scram_sha256_password BY 'x'" +$CLICKHOUSE_CLIENT --query "CREATE USER ${USER_PLAIN} IDENTIFIED WITH plaintext_password BY 'x'" + +CLICKHOUSE_PORT_POSTGRESQL="$CLICKHOUSE_PORT_POSTGRESQL" USER_SCRAM="$USER_SCRAM" USER_PLAIN="$USER_PLAIN" python3 - <<'PYTHON' +import os +import socket +import struct + +port = int(os.environ["CLICKHOUSE_PORT_POSTGRESQL"]) +user_scram = os.environ["USER_SCRAM"] +user_plain = os.environ["USER_PLAIN"] + + +def connect(): + sock = socket.create_connection(("127.0.0.1", port), timeout=30) + sock.settimeout(30) + return sock + + +def startup(sock, user): + payload = ("user\x00" + user + "\x00\x00").encode() + sock.sendall(struct.pack(">ii", 8 + len(payload), 196608) + payload) + + +def outcome(sock): + """Drain the reply and describe it: a rejected message gets an error response, an accepted one + that is never completed leaves the server waiting for the payload.""" + data = b"" + while True: + try: + chunk = sock.recv(4096) + except socket.timeout: + return "TIMEOUT" + except ConnectionResetError: + break + if not chunk: + break + data += chunk + if data[0:1] == b"E": + return "error response" + if not data: + return "NO REPLY" + return "UNEXPECTED REPLY " + repr(data[:64]) + + +def first_reply(sock): + """Describe the first reply message, for a case where the server rejects the message but keeps + the connection open, so waiting for it to close would only hit the timeout.""" + try: + data = sock.recv(4096) + except (socket.timeout, ConnectionResetError): + return "TIMEOUT" + if data[0:1] == b"E": + return "error response" + if not data: + return "NO REPLY" + return "UNEXPECTED REPLY " + repr(data[:64]) + + +# A SASL initial response that declares a two-gigabyte mechanism in a thirty-byte message. +sock = connect() +startup(sock, user_scram) +sock.recv(4096) +body = b"SCRAM-SHA-256\x00" + struct.pack(">i", 0x7FFFFFFF) +sock.sendall(b"p" + struct.pack(">i", 4 + len(body)) + body) +print("SASL mechanism longer than the message:", outcome(sock)) +sock.close() + +# A password message that declares a small size and then streams a password without a terminator. +sock = connect() +startup(sock, user_plain) +sock.recv(4096) +sock.sendall(b"p" + struct.pack(">i", 4 + 10)) +try: + sock.sendall(b"x" * 200000) +except (BrokenPipeError, ConnectionResetError, socket.timeout): + pass +print("password without a terminator:", outcome(sock)) +sock.close() + +# A well-formed exchange still works: authenticate and run a query. +def read_until_ready(sock): + data = b"" + while not data.endswith(b"Z\x00\x00\x00\x05I"): + chunk = sock.recv(4096) + if not chunk: + break + data += chunk + return data + + +sock = connect() +startup(sock, user_plain) +sock.recv(4096) +sock.sendall(b"p" + struct.pack(">i", 4 + 2) + b"x\x00") +read_until_ready(sock) +sock.sendall(b"Q" + struct.pack(">i", 4 + 9) + b"SELECT 1\x00") +data = read_until_ready(sock) +print("well-formed exchange:", "query result received" if b"C\x00\x00\x00\rSELECT 1\x00" in data else "UNEXPECTED REPLY " + repr(data[:96])) +sock.close() + +# A `Query` that terminates its string early and smuggles a whole second `Query` into the tail of the +# same declared frame. The declared length is a frame boundary, so the tail must be rejected instead +# of being read as the next message. +sock = connect() +startup(sock, user_plain) +sock.recv(4096) +sock.sendall(b"p" + struct.pack(">i", 4 + 2) + b"x\x00") +read_until_ready(sock) +body = b"SELECT 1\x00" + b"SELECT 2\x00" +sock.sendall(b"Q" + struct.pack(">i", 4 + len(body)) + body) +print("query with a smuggled tail:", first_reply(sock)) +sock.close() + +# A `Query` that declares more bytes than the client sends and then closes the write side of the +# connection. The declared length is a frame boundary in both directions, so the message must be +# rejected instead of being executed with the part of the payload that did arrive. +sock = connect() +startup(sock, user_plain) +sock.recv(4096) +sock.sendall(b"p" + struct.pack(">i", 4 + 2) + b"x\x00") +read_until_ready(sock) +body = b"SELECT 1\x00" +sock.sendall(b"Q" + struct.pack(">i", 4 + len(body) + 1000) + body) +sock.shutdown(socket.SHUT_WR) +print("query shorter than declared:", outcome(sock)) +sock.close() + +# An oversized `Sync`, whose parser reads nothing at all: its payload must not survive the message +# boundary and be reinterpreted as the next message. +sock = connect() +startup(sock, user_plain) +sock.recv(4096) +sock.sendall(b"p" + struct.pack(">i", 4 + 2) + b"x\x00") +read_until_ready(sock) +body = b"Q" + struct.pack(">i", 4 + 9) + b"SELECT 2\x00" +sock.sendall(b"S" + struct.pack(">i", 4 + len(body)) + body) +print("oversized sync:", outcome(sock)) +sock.close() +PYTHON + +$CLICKHOUSE_CLIENT --query "DROP USER ${USER_SCRAM}" +$CLICKHOUSE_CLIENT --query "DROP USER ${USER_PLAIN}" diff --git a/tests/queries/0_stateless/05055_aggregation_in_order_limit_order_by_prefix.reference b/tests/queries/0_stateless/05055_aggregation_in_order_limit_order_by_prefix.reference new file mode 100644 index 000000000000..cebd374be5e1 --- /dev/null +++ b/tests/queries/0_stateless/05055_aggregation_in_order_limit_order_by_prefix.reference @@ -0,0 +1,53 @@ +1 +1 +1 +1 +1 +1 +1 1 10 +1 2 10 +1 3 10 +1 4 11 +1 5 11 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +PUSHDOWN_FIRES +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +PUSHDOWN_FIRES diff --git a/tests/queries/0_stateless/05055_aggregation_in_order_limit_order_by_prefix.sql b/tests/queries/0_stateless/05055_aggregation_in_order_limit_order_by_prefix.sql new file mode 100644 index 000000000000..17ea64ae2024 --- /dev/null +++ b/tests/queries/0_stateless/05055_aggregation_in_order_limit_order_by_prefix.sql @@ -0,0 +1,208 @@ +-- Regression test for issue #116849: with `optimize_aggregation_in_order_limit` +-- enabled, a query whose `ORDER BY` is a strict prefix of the `GROUP BY` key +-- returned incomplete aggregate values when groups tie on that prefix and a +-- group's rows span more than one part. Each in-order stream stopped as soon as +-- it had emitted `LIMIT` groups, at a boundary of the full group key; now it +-- stops only at a boundary of the `ORDER BY` prefix, so the groups it drops +-- always sort after at least `LIMIT` complete groups. + +DROP TABLE IF EXISTS t_agg_in_order_limit_prefix; + +CREATE TABLE t_agg_in_order_limit_prefix (a UInt32, b UInt32, x UInt32) +ENGINE = MergeTree ORDER BY (a, b); + +SYSTEM STOP MERGES t_agg_in_order_limit_prefix; + +-- Two parts; groups with b in 4..20 span both parts. +INSERT INTO t_agg_in_order_limit_prefix SELECT 1, number, 10 FROM numbers(1, 20); +INSERT INTO t_agg_in_order_limit_prefix SELECT 1, number, 1 FROM numbers(4, 17); + +-- Ground truth: sum(x) is 10 for groups with b in 1..3 and 11 for groups with b in 4..20. +-- All groups tie on `a`, so any three groups are a legal answer; assert that every +-- returned group carries its complete aggregate value. The check is done in the +-- projection on purpose: wrapping the query into a subquery changes the plan +-- (the aggregation is no longer executed in order), which hides the bug. The block +-- settings are pinned because tiny blocks also hide it, and `max_threads` because with a +-- single thread and `read_in_order_two_level_merge_threshold` at most the number of parts +-- the parts are merged into one stream before the aggregation, where the bug cannot occur. +SELECT sum(x) = if(b <= 3, 10, 11) +FROM t_agg_in_order_limit_prefix +GROUP BY a, b +ORDER BY a +LIMIT 3 +SETTINGS optimize_aggregation_in_order = 1, optimize_aggregation_in_order_limit = 1, + max_threads = 2, max_block_size = 65409, aggregation_in_order_max_block_bytes = 50000000; + +-- Same with OFFSET. +SELECT sum(x) = if(b <= 3, 10, 11) +FROM t_agg_in_order_limit_prefix +GROUP BY a, b +ORDER BY a +LIMIT 3 OFFSET 2 +SETTINGS optimize_aggregation_in_order = 1, optimize_aggregation_in_order_limit = 1, + max_threads = 2, max_block_size = 65409, aggregation_in_order_max_block_bytes = 50000000; + +-- The full-key `ORDER BY` still admits the push-down and stays correct. +SELECT a, b, sum(x) +FROM t_agg_in_order_limit_prefix +GROUP BY a, b +ORDER BY a, b +LIMIT 5 +SETTINGS optimize_aggregation_in_order = 1, optimize_aggregation_in_order_limit = 1; + +DROP TABLE t_agg_in_order_limit_prefix; + +-- With enough distinct values of the prefix the push-down still stops the streams +-- early, and the groups it returns are complete even though every group spans two parts. + +DROP TABLE IF EXISTS t_agg_in_order_limit_prefix_reads; + +CREATE TABLE t_agg_in_order_limit_prefix_reads (a UInt32, b UInt32, x UInt32) +ENGINE = MergeTree ORDER BY (a, b) +SETTINGS index_granularity = 8; + +SYSTEM STOP MERGES t_agg_in_order_limit_prefix_reads; + +-- Two parts, 100 values of `a` with 10 values of `b` each; sum(x) is 3 for every group. +INSERT INTO t_agg_in_order_limit_prefix_reads SELECT intDiv(number, 10), number % 10, 1 FROM numbers(1000); +INSERT INTO t_agg_in_order_limit_prefix_reads SELECT intDiv(number, 10), number % 10, 2 FROM numbers(1000); + +-- Every returned group must be complete. +SELECT sum(x) = 3 +FROM t_agg_in_order_limit_prefix_reads +GROUP BY a, b +ORDER BY a +LIMIT 5 +SETTINGS optimize_aggregation_in_order = 1, optimize_aggregation_in_order_limit = 1, + max_threads = 2, max_block_size = 65409, aggregation_in_order_max_block_bytes = 50000000; + +SELECT sum(x) = 3 +FROM t_agg_in_order_limit_prefix_reads +GROUP BY a, b +ORDER BY a +LIMIT 5 OFFSET 7 +SETTINGS optimize_aggregation_in_order = 1, optimize_aggregation_in_order_limit = 1, + max_threads = 2, max_block_size = 65409, aggregation_in_order_max_block_bytes = 50000000; + +-- The push-down is observed through `read_rows`. The small-block settings expose the +-- effect on a 2000-row table; `enable_parallel_replicas = 0` is required because +-- `read_rows` is accounted per reading node, and `read_in_order_two_level_merge_threshold` +-- keeps the two parts in separate streams instead of merging them before the aggregation. +SELECT sum(x) = 3 +FROM t_agg_in_order_limit_prefix_reads +GROUP BY a, b +ORDER BY a +LIMIT 5 +SETTINGS optimize_aggregation_in_order = 1, optimize_aggregation_in_order_limit = 1, + max_threads = 1, max_block_size = 16, read_in_order_two_level_merge_threshold = 100, + merge_tree_min_rows_for_concurrent_read = 0, merge_tree_min_bytes_for_concurrent_read = 0, + merge_tree_min_rows_for_seek = 0, + enable_parallel_replicas = 0, + log_comment = '05055_prefix_pushdown_on'; + +SELECT sum(x) = 3 +FROM t_agg_in_order_limit_prefix_reads +GROUP BY a, b +ORDER BY a +LIMIT 5 +SETTINGS optimize_aggregation_in_order = 1, optimize_aggregation_in_order_limit = 0, + max_threads = 1, max_block_size = 16, read_in_order_two_level_merge_threshold = 100, + merge_tree_min_rows_for_concurrent_read = 0, merge_tree_min_bytes_for_concurrent_read = 0, + merge_tree_min_rows_for_seek = 0, + enable_parallel_replicas = 0, + log_comment = '05055_prefix_pushdown_off'; + +SYSTEM FLUSH LOGS query_log; + +SELECT if(on_reads < off_reads, 'PUSHDOWN_FIRES', format('FAIL: on={} off={}', on_reads, off_reads)) +FROM +( + SELECT + anyIf(read_rows, log_comment = '05055_prefix_pushdown_on') AS on_reads, + anyIf(read_rows, log_comment = '05055_prefix_pushdown_off') AS off_reads + FROM system.query_log + WHERE current_database = currentDatabase() + AND log_comment IN ('05055_prefix_pushdown_on', '05055_prefix_pushdown_off') + AND type = 'QueryFinish' + AND event_date >= yesterday() + AND event_time >= now() - 600 +); + +DROP TABLE t_agg_in_order_limit_prefix_reads; + +-- The table is sorted only by `a`, so the in-order aggregation of `GROUP BY a, b` keeps the +-- groups of one `a` run in a hash table (the `group_by_key` path of `AggregatingInOrderTransform`). +-- The stream must count the groups it accumulated, not the `a` runs: a single run already +-- holds 10 complete groups, so a `LIMIT 5` stream stops right after the first run. + +DROP TABLE IF EXISTS t_agg_in_order_limit_partial_key; + +CREATE TABLE t_agg_in_order_limit_partial_key (a UInt32, b UInt32, x UInt32) +ENGINE = MergeTree ORDER BY a +SETTINGS index_granularity = 8; + +SYSTEM STOP MERGES t_agg_in_order_limit_partial_key; + +-- Two parts, 100 values of `a` with 10 values of `b` each; sum(x) is 3 for every group. +INSERT INTO t_agg_in_order_limit_partial_key SELECT intDiv(number, 10), number % 10, 1 FROM numbers(1000); +INSERT INTO t_agg_in_order_limit_partial_key SELECT intDiv(number, 10), number % 10, 2 FROM numbers(1000); + +SELECT sum(x) = 3 +FROM t_agg_in_order_limit_partial_key +GROUP BY a, b +ORDER BY a +LIMIT 5 +SETTINGS optimize_aggregation_in_order = 1, optimize_aggregation_in_order_limit = 1, + max_threads = 2, max_block_size = 65409, aggregation_in_order_max_block_bytes = 50000000; + +SELECT sum(x) = 3 +FROM t_agg_in_order_limit_partial_key +GROUP BY a, b +ORDER BY a, b +LIMIT 5 OFFSET 7 +SETTINGS optimize_aggregation_in_order = 1, optimize_aggregation_in_order_limit = 1, + max_threads = 2, max_block_size = 65409, aggregation_in_order_max_block_bytes = 50000000; + +SELECT sum(x) = 3 +FROM t_agg_in_order_limit_partial_key +GROUP BY a, b +ORDER BY a +LIMIT 5 +SETTINGS optimize_aggregation_in_order = 1, optimize_aggregation_in_order_limit = 1, + max_threads = 1, max_block_size = 16, read_in_order_two_level_merge_threshold = 100, + merge_tree_min_rows_for_concurrent_read = 0, merge_tree_min_bytes_for_concurrent_read = 0, + merge_tree_min_rows_for_seek = 0, + enable_parallel_replicas = 0, + log_comment = '05055_partial_key_pushdown_on'; + +SELECT sum(x) = 3 +FROM t_agg_in_order_limit_partial_key +GROUP BY a, b +ORDER BY a +LIMIT 5 +SETTINGS optimize_aggregation_in_order = 1, optimize_aggregation_in_order_limit = 0, + max_threads = 1, max_block_size = 16, read_in_order_two_level_merge_threshold = 100, + merge_tree_min_rows_for_concurrent_read = 0, merge_tree_min_bytes_for_concurrent_read = 0, + merge_tree_min_rows_for_seek = 0, + enable_parallel_replicas = 0, + log_comment = '05055_partial_key_pushdown_off'; + +SYSTEM FLUSH LOGS query_log; + +-- Stopping after the first `a` run reads a small fraction of the table; counting `a` runs +-- instead of groups would need five runs, i.e. several times more rows. +SELECT if(on_reads * 20 <= off_reads, 'PUSHDOWN_FIRES', format('FAIL: on={} off={}', on_reads, off_reads)) +FROM +( + SELECT + anyIf(read_rows, log_comment = '05055_partial_key_pushdown_on') AS on_reads, + anyIf(read_rows, log_comment = '05055_partial_key_pushdown_off') AS off_reads + FROM system.query_log + WHERE current_database = currentDatabase() + AND log_comment IN ('05055_partial_key_pushdown_on', '05055_partial_key_pushdown_off') + AND type = 'QueryFinish' + AND event_date >= yesterday() + AND event_time >= now() - 600 +); + +DROP TABLE t_agg_in_order_limit_partial_key; diff --git a/tests/queries/0_stateless/05059_mergeTreeCodecBlockCounts_access.reference b/tests/queries/0_stateless/05059_mergeTreeCodecBlockCounts_access.reference new file mode 100644 index 000000000000..b0fe7054c2e5 --- /dev/null +++ b/tests/queries/0_stateless/05059_mergeTreeCodecBlockCounts_access.reference @@ -0,0 +1,52 @@ +Without SELECT on the source table +ACCESS_DENIED +Without SELECT on the source table, structure only +ACCESS_DENIED +With SELECT on a single column of the source table +ACCESS_DENIED +ACCESS_DENIED +With SELECT on every column of the source table +a ['LZ4'] +b ['LZ4'] +part_name +column +substream +data_compressed_bytes +data_uncompressed_bytes +codec_block_counts +Non-MergeTree source table, without SELECT on it +ACCESS_DENIED +Non-MergeTree source table, with SELECT on it +BAD_ARGUMENTS +Hidden source table, before it is resolved +ACCESS_DENIED +Missing source table +ACCESS_DENIED +Missing source table, for a user who can see the database +UNKNOWN_TABLE +Hidden source table, before it is resolved, on the read path +ACCESS_DENIED +Missing source table, on the read path +ACCESS_DENIED +Missing source table, on the read path, for a user who can see the database +UNKNOWN_TABLE +EXPLAIN QUERY TREE is the same for a readable, a hidden and a missing source table +EXPLAIN SYNTAX is the same for a readable, a hidden and a missing source table +EXPLAIN PLAN of a hidden source table +ACCESS_DENIED +EXPLAIN PIPELINE of a non-MergeTree source table +BAD_ARGUMENTS +Nested in remote(...), with SELECT on a single column of the source table +grant SELECT +ACCESS_DENIED +Nested in a Remote table engine, with SELECT on a single column of the source table +grant SELECT +ACCESS_DENIED +Nested in remote(...), over a hidden source table +grant SHOW TABLES +ACCESS_DENIED +Nested in a Remote table engine, over a hidden source table +grant SHOW TABLES +ACCESS_DENIED +Number of tables the refused carriers left behind +0 diff --git a/tests/queries/0_stateless/05059_mergeTreeCodecBlockCounts_access.sh b/tests/queries/0_stateless/05059_mergeTreeCodecBlockCounts_access.sh new file mode 100755 index 000000000000..fcebc1a3d35e --- /dev/null +++ b/tests/queries/0_stateless/05059_mergeTreeCodecBlockCounts_access.sh @@ -0,0 +1,205 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# Every column of `mergeTreeCodecBlockCounts` is derived from the source table's data, so reading any of +# them requires `SELECT` on all of its columns. Resolving the structure of the function requires the same. + +username="user_${CLICKHOUSE_TEST_UNIQUE_NAME}" + +${CLICKHOUSE_CLIENT} -m --query " + DROP USER IF EXISTS ${username}; + DROP TABLE IF EXISTS t_codec_access; + DROP TABLE IF EXISTS t_codec_access_log; + DROP TABLE IF EXISTS t_codec_access_hidden; + DROP TABLE IF EXISTS t_codec_access_partial; + DROP TABLE IF EXISTS t_codec_access_dst; + + -- Explicit codecs, CI randomises the server-level default compression codec. + CREATE TABLE t_codec_access (a UInt64 CODEC(LZ4), b UInt64 CODEC(LZ4)) + ENGINE = MergeTree ORDER BY tuple() + SETTINGS min_bytes_for_wide_part = 0; + + INSERT INTO t_codec_access SELECT number, number FROM numbers(1000); + + CREATE TABLE t_codec_access_log (a UInt64) ENGINE = Log; + + -- Never granted to the test user, so it stays invisible to it. + CREATE TABLE t_codec_access_hidden (a UInt64) ENGINE = MergeTree ORDER BY tuple(); + + CREATE USER ${username} NOT IDENTIFIED; +" + +echo "Without SELECT on the source table" +${CLICKHOUSE_CLIENT} --user="${username}" --query \ + "SELECT count() FROM mergeTreeCodecBlockCounts(currentDatabase(), t_codec_access);" 2>&1 | + grep -o "ACCESS_DENIED" | uniq + +echo "Without SELECT on the source table, structure only" +${CLICKHOUSE_CLIENT} --user="${username}" --query \ + "DESCRIBE mergeTreeCodecBlockCounts(currentDatabase(), t_codec_access);" 2>&1 | + grep -o "ACCESS_DENIED" | uniq + +echo "With SELECT on a single column of the source table" +${CLICKHOUSE_CLIENT} --query "GRANT SELECT(a) ON t_codec_access TO ${username};" +${CLICKHOUSE_CLIENT} --user="${username}" --query \ + "SELECT count() FROM mergeTreeCodecBlockCounts(currentDatabase(), t_codec_access);" 2>&1 | + grep -o "ACCESS_DENIED" | uniq +${CLICKHOUSE_CLIENT} --user="${username}" --query \ + "DESCRIBE mergeTreeCodecBlockCounts(currentDatabase(), t_codec_access);" 2>&1 | + grep -o "ACCESS_DENIED" | uniq + +echo "With SELECT on every column of the source table" +${CLICKHOUSE_CLIENT} --query "GRANT SELECT(b) ON t_codec_access TO ${username};" +${CLICKHOUSE_CLIENT} --user="${username}" --query \ + "SELECT DISTINCT column, mapKeys(codec_block_counts) FROM mergeTreeCodecBlockCounts(currentDatabase(), t_codec_access) ORDER BY column;" +${CLICKHOUSE_CLIENT} --user="${username}" --query \ + "DESCRIBE mergeTreeCodecBlockCounts(currentDatabase(), t_codec_access) FORMAT TSV" | cut -f 1 + +# The engine of the source table is disclosed by `SHOW CREATE TABLE`, which requires `SHOW COLUMNS` on it, so +# it is not something the user below is allowed to learn: at this point it holds neither that privilege nor +# `SELECT` on the table. The access check therefore has to run before the check that rejects a table of another +# engine with `BAD_ARGUMENTS`. + +echo "Non-MergeTree source table, without SELECT on it" +${CLICKHOUSE_CLIENT} --user="${username}" --query \ + "DESCRIBE mergeTreeCodecBlockCounts(currentDatabase(), t_codec_access_log);" 2>&1 | + grep -o "ACCESS_DENIED\|BAD_ARGUMENTS" | uniq + +echo "Non-MergeTree source table, with SELECT on it" +${CLICKHOUSE_CLIENT} --query "GRANT SELECT ON t_codec_access_log TO ${username};" +${CLICKHOUSE_CLIENT} --user="${username}" --query \ + "DESCRIBE mergeTreeCodecBlockCounts(currentDatabase(), t_codec_access_log);" 2>&1 | + grep -o "ACCESS_DENIED\|BAD_ARGUMENTS" | uniq + +# Which tables exist is not something a user without any privilege on them is allowed to learn, so the check +# on the name has to run before the source table is resolved: an inaccessible table and a missing one answer alike. + +echo "Hidden source table, before it is resolved" +${CLICKHOUSE_CLIENT} --user="${username}" --query \ + "DESCRIBE mergeTreeCodecBlockCounts(currentDatabase(), t_codec_access_hidden);" 2>&1 | + grep -o "ACCESS_DENIED\|UNKNOWN_TABLE" | uniq + +echo "Missing source table" +${CLICKHOUSE_CLIENT} --user="${username}" --query \ + "DESCRIBE mergeTreeCodecBlockCounts(currentDatabase(), t_codec_access_missing);" 2>&1 | + grep -o "ACCESS_DENIED\|UNKNOWN_TABLE" | uniq + +echo "Missing source table, for a user who can see the database" +${CLICKHOUSE_CLIENT} --query \ + "DESCRIBE mergeTreeCodecBlockCounts(currentDatabase(), t_codec_access_missing);" 2>&1 | + grep -o "ACCESS_DENIED\|UNKNOWN_TABLE" | uniq + +# The same for an ordinary read: the check on the name runs in `StorageMergeTreeCodecBlockCounts::read` as +# well, not only when the structure is resolved, so a plain `SELECT` is not an existence oracle either. + +echo "Hidden source table, before it is resolved, on the read path" +${CLICKHOUSE_CLIENT} --user="${username}" --query \ + "SELECT count() FROM mergeTreeCodecBlockCounts(currentDatabase(), t_codec_access_hidden);" 2>&1 | + grep -o "ACCESS_DENIED\|UNKNOWN_TABLE" | uniq + +echo "Missing source table, on the read path" +${CLICKHOUSE_CLIENT} --user="${username}" --query \ + "SELECT count() FROM mergeTreeCodecBlockCounts(currentDatabase(), t_codec_access_missing);" 2>&1 | + grep -o "ACCESS_DENIED\|UNKNOWN_TABLE" | uniq + +echo "Missing source table, on the read path, for a user who can see the database" +${CLICKHOUSE_CLIENT} --query \ + "SELECT count() FROM mergeTreeCodecBlockCounts(currentDatabase(), t_codec_access_missing);" 2>&1 | + grep -o "ACCESS_DENIED\|UNKNOWN_TABLE" | uniq + +# Analysis-only entrypoints (`EXPLAIN QUERY TREE`, `EXPLAIN SYNTAX`) resolve the table function without reading +# it, so the source table is not resolved on that path and no check runs there. That discloses nothing: the +# structure of this function is a fixed constant and nothing on that path consults the catalog, so the answer is +# the same whether the source table is readable, hidden, or missing. Pinned below, one arm per entrypoint. + +# `EXPLAIN QUERY TREE` exists only with the analyzer, and a configuration that turns it off rejects the +# statement with a message that names the source table - which is what the arm below compares. The setting +# is pinned so that the comparison stays about the disclosure and not about which rejection arrived. +explain_as_user() { + ${CLICKHOUSE_CLIENT} --user="${username}" --enable_analyzer 1 --query \ + "EXPLAIN $1 SELECT * FROM mergeTreeCodecBlockCounts(currentDatabase(), $2);" 2>&1 | sed "s/$2/SOURCE/g" +} + +for kind in "QUERY TREE" "SYNTAX"; do + readable=$(explain_as_user "${kind}" t_codec_access) + hidden=$(explain_as_user "${kind}" t_codec_access_hidden) + missing=$(explain_as_user "${kind}" t_codec_access_missing) + if [ "${readable}" = "${hidden}" ] && [ "${hidden}" = "${missing}" ]; then + echo "EXPLAIN ${kind} is the same for a readable, a hidden and a missing source table" + else + echo "EXPLAIN ${kind} tells a readable, a hidden and a missing source table apart" + fi +done + +# `EXPLAIN PLAN` and `EXPLAIN PIPELINE` do build the read plan, so they go through the checks. + +echo "EXPLAIN PLAN of a hidden source table" +${CLICKHOUSE_CLIENT} --user="${username}" --query \ + "EXPLAIN PLAN SELECT * FROM mergeTreeCodecBlockCounts(currentDatabase(), t_codec_access_hidden);" 2>&1 | + grep -o "ACCESS_DENIED\|UNKNOWN_TABLE" | uniq + +echo "EXPLAIN PIPELINE of a non-MergeTree source table" +${CLICKHOUSE_CLIENT} --user="${username}" --query \ + "EXPLAIN PIPELINE SELECT * FROM mergeTreeCodecBlockCounts(currentDatabase(), t_codec_access_log);" 2>&1 | + grep -o "ACCESS_DENIED\|BAD_ARGUMENTS" | uniq + +# A table function that `canBeUsedToCreateTable` refuses for `CREATE TABLE ... AS f(...)` can still reach a +# persisted definition nested in an argument of another one: both `remote(..., f(...))` and +# `ENGINE = Remote(..., f(...))` keep it in `remote_table_function_ptr`. When the cluster has a local shard, +# both forms resolve the nested function under the creating user's context, through +# `getStructureOfRemoteTableInShard` -> `getActualTableStructureWithAccess`, which is the seam this change adds +# the check to. So the carrier is closed by the check on the source table rather than by the veto, and the arms +# below pin which privilege refuses it, one per tier and one per form. Everything the carrier itself needs is +# granted first, so that what refuses is the source table and not the carrier; `CREATE TABLE` is granted on the +# destination name alone, because a grant on the database would imply `SHOW TABLES` on the hidden table too. + +${CLICKHOUSE_CLIENT} -m --query " + CREATE TABLE t_codec_access_partial (a UInt64 CODEC(LZ4), b UInt64 CODEC(LZ4)) + ENGINE = MergeTree ORDER BY tuple() + SETTINGS min_bytes_for_wide_part = 0; + + INSERT INTO t_codec_access_partial SELECT number, number FROM numbers(1000); + + GRANT CREATE TABLE, DROP TABLE ON t_codec_access_dst TO ${username}; + GRANT READ, WRITE ON REMOTE TO ${username}; + GRANT TABLE ENGINE ON Remote TO ${username}; + GRANT TABLE ENGINE ON Distributed TO ${username}; + GRANT SELECT(a) ON t_codec_access_partial TO ${username}; +" + +# Reports the privilege the carrier demanded and the error code, one line each, from a single attempt. +carrier_as_user() { + local out + out=$(${CLICKHOUSE_CLIENT} --user="${username}" --query "$1" 2>&1) + echo "${out}" | grep -o "grant SHOW TABLES\|grant SELECT" | uniq + echo "${out}" | grep -o "ACCESS_DENIED\|BAD_ARGUMENTS" | uniq +} + +echo "Nested in remote(...), with SELECT on a single column of the source table" +carrier_as_user "CREATE TABLE t_codec_access_dst AS remote('127.0.0.1:${CLICKHOUSE_PORT_TCP}', mergeTreeCodecBlockCounts(currentDatabase(), t_codec_access_partial));" + +echo "Nested in a Remote table engine, with SELECT on a single column of the source table" +carrier_as_user "CREATE TABLE t_codec_access_dst ENGINE = Remote('127.0.0.1:${CLICKHOUSE_PORT_TCP}', mergeTreeCodecBlockCounts(currentDatabase(), t_codec_access_partial));" + +# The check on the name runs on this path too, so a carrier is not an existence oracle either. + +echo "Nested in remote(...), over a hidden source table" +carrier_as_user "CREATE TABLE t_codec_access_dst AS remote('127.0.0.1:${CLICKHOUSE_PORT_TCP}', mergeTreeCodecBlockCounts(currentDatabase(), t_codec_access_hidden));" + +echo "Nested in a Remote table engine, over a hidden source table" +carrier_as_user "CREATE TABLE t_codec_access_dst ENGINE = Remote('127.0.0.1:${CLICKHOUSE_PORT_TCP}', mergeTreeCodecBlockCounts(currentDatabase(), t_codec_access_hidden));" + +echo "Number of tables the refused carriers left behind" +${CLICKHOUSE_CLIENT} --query \ + "SELECT count() FROM system.tables WHERE database = currentDatabase() AND name = 't_codec_access_dst';" + +${CLICKHOUSE_CLIENT} -m --query " + DROP USER ${username}; + DROP TABLE t_codec_access; + DROP TABLE t_codec_access_log; + DROP TABLE t_codec_access_hidden; + DROP TABLE t_codec_access_partial; +" diff --git a/tests/queries/0_stateless/05137_join_use_nulls_right_key_restored.reference b/tests/queries/0_stateless/05137_join_use_nulls_right_key_restored.reference new file mode 100644 index 000000000000..13c43b066be5 --- /dev/null +++ b/tests/queries/0_stateless/05137_join_use_nulls_right_key_restored.reference @@ -0,0 +1,224 @@ +-- right side of the plan carries only the Nullable key +Header: __table2.k Nullable(String) +Header: __table2.k Nullable(String) +__table2.k Nullable(String) +toNullable(__table2.k) Nullable(String) +Expression (Right Pre Join Actions) +Header: toNullable(__table2.k) Nullable(String) +Header: __table2.k String +-- LEFT +a a 10 +b \N \N +c c 30 +c c 30 +c c 31 +c c 31 +Nullable(String) 6 +-- LEFT, key only +a +c +c +c +c +\N +-- LEFT ANY +a a +b \N +c c +c c +-- LEFT SEMI +a a +c c +c c +-- LEFT ANTI +b \N +-- FULL +a a 10 +b \N \N +c c 30 +c c 30 +c c 31 +c c 31 +\N d 40 +-- LEFT with residual ON condition +a \N \N +b \N \N +c c 31 +c c 31 +-- LEFT with a residual ON condition on the right key itself +a \N \N +b \N \N +c c 30 +c c 30 +c c 31 +c c 31 +-- LEFT with WHERE on the right key +b \N +c c +c c +c c +c c +-- null-safe key +a a +b \N +c c +c c +c c +c c +-- parallel_hash +a a 10 +b \N \N +c c 30 +c c 30 +c c 31 +c c 31 +a a 10 +b \N \N +c c 30 +c c 30 +c c 31 +c c 31 +\N d 40 +-- grace_hash +a a 10 +b \N \N +c c 30 +c c 30 +c c 31 +c c 31 +a a 10 +b \N \N +c c 30 +c c 30 +c c 31 +c c 31 +\N d 40 +-- spilling wrapper +a a 10 +b \N \N +c c 30 +c c 30 +c c 31 +c c 31 +-- partial_merge +Header: __table2.k Nullable(String) +Header: __table2.k Nullable(String) +__table2.k Nullable(String) +toNullable(__table2.k) Nullable(String) +Expression (Right Pre Join Actions) +Header: toNullable(__table2.k) Nullable(String) +Header: __table2.k String +a a 10 +b \N \N +c c 30 +c c 30 +c c 31 +c c 31 +a a +b \N +c c +c c +a a 10 +b \N \N +c c 30 +c c 30 +c c 31 +c c 31 +\N d 40 +a a +b \N +c c +c c +c c +c c +LowCardinality(Nullable(String)) +-- auto, switched to partial_merge +Header: __table2.k Nullable(String) +Header: __table2.k Nullable(String) +__table2.k Nullable(String) +toNullable(__table2.k) Nullable(String) +Expression (Right Pre Join Actions) +Header: toNullable(__table2.k) Nullable(String) +Header: __table2.k String +a a 10 +b \N \N +c c 30 +c c 30 +c c 31 +c c 31 +a a 10 +b \N \N +c c 30 +c c 30 +c c 31 +c c 31 +\N d 40 +a a +b \N +c c +c c +c c +c c +-- full_sorting_merge +Header: __table2.k Nullable(String) +Header: __table2.k Nullable(String) +__table2.k Nullable(String) +toNullable(__table2.k) Nullable(String) +Header: toNullable(__table2.k) Nullable(String) +Expression (Right Pre Join Actions) +Header: toNullable(__table2.k) Nullable(String) +Header: __table2.k String +a a 10 +b \N \N +c c 30 +c c 30 +c c 31 +c c 31 +-- LowCardinality key +a a +b \N +c c +c c +c c +c c +LowCardinality(Nullable(String)) +-- key with a type conversion +1 10 +2 \N +3 30 +4 40 +1 10 +2 \N +3 30 +4 40 +-- derived right key +a a +b \N +c c +c c +c c +c c +-- WITH TOTALS +a 1 +c 14 +\N 2 + +\N 17 +a 1 10 +b 2 \N +c 7 61 +d \N 40 + +\N 10 111 +a a +b \N +c c + + +-- swapped +a a 10 +b \N \N +c c 30 +c c 30 +c c 31 +c c 31 diff --git a/tests/queries/0_stateless/05137_join_use_nulls_right_key_restored.sql b/tests/queries/0_stateless/05137_join_use_nulls_right_key_restored.sql new file mode 100644 index 000000000000..e4a19097ef0a --- /dev/null +++ b/tests/queries/0_stateless/05137_join_use_nulls_right_key_restored.sql @@ -0,0 +1,99 @@ +-- Tags: no-parallel-replicas +-- The EXPLAIN queries filter plan headers by column name, and parallel replicas add extra plan steps that print the same header. + +-- https://github.com/ClickHouse/ClickHouse/issues/118738 +-- With join_use_nulls, a selected right join key is joined on as its Nullable output column, so the right +-- side carries one column that the join restores from the left key, instead of the plain key plus the +-- Nullable wrapper as a payload column. + +SET join_use_nulls = 1; +SET enable_analyzer = 1; + +DROP TABLE IF EXISTS l; +DROP TABLE IF EXISTS r; +CREATE TABLE l (k String, v UInt32) ENGINE = MergeTree ORDER BY tuple(); +CREATE TABLE r (k String, w UInt32) ENGINE = MergeTree ORDER BY tuple(); +INSERT INTO l VALUES ('a', 1), ('b', 2), ('c', 3), ('c', 4); +INSERT INTO r VALUES ('a', 10), ('c', 30), ('c', 31), ('d', 40); + +SELECT '-- right side of the plan carries only the Nullable key'; +SELECT trimLeft(explain) FROM ( + EXPLAIN header = 1 + SELECT r.k, sum(l.v) FROM l LEFT JOIN r ON l.k = r.k GROUP BY r.k + SETTINGS explain_query_plan_default = 'legacy', query_plan_join_swap_table = 0, join_algorithm = 'hash' +) WHERE explain LIKE '%Right Pre Join Actions%' OR explain LIKE '%__table2.k%'; + +SELECT '-- LEFT'; +SELECT l.k, r.k, r.w FROM l LEFT JOIN r ON l.k = r.k ORDER BY ALL SETTINGS join_algorithm = 'hash'; +SELECT toTypeName(r.k), count() FROM l LEFT JOIN r ON l.k = r.k GROUP BY 1 SETTINGS join_algorithm = 'hash'; +SELECT '-- LEFT, key only'; +SELECT r.k FROM l LEFT JOIN r ON l.k = r.k ORDER BY ALL SETTINGS join_algorithm = 'hash'; +SELECT '-- LEFT ANY'; +SELECT l.k, r.k FROM l LEFT ANY JOIN r ON l.k = r.k ORDER BY ALL SETTINGS join_algorithm = 'hash'; +SELECT '-- LEFT SEMI'; +SELECT l.k, r.k FROM l LEFT SEMI JOIN r ON l.k = r.k ORDER BY ALL SETTINGS join_algorithm = 'hash'; +SELECT '-- LEFT ANTI'; +SELECT l.k, r.k FROM l LEFT ANTI JOIN r ON l.k = r.k ORDER BY ALL SETTINGS join_algorithm = 'hash'; +SELECT '-- FULL'; +SELECT l.k, r.k, r.w FROM l FULL JOIN r ON l.k = r.k ORDER BY ALL SETTINGS join_algorithm = 'hash'; +SELECT '-- LEFT with residual ON condition'; +SELECT l.k, r.k, r.w FROM l LEFT JOIN r ON l.k = r.k AND r.w > 30 ORDER BY ALL SETTINGS join_algorithm = 'hash'; +SELECT '-- LEFT with a residual ON condition on the right key itself'; +SELECT l.k, r.k, r.w FROM l LEFT JOIN r ON l.k = r.k AND r.k != 'a' ORDER BY ALL SETTINGS join_algorithm = 'hash'; +SELECT '-- LEFT with WHERE on the right key'; +SELECT l.k, r.k FROM l LEFT JOIN r ON l.k = r.k WHERE r.k IS NULL ORDER BY ALL SETTINGS join_algorithm = 'hash'; +SELECT l.k, r.k FROM l LEFT JOIN r ON l.k = r.k WHERE r.k = 'c' ORDER BY ALL SETTINGS join_algorithm = 'hash'; +SELECT '-- null-safe key'; +SELECT l.k, r.k FROM l LEFT JOIN r ON l.k IS NOT DISTINCT FROM r.k ORDER BY ALL SETTINGS join_algorithm = 'hash'; +SELECT '-- parallel_hash'; +SELECT l.k, r.k, r.w FROM l LEFT JOIN r ON l.k = r.k ORDER BY ALL SETTINGS join_algorithm = 'parallel_hash'; +SELECT l.k, r.k, r.w FROM l FULL JOIN r ON l.k = r.k ORDER BY ALL SETTINGS join_algorithm = 'parallel_hash'; +SELECT '-- grace_hash'; +SELECT l.k, r.k, r.w FROM l LEFT JOIN r ON l.k = r.k ORDER BY ALL SETTINGS join_algorithm = 'grace_hash', grace_hash_join_initial_buckets = 4; +SELECT l.k, r.k, r.w FROM l FULL JOIN r ON l.k = r.k ORDER BY ALL SETTINGS join_algorithm = 'grace_hash', grace_hash_join_initial_buckets = 4; +SELECT '-- spilling wrapper'; +SELECT l.k, r.k, r.w FROM l LEFT JOIN r ON l.k = r.k ORDER BY ALL SETTINGS join_algorithm = 'hash', max_bytes_before_external_join = 100000000; +SELECT '-- partial_merge'; +SELECT trimLeft(explain) FROM ( + EXPLAIN header = 1 + SELECT r.k, sum(l.v) FROM l LEFT JOIN r ON l.k = r.k GROUP BY r.k + SETTINGS explain_query_plan_default = 'legacy', query_plan_join_swap_table = 0, join_algorithm = 'partial_merge' +) WHERE explain LIKE '%Right Pre Join Actions%' OR explain LIKE '%__table2.k%'; +SELECT l.k, r.k, r.w FROM l LEFT JOIN r ON l.k = r.k ORDER BY ALL SETTINGS join_algorithm = 'partial_merge'; +SELECT l.k, r.k FROM l LEFT ANY JOIN r ON l.k = r.k ORDER BY ALL SETTINGS join_algorithm = 'partial_merge'; +SELECT l.k, r.k, r.w FROM l FULL JOIN r ON l.k = r.k ORDER BY ALL SETTINGS join_algorithm = 'partial_merge'; +SELECT l.k, r.k FROM l LEFT JOIN (SELECT toLowCardinality(k) AS k FROM r) AS r ON l.k = r.k ORDER BY ALL SETTINGS join_algorithm = 'partial_merge'; +SELECT toTypeName(r.k) FROM l LEFT JOIN (SELECT toLowCardinality(k) AS k FROM r) AS r ON l.k = r.k LIMIT 1 SETTINGS join_algorithm = 'partial_merge'; +SELECT '-- auto, switched to partial_merge'; +SELECT trimLeft(explain) FROM ( + EXPLAIN header = 1 + SELECT r.k, sum(l.v) FROM l LEFT JOIN r ON l.k = r.k GROUP BY r.k + SETTINGS explain_query_plan_default = 'legacy', query_plan_join_swap_table = 0, join_algorithm = 'auto' +) WHERE explain LIKE '%Right Pre Join Actions%' OR explain LIKE '%__table2.k%'; +SELECT l.k, r.k, r.w FROM l LEFT JOIN r ON l.k = r.k ORDER BY ALL SETTINGS join_algorithm = 'auto', max_rows_in_join = 2, join_overflow_mode = 'break'; +SELECT l.k, r.k, r.w FROM l FULL JOIN r ON l.k = r.k ORDER BY ALL SETTINGS join_algorithm = 'auto', max_rows_in_join = 2, join_overflow_mode = 'break'; +SELECT l.k, r.k FROM l LEFT JOIN (SELECT toLowCardinality(k) AS k FROM r) AS r ON l.k = r.k ORDER BY ALL SETTINGS join_algorithm = 'auto', max_rows_in_join = 2, join_overflow_mode = 'break'; +SELECT '-- full_sorting_merge'; +SELECT trimLeft(explain) FROM ( + EXPLAIN header = 1 + SELECT r.k, sum(l.v) FROM l LEFT JOIN r ON l.k = r.k GROUP BY r.k + SETTINGS explain_query_plan_default = 'legacy', query_plan_join_swap_table = 0, join_algorithm = 'full_sorting_merge' +) WHERE explain LIKE '%Right Pre Join Actions%' OR explain LIKE '%__table2.k%'; +SELECT l.k, r.k, r.w FROM l LEFT JOIN r ON l.k = r.k ORDER BY ALL SETTINGS join_algorithm = 'full_sorting_merge'; +SELECT '-- LowCardinality key'; +SELECT l.k, r.k FROM l LEFT JOIN (SELECT toLowCardinality(k) AS k FROM r) AS r ON l.k = r.k ORDER BY ALL SETTINGS join_algorithm = 'hash'; +SELECT toTypeName(r.k) FROM l LEFT JOIN (SELECT toLowCardinality(k) AS k FROM r) AS r ON l.k = r.k LIMIT 1 SETTINGS join_algorithm = 'hash'; +SELECT '-- key with a type conversion'; +SELECT l.v, r.w FROM l LEFT JOIN r ON l.v * 10 = r.w ORDER BY ALL SETTINGS join_algorithm = 'hash'; +SELECT l.v, r.w FROM l LEFT JOIN r ON toInt64(l.v) * 10 = r.w ORDER BY ALL SETTINGS join_algorithm = 'hash'; +SELECT '-- derived right key'; +SELECT l.k, r.k FROM l LEFT JOIN r ON l.k = lower(upper(r.k)) ORDER BY ALL SETTINGS join_algorithm = 'hash'; +SELECT '-- WITH TOTALS'; +SELECT r.k, sum(l.v) FROM l LEFT JOIN r ON l.k = r.k GROUP BY r.k WITH TOTALS ORDER BY ALL SETTINGS join_algorithm = 'hash'; +SELECT k, sum(v), sum(w) FROM (SELECT k, sum(v) AS v FROM l GROUP BY k WITH TOTALS) AS l FULL JOIN (SELECT k, sum(w) AS w FROM r GROUP BY k WITH TOTALS) AS r USING (k) GROUP BY k WITH TOTALS ORDER BY ALL SETTINGS join_algorithm = 'hash'; +SELECT l.k, r.k FROM (SELECT k FROM l GROUP BY k WITH TOTALS) AS l LEFT JOIN (SELECT k FROM r GROUP BY k WITH TOTALS) AS r ON l.k = r.k ORDER BY ALL SETTINGS join_algorithm = 'hash'; +SELECT '-- swapped'; +SELECT l.k, r.k, r.w FROM l LEFT JOIN r ON l.k = r.k ORDER BY ALL SETTINGS join_algorithm = 'hash', query_plan_join_swap_table = 1; + +DROP TABLE l; +DROP TABLE r; diff --git a/tests/queries/0_stateless/05153_text_index_mergeTreeTextIndex_create_table_as.reference b/tests/queries/0_stateless/05153_text_index_mergeTreeTextIndex_create_table_as.reference new file mode 100644 index 000000000000..573541ac9702 --- /dev/null +++ b/tests/queries/0_stateless/05153_text_index_mergeTreeTextIndex_create_table_as.reference @@ -0,0 +1 @@ +0 diff --git a/tests/queries/0_stateless/05153_text_index_mergeTreeTextIndex_create_table_as.sql b/tests/queries/0_stateless/05153_text_index_mergeTreeTextIndex_create_table_as.sql new file mode 100644 index 000000000000..09357fa59d88 --- /dev/null +++ b/tests/queries/0_stateless/05153_text_index_mergeTreeTextIndex_create_table_as.sql @@ -0,0 +1,5 @@ +-- A persisted table over `mergeTreeTextIndex` would resolve the source table without the reader's grants. +CREATE TABLE tab (s String, INDEX idx_s s TYPE text(tokenizer = 'splitByNonAlpha')) ENGINE = MergeTree ORDER BY tuple(); +CREATE TABLE tab_index AS mergeTreeTextIndex(currentDatabase(), tab, idx_s); -- { serverError BAD_ARGUMENTS } +CREATE TABLE tab_index (part_name String, token String) AS mergeTreeTextIndex(currentDatabase(), tab, idx_s); -- { serverError BAD_ARGUMENTS } +SELECT count() FROM system.tables WHERE database = currentDatabase() AND name = 'tab_index'; diff --git a/tests/queries/0_stateless/05153_text_index_mergeTreeTextIndex_interserver.reference b/tests/queries/0_stateless/05153_text_index_mergeTreeTextIndex_interserver.reference new file mode 100644 index 000000000000..8734b9252842 --- /dev/null +++ b/tests/queries/0_stateless/05153_text_index_mergeTreeTextIndex_interserver.reference @@ -0,0 +1,2 @@ +ACCESS_DENIED +['hidden','token'] diff --git a/tests/queries/0_stateless/05153_text_index_mergeTreeTextIndex_interserver.sh b/tests/queries/0_stateless/05153_text_index_mergeTreeTextIndex_interserver.sh new file mode 100755 index 000000000000..e6052c5ed95c --- /dev/null +++ b/tests/queries/0_stateless/05153_text_index_mergeTreeTextIndex_interserver.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# The interserver secret needs the SSL library, which the fast test build does not have. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# Through an interserver connection the shard authenticates the initiating user, so the checks of +# `mergeTreeTextIndex` apply to that user instead of refusing the query as over an ordinary connection. + +user_name="${CLICKHOUSE_DATABASE}_user_05153" + +$CLICKHOUSE_CLIENT -q " +DROP USER IF EXISTS $user_name; + +CREATE TABLE tab (s String, INDEX idx_s s TYPE text(tokenizer = 'splitByNonAlpha')) ENGINE = MergeTree ORDER BY tuple(); +INSERT INTO tab VALUES ('hidden token'); + +CREATE USER $user_name IDENTIFIED WITH plaintext_password BY 'password'; +GRANT REMOTE ON *.* TO $user_name; +GRANT CREATE TEMPORARY TABLE ON *.* TO $user_name; +" + +function run_as_user() +{ + local output + if output=$($CLICKHOUSE_CLIENT --user "$user_name" --password "password" -q "$1" 2>&1); then + echo "$output" + else + echo "$output" | grep -oE '\([A-Z_]+\)' | tail -1 | tr -d '()' + fi +} + +query="SELECT arraySort(groupUniqArray(token)) FROM cluster('test_cluster_interserver_secret', mergeTreeTextIndex('$CLICKHOUSE_DATABASE', 'tab', 'idx_s')) SETTINGS prefer_localhost_replica = 0" + +run_as_user "$query" + +$CLICKHOUSE_CLIENT -q "GRANT SELECT ON $CLICKHOUSE_DATABASE.tab TO $user_name" + +run_as_user "$query" + +$CLICKHOUSE_CLIENT -q " +DROP TABLE tab; +DROP USER $user_name; +" diff --git a/tests/queries/0_stateless/05153_text_index_mergeTreeTextIndex_metadata_access.reference b/tests/queries/0_stateless/05153_text_index_mergeTreeTextIndex_metadata_access.reference new file mode 100644 index 000000000000..361650ff1a48 --- /dev/null +++ b/tests/queries/0_stateless/05153_text_index_mergeTreeTextIndex_metadata_access.reference @@ -0,0 +1,10 @@ +ACCESS_DENIED +ACCESS_DENIED +ACCESS_DENIED +ACCESS_DENIED +ACCESS_DENIED +ACCESS_DENIED +ACCESS_DENIED +BAD_ARGUMENTS +BAD_ARGUMENTS +NOT_IMPLEMENTED diff --git a/tests/queries/0_stateless/05153_text_index_mergeTreeTextIndex_metadata_access.sh b/tests/queries/0_stateless/05153_text_index_mergeTreeTextIndex_metadata_access.sh new file mode 100755 index 000000000000..f49d9906b7ba --- /dev/null +++ b/tests/queries/0_stateless/05153_text_index_mergeTreeTextIndex_metadata_access.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# Without any grant on the table, `mergeTreeTextIndex` must not reveal whether the table +# is a MergeTree table or which indexes it has: every error must be ACCESS_DENIED. + +user_name="${CLICKHOUSE_DATABASE}_user_05153" + +$CLICKHOUSE_CLIENT -q " +DROP USER IF EXISTS $user_name; + +CREATE TABLE tab +( + a String, + b String, + INDEX idx_text a TYPE text(tokenizer = 'splitByNonAlpha'), + INDEX idx_set b TYPE set(0) +) +ENGINE = MergeTree +ORDER BY tuple(); + +CREATE TABLE tab_memory (a String) ENGINE = Memory; + +INSERT INTO tab VALUES ('hello', 'world'); + +CREATE USER $user_name IDENTIFIED WITH plaintext_password BY 'password'; +" + +function run_as_user() +{ + local output + if output=$($CLICKHOUSE_CLIENT --user "$user_name" --password "password" -q "$1" 2>&1); then + echo "OK" + else + echo "$output" | grep -oE '\([A-Z_]+\)' | tail -1 | tr -d '()' + fi +} + +run_as_user "SELECT * FROM mergeTreeTextIndex(currentDatabase(), tab, idx_text)" +run_as_user "SELECT * FROM mergeTreeTextIndex(currentDatabase(), tab, idx_set)" +run_as_user "SELECT * FROM mergeTreeTextIndex(currentDatabase(), tab, idx_missing)" +run_as_user "SELECT * FROM mergeTreeTextIndex(currentDatabase(), tab_memory, idx_text)" +run_as_user "SELECT * FROM mergeTreeTextIndex(currentDatabase(), tab_missing, idx_text)" +run_as_user "INSERT INTO FUNCTION mergeTreeTextIndex(currentDatabase(), tab, idx_text) SELECT 'p', 'x', 'raw', 1, 1, 0, 0, 0" + +# A grant on any column implies SHOW TABLES, so index metadata becomes visible, +# while reading the index still requires SELECT on the indexed column. +$CLICKHOUSE_CLIENT -q "GRANT SELECT(b) ON $CLICKHOUSE_DATABASE.tab TO $user_name" + +run_as_user "SELECT * FROM mergeTreeTextIndex(currentDatabase(), tab, idx_text)" +run_as_user "SELECT * FROM mergeTreeTextIndex(currentDatabase(), tab, idx_set)" +run_as_user "SELECT * FROM mergeTreeTextIndex(currentDatabase(), tab, idx_missing)" + +# The index is read-only: the access check runs before the write is rejected. +$CLICKHOUSE_CLIENT -q "INSERT INTO FUNCTION mergeTreeTextIndex(currentDatabase(), tab, idx_text) SELECT 'p', 'x', 'raw', 1, 1, 0, 0, 0" 2>&1 | grep -oE '\([A-Z_]+\)' | tail -1 | tr -d '()' + +$CLICKHOUSE_CLIENT -q " +DROP TABLE tab; +DROP TABLE tab_memory; +DROP USER $user_name; +" diff --git a/tests/queries/0_stateless/05153_text_index_mergeTreeTextIndex_remote.reference b/tests/queries/0_stateless/05153_text_index_mergeTreeTextIndex_remote.reference new file mode 100644 index 000000000000..80d7d44bd466 --- /dev/null +++ b/tests/queries/0_stateless/05153_text_index_mergeTreeTextIndex_remote.reference @@ -0,0 +1,9 @@ +ACCESS_DENIED +ACCESS_DENIED +ACCESS_DENIED +ACCESS_DENIED +ACCESS_DENIED +ACCESS_DENIED +2 +2 +2 diff --git a/tests/queries/0_stateless/05153_text_index_mergeTreeTextIndex_remote.sh b/tests/queries/0_stateless/05153_text_index_mergeTreeTextIndex_remote.sh new file mode 100755 index 000000000000..0bd207645ea7 --- /dev/null +++ b/tests/queries/0_stateless/05153_text_index_mergeTreeTextIndex_remote.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# `remote` over a local shard validates the access of the current user while inferring the structure of a nested +# table function, and may then route the query over loopback with other credentials. `mergeTreeTextIndex` has a +# static structure, so it has to check the access of the user itself there. + +user_name="${CLICKHOUSE_DATABASE}_user_05153" + +$CLICKHOUSE_CLIENT -q " +DROP USER IF EXISTS $user_name; + +CREATE TABLE tab (s String, INDEX idx_s s TYPE text(tokenizer = 'splitByNonAlpha')) ENGINE = MergeTree ORDER BY tuple(); +INSERT INTO tab VALUES ('hidden token'); + +CREATE USER $user_name IDENTIFIED WITH plaintext_password BY 'password'; +GRANT REMOTE ON *.* TO $user_name; +GRANT CREATE TEMPORARY TABLE ON *.* TO $user_name; +" + +function run_as_user() +{ + local output + if output=$($CLICKHOUSE_CLIENT --user "$user_name" --password "password" -q "$1" 2>&1); then + echo "$output" + else + echo "$output" | grep -oE '\([A-Z_]+\)' | tail -1 | tr -d '()' + fi +} + +index="mergeTreeTextIndex('$CLICKHOUSE_DATABASE', 'tab', 'idx_s')" +query="SELECT count() FROM remote('127.0.0.1:$CLICKHOUSE_PORT_TCP', $index)" +same_user_query="SELECT count() FROM remote('127.0.0.1:$CLICKHOUSE_PORT_TCP', $index, '$user_name', 'password')" + +function run_remote_as_user() +{ + for localhost_replica in 0 1; do + run_as_user "$query SETTINGS prefer_localhost_replica = $localhost_replica" + done + for localhost_replica in 0 1; do + run_as_user "$same_user_query SETTINGS prefer_localhost_replica = $localhost_replica" + done +} + +run_as_user "DESCRIBE TABLE $index" +run_remote_as_user + +$CLICKHOUSE_CLIENT -q "GRANT SELECT ON $CLICKHOUSE_DATABASE.tab TO $user_name" + +# Over an ordinary connection the shard runs the query as the user of the connection, which the function refuses +# unless that is the initiating user; with the local shortcut it runs as the user itself. +run_remote_as_user + +$CLICKHOUSE_CLIENT -q " +DROP TABLE tab; +DROP USER $user_name; +" diff --git a/tests/queries/0_stateless/05153_text_index_mergeTreeTextIndex_row_policy_any_column.reference b/tests/queries/0_stateless/05153_text_index_mergeTreeTextIndex_row_policy_any_column.reference new file mode 100644 index 000000000000..2efecfaa4921 --- /dev/null +++ b/tests/queries/0_stateless/05153_text_index_mergeTreeTextIndex_row_policy_any_column.reference @@ -0,0 +1,4 @@ +['visible apple phrase'] +0 +ACCESS_DENIED +['apple','hidden','phrase','visible','zebra'] diff --git a/tests/queries/0_stateless/05153_text_index_mergeTreeTextIndex_row_policy_any_column.sh b/tests/queries/0_stateless/05153_text_index_mergeTreeTextIndex_row_policy_any_column.sh new file mode 100755 index 000000000000..d5a3aa569869 --- /dev/null +++ b/tests/queries/0_stateless/05153_text_index_mergeTreeTextIndex_row_policy_any_column.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# A row policy on a column that is not indexed must still deny reading the text index: +# the dictionary contains the tokens of the rows the policy hides. + +user_name="${CLICKHOUSE_DATABASE}_user_05153" + +$CLICKHOUSE_CLIENT -q " +DROP USER IF EXISTS $user_name; + +CREATE TABLE tab +( + tenant_id String, + doc_id UInt64, + secret_text String, + INDEX idx_secret secret_text TYPE text(tokenizer = 'splitByNonAlpha') GRANULARITY 1 +) +ENGINE = MergeTree +ORDER BY doc_id +SETTINGS index_granularity = 1; + +INSERT INTO tab VALUES ('tenant_a', 1, 'visible apple phrase'), ('tenant_b', 2, 'hidden zebra phrase'); + +CREATE USER $user_name IDENTIFIED WITH plaintext_password BY 'password'; +GRANT SELECT ON $CLICKHOUSE_DATABASE.tab TO $user_name; +CREATE ROW POLICY p_05153 ON $CLICKHOUSE_DATABASE.tab FOR SELECT USING tenant_id = 'tenant_a' TO $user_name; +" + +function run_as_user() +{ + local output + if output=$($CLICKHOUSE_CLIENT --user "$user_name" --password "password" -q "$1" 2>&1); then + echo "$output" + elif echo "$output" | grep -q "ACCESS_DENIED"; then + echo "ACCESS_DENIED" + else + echo "$output" + fi +} + +run_as_user "SELECT groupArray(secret_text) FROM tab" +run_as_user "SELECT count() FROM tab WHERE hasToken(secret_text, 'zebra')" +run_as_user "SELECT arraySort(groupArray(token)) FROM mergeTreeTextIndex(currentDatabase(), tab, idx_secret)" + +$CLICKHOUSE_CLIENT -q "DROP ROW POLICY p_05153 ON $CLICKHOUSE_DATABASE.tab" + +run_as_user "SELECT arraySort(groupArray(token)) FROM mergeTreeTextIndex(currentDatabase(), tab, idx_secret)" + +$CLICKHOUSE_CLIENT -q " +DROP TABLE tab; +DROP USER $user_name; +" diff --git a/tests/queries/0_stateless/05153_variant_truncated_compact_discriminators.reference b/tests/queries/0_stateless/05153_variant_truncated_compact_discriminators.reference new file mode 100644 index 000000000000..91425ed846bb --- /dev/null +++ b/tests/queries/0_stateless/05153_variant_truncated_compact_discriminators.reference @@ -0,0 +1,30 @@ +42 +\N +\N +\N +42 +\N +0 +\N +2 +\N +4 +\N +6 +\N +0 +\N +2 +\N +4 +\N +6 +\N +0 +1 +0 +1 +0 +1 +0 +1 diff --git a/tests/queries/0_stateless/05153_variant_truncated_compact_discriminators.sql b/tests/queries/0_stateless/05153_variant_truncated_compact_discriminators.sql new file mode 100644 index 000000000000..a42982dfe579 --- /dev/null +++ b/tests/queries/0_stateless/05153_variant_truncated_compact_discriminators.sql @@ -0,0 +1,18 @@ +-- Truncated `COMPACT` `Variant` discriminator streams must be rejected, not read past the buffer. +SELECT * FROM format(Native, 'v Variant(UInt8)', concat('\x01\x64\x01v\x0eVariant(UInt8)', unhex('01000000000000006400'))); -- { serverError CANNOT_READ_ALL_DATA } +SELECT * FROM format(Native, 'v Variant(UInt8)', concat('\x01\x02\x01v\x0eVariant(UInt8)', unhex('01000000000000000200ff'))); -- { serverError CANNOT_READ_ALL_DATA } +SELECT * FROM format(Native, 'v Variant(UInt8)', concat('\x01\x6c\x01v\x0eVariant(UInt8)', unhex('01000000000000000801ff6400ff'))); -- { serverError CANNOT_READ_ALL_DATA } + +-- Complete streams still decode correctly. +SELECT * FROM format(Native, 'v Variant(UInt8)', concat('\x01\x02\x01v\x0eVariant(UInt8)', unhex('0100000000000000020000ff2a'))); +SELECT * FROM format(Native, 'v Variant(UInt8)', concat('\x01\x04\x01v\x0eVariant(UInt8)', unhex('01000000000000000201ff020000ff2a'))); + +CREATE TABLE variant_compact_granules (id UInt8, v Variant(UInt8, String)) +ENGINE = MergeTree ORDER BY id +SETTINGS use_compact_variant_discriminators_serialization = 1, index_granularity = 8, + min_rows_for_wide_part = 0, min_bytes_for_wide_part = 0; +INSERT INTO variant_compact_granules SELECT number, if(number % 2 = 0, number::UInt8, NULL) FROM numbers(8); +SELECT v FROM variant_compact_granules ORDER BY id SETTINGS max_block_size = 1, max_threads = 1; +SELECT v.UInt8 FROM variant_compact_granules ORDER BY id SETTINGS max_block_size = 1, max_threads = 1; +SELECT v.UInt8.null FROM variant_compact_granules ORDER BY id SETTINGS max_block_size = 1, max_threads = 1; +DROP TABLE variant_compact_granules; diff --git a/tests/queries/0_stateless/05182_limit_shrinks_block_size_with_array_join.reference b/tests/queries/0_stateless/05182_limit_shrinks_block_size_with_array_join.reference new file mode 100644 index 000000000000..e38f471d7a81 --- /dev/null +++ b/tests/queries/0_stateless/05182_limit_shrinks_block_size_with_array_join.reference @@ -0,0 +1,7 @@ +1 +256 +0 +9000 +9001 +9002 +1 diff --git a/tests/queries/0_stateless/05182_limit_shrinks_block_size_with_array_join.sql b/tests/queries/0_stateless/05182_limit_shrinks_block_size_with_array_join.sql new file mode 100644 index 000000000000..36d40033f6ce --- /dev/null +++ b/tests/queries/0_stateless/05182_limit_shrinks_block_size_with_array_join.sql @@ -0,0 +1,36 @@ +-- LIMIT still shrinks the block size with arrayJoin, only the source-side limit stays off (#82279) +DROP TABLE IF EXISTS t_aj_limit; +CREATE TABLE t_aj_limit (k UInt64, a Array(UInt64)) ENGINE = MergeTree ORDER BY k SETTINGS index_granularity = 8; +-- the 5000 bound below needs an un-shrunk read above it, and marks = rows / index_granularity +INSERT INTO t_aj_limit SELECT number, [number] FROM numbers(20000); + +-- the initiator's read_rows sums every replica's read, so the bound below needs a single reader +SELECT arrayJoin(a) FROM t_aj_limit LIMIT 1 FORMAT Null SETTINGS max_threads = 8, enable_parallel_replicas = 0, log_comment = '05182_limit'; + +SYSTEM FLUSH LOGS query_log; +SELECT argMax(read_rows, event_time_microseconds) < 5000 FROM system.query_log +WHERE current_database = currentDatabase() AND type = 'QueryFinish' AND log_comment = '05182_limit'; + +-- the block does not shrink below a few hundred rows, so a long run of empty arrays is not streamed one row at a time +SELECT DISTINCT bs FROM (SELECT arrayJoin(a), blockSize() AS bs FROM t_aj_limit LIMIT 3 SETTINGS max_threads = 1, enable_parallel_replicas = 0); + +-- the prefetched pool sizes its reads by marks, not by the block size, so a small LIMIT keeps it off +-- it is only a candidate for a single-node read that is multi-stream or all-remote, and the local +-- read settings gate an all-local read while the remote ones gate an all-remote read +SELECT countIf(explain LIKE '%PrefetchedReadPool%') FROM (EXPLAIN PIPELINE SELECT arrayJoin(a) FROM t_aj_limit LIMIT 1 SETTINGS allow_prefetched_read_pool_for_local_filesystem = 1, local_filesystem_read_method = 'pread_threadpool', allow_prefetched_read_pool_for_remote_filesystem = 1, remote_filesystem_read_method = 'threadpool', max_threads = 8, max_threads_min_free_memory_per_thread = 0, merge_tree_min_rows_for_concurrent_read = 1, merge_tree_min_bytes_for_concurrent_read = 1, enable_parallel_replicas = 0); + +DROP TABLE t_aj_limit; + +-- a long empty-array prefix: the LIMIT is tiny but the source has to get past the prefix +DROP TABLE IF EXISTS t_aj_sparse; +CREATE TABLE t_aj_sparse (k UInt64, a Array(UInt64)) ENGINE = MergeTree ORDER BY k SETTINGS index_granularity = 8; +INSERT INTO t_aj_sparse SELECT number, if(number < 9000, [], [number]) FROM numbers(10000); + +-- one local stream reads in key order; with parallel replicas the order and the initiator's read_rows are not deterministic +SELECT arrayJoin(a) FROM t_aj_sparse LIMIT 3 SETTINGS max_threads = 1, enable_parallel_replicas = 0, log_comment = '05182_sparse'; + +SYSTEM FLUSH LOGS query_log; +SELECT argMax(read_rows, event_time_microseconds) >= 9000 FROM system.query_log +WHERE current_database = currentDatabase() AND type = 'QueryFinish' AND log_comment = '05182_sparse'; + +DROP TABLE t_aj_sparse; diff --git a/tests/queries/0_stateless/05182_parameterized_view_insert_select_parallel_distributed.reference b/tests/queries/0_stateless/05182_parameterized_view_insert_select_parallel_distributed.reference new file mode 100644 index 000000000000..573541ac9702 --- /dev/null +++ b/tests/queries/0_stateless/05182_parameterized_view_insert_select_parallel_distributed.reference @@ -0,0 +1 @@ +0 diff --git a/tests/queries/0_stateless/05182_parameterized_view_insert_select_parallel_distributed.sql b/tests/queries/0_stateless/05182_parameterized_view_insert_select_parallel_distributed.sql new file mode 100644 index 000000000000..9561e1a60a5c --- /dev/null +++ b/tests/queries/0_stateless/05182_parameterized_view_insert_select_parallel_distributed.sql @@ -0,0 +1,14 @@ +-- INSERT SELECT from a parameterized view resolved the view through the legacy interpreter when +-- parallel_distributed_insert_select was enabled, so views only the analyzer can resolve failed. +SET enable_analyzer = 1; +SET parallel_distributed_insert_select = 2; + +CREATE TABLE t (id UInt8, p String) ENGINE = MergeTree ORDER BY id; +-- `SELECT i.*` over two joins yields columns named `i.p` in the legacy interpreter. +CREATE VIEW v AS SELECT count() AS c FROM (SELECT i.* FROM t AS i LEFT JOIN t AS a ON i.p = a.p LEFT JOIN t AS b ON i.p = b.p WHERE i.id = {id:UInt8}) AS i WHERE i.p = 'a'; +CREATE TABLE d (c UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{database}/d', '1') ORDER BY c; + +INSERT INTO d SELECT * FROM v(id = 1); +SELECT * FROM d; + +DROP TABLE d SYNC; diff --git a/tests/queries/0_stateless/05211_arrow_output_record_batch_coalescing.reference b/tests/queries/0_stateless/05211_arrow_output_record_batch_coalescing.reference new file mode 100644 index 000000000000..ec8e6ce9d8bc --- /dev/null +++ b/tests/queries/0_stateless/05211_arrow_output_record_batch_coalescing.reference @@ -0,0 +1,65 @@ +--- one batch per block by default --- +batches: 64 +64 2016 1 +--- both targets set: a result below both leaves as one batch --- +batches: 1 +64 2016 1 +--- row target alone: 64 rows in batches of 16 --- +batches: 4 +64 2016 1 +--- byte target alone: 8 bytes per row, so batches of 8 rows --- +batches: 8 +64 2016 1 +--- a combined batch may exceed the target: two 40-row blocks under a 41-row target --- +batches: 1, rows per batch: [80] +80 3160 1 +--- blocks that already reach the target are neither merged nor split --- +batches: 4, rows per batch: [65409, 65409, 65409, 3773] +200000 19999900000 1 +--- a block reaching the target flushes what is staged instead of absorbing it --- +batches: 3, rows per batch: [3, 65409, 254] +65666 6450733123 1 +--- a multi-column result keeps every value, in order, when combined --- +batches: 1 +7 0 \N +7 1 1 +7 2 2 +7 0 3 +7 1 \N +7 2 5 +7 0 6 +7 1 7 +--- dictionary-encoded LowCardinality spread over several combined batches --- +batches: 4, rows per batch: [4, 4, 4, 4] +0 2 +1 2 +2 2 +3 2 +shared 8 +--- the same dictionary column combined into one batch --- +batches: 1, rows per batch: [16] +0 2 +1 2 +2 2 +3 2 +shared 8 +--- the byte target counts the bytes the block holds, which for LowCardinality is a deduplicated value --- +batches: 1, rows per batch: [100] +100 1 100000 +batches: 50 +--- the row criterion still applies when the byte criterion is set, and bounds that batch --- +batches: 13, rows per batch: [8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 4] +100 1 100000 +--- a block keeping the dictionary of the block it was filtered out of reaches the byte target alone --- +batches: 4, rows per batch: [1, 1, 1, 1] +4 4 1000 +batches: 1, rows per batch: [4] +--- the row target alone combines those same blocks --- +batches: 1, rows per batch: [4] +4 4 1000 +--- the Arrow file format coalesces too, and its footer stays consistent --- +batches: 1 +64 2016 1 +--- an Arrow file footer with one Block per combined batch --- +batches: 4, rows per batch: [16, 16, 16, 16] +64 2016 1 diff --git a/tests/queries/0_stateless/05211_arrow_output_record_batch_coalescing.sh b/tests/queries/0_stateless/05211_arrow_output_record_batch_coalescing.sh new file mode 100755 index 000000000000..1398fdf84995 --- /dev/null +++ b/tests/queries/0_stateless/05211_arrow_output_record_batch_coalescing.sh @@ -0,0 +1,160 @@ +#!/usr/bin/env bash +# Tags: no-fasttest + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# Tests `output_format_arrow_record_batch_size` and `output_format_arrow_record_batch_size_bytes`, which +# combine consecutive blocks that are individually smaller than the target into one Arrow IPC record batch. +# Every arm asserts an exact batch count plus the rows read back, so that combining cannot lose, duplicate +# or reorder rows. +# +# `numbers` (never `numbers_mt`) with `max_block_size` pinned in the query's own SETTINGS, which beats +# the test runner's randomization, is what makes the sequence of blocks reaching the writer, and therefore +# the batch counts, reproducible. `max_threads = 1` keeps the source and the read-back single-stream, so +# that neither the batch counts nor the `in_order` check below can depend on scheduling. +# +# The result is written with `FORMAT ArrowStream` redirected to a file, never with +# `INSERT INTO FUNCTION file(...)`: `ArrowStream` prefers large blocks, so the INSERT path would squash to +# `min_insert_block_size_rows` and deliver the whole fixture as one block, leaving every arm at one batch +# whether or not the feature works. + +FILE="${CLICKHOUSE_TMP}/05211_arrow_batch_coalescing.arrows" + +# An older pyarrow cannot read the sentinel written for an uncompressed lz4_frame body, and the batch +# counts must not depend on the reader's codec support. +COMMON="max_threads = 1, output_format_arrow_compression_method = 'none'" + +# Prints the number of record batches, and with a second argument also the rows in each of them. +arrow_batches() { + python3 - "$FILE" "$1" "${2-}" <<'PY' +import sys +import pyarrow as pa + +path, mode, detail = sys.argv[1], sys.argv[2], sys.argv[3] +with pa.OSFile(path, "rb") as source: + reader = pa.ipc.open_file(source) if mode == "file" else pa.ipc.open_stream(source) + if mode == "file": + rows = [reader.get_batch(i).num_rows for i in range(reader.num_record_batches)] + else: + rows = [batch.num_rows for batch in reader] +out = "batches: %d" % len(rows) +if detail: + out += ", rows per batch: %s" % rows +print(out) +PY +} + +# The count and the checksum prove no row was lost or duplicated; `in_order` proves that combining +# appended rows instead of reordering them, since every fixture below is produced in ascending order. +oracle() { + ${CLICKHOUSE_LOCAL} --query "SELECT count(), sum(number), groupArray(number) = arraySort(groupArray(number)) AS in_order FROM file('${FILE}', '${1-ArrowStream}') SETTINGS max_threads = 1" +} + +# The same, for the `LowCardinality(String)` arms whose fixture is not a sequence of numbers: the row count, +# the distinct values and their total length prove the combined batch still describes every row. +string_oracle() { + ${CLICKHOUSE_LOCAL} --query "SELECT count(), uniqExact(s), sum(length(s)) FROM file('${FILE}', 'ArrowStream') SETTINGS max_threads = 1" +} + +echo "--- one batch per block by default ---" +${CLICKHOUSE_LOCAL} --query "SELECT number FROM numbers(64) SETTINGS max_block_size = 1, ${COMMON} FORMAT ArrowStream" > "${FILE}" +arrow_batches stream +oracle + +echo "--- both targets set: a result below both leaves as one batch ---" +${CLICKHOUSE_LOCAL} --query "SELECT number FROM numbers(64) SETTINGS max_block_size = 1, ${COMMON}, output_format_arrow_record_batch_size = 65409, output_format_arrow_record_batch_size_bytes = 1048576 FORMAT ArrowStream" > "${FILE}" +arrow_batches stream +oracle + +echo "--- row target alone: 64 rows in batches of 16 ---" +${CLICKHOUSE_LOCAL} --query "SELECT number FROM numbers(64) SETTINGS max_block_size = 1, ${COMMON}, output_format_arrow_record_batch_size = 16, output_format_arrow_record_batch_size_bytes = 0 FORMAT ArrowStream" > "${FILE}" +arrow_batches stream +oracle + +echo "--- byte target alone: 8 bytes per row, so batches of 8 rows ---" +${CLICKHOUSE_LOCAL} --query "SELECT number FROM numbers(64) SETTINGS max_block_size = 1, ${COMMON}, output_format_arrow_record_batch_size = 0, output_format_arrow_record_batch_size_bytes = 64 FORMAT ArrowStream" > "${FILE}" +arrow_batches stream +oracle + +echo "--- a combined batch may exceed the target: two 40-row blocks under a 41-row target ---" +${CLICKHOUSE_LOCAL} --query "SELECT number FROM numbers(80) SETTINGS max_block_size = 40, ${COMMON}, output_format_arrow_record_batch_size = 41, output_format_arrow_record_batch_size_bytes = 0 FORMAT ArrowStream" > "${FILE}" +arrow_batches stream detail +oracle + +echo "--- blocks that already reach the target are neither merged nor split ---" +${CLICKHOUSE_LOCAL} --query "SELECT number FROM numbers(200000) SETTINGS max_block_size = 65409, ${COMMON}, output_format_arrow_record_batch_size = 65409, output_format_arrow_record_batch_size_bytes = 0 FORMAT ArrowStream" > "${FILE}" +arrow_batches stream detail +oracle + +echo "--- a block reaching the target flushes what is staged instead of absorbing it ---" +# The three blocks are 3, 65409 and 254 rows. `toString(number) IN (...)` rather than `number < 3` keeps +# the condition out of reach of the range analysis of `numbers`, which would otherwise generate exactly +# the matching rows and deliver them as two full blocks, leaving no small block in front of the big one. +${CLICKHOUSE_LOCAL} --query "SELECT number FROM numbers(131072) WHERE toString(number) IN ('0', '1', '2') OR number > 65408 SETTINGS max_block_size = 65409, ${COMMON}, output_format_arrow_record_batch_size = 1000, output_format_arrow_record_batch_size_bytes = 0 FORMAT ArrowStream" > "${FILE}" +arrow_batches stream detail +oracle + +echo "--- a multi-column result keeps every value, in order, when combined ---" +${CLICKHOUSE_LOCAL} --query "SELECT 7 AS c, toLowCardinality(toString(number % 3)) AS lc, if(number % 4 = 0, NULL, number)::Nullable(UInt64) AS n FROM numbers(8) SETTINGS max_block_size = 1, ${COMMON}, output_format_arrow_record_batch_size = 65409 FORMAT ArrowStream" > "${FILE}" +arrow_batches stream +${CLICKHOUSE_LOCAL} --query "SELECT * FROM file('${FILE}', 'ArrowStream') SETTINGS max_threads = 1" + +echo "--- dictionary-encoded LowCardinality spread over several combined batches ---" +# Every batch after the first mixes a value the stream's dictionary already carries with one it does not, so +# each emits a dictionary delta and has the indexes of a merged column remapped against the accumulated dictionary. +DICT_FIXTURE="toLowCardinality(if(number % 4 < 2, 'shared', toString(intDiv(number, 4)))) AS lc FROM numbers(16)" +DICT_READBACK="SELECT lc, count() FROM file('${FILE}', 'ArrowStream') GROUP BY lc ORDER BY lc SETTINGS max_threads = 1" +${CLICKHOUSE_LOCAL} --query "SELECT ${DICT_FIXTURE} SETTINGS max_block_size = 1, ${COMMON}, output_format_arrow_low_cardinality_as_dictionary = 1, output_format_arrow_record_batch_size = 4 FORMAT ArrowStream" > "${FILE}" +arrow_batches stream detail +${CLICKHOUSE_LOCAL} --query "${DICT_READBACK}" + +echo "--- the same dictionary column combined into one batch ---" +${CLICKHOUSE_LOCAL} --query "SELECT ${DICT_FIXTURE} SETTINGS max_block_size = 1, ${COMMON}, output_format_arrow_low_cardinality_as_dictionary = 1, output_format_arrow_record_batch_size = 65409 FORMAT ArrowStream" > "${FILE}" +arrow_batches stream detail +${CLICKHOUSE_LOCAL} --query "${DICT_READBACK}" + +echo "--- the byte target counts the bytes the block holds, which for LowCardinality is a deduplicated value ---" +# The byte target counts one index per row plus the dictionary once (`ColumnLowCardinality::byteSize`), while +# the encoder writes one value per row: documented on the setting, hence asserted here rather than filed as a bug. +${CLICKHOUSE_LOCAL} --query "SELECT toLowCardinality(repeat('x', 1000)) AS s FROM numbers(100) SETTINGS max_block_size = 1, ${COMMON}, output_format_arrow_low_cardinality_as_dictionary = 0, output_format_arrow_record_batch_size = 0, output_format_arrow_record_batch_size_bytes = 1500 FORMAT ArrowStream" > "${FILE}" +arrow_batches stream detail +string_oracle +${CLICKHOUSE_LOCAL} --query "SELECT repeat('x', 1000) AS s FROM numbers(100) SETTINGS max_block_size = 1, ${COMMON}, output_format_arrow_record_batch_size = 0, output_format_arrow_record_batch_size_bytes = 1500 FORMAT ArrowStream" > "${FILE}" +arrow_batches stream + +echo "--- the row criterion still applies when the byte criterion is set, and bounds that batch ---" +${CLICKHOUSE_LOCAL} --query "SELECT toLowCardinality(repeat('x', 1000)) AS s FROM numbers(100) SETTINGS max_block_size = 1, ${COMMON}, output_format_arrow_low_cardinality_as_dictionary = 0, output_format_arrow_record_batch_size = 8, output_format_arrow_record_batch_size_bytes = 1500 FORMAT ArrowStream" > "${FILE}" +arrow_batches stream detail +string_oracle + +echo "--- a block keeping the dictionary of the block it was filtered out of reaches the byte target alone ---" +# `ColumnLowCardinality::filter` keeps the whole source dictionary, so each surviving row measures the +# ~64 KB dictionary its 250-row block built and is written as its own record batch, which also ends the +# batches the row target set here would otherwise have combined. The `OR` is what keeps the filter above +# the projection that builds `s`, so the dictionary is built for the whole block; `length(s) = 0` is +# never true. The `String` arm below has the same values and the same targets and is combined. +LC_SOURCE_DICT="SELECT s FROM (SELECT toLowCardinality(repeat(toString(number), 100)) AS s, number AS n FROM numbers(1000)) WHERE (n % 250 = 0) OR (length(s) = 0)" +${CLICKHOUSE_LOCAL} --query "${LC_SOURCE_DICT} SETTINGS max_block_size = 250, ${COMMON}, output_format_arrow_low_cardinality_as_dictionary = 0, output_format_arrow_record_batch_size = 65409, output_format_arrow_record_batch_size_bytes = 8192 FORMAT ArrowStream" > "${FILE}" +arrow_batches stream detail +string_oracle +${CLICKHOUSE_LOCAL} --query "SELECT s FROM (SELECT repeat(toString(number), 100) AS s, number AS n FROM numbers(1000)) WHERE (n % 250 = 0) OR (length(s) = 0) SETTINGS max_block_size = 250, ${COMMON}, output_format_arrow_record_batch_size = 65409, output_format_arrow_record_batch_size_bytes = 8192 FORMAT ArrowStream" > "${FILE}" +arrow_batches stream detail + +echo "--- the row target alone combines those same blocks ---" +${CLICKHOUSE_LOCAL} --query "${LC_SOURCE_DICT} SETTINGS max_block_size = 250, ${COMMON}, output_format_arrow_low_cardinality_as_dictionary = 0, output_format_arrow_record_batch_size = 65409, output_format_arrow_record_batch_size_bytes = 0 FORMAT ArrowStream" > "${FILE}" +arrow_batches stream detail +string_oracle + +echo "--- the Arrow file format coalesces too, and its footer stays consistent ---" +${CLICKHOUSE_LOCAL} --query "SELECT number FROM numbers(64) SETTINGS max_block_size = 1, ${COMMON}, output_format_arrow_record_batch_size = 65409 FORMAT Arrow" > "${FILE}" +arrow_batches file +oracle Arrow + +echo "--- an Arrow file footer with one Block per combined batch ---" +${CLICKHOUSE_LOCAL} --query "SELECT number FROM numbers(64) SETTINGS max_block_size = 1, ${COMMON}, output_format_arrow_record_batch_size = 16 FORMAT Arrow" > "${FILE}" +arrow_batches file detail +oracle Arrow + +rm -f "${FILE}" diff --git a/tests/queries/0_stateless/05211_local_object_storage_path_embedded_nul.reference b/tests/queries/0_stateless/05211_local_object_storage_path_embedded_nul.reference new file mode 100644 index 000000000000..9f845b036642 --- /dev/null +++ b/tests/queries/0_stateless/05211_local_object_storage_path_embedded_nul.reference @@ -0,0 +1,3 @@ +-- a path with an embedded NUL is rejected +-- the same for icebergLocal +-- a path inside user_files is still accepted (and fails for its own reason) diff --git a/tests/queries/0_stateless/05211_local_object_storage_path_embedded_nul.sh b/tests/queries/0_stateless/05211_local_object_storage_path_embedded_nul.sh new file mode 100755 index 000000000000..492ed2d68abd --- /dev/null +++ b/tests/queries/0_stateless/05211_local_object_storage_path_embedded_nul.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, no-msan +# Tag no-fasttest: Requires DeltaLake +# Tag no-msan: DeltaKernel is not compiled with msan + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# A path with an embedded NUL byte must not pass the `user_files` containment check: the check sees the +# whole string, while every syscall the path is later passed to stops at the NUL. The path below is +# normalized into `user_files` as a whole string, but addresses `/../probe_$CLICKHOUSE_DATABASE` +# once truncated at the NUL. + +TAIL="../$(basename "${USER_FILES_PATH}")/nul_${CLICKHOUSE_DATABASE}" + +echo "-- a path with an embedded NUL is rejected" +${CLICKHOUSE_CLIENT} --query " + SELECT * FROM deltaLakeLocal('${USER_FILES_PATH}/../probe_${CLICKHOUSE_DATABASE}\0/${TAIL}') -- { serverError PATH_ACCESS_DENIED } +" + +echo "-- the same for icebergLocal" +${CLICKHOUSE_CLIENT} --query " + SELECT * FROM icebergLocal('${USER_FILES_PATH}/../probe_${CLICKHOUSE_DATABASE}\0/${TAIL}') -- { serverError PATH_ACCESS_DENIED } +" + +echo "-- a path inside user_files is still accepted (and fails for its own reason)" +mkdir -p "${USER_FILES_PATH}/nul_${CLICKHOUSE_DATABASE}" +${CLICKHOUSE_CLIENT} --query " + SELECT * FROM deltaLakeLocal('${USER_FILES_PATH}/nul_${CLICKHOUSE_DATABASE}') -- { serverError DELTA_KERNEL_ERROR } +" +rmdir "${USER_FILES_PATH}/nul_${CLICKHOUSE_DATABASE}" diff --git a/tests/queries/0_stateless/05212_parallel_with_access_checks.reference b/tests/queries/0_stateless/05212_parallel_with_access_checks.reference new file mode 100644 index 000000000000..10f1739ece85 --- /dev/null +++ b/tests/queries/0_stateless/05212_parallel_with_access_checks.reference @@ -0,0 +1,12 @@ +-- without the CREATE TABLE privilege, directly +ACCESS_DENIED +-- without the CREATE TABLE privilege, inside PARALLEL WITH +ACCESS_DENIED +-- CREATE DATABASE inside PARALLEL WITH is checked too +ACCESS_DENIED +-- nothing was created +0 +0 +-- with the CREATE TABLE privilege, PARALLEL WITH still works +t_parallel_1 +t_parallel_2 diff --git a/tests/queries/0_stateless/05212_parallel_with_access_checks.sh b/tests/queries/0_stateless/05212_parallel_with_access_checks.sh new file mode 100755 index 000000000000..d66dceb563cf --- /dev/null +++ b/tests/queries/0_stateless/05212_parallel_with_access_checks.sh @@ -0,0 +1,63 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# `PARALLEL WITH` must not bypass access checks of its subqueries. + +user="user_${CLICKHOUSE_DATABASE}" + +${CLICKHOUSE_CLIENT} --query "DROP USER IF EXISTS ${user}" +${CLICKHOUSE_CLIENT} --query "CREATE USER ${user} IDENTIFIED WITH plaintext_password BY 'password'" +${CLICKHOUSE_CLIENT} --query "GRANT SELECT ON ${CLICKHOUSE_DATABASE}.* TO ${user}" +# `TABLE ENGINE` is granted up front so that the only privilege missing below is `CREATE TABLE`. +# `access_control_improvements.table_engines_require_grant` is enabled in the test configuration, and +# the `TABLE ENGINE` check in `getTablePropertiesAndNormalizeCreateQuery` does not consult `internal` - +# without this grant a `CREATE TABLE` would be refused by that check even when its own access check is +# skipped, which would hide the very bypass this test is about. +${CLICKHOUSE_CLIENT} --query "GRANT TABLE ENGINE ON Memory TO ${user}" + +CLIENT_AS_USER="${CLICKHOUSE_CLIENT} --user ${user} --password password" + +# The test database may be reused across runs, so only the objects of this test are counted, and they +# are dropped both before and after the run. +tables="'t_direct', 't_parallel_1', 't_parallel_2'" +${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS ${CLICKHOUSE_DATABASE}.t_direct, ${CLICKHOUSE_DATABASE}.t_parallel_1, ${CLICKHOUSE_DATABASE}.t_parallel_2" +${CLICKHOUSE_CLIENT} --query "DROP DATABASE IF EXISTS ${CLICKHOUSE_DATABASE}_db_1" +${CLICKHOUSE_CLIENT} --query "DROP DATABASE IF EXISTS ${CLICKHOUSE_DATABASE}_db_2" + +echo "-- without the CREATE TABLE privilege, directly" +${CLIENT_AS_USER} --query " + CREATE TABLE ${CLICKHOUSE_DATABASE}.t_direct (x UInt8) ENGINE = Memory +" 2>&1 | grep -q "ACCESS_DENIED" && echo "ACCESS_DENIED" || echo "ALLOWED" + +echo "-- without the CREATE TABLE privilege, inside PARALLEL WITH" +${CLIENT_AS_USER} --query " + CREATE TABLE ${CLICKHOUSE_DATABASE}.t_parallel_1 (x UInt8) ENGINE = Memory + PARALLEL WITH + CREATE TABLE ${CLICKHOUSE_DATABASE}.t_parallel_2 (x UInt8) ENGINE = Memory +" 2>&1 | grep -q "ACCESS_DENIED" && echo "ACCESS_DENIED" || echo "ALLOWED" + +echo "-- CREATE DATABASE inside PARALLEL WITH is checked too" +${CLIENT_AS_USER} --query " + CREATE DATABASE ${CLICKHOUSE_DATABASE}_db_1 + PARALLEL WITH + CREATE DATABASE ${CLICKHOUSE_DATABASE}_db_2 +" 2>&1 | grep -q "ACCESS_DENIED" && echo "ACCESS_DENIED" || echo "ALLOWED" + +echo "-- nothing was created" +${CLICKHOUSE_CLIENT} --query "SELECT count() FROM system.tables WHERE database = '${CLICKHOUSE_DATABASE}' AND name IN (${tables})" +${CLICKHOUSE_CLIENT} --query "SELECT count() FROM system.databases WHERE name IN ('${CLICKHOUSE_DATABASE}_db_1', '${CLICKHOUSE_DATABASE}_db_2')" + +echo "-- with the CREATE TABLE privilege, PARALLEL WITH still works" +${CLICKHOUSE_CLIENT} --query "GRANT CREATE TABLE ON ${CLICKHOUSE_DATABASE}.* TO ${user}" +${CLIENT_AS_USER} --query " + CREATE TABLE ${CLICKHOUSE_DATABASE}.t_parallel_1 (x UInt8) ENGINE = Memory + PARALLEL WITH + CREATE TABLE ${CLICKHOUSE_DATABASE}.t_parallel_2 (x UInt8) ENGINE = Memory +" +${CLICKHOUSE_CLIENT} --query "SELECT name FROM system.tables WHERE database = '${CLICKHOUSE_DATABASE}' AND name IN (${tables}) ORDER BY name" + +${CLICKHOUSE_CLIENT} --query "DROP TABLE ${CLICKHOUSE_DATABASE}.t_parallel_1, ${CLICKHOUSE_DATABASE}.t_parallel_2" +${CLICKHOUSE_CLIENT} --query "DROP USER ${user}" diff --git a/tests/queries/0_stateless/05212_path_containment_embedded_nul.reference b/tests/queries/0_stateless/05212_path_containment_embedded_nul.reference new file mode 100644 index 000000000000..056f0839d8ec --- /dev/null +++ b/tests/queries/0_stateless/05212_path_containment_embedded_nul.reference @@ -0,0 +1,4 @@ +-- the file function checks with fileOrSymlinkPathStartsWith only +data +-- the Filesystem database engine checks with pathStartsWith only +data diff --git a/tests/queries/0_stateless/05212_path_containment_embedded_nul.sh b/tests/queries/0_stateless/05212_path_containment_embedded_nul.sh new file mode 100755 index 000000000000..978a90a4ccba --- /dev/null +++ b/tests/queries/0_stateless/05212_path_containment_embedded_nul.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# A path with an embedded NUL byte must be rejected by every filesystem containment check: the check sees +# the whole string, while every syscall the path is later passed to stops at the NUL. Each case below names +# an existing entry inside `user_files` followed by a NUL and a suffix, so that a check which ignored the +# NUL would let the truncated path through and the query would succeed. + +DIR="${USER_FILES_PATH}/nul_${CLICKHOUSE_DATABASE}" +mkdir -p "${DIR}" +printf 'data' > "${DIR}/file.csv" + +echo "-- the file function checks with fileOrSymlinkPathStartsWith only" +${CLICKHOUSE_CLIENT} --query "SELECT file('nul_${CLICKHOUSE_DATABASE}/file.csv')" +${CLICKHOUSE_CLIENT} --query "SELECT file('nul_${CLICKHOUSE_DATABASE}/file.csv\0suffix') -- { serverError DATABASE_ACCESS_DENIED }" +${CLICKHOUSE_CLIENT} --query "SELECT file('${DIR}/file.csv\0suffix') -- { serverError DATABASE_ACCESS_DENIED }" + +echo "-- the Filesystem database engine checks with pathStartsWith only" +${CLICKHOUSE_CLIENT} --query "CREATE DATABASE nul_${CLICKHOUSE_DATABASE} ENGINE = Filesystem('${DIR}\0suffix') -- { serverError BAD_ARGUMENTS }" +${CLICKHOUSE_CLIENT} --query "CREATE DATABASE nul_${CLICKHOUSE_DATABASE} ENGINE = Filesystem('nul_${CLICKHOUSE_DATABASE}\0suffix') -- { serverError BAD_ARGUMENTS }" +${CLICKHOUSE_CLIENT} --query "CREATE DATABASE nul_${CLICKHOUSE_DATABASE} ENGINE = Filesystem('${DIR}')" +${CLICKHOUSE_CLIENT} --query "SELECT * FROM nul_${CLICKHOUSE_DATABASE}.\`file.csv\`" +${CLICKHOUSE_CLIENT} --query "DROP DATABASE nul_${CLICKHOUSE_DATABASE}" + +rm -r "${DIR}" diff --git a/tests/queries/0_stateless/05213_execute_as_access_checks.reference b/tests/queries/0_stateless/05213_execute_as_access_checks.reference new file mode 100644 index 000000000000..6454eeb3e162 --- /dev/null +++ b/tests/queries/0_stateless/05213_execute_as_access_checks.reference @@ -0,0 +1,11 @@ +-- target has no CREATE TABLE privilege +ACCESS_DENIED +-- target has no CREATE DATABASE privilege +ACCESS_DENIED +-- two layers of nesting: PARALLEL WITH inside EXECUTE AS is checked too +ACCESS_DENIED +-- nothing was created +0 +0 +-- once the target has the privilege, EXECUTE AS still works +t_execute_as diff --git a/tests/queries/0_stateless/05213_execute_as_access_checks.sh b/tests/queries/0_stateless/05213_execute_as_access_checks.sh new file mode 100755 index 000000000000..8c36ae883e7a --- /dev/null +++ b/tests/queries/0_stateless/05213_execute_as_access_checks.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# `EXECUTE AS` must not bypass access checks of the impersonated statement: the statement runs with +# the privileges of the target user, so a target without the `CREATE TABLE` privilege must be denied. + +caller="caller_${CLICKHOUSE_DATABASE}" +target="target_${CLICKHOUSE_DATABASE}" + +${CLICKHOUSE_CLIENT} --query "DROP USER IF EXISTS ${caller}, ${target}" +${CLICKHOUSE_CLIENT} --query "CREATE USER ${caller} IDENTIFIED WITH plaintext_password BY 'password'" +${CLICKHOUSE_CLIENT} --query "CREATE USER ${target}" +${CLICKHOUSE_CLIENT} --query "GRANT SELECT ON ${CLICKHOUSE_DATABASE}.* TO ${caller}, ${target}" +${CLICKHOUSE_CLIENT} --query "GRANT IMPERSONATE ON ${target} TO ${caller}" +# See the note in `05212_parallel_with_access_checks.sh`: `TABLE ENGINE` is granted up front so that +# the only privilege missing below is `CREATE TABLE`, otherwise the `TABLE ENGINE` check - which does +# not consult `internal` - would hide the bypass this test is about. +${CLICKHOUSE_CLIENT} --query "GRANT TABLE ENGINE ON Memory TO ${target}" + +CLIENT_AS_CALLER="${CLICKHOUSE_CLIENT} --user ${caller} --password password" + +# The test database may be reused across runs, so only the objects of this test are counted, and they +# are dropped both before and after the run. +tables="'t_execute_as', 't_nested_1', 't_nested_2'" +${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS ${CLICKHOUSE_DATABASE}.t_execute_as, ${CLICKHOUSE_DATABASE}.t_nested_1, ${CLICKHOUSE_DATABASE}.t_nested_2" +${CLICKHOUSE_CLIENT} --query "DROP DATABASE IF EXISTS ${CLICKHOUSE_DATABASE}_db" + +echo "-- target has no CREATE TABLE privilege" +${CLIENT_AS_CALLER} --query " + EXECUTE AS ${target} CREATE TABLE ${CLICKHOUSE_DATABASE}.t_execute_as (x UInt8) ENGINE = Memory +" 2>&1 | grep -q "ACCESS_DENIED" && echo "ACCESS_DENIED" || echo "ALLOWED" + +echo "-- target has no CREATE DATABASE privilege" +${CLIENT_AS_CALLER} --query " + EXECUTE AS ${target} CREATE DATABASE ${CLICKHOUSE_DATABASE}_db +" 2>&1 | grep -q "ACCESS_DENIED" && echo "ACCESS_DENIED" || echo "ALLOWED" + +echo "-- two layers of nesting: PARALLEL WITH inside EXECUTE AS is checked too" +${CLIENT_AS_CALLER} --query " + EXECUTE AS ${target} + CREATE TABLE ${CLICKHOUSE_DATABASE}.t_nested_1 (x UInt8) ENGINE = Memory + PARALLEL WITH + CREATE TABLE ${CLICKHOUSE_DATABASE}.t_nested_2 (x UInt8) ENGINE = Memory +" 2>&1 | grep -q "ACCESS_DENIED" && echo "ACCESS_DENIED" || echo "ALLOWED" + +echo "-- nothing was created" +${CLICKHOUSE_CLIENT} --query "SELECT count() FROM system.tables WHERE database = '${CLICKHOUSE_DATABASE}' AND name IN (${tables})" +${CLICKHOUSE_CLIENT} --query "SELECT count() FROM system.databases WHERE name = '${CLICKHOUSE_DATABASE}_db'" + +echo "-- once the target has the privilege, EXECUTE AS still works" +${CLICKHOUSE_CLIENT} --query "GRANT CREATE TABLE ON ${CLICKHOUSE_DATABASE}.* TO ${target}" +${CLIENT_AS_CALLER} --query " + EXECUTE AS ${target} CREATE TABLE ${CLICKHOUSE_DATABASE}.t_execute_as (x UInt8) ENGINE = Memory +" +${CLICKHOUSE_CLIENT} --query "SELECT name FROM system.tables WHERE database = '${CLICKHOUSE_DATABASE}' AND name IN (${tables}) ORDER BY name" + +${CLICKHOUSE_CLIENT} --query "DROP TABLE ${CLICKHOUSE_DATABASE}.t_execute_as" +${CLICKHOUSE_CLIENT} --query "DROP USER ${caller}, ${target}" diff --git a/tests/queries/0_stateless/05213_storage_file_embedded_nul.reference b/tests/queries/0_stateless/05213_storage_file_embedded_nul.reference new file mode 100644 index 000000000000..3fab3d2e8c89 --- /dev/null +++ b/tests/queries/0_stateless/05213_storage_file_embedded_nul.reference @@ -0,0 +1,5 @@ +-- the file table function +data +-- the File engine +data +data diff --git a/tests/queries/0_stateless/05213_storage_file_embedded_nul.sh b/tests/queries/0_stateless/05213_storage_file_embedded_nul.sh new file mode 100755 index 000000000000..79adfea69744 --- /dev/null +++ b/tests/queries/0_stateless/05213_storage_file_embedded_nul.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# The `file` table function and the `File` engine stat the path and expand its globs on the filesystem before +# the matched paths are checked for containment in `user_files`, and every syscall stops at the first NUL byte. +# A path with an embedded NUL must therefore be rejected up front, before anything touches the filesystem. +# Each case names an existing entry followed by a NUL and a suffix, so that the truncated path would succeed. + +DIR="${USER_FILES_PATH}/nul_${CLICKHOUSE_DATABASE}" +mkdir -p "${DIR}" +printf 'data\n' > "${DIR}/file.csv" + +echo "-- the file table function" +${CLICKHOUSE_CLIENT} --query "SELECT * FROM file('nul_${CLICKHOUSE_DATABASE}/file.csv', CSV, 'x String')" +${CLICKHOUSE_CLIENT} --query "SELECT * FROM file('nul_${CLICKHOUSE_DATABASE}/file.csv\0suffix', CSV, 'x String') -- { serverError BAD_ARGUMENTS }" +${CLICKHOUSE_CLIENT} --query "SELECT * FROM file('${DIR}/file.csv\0suffix', CSV, 'x String') -- { serverError BAD_ARGUMENTS }" +${CLICKHOUSE_CLIENT} --query "SELECT * FROM file('${DIR}/*\0suffix', CSV, 'x String') -- { serverError BAD_ARGUMENTS }" +${CLICKHOUSE_CLIENT} --query "SELECT * FROM file('${DIR}/file.csv\0suffix.zip :: file.csv', CSV, 'x String') -- { serverError BAD_ARGUMENTS }" +${CLICKHOUSE_CLIENT} --query "INSERT INTO FUNCTION file('${DIR}/file.csv\0suffix', CSV, 'x String') VALUES ('more') -- { serverError BAD_ARGUMENTS }" + +echo "-- the File engine" +${CLICKHOUSE_CLIENT} --query "CREATE TABLE nul_${CLICKHOUSE_DATABASE} (x String) ENGINE = File(CSV, 'nul_${CLICKHOUSE_DATABASE}/file.csv\0suffix') -- { serverError BAD_ARGUMENTS }" +${CLICKHOUSE_CLIENT} --query "CREATE TABLE nul_${CLICKHOUSE_DATABASE} (x String) ENGINE = File(CSV, '${DIR}/file.csv\0suffix') -- { serverError BAD_ARGUMENTS }" +${CLICKHOUSE_CLIENT} --query "CREATE TABLE nul_${CLICKHOUSE_DATABASE} (x String) ENGINE = File(CSV, 'nul_${CLICKHOUSE_DATABASE}/file.csv')" +${CLICKHOUSE_CLIENT} --query "SELECT * FROM nul_${CLICKHOUSE_DATABASE}" +${CLICKHOUSE_CLIENT} --query "DROP TABLE nul_${CLICKHOUSE_DATABASE}" + +cat "${DIR}/file.csv" +rm -r "${DIR}" diff --git a/tests/queries/0_stateless/05214_query_runner_access_checks.reference b/tests/queries/0_stateless/05214_query_runner_access_checks.reference new file mode 100644 index 000000000000..50d1f2894c73 --- /dev/null +++ b/tests/queries/0_stateless/05214_query_runner_access_checks.reference @@ -0,0 +1,15 @@ +-- without the CREATE DATABASE privilege, directly +ACCESS_DENIED +-- without the CREATE DATABASE privilege, queued into the QueryRunner table: the job is denied +ExceptionBeforeStart ACCESS_DENIED +-- queued through two layers of QueryRunner tables: the inner job is denied as well +QueryFinish OK +ExceptionBeforeStart ACCESS_DENIED +ExceptionBeforeStart ACCESS_DENIED +-- the database was not created +0 +-- once the privilege is granted, the queued job runs +ExceptionBeforeStart ACCESS_DENIED +ExceptionBeforeStart ACCESS_DENIED +QueryFinish OK +1 diff --git a/tests/queries/0_stateless/05214_query_runner_access_checks.sh b/tests/queries/0_stateless/05214_query_runner_access_checks.sh new file mode 100755 index 000000000000..646a4c82d8cb --- /dev/null +++ b/tests/queries/0_stateless/05214_query_runner_access_checks.sh @@ -0,0 +1,84 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# A job queued into a `QueryRunner` table must not bypass the access checks of the principal it runs +# as. With `SQL SECURITY INVOKER` that principal is the user who inserted the job, so a user without +# the `CREATE DATABASE` privilege must not be able to create a database by queueing it. +# +# The `INSERT` into the table succeeds no matter how the job ends - a failed job is only logged - and +# the mere absence of the database would also be observed if the job never ran at all. Hence the +# outcome of every job is asserted through `system.query_log`, where the jobs are recorded as +# internal queries of the `ClickHouse QueryRunner` client running on behalf of the user. + +# The user and the database carry a random suffix so that the rows this run reads from the shared +# `system.query_log` are its own: a previous run on the same server may have used the same test +# database, and its jobs would otherwise be printed again. +run_id=$(random_str 8) +user="user_${CLICKHOUSE_DATABASE}_${run_id}" +db="${CLICKHOUSE_DATABASE}_db_${run_id}" + +${CLICKHOUSE_CLIENT} --query "DROP USER IF EXISTS ${user}" +${CLICKHOUSE_CLIENT} --query "CREATE USER ${user} IDENTIFIED WITH plaintext_password BY 'password'" +${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS ${CLICKHOUSE_DATABASE}.runner" +${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS ${CLICKHOUSE_DATABASE}.outer_runner" +${CLICKHOUSE_CLIENT} --query " + CREATE TABLE ${CLICKHOUSE_DATABASE}.runner (query String, database String) ENGINE = QueryRunner + SETTINGS mode = 'synchronous', threads = 1 SQL SECURITY INVOKER +" +${CLICKHOUSE_CLIENT} --query " + CREATE TABLE ${CLICKHOUSE_DATABASE}.outer_runner (query String, database String) ENGINE = QueryRunner + SETTINGS mode = 'synchronous', threads = 1 SQL SECURITY INVOKER +" +${CLICKHOUSE_CLIENT} --query "GRANT SELECT, INSERT ON ${CLICKHOUSE_DATABASE}.* TO ${user}" + +CLIENT_AS_USER="${CLICKHOUSE_CLIENT} --user ${user} --password password" + +# Prints how the jobs whose query text starts with $1 ended: the log entry type and the error code. +# Only the jobs of this test's user run by the `QueryRunner` client are considered, so the direct +# queries of the user and the `INSERT` queries that queued the jobs are not mixed in, and the user's +# name is unique to this run. Every job below names the database of this test, which is what puts it +# in `current_database`. +function jobs_outcome() +{ + ${CLICKHOUSE_CLIENT} --query "SYSTEM FLUSH LOGS query_log" + ${CLICKHOUSE_CLIENT} --query " + SELECT type, errorCodeToName(exception_code) + FROM system.query_log + WHERE event_date >= yesterday() AND is_internal AND client_name = 'ClickHouse QueryRunner' + AND current_database = currentDatabase() + AND user = '${user}' AND startsWith(query, '${1}') AND type != 'QueryStart' + ORDER BY event_time_microseconds + " +} + +echo "-- without the CREATE DATABASE privilege, directly" +${CLIENT_AS_USER} --query "CREATE DATABASE ${db}" 2>&1 | grep -q "ACCESS_DENIED" && echo "ACCESS_DENIED" || echo "ALLOWED" + +echo "-- without the CREATE DATABASE privilege, queued into the QueryRunner table: the job is denied" +${CLIENT_AS_USER} --query "INSERT INTO ${CLICKHOUSE_DATABASE}.runner VALUES ('CREATE DATABASE ${db}', '${CLICKHOUSE_DATABASE}')" +jobs_outcome "CREATE DATABASE ${db}" + +echo "-- queued through two layers of QueryRunner tables: the inner job is denied as well" +${CLIENT_AS_USER} --query " + INSERT INTO ${CLICKHOUSE_DATABASE}.outer_runner + VALUES ('INSERT INTO ${CLICKHOUSE_DATABASE}.runner VALUES (''CREATE DATABASE ${db}'', ''${CLICKHOUSE_DATABASE}'')', '${CLICKHOUSE_DATABASE}') +" +jobs_outcome "INSERT INTO ${CLICKHOUSE_DATABASE}.runner" +jobs_outcome "CREATE DATABASE ${db}" + +echo "-- the database was not created" +${CLICKHOUSE_CLIENT} --query "SELECT count() FROM system.databases WHERE name = '${db}'" + +echo "-- once the privilege is granted, the queued job runs" +${CLICKHOUSE_CLIENT} --query "GRANT CREATE DATABASE ON ${db}.* TO ${user}" +${CLIENT_AS_USER} --query "INSERT INTO ${CLICKHOUSE_DATABASE}.runner VALUES ('CREATE DATABASE ${db}', '${CLICKHOUSE_DATABASE}')" +jobs_outcome "CREATE DATABASE ${db}" +${CLICKHOUSE_CLIENT} --query "SELECT count() FROM system.databases WHERE name = '${db}'" + +${CLICKHOUSE_CLIENT} --query "DROP DATABASE IF EXISTS ${db}" +${CLICKHOUSE_CLIENT} --query "DROP TABLE ${CLICKHOUSE_DATABASE}.outer_runner" +${CLICKHOUSE_CLIENT} --query "DROP TABLE ${CLICKHOUSE_DATABASE}.runner" +${CLICKHOUSE_CLIENT} --query "DROP USER ${user}" diff --git a/tests/queries/0_stateless/05215_parallel_with_access_checks_replicated_database.reference b/tests/queries/0_stateless/05215_parallel_with_access_checks_replicated_database.reference new file mode 100644 index 000000000000..8e3a9dbe8fd4 --- /dev/null +++ b/tests/queries/0_stateless/05215_parallel_with_access_checks_replicated_database.reference @@ -0,0 +1,9 @@ +-- without the CREATE TABLE privilege, directly +ACCESS_DENIED +-- without the CREATE TABLE privilege, inside PARALLEL WITH +ACCESS_DENIED +-- nothing was created +0 +-- with the CREATE TABLE privilege, PARALLEL WITH still works +t_parallel_1 +t_parallel_2 diff --git a/tests/queries/0_stateless/05215_parallel_with_access_checks_replicated_database.sh b/tests/queries/0_stateless/05215_parallel_with_access_checks_replicated_database.sh new file mode 100755 index 000000000000..dff7b96ce9cb --- /dev/null +++ b/tests/queries/0_stateless/05215_parallel_with_access_checks_replicated_database.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash +# Tags: zookeeper + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# `PARALLEL WITH` must not bypass the access checks of its subqueries in a `Replicated` database either. +# There a `CREATE TABLE` is additionally replayed through the replicated DDL log, and the replay of a +# statement written by the user is subject to the same rules as the replay of a direct statement. + +# The user, the database and its ZooKeeper path carry a random suffix, and the leftovers of a previous +# run that died halfway are dropped up front, so that the test can be rerun on the same server. +run_id=$(random_str 8) +user="user_${CLICKHOUSE_DATABASE}_${run_id}" +db="rdb_${CLICKHOUSE_DATABASE}_${run_id}" + +${CLICKHOUSE_CLIENT} --query "DROP DATABASE IF EXISTS ${db} SYNC" +${CLICKHOUSE_CLIENT} --query "CREATE DATABASE ${db} ENGINE = Replicated('/test/${CLICKHOUSE_TEST_ZOOKEEPER_PREFIX}/rdb_${run_id}', '1', '1')" +${CLICKHOUSE_CLIENT} --query "DROP USER IF EXISTS ${user}" +${CLICKHOUSE_CLIENT} --query "CREATE USER ${user} IDENTIFIED WITH plaintext_password BY 'password'" +${CLICKHOUSE_CLIENT} --query "GRANT SELECT ON ${db}.* TO ${user}" +# `TABLE ENGINE` is granted up front so that the only privilege missing below is `CREATE TABLE`, +# see `05212_parallel_with_access_checks.sh`. +${CLICKHOUSE_CLIENT} --query "GRANT TABLE ENGINE ON Memory TO ${user}" + +CLIENT_AS_USER="${CLICKHOUSE_CLIENT} --user ${user} --password password --distributed_ddl_output_mode none" + +echo "-- without the CREATE TABLE privilege, directly" +${CLIENT_AS_USER} --query " + CREATE TABLE ${db}.t_direct (x UInt8) ENGINE = Memory +" 2>&1 | grep -q "ACCESS_DENIED" && echo "ACCESS_DENIED" || echo "ALLOWED" + +echo "-- without the CREATE TABLE privilege, inside PARALLEL WITH" +${CLIENT_AS_USER} --query " + CREATE TABLE ${db}.t_parallel_1 (x UInt8) ENGINE = Memory + PARALLEL WITH + CREATE TABLE ${db}.t_parallel_2 (x UInt8) ENGINE = Memory +" 2>&1 | grep -q "ACCESS_DENIED" && echo "ACCESS_DENIED" || echo "ALLOWED" + +echo "-- nothing was created" +${CLICKHOUSE_CLIENT} --query " + SELECT count() FROM system.tables + WHERE database = '${db}' AND name IN ('t_direct', 't_parallel_1', 't_parallel_2') +" + +echo "-- with the CREATE TABLE privilege, PARALLEL WITH still works" +${CLICKHOUSE_CLIENT} --query "GRANT CREATE TABLE ON ${db}.* TO ${user}" +${CLIENT_AS_USER} --query " + CREATE TABLE ${db}.t_parallel_1 (x UInt8) ENGINE = Memory + PARALLEL WITH + CREATE TABLE ${db}.t_parallel_2 (x UInt8) ENGINE = Memory +" +${CLICKHOUSE_CLIENT} --query " + SELECT name FROM system.tables + WHERE database = '${db}' AND name IN ('t_direct', 't_parallel_1', 't_parallel_2') + ORDER BY name +" + +${CLICKHOUSE_CLIENT} --query "DROP DATABASE ${db} SYNC" +${CLICKHOUSE_CLIENT} --query "DROP USER ${user}" diff --git a/tests/queries/0_stateless/05218_dictionary_invalidate_query_access_checks.reference b/tests/queries/0_stateless/05218_dictionary_invalidate_query_access_checks.reference new file mode 100644 index 000000000000..97c9358c2bd0 --- /dev/null +++ b/tests/queries/0_stateless/05218_dictionary_invalidate_query_access_checks.reference @@ -0,0 +1,9 @@ +-- a non-SELECT invalidate query of a local source is rejected, so it does not run as internal +1 +-- reloaded after the invalidate query was evaluated +1 +1 +-- nothing was created +0 +-- a SELECT invalidate query keeps working +2 diff --git a/tests/queries/0_stateless/05218_dictionary_invalidate_query_access_checks.sh b/tests/queries/0_stateless/05218_dictionary_invalidate_query_access_checks.sh new file mode 100755 index 000000000000..c3b179a57cbb --- /dev/null +++ b/tests/queries/0_stateless/05218_dictionary_invalidate_query_access_checks.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# The `invalidate_query` of a local `ClickHouse` dictionary source is executed as an `internal` query +# on behalf of the source user. Its text comes from `CREATE DICTIONARY`, so a user who may create a +# dictionary but not a table must not be able to smuggle a `CREATE TABLE` through it: only a `SELECT` +# is accepted, the same way as for the main dictionary query. + +user="user_${CLICKHOUSE_DATABASE}" +db="${CLICKHOUSE_DATABASE}" + +${CLICKHOUSE_CLIENT} --query "DROP USER IF EXISTS ${user}" +${CLICKHOUSE_CLIENT} --query "CREATE USER ${user} IDENTIFIED WITH plaintext_password BY 'password'" +${CLICKHOUSE_CLIENT} --query "GRANT SELECT, CREATE DICTIONARY, DROP DICTIONARY, dictGet ON ${db}.* TO ${user}" + +CLIENT_AS_USER="${CLICKHOUSE_CLIENT} --user ${user} --password password" + +# The test database may be reused across runs, so the objects of this test are dropped both before +# and after the run. +${CLICKHOUSE_CLIENT} --query "DROP DICTIONARY IF EXISTS ${db}.dict_invalidate_ddl, ${db}.dict_invalidate_select" +${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS ${db}.t_smuggled" + +echo "-- a non-SELECT invalidate query of a local source is rejected, so it does not run as internal" +${CLIENT_AS_USER} --query " + CREATE DICTIONARY ${db}.dict_invalidate_ddl (id UInt64, value UInt64) PRIMARY KEY id + SOURCE(CLICKHOUSE(QUERY 'SELECT 1 AS id, 1 AS value' INVALIDATE_QUERY 'CREATE TABLE ${db}.t_smuggled (x UInt8) ENGINE = Memory')) + LAYOUT(FLAT()) LIFETIME(MIN 1 MAX 1) +" +${CLIENT_AS_USER} --query "SELECT dictGet(${db}.dict_invalidate_ddl, 'value', toUInt64(1))" + +# The first load does not run the invalidate query; it is run by the periodic check (every 5 seconds) +# once the lifetime has passed, and it decides whether the dictionary is reloaded. Wait for that +# check to happen: the rejected invalidate query counts as "modified", so the dictionary is reloaded +# and `last_successful_update_time` advances. +first_update=$(${CLICKHOUSE_CLIENT} --query "SELECT toUnixTimestamp(last_successful_update_time) FROM system.dictionaries WHERE database = '${db}' AND name = 'dict_invalidate_ddl'") +for _ in $(seq 1 120) +do + update=$(${CLICKHOUSE_CLIENT} --query "SELECT toUnixTimestamp(last_successful_update_time) FROM system.dictionaries WHERE database = '${db}' AND name = 'dict_invalidate_ddl'") + if [[ "${update}" -gt "${first_update}" ]] + then + break + fi + sleep 0.5 +done +echo "-- reloaded after the invalidate query was evaluated" +${CLICKHOUSE_CLIENT} --query "SELECT toUnixTimestamp(last_successful_update_time) > ${first_update} FROM system.dictionaries WHERE database = '${db}' AND name = 'dict_invalidate_ddl'" +${CLIENT_AS_USER} --query "SELECT dictGet(${db}.dict_invalidate_ddl, 'value', toUInt64(1))" + +echo "-- nothing was created" +${CLICKHOUSE_CLIENT} --query "SELECT count() FROM system.tables WHERE database = '${db}' AND name = 't_smuggled'" + +echo "-- a SELECT invalidate query keeps working" +${CLIENT_AS_USER} --query " + CREATE DICTIONARY ${db}.dict_invalidate_select (id UInt64, value UInt64) PRIMARY KEY id + SOURCE(CLICKHOUSE(QUERY 'SELECT 1 AS id, 2 AS value' INVALIDATE_QUERY 'SELECT 1')) + LAYOUT(FLAT()) LIFETIME(MIN 1 MAX 1) +" +${CLIENT_AS_USER} --query "SELECT dictGet(${db}.dict_invalidate_select, 'value', toUInt64(1))" + +${CLICKHOUSE_CLIENT} --query "DROP DICTIONARY ${db}.dict_invalidate_ddl, ${db}.dict_invalidate_select" +${CLICKHOUSE_CLIENT} --query "DROP USER ${user}" diff --git a/tests/queries/0_stateless/05218_trivial_count_sparsity_filter_after_metadata_only_alter.reference b/tests/queries/0_stateless/05218_trivial_count_sparsity_filter_after_metadata_only_alter.reference new file mode 100644 index 000000000000..c1b0a0d45e86 --- /dev/null +++ b/tests/queries/0_stateless/05218_trivial_count_sparsity_filter_after_metadata_only_alter.reference @@ -0,0 +1,6 @@ +0 +100 +0 +100 +0 +100 diff --git a/tests/queries/0_stateless/05218_trivial_count_sparsity_filter_after_metadata_only_alter.sql b/tests/queries/0_stateless/05218_trivial_count_sparsity_filter_after_metadata_only_alter.sql new file mode 100644 index 000000000000..bce113e7f214 --- /dev/null +++ b/tests/queries/0_stateless/05218_trivial_count_sparsity_filter_after_metadata_only_alter.sql @@ -0,0 +1,20 @@ +-- A metadata-only widening to Nullable does not rewrite the part, so the recorded number of +-- default values still counts zeros (or empty strings), not NULLs. + +CREATE TABLE t (k UInt64, v UInt64, s String) ENGINE = MergeTree ORDER BY tuple() SETTINGS auto_statistics_types = ''; +INSERT INTO t SELECT number, number % 10, if(number % 10 = 0, '', 'x') FROM numbers(100); + +ALTER TABLE t MODIFY COLUMN v Nullable(UInt64) SETTINGS mutations_sync = 2; +ALTER TABLE t MODIFY COLUMN s Nullable(String) SETTINGS mutations_sync = 2; + +SELECT count() FROM t WHERE v IS NULL; +SELECT count() FROM t WHERE v IS NOT NULL; +SELECT count() FROM t WHERE s IS NULL; +SELECT count() FROM t WHERE s IS NOT NULL; + +-- The stats are usable again once the part is rewritten with the new type. +OPTIMIZE TABLE t FINAL; +SELECT count() FROM t WHERE v IS NULL; +SELECT count() FROM t WHERE v IS NOT NULL; + +DROP TABLE t; diff --git a/tests/queries/0_stateless/05227_load_outdated_parts_retryable_error.reference b/tests/queries/0_stateless/05227_load_outdated_parts_retryable_error.reference new file mode 100644 index 000000000000..1d7c2d9d3d3d --- /dev/null +++ b/tests/queries/0_stateless/05227_load_outdated_parts_retryable_error.reference @@ -0,0 +1,4 @@ +3 +3 +0 +3 diff --git a/tests/queries/0_stateless/05227_load_outdated_parts_retryable_error.sql b/tests/queries/0_stateless/05227_load_outdated_parts_retryable_error.sql new file mode 100644 index 000000000000..4ff781a873d4 --- /dev/null +++ b/tests/queries/0_stateless/05227_load_outdated_parts_retryable_error.sql @@ -0,0 +1,33 @@ +-- Tags: no-shared-merge-tree, no-parallel +-- no-shared-merge-tree: SharedMergeTree doesn't load inactive parts to memory after restart +-- no-parallel: SYSTEM ENABLE FAILPOINT is process-wide, and another copy of this test +-- toggling the failpoint would change the number of parts loaded in this one. + +DROP TABLE IF EXISTS t_load_outdated_parts; + +-- Outdated parts must survive DETACH/ATTACH to be loaded in the background after ATTACH. +CREATE TABLE t_load_outdated_parts (x UInt64) ENGINE = MergeTree ORDER BY x SETTINGS old_parts_lifetime = 600; + +INSERT INTO t_load_outdated_parts VALUES (1); +INSERT INTO t_load_outdated_parts VALUES (2); +INSERT INTO t_load_outdated_parts VALUES (3); +OPTIMIZE TABLE t_load_outdated_parts FINAL; + +SELECT count() FROM system.parts WHERE database = currentDatabase() AND table = 't_load_outdated_parts' AND NOT active; + +DETACH TABLE t_load_outdated_parts; + +-- Every attempt to load an outdated part fails with a retryable error. +-- The server must not terminate, the loading must be retried later. +SYSTEM ENABLE FAILPOINT merge_tree_load_outdated_parts_retryable_error; +ATTACH TABLE t_load_outdated_parts; + +SELECT count() FROM t_load_outdated_parts; +SELECT count() FROM system.parts WHERE database = currentDatabase() AND table = 't_load_outdated_parts' AND NOT active; + +SYSTEM DISABLE FAILPOINT merge_tree_load_outdated_parts_retryable_error; +SYSTEM WAIT LOADING PARTS t_load_outdated_parts; + +SELECT count() FROM system.parts WHERE database = currentDatabase() AND table = 't_load_outdated_parts' AND NOT active; + +DROP TABLE t_load_outdated_parts; diff --git a/tests/queries/0_stateless/05227_parameterized_view_own_name_qualifier.reference b/tests/queries/0_stateless/05227_parameterized_view_own_name_qualifier.reference new file mode 100644 index 000000000000..359b94fc535a --- /dev/null +++ b/tests/queries/0_stateless/05227_parameterized_view_own_name_qualifier.reference @@ -0,0 +1,33 @@ +-- columns qualified by the view name (issue case 1) +x +x +x +-- qualifier inside IN (issue case 2) +dev +dev +-- unaliased JOIN operand (issue case 3) +4 4 4 +3 3 3 +2 2 2 +1 1 1 +0 0 0 +5 +5 +5 +-- qualified matcher and matcher-expanded column names +0 0 +c1 c2 k1 pv1.c1 +t1.c1 c2 k1 m1 t2.c1 c3 +t1.c1 t1.c2 pv3.k1 pv3.m1 t2.c1 t2.c3 +0 0 +1 +-- arguments still survive per call (issue 112148 must stay fixed) +1 +1 +-- controls: regular table functions gain nothing +UNKNOWN_IDENTIFIER +ALIAS_REQUIRED +UNKNOWN_IDENTIFIER +ALIAS_REQUIRED +c1 c2 c1 +t1.c1 t1.c2 t2.c1 t2.c3 c1 diff --git a/tests/queries/0_stateless/05227_parameterized_view_own_name_qualifier.sh b/tests/queries/0_stateless/05227_parameterized_view_own_name_qualifier.sh new file mode 100755 index 000000000000..e1f90de28154 --- /dev/null +++ b/tests/queries/0_stateless/05227_parameterized_view_own_name_qualifier.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash + +# A parameterized view call is resolved as a `TableFunctionNode`, but unlike a real table function it has a +# name of its own: its columns bind by that name and it needs no alias in a JOIN. +# https://github.com/ClickHouse/ClickHouse/issues/119837 + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +# The fix is in the analyzer and the test pins it: with enable_analyzer = 0 a dotted `db.view(...)` call fails with UNKNOWN_FUNCTION. +CLIENT="$CLICKHOUSE_CLIENT --enable_analyzer=1 --joined_subquery_requires_alias=1" +DB=$CLICKHOUSE_DATABASE + +$CLIENT --query " +CREATE TABLE t1 (c1 String, c2 Float64) ENGINE = ReplacingMergeTree ORDER BY c1; +INSERT INTO t1 SELECT toString(number), toFloat64(number) FROM numbers(5); +CREATE TABLE t2 (c1 String, c3 UInt8) ENGINE = MergeTree ORDER BY c1; +INSERT INTO t2 SELECT toString(number), number FROM numbers(3); +CREATE VIEW pv1 AS SELECT {p1:String} AS k1, 'x' AS c1; +CREATE VIEW pv2 AS SELECT arrayJoin({p1:Array(Int64)}) AS k1, 'dev' AS c1; +CREATE VIEW pv3 AS SELECT c1 AS k1, sum(c2) AS m1 FROM t1 WHERE length({p1:Array(String)}) = 0 OR c1 IN ({p1:Array(String)}) GROUP BY c1; +" + +echo '-- columns qualified by the view name (issue case 1)' +$CLIENT --query "SELECT pv1.c1 FROM \`$DB.pv1\`(p1 = 'a') LIMIT 1" +$CLIENT --query "SELECT pv1.c1 FROM pv1(p1 = 'a')" +$CLIENT --query "SELECT $DB.pv1.c1 FROM $DB.pv1(p1 = 'a')" + +echo '-- qualifier inside IN (issue case 2)' +$CLIENT --query "SELECT c1 FROM \`$DB.pv2\`(p1 = [157]) WHERE pv2.k1 IN (157) LIMIT 100" +$CLIENT --query "SELECT c1 FROM pv2(p1 = [157]) WHERE $DB.pv2.k1 IN (157)" + +echo '-- unaliased JOIN operand (issue case 3)' +$CLIENT --query "SELECT $DB.t1.c1, $DB.pv3.k1, $DB.pv3.m1 FROM $DB.t1 FINAL INNER JOIN \`$DB.pv3\`(p1 = []) ON $DB.t1.c1 = $DB.pv3.k1 ORDER BY m1 DESC LIMIT 200" +$CLIENT --query "SELECT count() FROM t1 INNER JOIN pv3(p1 = []) ON t1.c1 = pv3.k1" +$CLIENT --query "SELECT count() FROM t1, pv3(p1 = ['1'])" +$CLIENT --query "SELECT count() FROM pv3(p1 = []) INNER JOIN t1 ON t1.c1 = pv3.k1" + +echo '-- qualified matcher and matcher-expanded column names' +$CLIENT --query "SELECT pv3.* FROM t1 INNER JOIN pv3(p1 = []) ON t1.c1 = pv3.k1 ORDER BY k1 LIMIT 1" +# `c1` of the view clashes with `t1.c1`, so the view's copy must be qualified with the view name +$CLIENT --query "SELECT * FROM t1 INNER JOIN pv1(p1 = 'a') ON t1.c1 = pv1.c1 LIMIT 0 SETTINGS analyzer_compatibility_multiple_joins_qualify_column_names = 0 FORMAT TSVWithNames" +$CLIENT --query "SELECT * FROM t1 INNER JOIN pv3(p1 = []) ON t1.c1 = pv3.k1 INNER JOIN t2 ON t1.c1 = t2.c1 LIMIT 0 SETTINGS analyzer_compatibility_multiple_joins_qualify_column_names = 0 FORMAT TSVWithNames" +$CLIENT --query "SELECT * FROM t1 INNER JOIN pv3(p1 = []) ON t1.c1 = pv3.k1 INNER JOIN t2 ON t1.c1 = t2.c1 LIMIT 0 SETTINGS analyzer_compatibility_multiple_joins_qualify_column_names = 1 FORMAT TSVWithNames" +$CLIENT --query "SELECT $DB.pv3.* FROM t1 INNER JOIN pv3(p1 = []) ON t1.c1 = pv3.k1 ORDER BY k1 LIMIT 1" +$CLIENT --query "SELECT count() FROM pv3(p1 = []) INNER JOIN pv3(p1 = ['1']) USING (k1)" + +echo '-- arguments still survive per call (issue 112148 must stay fixed)' +$CLIENT --query "SELECT k1 FROM pv3(p1 = ['1'])" +$CLIENT --query "SELECT count() FROM pv3(p1 = []) AS a INNER JOIN pv3(p1 = ['1']) AS b USING (k1)" + +echo '-- controls: regular table functions gain nothing' +$CLIENT --query "SELECT numbers.number FROM numbers(3)" 2>&1 | grep -o -m1 'UNKNOWN_IDENTIFIER' +$CLIENT --query "SELECT count() FROM t1 INNER JOIN numbers(3) ON 1 = 1" 2>&1 | grep -o -m1 'ALIAS_REQUIRED' +$CLIENT --query "SELECT view.dummy FROM view(SELECT 1 AS dummy)" 2>&1 | grep -o -m1 'UNKNOWN_IDENTIFIER' +$CLIENT --query "SELECT count() FROM t1 INNER JOIN view(SELECT 1 AS dummy) ON 1 = 1" 2>&1 | grep -o -m1 'ALIAS_REQUIRED' +# a clashing column of `view(...)` has no name to be qualified with, in both qualification modes +$CLIENT --query "SELECT * FROM t1, view(SELECT '0' AS c1) LIMIT 0 SETTINGS joined_subquery_requires_alias = 0, analyzer_compatibility_multiple_joins_qualify_column_names = 0 FORMAT TSVWithNames" +$CLIENT --query "SELECT * FROM t1 INNER JOIN t2 ON t1.c1 = t2.c1, view(SELECT '0' AS c1) LIMIT 0 SETTINGS joined_subquery_requires_alias = 0, analyzer_compatibility_multiple_joins_qualify_column_names = 1 FORMAT TSVWithNames" diff --git a/tests/queries/0_stateless/05228_load_outdated_parts_cancel_on_detach.reference b/tests/queries/0_stateless/05228_load_outdated_parts_cancel_on_detach.reference new file mode 100644 index 000000000000..74452ce6831b --- /dev/null +++ b/tests/queries/0_stateless/05228_load_outdated_parts_cancel_on_detach.reference @@ -0,0 +1,3 @@ +DETACH finished +All outdated parts are loaded +100 diff --git a/tests/queries/0_stateless/05228_load_outdated_parts_cancel_on_detach.sh b/tests/queries/0_stateless/05228_load_outdated_parts_cancel_on_detach.sh new file mode 100755 index 000000000000..5f4e63145a3e --- /dev/null +++ b/tests/queries/0_stateless/05228_load_outdated_parts_cancel_on_detach.sh @@ -0,0 +1,76 @@ +#!/usr/bin/env bash +# Tags: no-shared-merge-tree, no-parallel +# no-shared-merge-tree: SharedMergeTree doesn't load inactive parts to memory after restart +# no-parallel: SYSTEM ENABLE FAILPOINT is process-wide, the pause failpoint would block the +# loading of outdated parts of the tables of other tests. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +function cleanup() +{ + $CLICKHOUSE_CLIENT --query "SYSTEM DISABLE FAILPOINT merge_tree_load_outdated_parts_pause" 2>/dev/null + $CLICKHOUSE_CLIENT --query "SYSTEM DISABLE FAILPOINT merge_tree_load_outdated_parts_retryable_error" 2>/dev/null +} +trap cleanup EXIT + +# The loading of outdated parts is cancelled by DETACH TABLE while the loading workers are in flight, +# and then the workers fail with a retryable error. The DETACH must not hang: the workers must not +# take the mutex the cancellation branch waits under, and the failed parts must be requeued only +# after all the workers finished. +# +# The number of parts is greater than the size of the loading thread pool plus its queue +# (`max_outdated_parts_loading_thread_pool_size`, 32 each), so the dispatcher still has parts +# to hand out when the cancellation arrives and the cancellation branch is really exercised. +$CLICKHOUSE_CLIENT --query " + DROP TABLE IF EXISTS t_load_outdated_parts_cancel; + CREATE TABLE t_load_outdated_parts_cancel (x UInt64) ENGINE = MergeTree ORDER BY x SETTINGS old_parts_lifetime = 600; + SYSTEM STOP MERGES t_load_outdated_parts_cancel; + INSERT INTO t_load_outdated_parts_cancel SELECT number FROM numbers(100) + SETTINGS max_block_size = 1, min_insert_block_size_rows = 1, min_insert_block_size_bytes = 1, max_insert_threads = 1; + SYSTEM START MERGES t_load_outdated_parts_cancel; + OPTIMIZE TABLE t_load_outdated_parts_cancel FINAL; +" + +outdated_before=$($CLICKHOUSE_CLIENT --query "SELECT count() FROM system.parts WHERE database = currentDatabase() AND table = 't_load_outdated_parts_cancel' AND NOT active") +[ "$outdated_before" -ge 100 ] || echo "Unexpected number of outdated parts: $outdated_before" + +$CLICKHOUSE_CLIENT --query " + DETACH TABLE t_load_outdated_parts_cancel; + SYSTEM ENABLE FAILPOINT merge_tree_load_outdated_parts_pause; + SYSTEM ENABLE FAILPOINT merge_tree_load_outdated_parts_retryable_error; + ATTACH TABLE t_load_outdated_parts_cancel; + SYSTEM WAIT FAILPOINT merge_tree_load_outdated_parts_pause PAUSE; +" + +# Cancel the loading while the workers are paused. DETACH waits for the loading task to stop. +$CLICKHOUSE_CLIENT --query "DETACH TABLE t_load_outdated_parts_cancel" & +detach_pid=$! + +for _ in $(seq 1 300); do + detach_running=$($CLICKHOUSE_CLIENT --query "SELECT count() FROM system.processes WHERE current_database = currentDatabase() AND query LIKE 'DETACH TABLE t_load_outdated_parts_cancel%'") + [ "$detach_running" = "1" ] && break + sleep 0.1 +done +# Give the DETACH a moment to reach the loading task and cancel it. +sleep 0.5 + +# Let the workers fail with the retryable error. +$CLICKHOUSE_CLIENT --query "SYSTEM DISABLE FAILPOINT merge_tree_load_outdated_parts_pause" + +wait $detach_pid +echo "DETACH finished" + +# Without the failpoints, all the outdated parts are loaded after ATTACH. +$CLICKHOUSE_CLIENT --query " + SYSTEM DISABLE FAILPOINT merge_tree_load_outdated_parts_retryable_error; + ATTACH TABLE t_load_outdated_parts_cancel; + SYSTEM WAIT LOADING PARTS t_load_outdated_parts_cancel; +" + +outdated_after=$($CLICKHOUSE_CLIENT --query "SELECT count() FROM system.parts WHERE database = currentDatabase() AND table = 't_load_outdated_parts_cancel' AND NOT active") +[ "$outdated_after" = "$outdated_before" ] && echo "All outdated parts are loaded" || echo "Outdated parts before: $outdated_before, after: $outdated_after" + +$CLICKHOUSE_CLIENT --query "SELECT count() FROM t_load_outdated_parts_cancel" +$CLICKHOUSE_CLIENT --query "DROP TABLE t_load_outdated_parts_cancel" diff --git a/tests/queries/0_stateless/05229_cast_array_of_map_to_variant.reference b/tests/queries/0_stateless/05229_cast_array_of_map_to_variant.reference new file mode 100644 index 000000000000..b4dbd49ecd01 --- /dev/null +++ b/tests/queries/0_stateless/05229_cast_array_of_map_to_variant.reference @@ -0,0 +1,7 @@ +[{'a':'x'}] +[{'k':['v']}] +[{'a':'x'}] +[[{'a':'x'}]] +[{'a':'x'}] +['{"a":"x"}'] +['{\'a\':\'x\'}'] diff --git a/tests/queries/0_stateless/05229_cast_array_of_map_to_variant.sql b/tests/queries/0_stateless/05229_cast_array_of_map_to_variant.sql new file mode 100644 index 000000000000..e472cff5b8a9 --- /dev/null +++ b/tests/queries/0_stateless/05229_cast_array_of_map_to_variant.sql @@ -0,0 +1,21 @@ +-- A `Map(K, V)` is physically an `Array(Tuple(K, V))` and `CAST` converts between the two spellings, +-- so a `Map` in one of the types may or may not be spelled out as an array by the other one. +-- Counting a `Map` as one dimension unconditionally made the nesting check of `CAST AS Array` reject +-- an `Array(Map(...))` whose target element type keeps the `Map` behind a `Variant`, a `Dynamic`, +-- a `JSON` or a `String`. + +SELECT CAST([map('a', 'x')], 'Array(Variant(String, Map(String, String)))'); +SELECT CAST([map('k', ['v'])], 'Array(Variant(String, Map(String, Array(String))))'); +SELECT CAST(materialize([map('a', 'x')]), 'Array(Variant(String, Map(String, String)))'); +SELECT CAST([[map('a', 'x')]], 'Array(Array(Variant(String, Map(String, String))))'); +SELECT CAST([map('a', 'x')], 'Array(Dynamic)'); +SELECT CAST([map('a', 'x')], 'Array(JSON)'); +SELECT CAST([map('a', 'x')], 'Array(String)'); + +-- What cannot be converted is reported by the element wrapper, which names the offending types. +SELECT CAST([map('a', 'x')], 'Array(Variant(String, UInt8))'); -- { serverError CANNOT_CONVERT_TYPE } +SELECT CAST([map('a', 'x')], 'Array(Tuple(String, String))'); -- { serverError TYPE_MISMATCH } + +-- A genuine nesting mismatch is still rejected. +SELECT CAST(['v'], 'Array(Array(String))'); -- { serverError TYPE_MISMATCH } +SELECT CAST([['v']], 'Array(String)'); -- { serverError TYPE_MISMATCH } diff --git a/tests/queries/0_stateless/05229_column_named_from_in_parenthesized_expression.reference b/tests/queries/0_stateless/05229_column_named_from_in_parenthesized_expression.reference new file mode 100644 index 000000000000..4cd327b6fa46 --- /dev/null +++ b/tests/queries/0_stateless/05229_column_named_from_in_parenthesized_expression.reference @@ -0,0 +1,31 @@ +1 +1 +1 +2 +1 +1 +2 +1 +2 +1 +1 a +2 b +1 +0 +1 +1 +1 +1 +3 +2 +1 +1 +1 +(1,2) +1 +SELECT count()\nFROM t\nWHERE (`from` IN (1)) +0 +0 +1 +3 +1 diff --git a/tests/queries/0_stateless/05229_column_named_from_in_parenthesized_expression.sql b/tests/queries/0_stateless/05229_column_named_from_in_parenthesized_expression.sql new file mode 100644 index 000000000000..6f9a4860cede --- /dev/null +++ b/tests/queries/0_stateless/05229_column_named_from_in_parenthesized_expression.sql @@ -0,0 +1,44 @@ +-- A column named `from` is an unquoted keyword identifier, which is allowed. Inside parentheses it is +-- ambiguous with a subquery in the FROM-first form (`(FROM t)` means `(SELECT * FROM t)`); the column +-- reading wins. + +DROP TABLE IF EXISTS t_column_named_from; +CREATE TABLE t_column_named_from (`from` Nullable(String), c1 UInt8) ENGINE = MergeTree ORDER BY c1; +INSERT INTO t_column_named_from VALUES ('a', 1), ('b', 2), (NULL, 3); + +SELECT count() FROM t_column_named_from WHERE (from IN ('a')); +SELECT count() FROM t_column_named_from WHERE (from NOT IN ('a')); +SELECT count() FROM t_column_named_from WHERE (from IS NULL); +SELECT count() FROM t_column_named_from WHERE (from IS NOT NULL); +SELECT count() FROM t_column_named_from WHERE (from LIKE 'a%'); +SELECT count() FROM t_column_named_from WHERE (from = 'a'); +SELECT count() FROM t_column_named_from WHERE (from BETWEEN 'a' AND 'b'); +SELECT count() FROM t_column_named_from WHERE ((from IS NOT NULL) AND (from != 'b')); +SELECT count() FROM t_column_named_from WHERE (from IS NULL) OR (from = 'a'); +SELECT count() FROM t_column_named_from WHERE ((((from IN ('a'))))); +SELECT c1, from FROM t_column_named_from WHERE (from IS NOT NULL) ORDER BY c1; + +DROP TABLE t_column_named_from; + +-- The same ambiguity without a table. +SELECT (from IN (1)) FROM (SELECT 1 AS `from`); +SELECT (from IS NULL) FROM (SELECT 1 AS `from`); +SELECT (from IS NOT NULL) FROM (SELECT 1 AS `from`); +SELECT (from BETWEEN 0 AND 2) FROM (SELECT 1 AS `from`); +SELECT (from AND 1) FROM (SELECT 1 AS `from`); +SELECT (from OR 0) FROM (SELECT 1 AS `from`); +SELECT (from IS NULL ? 2 : 3) FROM (SELECT 1 AS `from`); +SELECT (from + 1) FROM (SELECT 1 AS `from`); +SELECT (from::String) FROM (SELECT 1 AS `from`); +SELECT (from.1) FROM (SELECT (1, 2) AS `from`); +SELECT (from[1]) FROM (SELECT [1, 2] AS `from`); +SELECT (from IN (1), from + 1) FROM (SELECT 1 AS `from`); +SELECT (from IN (1) AS in_a) FROM (SELECT 1 AS `from`); +SELECT formatQuery('SELECT count() FROM t WHERE (from IN (1))'); + +-- A subquery in the FROM-first form keeps its reading in the same positions. +SELECT 1 IN (FROM system.one); +SELECT * FROM (FROM system.one); +SELECT 1 IN (FROM (SELECT 1)); +SELECT (FROM numbers(3) |> SELECT count()); +SELECT (FROM system.one |> SELECT 1); diff --git a/tests/queries/0_stateless/05229_text_index_local_postings_cache_no_rebuilds.reference b/tests/queries/0_stateless/05229_text_index_local_postings_cache_no_rebuilds.reference new file mode 100644 index 000000000000..363198023a73 --- /dev/null +++ b/tests/queries/0_stateless/05229_text_index_local_postings_cache_no_rebuilds.reference @@ -0,0 +1,3 @@ +parts 1 +16384 +4096 1 diff --git a/tests/queries/0_stateless/05229_text_index_local_postings_cache_no_rebuilds.sql b/tests/queries/0_stateless/05229_text_index_local_postings_cache_no_rebuilds.sql new file mode 100644 index 000000000000..bca6fc376403 --- /dev/null +++ b/tests/queries/0_stateless/05229_text_index_local_postings_cache_no_rebuilds.sql @@ -0,0 +1,74 @@ +-- Tags: no-parallel-replicas +-- no-parallel-replicas: parallel replicas split the part between replicas, each with its own query-local +-- cache, so the segments at the replica boundaries are built more than once and the count below is not +-- deterministic. + +-- The postings cache that a query creates for itself when `use_text_index_postings_cache = 0` is sized as 10% +-- of `max_memory_usage`. The cache used to be SLRU with the protected queue as large as the whole cache: a +-- segment that was prepared twice was promoted to the protected queue and stayed there for good, and once the +-- promoted entries filled the cache every later insert was evicted right after it landed in the probationary +-- queue, so every later segment was read from disk again on its next preparation. With plain LRU a segment is +-- built exactly once even when the segments of the query do not fit into the cache. +-- +-- The query below prepares almost every segment twice regardless of how the read pool splits the part: the two +-- `hasAllTokens` are two independent search queries, the reader keeps a separate posting list cursor per token +-- per search query, and the 31 tokens the two share are walked twice over the same rows. `segments_reused` +-- asserts that the second walk really goes through the cache instead of the disk. + +SET enable_full_text_index = 1; +SET log_queries = 1; +-- A single part: the segment count below assumes that every token has exactly 16384 postings in one part. +SET max_insert_threads = 1; + +DROP TABLE IF EXISTS tab_local_postings_cache; + +-- Every row has all 32 tokens, so each token has 16384 / 128 = 128 segments of 128 rows. The 4096 segments +-- weigh about 4.8 MB in the cache, while `max_memory_usage = 30 MB` gives the query-local cache 3 MB: enough +-- for the segments of the granule being read, but not for all of them, so a policy that pins entries runs out +-- of space halfway through the part. +CREATE TABLE tab_local_postings_cache +( + k UInt64, + s String, + INDEX idx s TYPE text(tokenizer = 'splitByNonAlpha', posting_list_codec = 'bitpacking', posting_list_block_size = 128) +) +ENGINE = MergeTree ORDER BY k +SETTINGS index_granularity = 32, index_granularity_bytes = '10M', min_bytes_for_wide_part = 0; + +INSERT INTO tab_local_postings_cache +SELECT number, arrayStringConcat(arrayMap(i -> 'tok' || toString(i), range(32)), ' ') +FROM numbers(16384); + +SELECT 'parts', count() FROM system.parts WHERE database = currentDatabase() AND table = 'tab_local_postings_cache' AND active; + +SELECT count() FROM tab_local_postings_cache +WHERE hasAllTokens(s, ['tok0', 'tok1', 'tok2', 'tok3', 'tok4', 'tok5', 'tok6', 'tok7', 'tok8', 'tok9', 'tok10', 'tok11', 'tok12', 'tok13', 'tok14', 'tok15', + 'tok16', 'tok17', 'tok18', 'tok19', 'tok20', 'tok21', 'tok22', 'tok23', 'tok24', 'tok25', 'tok26', 'tok27', 'tok28', 'tok29', 'tok30', 'tok31']) + AND hasAllTokens(s, ['tok1', 'tok2', 'tok3', 'tok4', 'tok5', 'tok6', 'tok7', 'tok8', 'tok9', 'tok10', 'tok11', 'tok12', 'tok13', 'tok14', 'tok15', + 'tok16', 'tok17', 'tok18', 'tok19', 'tok20', 'tok21', 'tok22', 'tok23', 'tok24', 'tok25', 'tok26', 'tok27', 'tok28', 'tok29', 'tok30', 'tok31']) +SETTINGS + text_index_posting_list_apply_mode = 'lazy', + query_plan_direct_read_from_text_index = 1, + use_skip_indexes_on_data_read = 1, + query_plan_optimize_count_from_text_index = 0, + use_query_condition_cache = 0, + use_text_index_postings_cache = 0, + merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 0, + max_threads = 1, + max_memory_usage = 30000000, + log_comment = '05229_local_postings_cache'; + +SYSTEM FLUSH LOGS query_log; + +-- Every segment was read and decoded exactly once (32 tokens * 128 segments), and the repeated preparations +-- of the 31 shared tokens were served from the query-local cache. +SELECT + ProfileEvents['TextIndexLazySegmentsBuilt'] AS segments_built, + ProfileEvents['TextIndexLazySegmentsPrepared'] > ProfileEvents['TextIndexLazySegmentsBuilt'] AS segments_reused +FROM system.query_log +WHERE event_date >= yesterday() AND event_time >= now() - 600 + AND current_database = currentDatabase() + AND type = 'QueryFinish' + AND log_comment = '05229_local_postings_cache'; + +DROP TABLE tab_local_postings_cache;