From 15e4092e90620b604b14105a0162b3fae1f2f3ae Mon Sep 17 00:00:00 2001 From: gargsaumya Date: Mon, 21 Sep 2026 11:56:17 +0530 Subject: [PATCH 01/31] FIX: Validate caller-controlled native buffer sizes --- mssql_python/cursor.py | 59 +++++++++--- mssql_python/pybind/ddbc_bindings.cpp | 126 +++++++++++++++++++++----- tests/test_004_cursor.py | 28 ++++++ tests/test_024_bulkcopy_arrow.py | 5 + 4 files changed, 181 insertions(+), 37 deletions(-) diff --git a/mssql_python/cursor.py b/mssql_python/cursor.py index db45462c8..b364f9091 100644 --- a/mssql_python/cursor.py +++ b/mssql_python/cursor.py @@ -51,6 +51,7 @@ SMALLMONEY_MAX: decimal.Decimal = decimal.Decimal("214748.3647") MONEY_MIN: decimal.Decimal = decimal.Decimal("-922337203685477.5808") MONEY_MAX: decimal.Decimal = decimal.Decimal("922337203685477.5807") +MAX_NATIVE_ROW_COUNT: int = 2_147_483_647 # SQL BIGINT is a signed 64-bit integer. Ints outside this range have no BIGINT # encoding and must be rejected at detect time on both paths (see _map_sql_type). BIGINT_MIN: int = -(2**63) @@ -385,9 +386,7 @@ def __init__(self, connection: "Connection", timeout: int = 0) -> None: ] ] = None self.rowcount: int = -1 - self.arraysize: int = ( - 1 # Default number of rows to fetch at a time is 1, user can change it - ) + self.arraysize = 1 self.buffer_length: int = 1024 # Default buffer length for string data self._result_set_empty: bool = False # Add this initialization self.last_executed_stmt: str = "" # Stores the last statement executed by this cursor @@ -1136,6 +1135,25 @@ def _ensure_pyarrow(self) -> Any: "pyarrow is required for Arrow fetch methods. Please install pyarrow." ) from e + @staticmethod + def _validate_native_row_count(value: int, name: str, allow_zero: bool) -> int: + minimum = 0 if allow_zero else 1 + if isinstance(value, bool) or not isinstance(value, int): + raise TypeError(f"{name} must be an integer") + if value < minimum or value > MAX_NATIVE_ROW_COUNT: + raise ValueError( + f"{name} must be between {minimum} and {MAX_NATIVE_ROW_COUNT}, got {value}" + ) + return value + + @property + def arraysize(self) -> int: + return self._arraysize + + @arraysize.setter + def arraysize(self, value: int) -> None: + self._arraysize = self._validate_native_row_count(value, "arraysize", allow_zero=False) + def setinputsizes(self, sizes: List[Union[int, tuple]]) -> None: """ Sets the type information to be used for parameters in execute and executemany. @@ -1188,7 +1206,11 @@ def setinputsizes(self, sizes: List[Union[int, tuple]]) -> None: sql_type, column_size, decimal_digits = size_info # Validate SQL type - if not isinstance(sql_type, int) or sql_type not in valid_sql_types: + if ( + isinstance(sql_type, bool) + or not isinstance(sql_type, int) + or sql_type not in valid_sql_types + ): raise ValueError( f"Invalid SQL type: {sql_type}. Must be a valid SQL type constant." ) @@ -1196,12 +1218,20 @@ def setinputsizes(self, sizes: List[Union[int, tuple]]) -> None: sql_type = ODBC3_TEMPORAL_SQL_TYPES.get(sql_type, sql_type) # Validate size and precision - if not isinstance(column_size, int) or column_size < 0: + if ( + isinstance(column_size, bool) + or not isinstance(column_size, int) + or column_size < 0 + ): raise ValueError( f"Invalid column size: {column_size}. Must be a non-negative integer." ) - if not isinstance(decimal_digits, int) or decimal_digits < 0: + if ( + isinstance(decimal_digits, bool) + or not isinstance(decimal_digits, int) + or decimal_digits < 0 + ): raise ValueError( f"Invalid decimal digits: {decimal_digits}. " f"Must be a non-negative integer." @@ -1220,7 +1250,11 @@ def setinputsizes(self, sizes: List[Union[int, tuple]]) -> None: sql_type = size_info # Validate SQL type - if not isinstance(sql_type, int) or sql_type not in valid_sql_types: + if ( + isinstance(sql_type, bool) + or not isinstance(sql_type, int) + or sql_type not in valid_sql_types + ): raise ValueError( f"Invalid SQL type: {sql_type}. Must be a valid SQL type constant." ) @@ -2852,8 +2886,10 @@ def fetchmany(self, size: Optional[int] = None) -> List[Row]: if size is None: size = self.arraysize + else: + size = self._validate_native_row_count(size, "size", allow_zero=True) - if size <= 0: + if size == 0: return [] char_decoding = self._get_decoding_settings(ddbc_sql_const.SQL_CHAR.value) @@ -2989,6 +3025,7 @@ def arrow_batch(self, batch_size: int = 8192) -> "pyarrow.RecordBatch": """ self._check_closed() # Check if the cursor is closed pyarrow = self._ensure_pyarrow() + batch_size = self._validate_native_row_count(batch_size, "batch_size", allow_zero=True) if not self._has_result_set and self.description: self._reset_rownumber() @@ -2996,9 +3033,7 @@ def arrow_batch(self, batch_size: int = 8192) -> "pyarrow.RecordBatch": capsules = [] char_decoding = self._get_decoding_settings(ddbc_sql_const.SQL_CHAR.value) char_c_type = char_decoding.get("ctype", ddbc_sql_const.SQL_WCHAR.value) - ret = ddbc_bindings.DDBCSQLFetchArrowBatch( - self.hstmt, capsules, max(batch_size, 0), char_c_type - ) + ret = ddbc_bindings.DDBCSQLFetchArrowBatch(self.hstmt, capsules, batch_size, char_c_type) check_error(ddbc_sql_const.SQL_HANDLE_STMT.value, self.hstmt, ret) batch = pyarrow.RecordBatch._import_from_c_capsule(*capsules) @@ -3383,7 +3418,7 @@ def _bulkcopy_core_and_validate(table_name, batch_size, timeout): logger.error("bulkcopy: Invalid table_name parameter") raise ValueError("table_name must be a non-empty string") - if not isinstance(batch_size, int): + if not isinstance(batch_size, int) or isinstance(batch_size, bool): raise TypeError( f"batch_size must be a non-negative integer, got {type(batch_size).__name__}" ) diff --git a/mssql_python/pybind/ddbc_bindings.cpp b/mssql_python/pybind/ddbc_bindings.cpp index 1f235f829..8f16d46b4 100644 --- a/mssql_python/pybind/ddbc_bindings.cpp +++ b/mssql_python/pybind/ddbc_bindings.cpp @@ -21,6 +21,7 @@ #include // For std::memcpy #include #include +#include #include // std::forward #include // CPython datetime API (PyDateTime_IMPORT, PyDateTime_GET_*, etc.) @@ -322,12 +323,37 @@ ParamType* AllocateParamBuffer(std::vector>& paramBuffers, template ParamType* AllocateParamBufferArray(std::vector>& paramBuffers, size_t count) { + if (count > std::numeric_limits::max() / sizeof(ParamType)) { + ThrowStdException("Parameter buffer size is too large"); + } std::shared_ptr buffer(new ParamType[count], std::default_delete()); ParamType* raw = buffer.get(); paramBuffers.push_back(buffer); return raw; } +size_t CheckedAddSize(size_t left, size_t right, const char* errorMessage) { + if (left > std::numeric_limits::max() - right) { + ThrowStdException(errorMessage); + } + return left + right; +} + +size_t CheckedMultiplySize(size_t left, size_t right, const char* errorMessage) { + if (left != 0 && right > std::numeric_limits::max() / left) { + ThrowStdException(errorMessage); + } + return left * right; +} + +template +std::unique_ptr AllocateUniqueArray(size_t count, const char* errorMessage) { + if (count > std::numeric_limits::max() / sizeof(ElementType)) { + ThrowStdException(errorMessage); + } + return std::make_unique(count); +} + std::string DescribeChar(unsigned char ch) { if (ch >= 32 && ch <= 126) { return std::string("'") + static_cast(ch) + "'"; @@ -460,6 +486,10 @@ SQLRETURN BindParameters(SqlHandle& handle, SQLHANDLE hStmt, const py::list& par "with %zu parameters", (void*)hStmt, params.size()); + if (params.size() != paramInfos.size()) { + ThrowStdException("Parameter count does not match parameter metadata count"); + } + // GH-627: resolve unknown NULL param SQL types before binding any param. PreResolveUnknownNullTypes(handle, hStmt, paramInfos, ¶ms); for (int paramIndex = 0; paramIndex < params.size(); paramIndex++) { @@ -537,8 +567,7 @@ SQLRETURN BindParameters(SqlHandle& handle, SQLHANDLE hStmt, const py::list& par break; } case SQL_C_BINARY: { - if (!py::isinstance(param) && !py::isinstance(param) && - !py::isinstance(param)) { + if (!py::isinstance(param) && !py::isinstance(param)) { ThrowStdException(MakeParamMismatchErrorStr(paramInfo.paramCType, paramIndex)); } if (paramInfo.isDAE) { @@ -2194,6 +2223,9 @@ SQLRETURN BindParameterArray(SqlHandle& handle, SQLHANDLE hStmt, const py::list& LOG("BindParameterArray: Starting column-wise array binding - " "param_count=%zu, param_set_size=%zu", columnwise_params.size(), paramSetSize); + if (columnwise_params.size() != paramInfos.size()) { + ThrowStdException("Parameter count does not match parameter metadata count"); + } std::vector> tempBuffers; @@ -2261,8 +2293,18 @@ SQLRETURN BindParameterArray(SqlHandle& handle, SQLHANDLE hStmt, const py::list& LOG("BindParameterArray: Binding SQL_C_WCHAR array - " "param_index=%d, count=%zu, column_size=%zu", paramIndex, paramSetSize, info.columnSize); + const size_t elementWidth = CheckedAddSize( + info.columnSize, 1, "Wide-character parameter size is too large"); + const size_t bufferBytes = CheckedMultiplySize( + elementWidth, sizeof(SQLWCHAR), + "Wide-character parameter length is too large"); + if (bufferBytes > static_cast(std::numeric_limits::max())) { + ThrowStdException("Wide-character parameter length is too large"); + } SQLWCHAR* wcharArray = AllocateParamBufferArray( - tempBuffers, paramSetSize * (info.columnSize + 1)); + tempBuffers, + CheckedMultiplySize(paramSetSize, elementWidth, + "Wide-character parameter buffer is too large")); strLenOrIndArray = AllocateParamBufferArray(tempBuffers, paramSetSize); for (size_t i = 0; i < paramSetSize; ++i) { if (columnValues[i].is_none()) { @@ -2287,7 +2329,7 @@ SQLRETURN BindParameterArray(SqlHandle& handle, SQLHANDLE hStmt, const py::list& "param_index=%d", paramIndex); dataPtr = wcharArray; - bufferLength = (info.columnSize + 1) * sizeof(SQLWCHAR); + bufferLength = static_cast(bufferBytes); break; } case SQL_C_TINYINT: @@ -2358,8 +2400,15 @@ SQLRETURN BindParameterArray(SqlHandle& handle, SQLHANDLE hStmt, const py::list& LOG("BindParameterArray: Binding SQL_C_CHAR/BINARY array - " "param_index=%d, count=%zu, column_size=%zu, encoding='%s'", paramIndex, paramSetSize, info.columnSize, charEncoding.c_str()); + const size_t elementWidth = CheckedAddSize( + info.columnSize, 1, "Character parameter size is too large"); + if (elementWidth > static_cast(std::numeric_limits::max())) { + ThrowStdException("Character parameter length is too large"); + } char* charArray = AllocateParamBufferArray( - tempBuffers, paramSetSize * (info.columnSize + 1)); + tempBuffers, + CheckedMultiplySize(paramSetSize, elementWidth, + "Character parameter buffer is too large")); strLenOrIndArray = AllocateParamBufferArray(tempBuffers, paramSetSize); for (size_t i = 0; i < paramSetSize; ++i) { if (columnValues[i].is_none()) { @@ -2412,7 +2461,7 @@ SQLRETURN BindParameterArray(SqlHandle& handle, SQLHANDLE hStmt, const py::list& "param_index=%d", paramIndex); dataPtr = charArray; - bufferLength = info.columnSize + 1; + bufferLength = static_cast(elementWidth); break; } case SQL_C_BIT: { @@ -3177,6 +3226,9 @@ py::object FetchLobColumnData(SQLHSTMT hStmt, SQLUSMALLINT colIndex, SQLSMALLINT return py::str(""); } if (isWideChar) { + if (buffer.size() % sizeof(SQLWCHAR) != 0) { + ThrowStdException("Wide-character LOB data has an invalid byte length"); + } size_t wcharCount = buffer.size() / sizeof(SQLWCHAR); std::vector alignedBuf(wcharCount); std::memcpy(alignedBuf.data(), buffer.data(), buffer.size()); @@ -4836,6 +4888,15 @@ SQLRETURN FetchArrowBatch_wrap(SqlHandlePtr StatementHandle, py::list& capsules, int arrowBatchSize, int charCtype) { PERF_TIMER("FetchArrowBatch_wrap"); + if (arrowBatchSize < 0) { + ThrowStdException("Arrow batch size must be non-negative"); + } + const size_t batchSize = static_cast(arrowBatchSize); + const size_t offsetCount = CheckedAddSize(batchSize, 1, "Arrow batch size is too large"); + const size_t initialVarDataSize = + CheckedMultiplySize(batchSize, 42, "Arrow batch size is too large"); + const size_t bitmapSize = + CheckedAddSize(batchSize, 7, "Arrow batch size is too large") / 8; // Fetch narrow char data as SQL_C_CHAR if on Linux/macOS and configured by the user charCtype = EffectiveCharCtypeForFetch(charCtype, "utf-8"); @@ -4908,8 +4969,9 @@ SQLRETURN FetchArrowBatch_wrap(SqlHandlePtr StatementHandle, py::list& capsules, case SQL_WLONGVARCHAR: case SQL_GUID: format = "U"; - arrowColumnProducer->varVal = std::make_unique(arrowBatchSize + 1); - arrowColumnProducer->varData.resize(arrowBatchSize * 42); + arrowColumnProducer->varVal = + AllocateUniqueArray(offsetCount, "Arrow offset buffer is too large"); + arrowColumnProducer->varData.resize(initialVarDataSize); columnVarLen[i] = true; // start at offset 0 arrowColumnProducer->varVal[0] = 0; @@ -4920,8 +4982,9 @@ SQLRETURN FetchArrowBatch_wrap(SqlHandlePtr StatementHandle, py::list& capsules, case SQL_VARBINARY: case SQL_LONGVARBINARY: format = "Z"; - arrowColumnProducer->varVal = std::make_unique(arrowBatchSize + 1); - arrowColumnProducer->varData.resize(arrowBatchSize * 42); + arrowColumnProducer->varVal = + AllocateUniqueArray(offsetCount, "Arrow offset buffer is too large"); + arrowColumnProducer->varData.resize(initialVarDataSize); columnVarLen[i] = true; // start at offset 0 arrowColumnProducer->varVal[0] = 0; @@ -4929,33 +4992,39 @@ SQLRETURN FetchArrowBatch_wrap(SqlHandlePtr StatementHandle, py::list& capsules, break; case SQL_TINYINT: format = "C"; - arrowColumnProducer->uint8Val = std::make_unique(arrowBatchSize); + arrowColumnProducer->uint8Val = + AllocateUniqueArray(batchSize, "Arrow value buffer is too large"); arrowColumnProducer->ptrValueBuffer = arrowColumnProducer->uint8Val.get(); break; case SQL_SMALLINT: format = "s"; - arrowColumnProducer->int16Val = std::make_unique(arrowBatchSize); + arrowColumnProducer->int16Val = + AllocateUniqueArray(batchSize, "Arrow value buffer is too large"); arrowColumnProducer->ptrValueBuffer = arrowColumnProducer->int16Val.get(); break; case SQL_INTEGER: format = "i"; - arrowColumnProducer->int32Val = std::make_unique(arrowBatchSize); + arrowColumnProducer->int32Val = + AllocateUniqueArray(batchSize, "Arrow value buffer is too large"); arrowColumnProducer->ptrValueBuffer = arrowColumnProducer->int32Val.get(); break; case SQL_BIGINT: format = "l"; - arrowColumnProducer->int64Val = std::make_unique(arrowBatchSize); + arrowColumnProducer->int64Val = + AllocateUniqueArray(batchSize, "Arrow value buffer is too large"); arrowColumnProducer->ptrValueBuffer = arrowColumnProducer->int64Val.get(); break; case SQL_REAL: format = "f"; - arrowColumnProducer->float32Val = std::make_unique(arrowBatchSize); + arrowColumnProducer->float32Val = + AllocateUniqueArray(batchSize, "Arrow value buffer is too large"); arrowColumnProducer->ptrValueBuffer = arrowColumnProducer->float32Val.get(); break; case SQL_FLOAT: case SQL_DOUBLE: format = "g"; - arrowColumnProducer->float64Val = std::make_unique(arrowBatchSize); + arrowColumnProducer->float64Val = + AllocateUniqueArray(batchSize, "Arrow value buffer is too large"); arrowColumnProducer->ptrValueBuffer = arrowColumnProducer->float64Val.get(); break; case SQL_DECIMAL: @@ -4968,7 +5037,8 @@ SQLRETURN FetchArrowBatch_wrap(SqlHandlePtr StatementHandle, py::list& capsules, arrowSchemaPrivateData[i]->format = std::make_unique(formatLen); std::memcpy(arrowSchemaPrivateData[i]->format.get(), formatStr.c_str(), formatLen); format = arrowSchemaPrivateData[i]->format.get(); - arrowColumnProducer->decimalVal = std::make_unique(arrowBatchSize); + arrowColumnProducer->decimalVal = + AllocateUniqueArray(batchSize, "Arrow value buffer is too large"); arrowColumnProducer->ptrValueBuffer = arrowColumnProducer->decimalVal.get(); break; } @@ -4976,28 +5046,33 @@ SQLRETURN FetchArrowBatch_wrap(SqlHandlePtr StatementHandle, py::list& capsules, case SQL_TYPE_TIMESTAMP: case SQL_DATETIME: format = "tsu:"; - arrowColumnProducer->tsMicroVal = std::make_unique(arrowBatchSize); + arrowColumnProducer->tsMicroVal = + AllocateUniqueArray(batchSize, "Arrow value buffer is too large"); arrowColumnProducer->ptrValueBuffer = arrowColumnProducer->tsMicroVal.get(); break; case SQL_SS_TIMESTAMPOFFSET: format = "tsu:+00:00"; - arrowColumnProducer->tsMicroVal = std::make_unique(arrowBatchSize); + arrowColumnProducer->tsMicroVal = + AllocateUniqueArray(batchSize, "Arrow value buffer is too large"); arrowColumnProducer->ptrValueBuffer = arrowColumnProducer->tsMicroVal.get(); break; case SQL_TYPE_DATE: format = "tdD"; - arrowColumnProducer->dateVal = std::make_unique(arrowBatchSize); + arrowColumnProducer->dateVal = + AllocateUniqueArray(batchSize, "Arrow value buffer is too large"); arrowColumnProducer->ptrValueBuffer = arrowColumnProducer->dateVal.get(); break; case SQL_SS_TIME2: format = "ttn"; - arrowColumnProducer->timeNanoVal = std::make_unique(arrowBatchSize); + arrowColumnProducer->timeNanoVal = + AllocateUniqueArray(batchSize, "Arrow value buffer is too large"); arrowColumnProducer->ptrValueBuffer = arrowColumnProducer->timeNanoVal.get(); break; case SQL_BIT: format = "b"; - arrowColumnProducer->bitVal = std::make_unique((arrowBatchSize + 7) / 8); - std::memset(arrowColumnProducer->bitVal.get(), 0, (arrowBatchSize + 7) / 8); + arrowColumnProducer->bitVal = + AllocateUniqueArray(bitmapSize, "Arrow bitmap is too large"); + std::memset(arrowColumnProducer->bitVal.get(), 0, bitmapSize); arrowColumnProducer->ptrValueBuffer = arrowColumnProducer->bitVal.get(); break; default: @@ -5018,9 +5093,10 @@ SQLRETURN FetchArrowBatch_wrap(SqlHandlePtr StatementHandle, py::list& capsules, std::memcpy(arrowSchemaPrivateData[i]->format.get(), format.c_str(), formatLen); } - arrowColumnProducer->valid = std::make_unique((arrowBatchSize + 7) / 8); + arrowColumnProducer->valid = + AllocateUniqueArray(bitmapSize, "Arrow bitmap is too large"); // Initialize validity bitmap to all valid - std::memset(arrowColumnProducer->valid.get(), 0xFF, (arrowBatchSize + 7) / 8); + std::memset(arrowColumnProducer->valid.get(), 0xFF, bitmapSize); } // Initialize column buffers diff --git a/tests/test_004_cursor.py b/tests/test_004_cursor.py index 40ea4d09d..8dda60476 100644 --- a/tests/test_004_cursor.py +++ b/tests/test_004_cursor.py @@ -1730,6 +1730,34 @@ def test_arraysize(cursor): assert cursor.arraysize == 5, "Arraysize mismatch after change" +@pytest.mark.parametrize("value", [0, -1, 2**31]) +def test_arraysize_rejects_out_of_range_values(value): + cursor = mssql_python.Cursor.__new__(mssql_python.Cursor) + with pytest.raises(ValueError, match="arraysize"): + cursor.arraysize = value + + +@pytest.mark.parametrize("value", [True, False, 1.5, "10"]) +def test_arraysize_rejects_non_integer_values(value): + cursor = mssql_python.Cursor.__new__(mssql_python.Cursor) + with pytest.raises(TypeError, match="arraysize"): + cursor.arraysize = value + + +@pytest.mark.parametrize( + "size_info", + [ + (True,), + (mssql_python.SQL_WVARCHAR, True, 0), + (mssql_python.SQL_DECIMAL, 18, True), + ], +) +def test_setinputsizes_rejects_boolean_sizes(size_info): + cursor = mssql_python.Cursor.__new__(mssql_python.Cursor) + with pytest.raises(ValueError): + cursor.setinputsizes([size_info]) + + def test_description(cursor): """Test description""" cursor.execute("SELECT * FROM #pytest_all_data_types WHERE id = 1") diff --git a/tests/test_024_bulkcopy_arrow.py b/tests/test_024_bulkcopy_arrow.py index a840853ba..ee9858262 100644 --- a/tests/test_024_bulkcopy_arrow.py +++ b/tests/test_024_bulkcopy_arrow.py @@ -202,6 +202,11 @@ def test_batch_size_negative(self): with pytest.raises(ValueError, match="batch_size"): _bare_cursor().bulkcopy_arrow("t", pa.table({"a": [1]}), batch_size=-1) + def test_batch_size_bool_rejected(self): + for flag in (False, True): + with pytest.raises(TypeError, match="batch_size"): + _bare_cursor().bulkcopy_arrow("t", pa.table({"a": [1]}), batch_size=flag) + def test_timeout_wrong_type(self): with pytest.raises(TypeError, match="timeout"): _bare_cursor().bulkcopy_arrow("t", pa.table({"a": [1]}), timeout="30") From 39dea1b8ec01f941dbc0601f601cdae898bdd5f8 Mon Sep 17 00:00:00 2001 From: gargsaumya Date: Mon, 21 Sep 2026 12:57:25 +0530 Subject: [PATCH 02/31] FIX: Update negative Arrow batch size test --- tests/test_004_cursor_arrow.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/test_004_cursor_arrow.py b/tests/test_004_cursor_arrow.py index 6a7d7b5d9..b815e524b 100644 --- a/tests/test_004_cursor_arrow.py +++ b/tests/test_004_cursor_arrow.py @@ -226,10 +226,9 @@ def test_arrow_empty_fetch(cursor: mssql_python.Cursor): def test_arrow_table_batchsize_negative(cursor: mssql_python.Cursor): - tbl = cursor.execute("select 1 a").arrow(batch_size=-42) - assert type(tbl) is pa.Table - assert tbl.num_rows == 0 - assert tbl.num_columns == 1 + cursor.execute("select 1 a") + with pytest.raises(ValueError, match="batch_size"): + cursor.arrow(batch_size=-42) assert cursor.fetchone()[0] == 1 From a02fdf41c37190fac0d3749bf00d4e8f64ae04f7 Mon Sep 17 00:00:00 2001 From: gargsaumya Date: Mon, 21 Sep 2026 14:54:35 +0530 Subject: [PATCH 03/31] FIX: Address native size review findings --- mssql_python/cursor.py | 3 +- mssql_python/pybind/ddbc_bindings.cpp | 253 +++++++++++++++++++------- tests/test_004_cursor.py | 15 +- tests/test_010_pybind_functions.py | 14 ++ 4 files changed, 221 insertions(+), 64 deletions(-) diff --git a/mssql_python/cursor.py b/mssql_python/cursor.py index b364f9091..9ed94cd90 100644 --- a/mssql_python/cursor.py +++ b/mssql_python/cursor.py @@ -51,7 +51,8 @@ SMALLMONEY_MAX: decimal.Decimal = decimal.Decimal("214748.3647") MONEY_MIN: decimal.Decimal = decimal.Decimal("-922337203685477.5808") MONEY_MAX: decimal.Decimal = decimal.Decimal("922337203685477.5807") -MAX_NATIVE_ROW_COUNT: int = 2_147_483_647 +# Bound each native fetch allocation; Arrow initially reserves 42 bytes per variable-width row. +MAX_NATIVE_ROW_COUNT: int = 1_000_000 # SQL BIGINT is a signed 64-bit integer. Ints outside this range have no BIGINT # encoding and must be rejected at detect time on both paths (see _map_sql_type). BIGINT_MIN: int = -(2**63) diff --git a/mssql_python/pybind/ddbc_bindings.cpp b/mssql_python/pybind/ddbc_bindings.cpp index 8f16d46b4..fd7f5ca41 100644 --- a/mssql_python/pybind/ddbc_bindings.cpp +++ b/mssql_python/pybind/ddbc_bindings.cpp @@ -346,6 +346,17 @@ size_t CheckedMultiplySize(size_t left, size_t right, const char* errorMessage) return left * right; } +constexpr int MAX_NATIVE_ROW_COUNT = 1000000; +constexpr size_t MAX_NATIVE_FETCH_BYTES = 256ULL * 1024 * 1024; + +void ValidateNativeRowCount(int value, const char* name, bool allowZero) { + const int minimum = allowZero ? 0 : 1; + if (value < minimum || value > MAX_NATIVE_ROW_COUNT) { + ThrowStdException(std::string(name) + " must be between " + std::to_string(minimum) + + " and " + std::to_string(MAX_NATIVE_ROW_COUNT)); + } +} + template std::unique_ptr AllocateUniqueArray(size_t count, const char* errorMessage) { if (count > std::numeric_limits::max() / sizeof(ElementType)) { @@ -354,6 +365,45 @@ std::unique_ptr AllocateUniqueArray(size_t count, const char* err return std::make_unique(count); } +void ReserveNativeFetchBytes(size_t& reservedBytes, size_t count, size_t elementSize) { + const size_t allocationBytes = + CheckedMultiplySize(count, elementSize, "Native fetch buffer size is too large"); + const size_t totalBytes = + CheckedAddSize(reservedBytes, allocationBytes, "Native fetch buffer size is too large"); + if (totalBytes > MAX_NATIVE_FETCH_BYTES) { + ThrowStdException("Native fetch buffers exceed the 256 MiB allocation limit"); + } + reservedBytes = totalBytes; +} + +template +void ResizeNativeFetchBuffer(std::vector& buffer, size_t count, + size_t& reservedBytes) { + ReserveNativeFetchBytes(reservedBytes, count, sizeof(ElementType)); + buffer.resize(count); +} + +template +void EnsureNativeFetchBufferSize(std::vector& buffer, size_t requiredSize, + size_t& reservedBytes) { + if (buffer.size() >= requiredSize) { + return; + } + size_t newSize = std::max(buffer.size(), 1); + while (newSize < requiredSize) { + newSize = CheckedMultiplySize(newSize, 2, "Native fetch buffer size is too large"); + } + ReserveNativeFetchBytes(reservedBytes, newSize - buffer.size(), sizeof(ElementType)); + buffer.resize(newSize); +} + +template +std::unique_ptr AllocateArrowArray(size_t count, size_t& reservedBytes, + const char* errorMessage) { + ReserveNativeFetchBytes(reservedBytes, count, sizeof(ElementType)); + return AllocateUniqueArray(count, errorMessage); +} + std::string DescribeChar(unsigned char ch) { if (ch >= 32 && ch <= 126) { return std::string("'") + static_cast(ch) + "'"; @@ -2416,6 +2466,12 @@ SQLRETURN BindParameterArray(SqlHandle& handle, SQLHANDLE hStmt, const py::list& std::memset(charArray + i * (info.columnSize + 1), 0, info.columnSize + 1); } else { + if (info.paramCType == SQL_C_BINARY && + !py::isinstance(columnValues[i]) && + !py::isinstance(columnValues[i])) { + ThrowStdException(MakeParamMismatchErrorStr(info.paramCType, + paramIndex)); + } std::string encodedStr; if (py::isinstance(columnValues[i])) { @@ -4037,8 +4093,18 @@ SQLRETURN SQLFetchScroll_wrap(SqlHandlePtr StatementHandle, SQLSMALLINT FetchOri // For column in the result set, binds a buffer to retrieve column data // TODO: Move to anonymous namespace, since it is not used outside this file SQLRETURN SQLBindColums(SQLHSTMT hStmt, ColumnBuffers& buffers, py::list& columnNames, - SQLUSMALLINT numCols, int fetchSize, int charCtype = SQL_C_WCHAR) { + SQLUSMALLINT numCols, int fetchSize, size_t& reservedBytes, + int charCtype = SQL_C_WCHAR) { PERF_TIMER("SQLBindColums"); + struct UnbindOnFailure { + SQLHSTMT hStmt; + bool active = true; + ~UnbindOnFailure() { + if (active) { + SQLFreeStmt_ptr(hStmt, SQL_UNBIND); + } + } + } unbindOnFailure{hStmt}; SQLRETURN ret = SQL_SUCCESS; const bool useWideChar = (charCtype == SQL_C_WCHAR); // Bind columns based on their data types @@ -4056,7 +4122,10 @@ SQLRETURN SQLBindColums(SQLHSTMT hStmt, ColumnBuffers& buffers, py::list& column // Bind VARCHAR columns as SQL_C_WCHAR so the ODBC driver // returns UTF-16 data, avoiding code-page decode issues. uint64_t fetchBufferSize = columnSize + 1 /*null-terminator*/; - buffers.wcharBuffers[col - 1].resize(fetchSize * fetchBufferSize); + ResizeNativeFetchBuffer(buffers.wcharBuffers[col - 1], + CheckedMultiplySize(fetchSize, fetchBufferSize, + "Native fetch buffer is too large"), + reservedBytes); ret = SQLBindCol_ptr( hStmt, col, SQL_C_WCHAR, buffers.wcharBuffers[col - 1].data(), fetchBufferSize * sizeof(SQLWCHAR), buffers.indicators[col - 1].data()); @@ -4067,7 +4136,10 @@ SQLRETURN SQLBindColums(SQLHSTMT hStmt, ColumnBuffers& buffers, py::list& column #else uint64_t fetchBufferSize = columnSize + 1 /*null-terminator*/; #endif - buffers.charBuffers[col - 1].resize(fetchSize * fetchBufferSize); + ResizeNativeFetchBuffer(buffers.charBuffers[col - 1], + CheckedMultiplySize(fetchSize, fetchBufferSize, + "Native fetch buffer is too large"), + reservedBytes); ret = SQLBindCol_ptr( hStmt, col, SQL_C_CHAR, buffers.charBuffers[col - 1].data(), fetchBufferSize * sizeof(SQLCHAR), buffers.indicators[col - 1].data()); @@ -4081,48 +4153,55 @@ SQLRETURN SQLBindColums(SQLHSTMT hStmt, ColumnBuffers& buffers, py::list& column // suffice HandleZeroColumnSizeAtFetch(columnSize); uint64_t fetchBufferSize = columnSize + 1 /*null-terminator*/; - buffers.wcharBuffers[col - 1].resize(fetchSize * fetchBufferSize); + ResizeNativeFetchBuffer(buffers.wcharBuffers[col - 1], + CheckedMultiplySize(fetchSize, fetchBufferSize, + "Native fetch buffer is too large"), + reservedBytes); ret = SQLBindCol_ptr(hStmt, col, SQL_C_WCHAR, buffers.wcharBuffers[col - 1].data(), fetchBufferSize * sizeof(SQLWCHAR), buffers.indicators[col - 1].data()); break; } case SQL_INTEGER: - buffers.intBuffers[col - 1].resize(fetchSize); + ResizeNativeFetchBuffer(buffers.intBuffers[col - 1], fetchSize, reservedBytes); ret = SQLBindCol_ptr(hStmt, col, SQL_C_SLONG, buffers.intBuffers[col - 1].data(), sizeof(SQLINTEGER), buffers.indicators[col - 1].data()); break; case SQL_SMALLINT: - buffers.smallIntBuffers[col - 1].resize(fetchSize); + ResizeNativeFetchBuffer(buffers.smallIntBuffers[col - 1], fetchSize, reservedBytes); ret = SQLBindCol_ptr(hStmt, col, SQL_C_SSHORT, buffers.smallIntBuffers[col - 1].data(), sizeof(SQLSMALLINT), buffers.indicators[col - 1].data()); break; case SQL_TINYINT: - buffers.charBuffers[col - 1].resize(fetchSize); + ResizeNativeFetchBuffer(buffers.charBuffers[col - 1], fetchSize, reservedBytes); ret = SQLBindCol_ptr(hStmt, col, SQL_C_TINYINT, buffers.charBuffers[col - 1].data(), sizeof(SQLCHAR), buffers.indicators[col - 1].data()); break; case SQL_BIT: - buffers.charBuffers[col - 1].resize(fetchSize); + ResizeNativeFetchBuffer(buffers.charBuffers[col - 1], fetchSize, reservedBytes); ret = SQLBindCol_ptr(hStmt, col, SQL_C_BIT, buffers.charBuffers[col - 1].data(), sizeof(SQLCHAR), buffers.indicators[col - 1].data()); break; case SQL_REAL: - buffers.realBuffers[col - 1].resize(fetchSize); + ResizeNativeFetchBuffer(buffers.realBuffers[col - 1], fetchSize, reservedBytes); ret = SQLBindCol_ptr(hStmt, col, SQL_C_FLOAT, buffers.realBuffers[col - 1].data(), sizeof(SQLREAL), buffers.indicators[col - 1].data()); break; case SQL_DECIMAL: case SQL_NUMERIC: - buffers.charBuffers[col - 1].resize(fetchSize * MAX_DIGITS_IN_NUMERIC); + ResizeNativeFetchBuffer( + buffers.charBuffers[col - 1], + CheckedMultiplySize(fetchSize, MAX_DIGITS_IN_NUMERIC, + "Native fetch buffer is too large"), + reservedBytes); ret = SQLBindCol_ptr(hStmt, col, SQL_C_CHAR, buffers.charBuffers[col - 1].data(), MAX_DIGITS_IN_NUMERIC * sizeof(SQLCHAR), buffers.indicators[col - 1].data()); break; case SQL_DOUBLE: case SQL_FLOAT: - buffers.doubleBuffers[col - 1].resize(fetchSize); + ResizeNativeFetchBuffer(buffers.doubleBuffers[col - 1], fetchSize, reservedBytes); ret = SQLBindCol_ptr(hStmt, col, SQL_C_DOUBLE, buffers.doubleBuffers[col - 1].data(), sizeof(SQLDOUBLE), buffers.indicators[col - 1].data()); @@ -4130,31 +4209,32 @@ SQLRETURN SQLBindColums(SQLHSTMT hStmt, ColumnBuffers& buffers, py::list& column case SQL_TIMESTAMP: case SQL_TYPE_TIMESTAMP: case SQL_DATETIME: - buffers.timestampBuffers[col - 1].resize(fetchSize); + ResizeNativeFetchBuffer(buffers.timestampBuffers[col - 1], fetchSize, + reservedBytes); ret = SQLBindCol_ptr( hStmt, col, SQL_C_TYPE_TIMESTAMP, buffers.timestampBuffers[col - 1].data(), sizeof(SQL_TIMESTAMP_STRUCT), buffers.indicators[col - 1].data()); break; case SQL_BIGINT: - buffers.bigIntBuffers[col - 1].resize(fetchSize); + ResizeNativeFetchBuffer(buffers.bigIntBuffers[col - 1], fetchSize, reservedBytes); ret = SQLBindCol_ptr(hStmt, col, SQL_C_SBIGINT, buffers.bigIntBuffers[col - 1].data(), sizeof(SQLBIGINT), buffers.indicators[col - 1].data()); break; case SQL_TYPE_DATE: - buffers.dateBuffers[col - 1].resize(fetchSize); + ResizeNativeFetchBuffer(buffers.dateBuffers[col - 1], fetchSize, reservedBytes); ret = SQLBindCol_ptr(hStmt, col, SQL_C_TYPE_DATE, buffers.dateBuffers[col - 1].data(), sizeof(SQL_DATE_STRUCT), buffers.indicators[col - 1].data()); break; case SQL_SS_TIME2: - buffers.timeBuffers[col - 1].resize(fetchSize); + ResizeNativeFetchBuffer(buffers.timeBuffers[col - 1], fetchSize, reservedBytes); ret = SQLBindCol_ptr(hStmt, col, SQL_C_SS_TIME2, buffers.timeBuffers[col - 1].data(), sizeof(SQL_SS_TIME2_STRUCT), buffers.indicators[col - 1].data()); break; case SQL_GUID: - buffers.guidBuffers[col - 1].resize(fetchSize); + ResizeNativeFetchBuffer(buffers.guidBuffers[col - 1], fetchSize, reservedBytes); ret = SQLBindCol_ptr(hStmt, col, SQL_C_GUID, buffers.guidBuffers[col - 1].data(), sizeof(SQLGUID), buffers.indicators[col - 1].data()); break; @@ -4165,12 +4245,16 @@ SQLRETURN SQLBindColums(SQLHSTMT hStmt, ColumnBuffers& buffers, py::list& column // TODO: handle variable length data correctly. This logic wont // suffice HandleZeroColumnSizeAtFetch(columnSize); - buffers.charBuffers[col - 1].resize(fetchSize * columnSize); + ResizeNativeFetchBuffer(buffers.charBuffers[col - 1], + CheckedMultiplySize(fetchSize, columnSize, + "Native fetch buffer is too large"), + reservedBytes); ret = SQLBindCol_ptr(hStmt, col, SQL_C_BINARY, buffers.charBuffers[col - 1].data(), columnSize, buffers.indicators[col - 1].data()); break; case SQL_SS_TIMESTAMPOFFSET: - buffers.datetimeoffsetBuffers[col - 1].resize(fetchSize); + ResizeNativeFetchBuffer(buffers.datetimeoffsetBuffers[col - 1], fetchSize, + reservedBytes); ret = SQLBindCol_ptr(hStmt, col, SQL_C_SS_TIMESTAMPOFFSET, buffers.datetimeoffsetBuffers[col - 1].data(), sizeof(DateTimeOffset) * fetchSize, @@ -4195,6 +4279,7 @@ SQLRETURN SQLBindColums(SQLHSTMT hStmt, ColumnBuffers& buffers, py::list& column return ret; } } + unbindOnFailure.active = false; return ret; } @@ -4678,6 +4763,7 @@ SQLRETURN FetchMany_wrap(SqlHandlePtr StatementHandle, py::list& rows, int fetch const std::string& wcharEncoding = "utf-16le", int charCtype = SQL_C_WCHAR) { PERF_TIMER("FetchMany_wrap"); + ValidateNativeRowCount(fetchSize, "Fetch size", false); // Issue #531: upgrade SQL_C_CHAR + utf-8 to SQL_C_WCHAR on Windows so the // driver does lossless UTF-16 conversion instead of returning ACP bytes. charCtype = EffectiveCharCtypeForFetch(charCtype, charEncoding); @@ -4733,10 +4819,16 @@ SQLRETURN FetchMany_wrap(SqlHandlePtr StatementHandle, py::list& rows, int fetch } // Initialize column buffers + size_t reservedBytes = 0; + ReserveNativeFetchBytes( + reservedBytes, + CheckedMultiplySize(static_cast(numCols), static_cast(fetchSize), + "Native fetch indicator buffer is too large"), + sizeof(SQLLEN)); ColumnBuffers buffers(numCols, fetchSize); // Bind columns - ret = SQLBindColums(hStmt, buffers, columnNames, numCols, fetchSize, charCtype); + ret = SQLBindColums(hStmt, buffers, columnNames, numCols, fetchSize, reservedBytes, charCtype); if (!SQL_SUCCEEDED(ret)) { LOG("FetchMany_wrap: Error when binding columns - SQLRETURN=%d", ret); return ret; @@ -4776,7 +4868,7 @@ SQLRETURN FetchMany_wrap(SqlHandlePtr StatementHandle, py::list& rows, int fetch // @return SQLRETURN: SQL_SUCCESS on success, or error code on failure template SQLRETURN GetDataVar(SQLHSTMT hStmt, SQLUSMALLINT colNumber, SQLSMALLINT cType, - std::vector& dataVec, SQLLEN* indicator) { + std::vector& dataVec, SQLLEN* indicator, size_t& reservedBytes) { size_t start = 0; size_t end = 0; @@ -4796,7 +4888,7 @@ SQLRETURN GetDataVar(SQLHSTMT hStmt, SQLUSMALLINT colNumber, SQLSMALLINT cType, // Ensure initial buffer has space for at least the null terminator if (dataVec.size() < sizeNullTerminator) { - dataVec.resize(sizeNullTerminator); + ResizeNativeFetchBuffer(dataVec, sizeNullTerminator, reservedBytes); } while (true) { @@ -4832,17 +4924,22 @@ SQLRETURN GetDataVar(SQLHSTMT hStmt, SQLUSMALLINT colNumber, SQLSMALLINT cType, // Determine how much more space we need if (localInd < 0) { // SQL_NO_TOTAL: driver doesn't know total size, double the buffer - end = dataVec.size() * 2; + end = CheckedMultiplySize(dataVec.size(), 2, + "Native fetch buffer size is too large"); } else { // Driver returned total size: allocate exactly what we need assert(localInd % sizeof(T) == 0); - end = start + static_cast(localInd) / sizeof(T) + sizeNullTerminator; + end = CheckedAddSize( + CheckedAddSize(start, static_cast(localInd) / sizeof(T), + "Native fetch buffer size is too large"), + sizeNullTerminator, "Native fetch buffer size is too large"); } // The next read starts where the null terminator would have been placed start = dataVec.size() - sizeNullTerminator; // Resize buffer for next iteration + ReserveNativeFetchBytes(reservedBytes, end - dataVec.size(), sizeof(T)); dataVec.resize(end); } else { // Unexpected return code @@ -4888,9 +4985,7 @@ SQLRETURN FetchArrowBatch_wrap(SqlHandlePtr StatementHandle, py::list& capsules, int arrowBatchSize, int charCtype) { PERF_TIMER("FetchArrowBatch_wrap"); - if (arrowBatchSize < 0) { - ThrowStdException("Arrow batch size must be non-negative"); - } + ValidateNativeRowCount(arrowBatchSize, "Arrow batch size", true); const size_t batchSize = static_cast(arrowBatchSize); const size_t offsetCount = CheckedAddSize(batchSize, 1, "Arrow batch size is too large"); const size_t initialVarDataSize = @@ -4926,6 +5021,7 @@ SQLRETURN FetchArrowBatch_wrap(SqlHandlePtr StatementHandle, py::list& capsules, std::vector columnNullable(numCols); std::vector columnVarLen(numCols, false); std::vector nullCounts(numCols, 0); + size_t reservedBytes = 0; std::vector> arrowArrayPrivateData(numCols); std::vector> arrowSchemaPrivateData(numCols); @@ -4970,8 +5066,10 @@ SQLRETURN FetchArrowBatch_wrap(SqlHandlePtr StatementHandle, py::list& capsules, case SQL_GUID: format = "U"; arrowColumnProducer->varVal = - AllocateUniqueArray(offsetCount, "Arrow offset buffer is too large"); - arrowColumnProducer->varData.resize(initialVarDataSize); + AllocateArrowArray(offsetCount, reservedBytes, + "Arrow offset buffer is too large"); + ResizeNativeFetchBuffer(arrowColumnProducer->varData, initialVarDataSize, + reservedBytes); columnVarLen[i] = true; // start at offset 0 arrowColumnProducer->varVal[0] = 0; @@ -4983,8 +5081,10 @@ SQLRETURN FetchArrowBatch_wrap(SqlHandlePtr StatementHandle, py::list& capsules, case SQL_LONGVARBINARY: format = "Z"; arrowColumnProducer->varVal = - AllocateUniqueArray(offsetCount, "Arrow offset buffer is too large"); - arrowColumnProducer->varData.resize(initialVarDataSize); + AllocateArrowArray(offsetCount, reservedBytes, + "Arrow offset buffer is too large"); + ResizeNativeFetchBuffer(arrowColumnProducer->varData, initialVarDataSize, + reservedBytes); columnVarLen[i] = true; // start at offset 0 arrowColumnProducer->varVal[0] = 0; @@ -4993,38 +5093,44 @@ SQLRETURN FetchArrowBatch_wrap(SqlHandlePtr StatementHandle, py::list& capsules, case SQL_TINYINT: format = "C"; arrowColumnProducer->uint8Val = - AllocateUniqueArray(batchSize, "Arrow value buffer is too large"); + AllocateArrowArray(batchSize, reservedBytes, + "Arrow value buffer is too large"); arrowColumnProducer->ptrValueBuffer = arrowColumnProducer->uint8Val.get(); break; case SQL_SMALLINT: format = "s"; arrowColumnProducer->int16Val = - AllocateUniqueArray(batchSize, "Arrow value buffer is too large"); + AllocateArrowArray(batchSize, reservedBytes, + "Arrow value buffer is too large"); arrowColumnProducer->ptrValueBuffer = arrowColumnProducer->int16Val.get(); break; case SQL_INTEGER: format = "i"; arrowColumnProducer->int32Val = - AllocateUniqueArray(batchSize, "Arrow value buffer is too large"); + AllocateArrowArray(batchSize, reservedBytes, + "Arrow value buffer is too large"); arrowColumnProducer->ptrValueBuffer = arrowColumnProducer->int32Val.get(); break; case SQL_BIGINT: format = "l"; arrowColumnProducer->int64Val = - AllocateUniqueArray(batchSize, "Arrow value buffer is too large"); + AllocateArrowArray(batchSize, reservedBytes, + "Arrow value buffer is too large"); arrowColumnProducer->ptrValueBuffer = arrowColumnProducer->int64Val.get(); break; case SQL_REAL: format = "f"; arrowColumnProducer->float32Val = - AllocateUniqueArray(batchSize, "Arrow value buffer is too large"); + AllocateArrowArray(batchSize, reservedBytes, + "Arrow value buffer is too large"); arrowColumnProducer->ptrValueBuffer = arrowColumnProducer->float32Val.get(); break; case SQL_FLOAT: case SQL_DOUBLE: format = "g"; arrowColumnProducer->float64Val = - AllocateUniqueArray(batchSize, "Arrow value buffer is too large"); + AllocateArrowArray(batchSize, reservedBytes, + "Arrow value buffer is too large"); arrowColumnProducer->ptrValueBuffer = arrowColumnProducer->float64Val.get(); break; case SQL_DECIMAL: @@ -5038,7 +5144,8 @@ SQLRETURN FetchArrowBatch_wrap(SqlHandlePtr StatementHandle, py::list& capsules, std::memcpy(arrowSchemaPrivateData[i]->format.get(), formatStr.c_str(), formatLen); format = arrowSchemaPrivateData[i]->format.get(); arrowColumnProducer->decimalVal = - AllocateUniqueArray(batchSize, "Arrow value buffer is too large"); + AllocateArrowArray(batchSize, reservedBytes, + "Arrow value buffer is too large"); arrowColumnProducer->ptrValueBuffer = arrowColumnProducer->decimalVal.get(); break; } @@ -5047,31 +5154,36 @@ SQLRETURN FetchArrowBatch_wrap(SqlHandlePtr StatementHandle, py::list& capsules, case SQL_DATETIME: format = "tsu:"; arrowColumnProducer->tsMicroVal = - AllocateUniqueArray(batchSize, "Arrow value buffer is too large"); + AllocateArrowArray(batchSize, reservedBytes, + "Arrow value buffer is too large"); arrowColumnProducer->ptrValueBuffer = arrowColumnProducer->tsMicroVal.get(); break; case SQL_SS_TIMESTAMPOFFSET: format = "tsu:+00:00"; arrowColumnProducer->tsMicroVal = - AllocateUniqueArray(batchSize, "Arrow value buffer is too large"); + AllocateArrowArray(batchSize, reservedBytes, + "Arrow value buffer is too large"); arrowColumnProducer->ptrValueBuffer = arrowColumnProducer->tsMicroVal.get(); break; case SQL_TYPE_DATE: format = "tdD"; arrowColumnProducer->dateVal = - AllocateUniqueArray(batchSize, "Arrow value buffer is too large"); + AllocateArrowArray(batchSize, reservedBytes, + "Arrow value buffer is too large"); arrowColumnProducer->ptrValueBuffer = arrowColumnProducer->dateVal.get(); break; case SQL_SS_TIME2: format = "ttn"; arrowColumnProducer->timeNanoVal = - AllocateUniqueArray(batchSize, "Arrow value buffer is too large"); + AllocateArrowArray(batchSize, reservedBytes, + "Arrow value buffer is too large"); arrowColumnProducer->ptrValueBuffer = arrowColumnProducer->timeNanoVal.get(); break; case SQL_BIT: format = "b"; arrowColumnProducer->bitVal = - AllocateUniqueArray(bitmapSize, "Arrow bitmap is too large"); + AllocateArrowArray(bitmapSize, reservedBytes, + "Arrow bitmap is too large"); std::memset(arrowColumnProducer->bitVal.get(), 0, bitmapSize); arrowColumnProducer->ptrValueBuffer = arrowColumnProducer->bitVal.get(); break; @@ -5094,16 +5206,22 @@ SQLRETURN FetchArrowBatch_wrap(SqlHandlePtr StatementHandle, py::list& capsules, } arrowColumnProducer->valid = - AllocateUniqueArray(bitmapSize, "Arrow bitmap is too large"); + AllocateArrowArray(bitmapSize, reservedBytes, "Arrow bitmap is too large"); // Initialize validity bitmap to all valid std::memset(arrowColumnProducer->valid.get(), 0xFF, bitmapSize); } // Initialize column buffers + ReserveNativeFetchBytes( + reservedBytes, + CheckedMultiplySize(static_cast(numCols), static_cast(fetchSize), + "Native fetch indicator buffer is too large"), + sizeof(SQLLEN)); ColumnBuffers buffers(numCols, fetchSize); if (!hasLobColumns && fetchSize > 0) { - ret = SQLBindColums(hStmt, buffers, columnNames, numCols, fetchSize, charCtype); + ret = SQLBindColums(hStmt, buffers, columnNames, numCols, fetchSize, reservedBytes, + charCtype); if (!SQL_SUCCEEDED(ret)) { LOG("Error when binding columns"); return ret; @@ -5153,7 +5271,7 @@ SQLRETURN FetchArrowBatch_wrap(SqlHandlePtr StatementHandle, py::list& capsules, case SQL_LONGVARBINARY: { ret = GetDataVar(hStmt, idxCol + 1, SQL_C_BINARY, buffers.charBuffers[idxCol], - buffers.indicators[idxCol].data()); + buffers.indicators[idxCol].data(), reservedBytes); if (!SQL_SUCCEEDED(ret)) { LOG("Error fetching BINARY LOB for column %d", idxCol + 1); return ret; @@ -5166,7 +5284,7 @@ SQLRETURN FetchArrowBatch_wrap(SqlHandlePtr StatementHandle, py::list& capsules, if (charCtype == SQL_C_CHAR) { ret = GetDataVar(hStmt, idxCol + 1, SQL_C_CHAR, buffers.charBuffers[idxCol], - buffers.indicators[idxCol].data()); + buffers.indicators[idxCol].data(), reservedBytes); if (!SQL_SUCCEEDED(ret)) { LOG("Error fetching CHAR LOB data for column %d", idxCol + 1); return ret; @@ -5181,7 +5299,7 @@ SQLRETURN FetchArrowBatch_wrap(SqlHandlePtr StatementHandle, py::list& capsules, case SQL_WLONGVARCHAR: { ret = GetDataVar(hStmt, idxCol + 1, SQL_C_WCHAR, buffers.wcharBuffers[idxCol], - buffers.indicators[idxCol].data()); + buffers.indicators[idxCol].data(), reservedBytes); if (!SQL_SUCCEEDED(ret)) { LOG("Error fetching WCHAR LOB data for column %d", idxCol + 1); return ret; @@ -5405,9 +5523,10 @@ SQLRETURN FetchArrowBatch_wrap(SqlHandlePtr StatementHandle, py::list& capsules, uint64_t fetchBufferSize = columnSize /* bytes are not null terminated */; auto target_vec = &arrowColumnProducer->varData; auto start = arrowColumnProducer->varVal[idxRowArrow]; - while (target_vec->size() < start + dataLen) { - target_vec->resize(target_vec->size() * 2); - } + EnsureNativeFetchBufferSize( + *target_vec, + CheckedAddSize(start, dataLen, "Arrow value buffer is too large"), + reservedBytes); std::memcpy(&(*target_vec)[start], &buffers.charBuffers[idxCol][idxRowSql * fetchBufferSize], @@ -5426,9 +5545,10 @@ SQLRETURN FetchArrowBatch_wrap(SqlHandlePtr StatementHandle, py::list& capsules, #endif auto target_vec = &arrowColumnProducer->varData; auto start = arrowColumnProducer->varVal[idxRowArrow]; - while (target_vec->size() < start + dataLen) { - target_vec->resize(target_vec->size() * 2); - } + EnsureNativeFetchBufferSize( + *target_vec, + CheckedAddSize(start, dataLen, "Arrow value buffer is too large"), + reservedBytes); std::memcpy(&(*target_vec)[start], &buffers.charBuffers[idxCol][idxRowSql * fetchBufferSize], @@ -5452,11 +5572,13 @@ SQLRETURN FetchArrowBatch_wrap(SqlHandlePtr StatementHandle, py::list& capsules, static_assert(sizeof(SQLWCHAR) == sizeof(char16_t)); static_assert(alignof(SQLWCHAR) == alignof(char16_t)); const auto* utf16Source = reinterpret_cast(wcharSource); - size_t maxUtf8Size = dataLenW * 3; - - while (target_vec->size() < start + maxUtf8Size) { - target_vec->resize(target_vec->size() * 2); - } + size_t maxUtf8Size = + CheckedMultiplySize(dataLenW, 3, "Arrow value buffer is too large"); + EnsureNativeFetchBufferSize( + *target_vec, + CheckedAddSize(start, maxUtf8Size, + "Arrow value buffer is too large"), + reservedBytes); size_t bytesWritten = simdutf::convert_utf16le_to_utf8_with_replacement( utf16Source, dataLenW, @@ -5473,9 +5595,10 @@ SQLRETURN FetchArrowBatch_wrap(SqlHandlePtr StatementHandle, py::list& capsules, auto start = arrowColumnProducer->varVal[idxRowArrow]; // Ensure buffer has space for the GUID string + null terminator - while (target_vec->size() < start + 37) { - target_vec->resize(target_vec->size() * 2); - } + EnsureNativeFetchBufferSize( + *target_vec, + CheckedAddSize(start, 37, "Arrow value buffer is too large"), + reservedBytes); // Get the GUID from the buffer const SQLGUID& guidValue = buffers.guidBuffers[idxCol][idxRowSql]; @@ -5917,10 +6040,16 @@ SQLRETURN FetchAll_wrap(SqlHandlePtr StatementHandle, py::list& rows, } LOG("FetchAll_wrap: Fetching data in batch sizes of %d", fetchSize); + size_t reservedBytes = 0; + ReserveNativeFetchBytes( + reservedBytes, + CheckedMultiplySize(static_cast(numCols), static_cast(fetchSize), + "Native fetch indicator buffer is too large"), + sizeof(SQLLEN)); ColumnBuffers buffers(numCols, fetchSize); // Bind columns - ret = SQLBindColums(hStmt, buffers, columnNames, numCols, fetchSize, charCtype); + ret = SQLBindColums(hStmt, buffers, columnNames, numCols, fetchSize, reservedBytes, charCtype); if (!SQL_SUCCEEDED(ret)) { LOG("FetchAll_wrap: Error when binding columns - SQLRETURN=%d", ret); return ret; diff --git a/tests/test_004_cursor.py b/tests/test_004_cursor.py index 8dda60476..54b5d3504 100644 --- a/tests/test_004_cursor.py +++ b/tests/test_004_cursor.py @@ -1730,7 +1730,7 @@ def test_arraysize(cursor): assert cursor.arraysize == 5, "Arraysize mismatch after change" -@pytest.mark.parametrize("value", [0, -1, 2**31]) +@pytest.mark.parametrize("value", [0, -1, 1_000_001]) def test_arraysize_rejects_out_of_range_values(value): cursor = mssql_python.Cursor.__new__(mssql_python.Cursor) with pytest.raises(ValueError, match="arraysize"): @@ -1758,6 +1758,19 @@ def test_setinputsizes_rejects_boolean_sizes(size_info): cursor.setinputsizes([size_info]) +def test_executemany_rejects_text_for_binary_parameter(cursor): + cursor.setinputsizes([(mssql_python.SQL_VARBINARY, 10, 0)]) + with pytest.raises(RuntimeError, match="object type does not match"): + cursor.executemany("SELECT CAST(? AS VARBINARY(10))", [("text",)]) + + +def test_fetchmany_rejects_excessive_native_buffer(cursor): + cursor.execute("SELECT CAST('x' AS VARCHAR(8000))") + with pytest.raises(RuntimeError, match="256 MiB allocation limit"): + cursor.fetchmany(100_000) + assert cursor.fetchone()[0] == "x" + + def test_description(cursor): """Test description""" cursor.execute("SELECT * FROM #pytest_all_data_types WHERE id = 1") diff --git a/tests/test_010_pybind_functions.py b/tests/test_010_pybind_functions.py index 106b64ca3..afed788e4 100644 --- a/tests/test_010_pybind_functions.py +++ b/tests/test_010_pybind_functions.py @@ -37,6 +37,20 @@ ) +@pytest.mark.skipif(not DDBC_AVAILABLE, reason="ddbc_bindings not available") +@pytest.mark.parametrize("fetch_size", [-1, 1_000_001]) +def test_fetchmany_rejects_unsafe_size_before_handle_access(fetch_size): + with pytest.raises(RuntimeError, match="Fetch size"): + ddbc.DDBCSQLFetchMany(None, [], fetch_size) + + +@pytest.mark.skipif(not DDBC_AVAILABLE, reason="ddbc_bindings not available") +@pytest.mark.parametrize("batch_size", [-1, 1_000_001]) +def test_arrow_batch_rejects_unsafe_size_before_handle_access(batch_size): + with pytest.raises(RuntimeError, match="Arrow batch size"): + ddbc.DDBCSQLFetchArrowBatch(None, [], batch_size, 0) + + @pytest.mark.skipif(not DDBC_AVAILABLE, reason="ddbc_bindings not available") class TestPybindModuleInfo: """Test module information and architecture detection.""" From 3038841dffbdb4ff9a4f593052237d32fc5b54e1 Mon Sep 17 00:00:00 2001 From: gargsaumya Date: Tue, 22 Sep 2026 11:12:18 +0530 Subject: [PATCH 04/31] FIX: Bound native parameter and Arrow source buffers --- mssql_python/cursor.py | 5 +- mssql_python/pybind/ddbc_bindings.cpp | 118 +++++++++++++++++++++++--- tests/test_004_cursor.py | 12 +++ 3 files changed, 120 insertions(+), 15 deletions(-) diff --git a/mssql_python/cursor.py b/mssql_python/cursor.py index 9ed94cd90..54dcbc174 100644 --- a/mssql_python/cursor.py +++ b/mssql_python/cursor.py @@ -53,6 +53,7 @@ MONEY_MAX: decimal.Decimal = decimal.Decimal("922337203685477.5807") # Bound each native fetch allocation; Arrow initially reserves 42 bytes per variable-width row. MAX_NATIVE_ROW_COUNT: int = 1_000_000 +MAX_NATIVE_PARAMETER_SIZE: int = 256 * 1024 * 1024 # SQL BIGINT is a signed 64-bit integer. Ints outside this range have no BIGINT # encoding and must be rejected at detect time on both paths (see _map_sql_type). BIGINT_MIN: int = -(2**63) @@ -1223,9 +1224,11 @@ def setinputsizes(self, sizes: List[Union[int, tuple]]) -> None: isinstance(column_size, bool) or not isinstance(column_size, int) or column_size < 0 + or column_size > MAX_NATIVE_PARAMETER_SIZE ): raise ValueError( - f"Invalid column size: {column_size}. Must be a non-negative integer." + f"Invalid column size: {column_size}. Must be a non-negative integer " + f"no greater than {MAX_NATIVE_PARAMETER_SIZE}." ) if ( diff --git a/mssql_python/pybind/ddbc_bindings.cpp b/mssql_python/pybind/ddbc_bindings.cpp index fd7f5ca41..aa5e276cf 100644 --- a/mssql_python/pybind/ddbc_bindings.cpp +++ b/mssql_python/pybind/ddbc_bindings.cpp @@ -348,6 +348,7 @@ size_t CheckedMultiplySize(size_t left, size_t right, const char* errorMessage) constexpr int MAX_NATIVE_ROW_COUNT = 1000000; constexpr size_t MAX_NATIVE_FETCH_BYTES = 256ULL * 1024 * 1024; +constexpr size_t MAX_NATIVE_PARAMETER_BYTES = 256ULL * 1024 * 1024; void ValidateNativeRowCount(int value, const char* name, bool allowZero) { const int minimum = allowZero ? 0 : 1; @@ -357,6 +358,79 @@ void ValidateNativeRowCount(int value, const char* name, bool allowZero) { } } +void ReserveNativeParameterBytes(size_t& reservedBytes, size_t count, size_t elementSize) { + const size_t allocationBytes = + CheckedMultiplySize(count, elementSize, "Parameter buffer size is too large"); + reservedBytes = + CheckedAddSize(reservedBytes, allocationBytes, "Parameter buffer size is too large"); + if (reservedBytes > MAX_NATIVE_PARAMETER_BYTES) { + ThrowStdException("Parameter buffers exceed the 256 MiB allocation limit"); + } +} + +size_t ParameterArrayElementSize(const ParamInfo& info) { + switch (info.paramCType) { + case SQL_C_LONG: + return sizeof(int); + case SQL_C_DOUBLE: + return sizeof(double); + case SQL_C_WCHAR: + return CheckedMultiplySize( + CheckedAddSize(info.columnSize, 1, "Wide-character parameter size is too large"), + sizeof(SQLWCHAR), "Wide-character parameter size is too large"); + case SQL_C_TINYINT: + case SQL_C_UTINYINT: + return sizeof(unsigned char); + case SQL_C_SHORT: + return sizeof(short); + case SQL_C_CHAR: + case SQL_C_BINARY: + return CheckedAddSize(info.columnSize, 1, "Character parameter size is too large"); + case SQL_C_BIT: + return sizeof(char); + case SQL_C_STINYINT: + case SQL_C_USHORT: + return sizeof(unsigned short); + case SQL_C_SBIGINT: + case SQL_C_SLONG: + case SQL_C_UBIGINT: + case SQL_C_ULONG: + return sizeof(int64_t); + case SQL_C_FLOAT: + return sizeof(float); + case SQL_C_TYPE_DATE: + return sizeof(SQL_DATE_STRUCT); + case SQL_C_TYPE_TIME: + return sizeof(SQL_TIME_STRUCT); + case SQL_C_TYPE_TIMESTAMP: + return sizeof(SQL_TIMESTAMP_STRUCT); + case SQL_C_SS_TIMESTAMPOFFSET: + return sizeof(DateTimeOffset); + case SQL_C_NUMERIC: + return sizeof(SQL_NUMERIC_STRUCT); + case SQL_C_GUID: + return sizeof(SQLGUID); + case SQL_C_DEFAULT: + return sizeof(char); + default: + ThrowStdException("Unsupported C type for parameter array allocation"); + } +} + +template +size_t CheckedArrowSourceOffset(const std::vector& buffer, size_t rowIndex, + size_t rowStride, size_t dataBytes) { + const size_t offset = + CheckedMultiplySize(rowIndex, rowStride, "Arrow source offset is too large"); + const size_t rowCapacity = CheckedMultiplySize( + rowStride, sizeof(ElementType), "Arrow source capacity is too large"); + if (offset > buffer.size() || rowStride > buffer.size() - offset || + dataBytes > rowCapacity) { + ThrowStdException("Driver data length exceeds the allocated fetch buffer"); + } + return offset; +} + template std::unique_ptr AllocateUniqueArray(size_t count, const char* errorMessage) { if (count > std::numeric_limits::max() / sizeof(ElementType)) { @@ -2282,6 +2356,12 @@ SQLRETURN BindParameterArray(SqlHandle& handle, SQLHANDLE hStmt, const py::list& try { // GH-627: resolve unknown NULL array param SQL types before binding any param. PreResolveUnknownNullTypes(handle, hStmt, paramInfos); + size_t reservedParameterBytes = 0; + for (const ParamInfo& info : paramInfos) { + ReserveNativeParameterBytes(reservedParameterBytes, paramSetSize, + ParameterArrayElementSize(info)); + ReserveNativeParameterBytes(reservedParameterBytes, paramSetSize, sizeof(SQLLEN)); + } for (int paramIndex = 0; paramIndex < columnwise_params.size(); ++paramIndex) { const py::list& columnValues = columnwise_params[paramIndex].cast(); ParamInfo& info = paramInfos[paramIndex]; @@ -5259,7 +5339,6 @@ SQLRETURN FetchArrowBatch_wrap(SqlHandlePtr StatementHandle, py::list& capsules, for (SQLUSMALLINT idxCol = 0; idxCol < numCols; idxCol++) { auto& arrowColumnProducer = arrowArrayPrivateData[idxCol]; auto dataType = dataTypes[idxCol]; - auto columnSize = columnSizes[idxCol]; if (hasLobColumns) { assert(idxRowSql == 0 && "GetData only works one row at a time"); @@ -5520,17 +5599,21 @@ SQLRETURN FetchArrowBatch_wrap(SqlHandlePtr StatementHandle, py::list& capsules, case SQL_BINARY: case SQL_VARBINARY: case SQL_LONGVARBINARY: { - uint64_t fetchBufferSize = columnSize /* bytes are not null terminated */; auto target_vec = &arrowColumnProducer->varData; auto start = arrowColumnProducer->varVal[idxRowArrow]; EnsureNativeFetchBufferSize( *target_vec, CheckedAddSize(start, dataLen, "Arrow value buffer is too large"), reservedBytes); + const size_t sourceStride = hasLobColumns + ? buffers.charBuffers[idxCol].size() + : buffers.charBuffers[idxCol].size() / + static_cast(fetchSize); + const size_t sourceOffset = CheckedArrowSourceOffset( + buffers.charBuffers[idxCol], idxRowSql, sourceStride, dataLen); std::memcpy(&(*target_vec)[start], - &buffers.charBuffers[idxCol][idxRowSql * fetchBufferSize], - dataLen); + &buffers.charBuffers[idxCol][sourceOffset], dataLen); arrowColumnProducer->varVal[idxRowArrow + 1] = start + dataLen; break; } @@ -5538,21 +5621,21 @@ SQLRETURN FetchArrowBatch_wrap(SqlHandlePtr StatementHandle, py::list& capsules, case SQL_VARCHAR: case SQL_LONGVARCHAR: { if (charCtype == SQL_C_CHAR) { -#if defined(__APPLE__) || defined(__linux__) - uint64_t fetchBufferSize = columnSize * 4 + 1 /*null-terminator*/; -#else - uint64_t fetchBufferSize = columnSize + 1 /*null-terminator*/; -#endif auto target_vec = &arrowColumnProducer->varData; auto start = arrowColumnProducer->varVal[idxRowArrow]; EnsureNativeFetchBufferSize( *target_vec, CheckedAddSize(start, dataLen, "Arrow value buffer is too large"), reservedBytes); + const size_t sourceStride = hasLobColumns + ? buffers.charBuffers[idxCol].size() + : buffers.charBuffers[idxCol].size() / + static_cast(fetchSize); + const size_t sourceOffset = CheckedArrowSourceOffset( + buffers.charBuffers[idxCol], idxRowSql, sourceStride, dataLen); std::memcpy(&(*target_vec)[start], - &buffers.charBuffers[idxCol][idxRowSql * fetchBufferSize], - dataLen); + &buffers.charBuffers[idxCol][sourceOffset], dataLen); arrowColumnProducer->varVal[idxRowArrow + 1] = start + dataLen; break; } @@ -5563,10 +5646,17 @@ SQLRETURN FetchArrowBatch_wrap(SqlHandlePtr StatementHandle, py::list& capsules, case SQL_WVARCHAR: case SQL_WLONGVARCHAR: { // We have previously fetched these as WCHARs, even for SQL_CHAR types. - assert(dataLen % sizeof(SQLWCHAR) == 0); + if (dataLen % sizeof(SQLWCHAR) != 0) { + ThrowStdException("Wide-character data has an invalid byte length"); + } auto dataLenW = dataLen / sizeof(SQLWCHAR); - auto wcharSource = - &buffers.wcharBuffers[idxCol][idxRowSql * (columnSize + 1)]; + const size_t sourceStride = hasLobColumns + ? buffers.wcharBuffers[idxCol].size() + : buffers.wcharBuffers[idxCol].size() / + static_cast(fetchSize); + const size_t sourceOffset = CheckedArrowSourceOffset( + buffers.wcharBuffers[idxCol], idxRowSql, sourceStride, dataLen); + auto wcharSource = &buffers.wcharBuffers[idxCol][sourceOffset]; auto start = arrowColumnProducer->varVal[idxRowArrow]; auto target_vec = &arrowColumnProducer->varData; static_assert(sizeof(SQLWCHAR) == sizeof(char16_t)); diff --git a/tests/test_004_cursor.py b/tests/test_004_cursor.py index 54b5d3504..4441308b9 100644 --- a/tests/test_004_cursor.py +++ b/tests/test_004_cursor.py @@ -1758,6 +1758,18 @@ def test_setinputsizes_rejects_boolean_sizes(size_info): cursor.setinputsizes([size_info]) +def test_setinputsizes_rejects_excessive_column_size(): + cursor = mssql_python.Cursor.__new__(mssql_python.Cursor) + with pytest.raises(ValueError, match="column size"): + cursor.setinputsizes([(mssql_python.SQL_VARCHAR, 1 << 40, 0)]) + + +def test_executemany_rejects_excessive_cumulative_parameter_buffers(cursor): + cursor.setinputsizes([(mssql_python.SQL_WVARCHAR, 50_000_000, 0)] * 3) + with pytest.raises(RuntimeError, match="Parameter buffers exceed the 256 MiB allocation limit"): + cursor.executemany("SELECT ?, ?, ?", [("", "", "")]) + + def test_executemany_rejects_text_for_binary_parameter(cursor): cursor.setinputsizes([(mssql_python.SQL_VARBINARY, 10, 0)]) with pytest.raises(RuntimeError, match="object type does not match"): From 710ff123aa06fc9556a6a021b18f15c07bc32772 Mon Sep 17 00:00:00 2001 From: gargsaumya Date: Fri, 25 Sep 2026 15:18:52 +0530 Subject: [PATCH 05/31] FIX: Address native size validation review --- mssql_python/cursor.py | 15 +++- mssql_python/pybind/ddbc_bindings.cpp | 110 +++++++++++++++++--------- tests/test_004_cursor.py | 7 +- tests/test_004_cursor_arrow.py | 15 ++++ tests/test_024_bulkcopy_arrow.py | 4 + 5 files changed, 109 insertions(+), 42 deletions(-) diff --git a/mssql_python/cursor.py b/mssql_python/cursor.py index 54dcbc174..4c90f52a3 100644 --- a/mssql_python/cursor.py +++ b/mssql_python/cursor.py @@ -1224,7 +1224,14 @@ def setinputsizes(self, sizes: List[Union[int, tuple]]) -> None: isinstance(column_size, bool) or not isinstance(column_size, int) or column_size < 0 - or column_size > MAX_NATIVE_PARAMETER_SIZE + or ( + sql_type + not in ( + ddbc_sql_const.SQL_DECIMAL.value, + ddbc_sql_const.SQL_NUMERIC.value, + ) + and column_size > MAX_NATIVE_PARAMETER_SIZE + ) ): raise ValueError( f"Invalid column size: {column_size}. Must be a non-negative integer " @@ -3426,8 +3433,10 @@ def _bulkcopy_core_and_validate(table_name, batch_size, timeout): raise TypeError( f"batch_size must be a non-negative integer, got {type(batch_size).__name__}" ) - if batch_size < 0: - raise ValueError(f"batch_size must be non-negative, got {batch_size}") + if batch_size < 0 or batch_size > MAX_NATIVE_ROW_COUNT: + raise ValueError( + f"batch_size must be between 0 and {MAX_NATIVE_ROW_COUNT}, got {batch_size}" + ) if not isinstance(timeout, int) or isinstance(timeout, bool): raise TypeError(f"timeout must be a non-negative integer, got {type(timeout).__name__}") diff --git a/mssql_python/pybind/ddbc_bindings.cpp b/mssql_python/pybind/ddbc_bindings.cpp index aa5e276cf..53087c370 100644 --- a/mssql_python/pybind/ddbc_bindings.cpp +++ b/mssql_python/pybind/ddbc_bindings.cpp @@ -415,6 +415,7 @@ size_t ParameterArrayElementSize(const ParamInfo& info) { default: ThrowStdException("Unsupported C type for parameter array allocation"); } + return 0; } template @@ -4736,77 +4737,96 @@ SQLRETURN FetchBatchData(SQLHSTMT hStmt, ColumnBuffers& buffers, py::list& colum // Given a list of columns that are a part of single row in the result set, // calculates the max size of the row // TODO: Move to anonymous namespace, since it is not used outside this file -size_t calculateRowSize(py::list& columnNames, SQLUSMALLINT numCols) { - size_t rowSize = 0; +size_t calculateRowSize(py::list& columnNames, SQLUSMALLINT numCols, + int charCtype = SQL_C_WCHAR) { + size_t rowSize = CheckedMultiplySize(numCols, sizeof(SQLLEN), + "Native fetch indicator row size is too large"); for (SQLUSMALLINT col = 1; col <= numCols; col++) { auto columnMeta = columnNames[col - 1].cast(); SQLSMALLINT dataType = columnMeta["DataType"].cast(); SQLULEN columnSize = columnMeta["ColumnSize"].cast(); + size_t columnBytes = 0; switch (dataType) { case SQL_CHAR: case SQL_VARCHAR: - case SQL_LONGVARCHAR: - rowSize += columnSize; + case SQL_LONGVARCHAR: { + HandleZeroColumnSizeAtFetch(columnSize); + if (charCtype == SQL_C_WCHAR) { + columnBytes = CheckedMultiplySize( + CheckedAddSize(columnSize, 1, "Native fetch row size is too large"), + sizeof(SQLWCHAR), "Native fetch row size is too large"); + } else { +#if defined(__APPLE__) || defined(__linux__) + columnBytes = CheckedAddSize( + CheckedMultiplySize(columnSize, 4, + "Native fetch row size is too large"), + 1, "Native fetch row size is too large"); +#else + columnBytes = + CheckedAddSize(columnSize, 1, "Native fetch row size is too large"); +#endif + } break; + } case SQL_SS_XML: case SQL_WCHAR: case SQL_WVARCHAR: case SQL_WLONGVARCHAR: - rowSize += columnSize * sizeof(SQLWCHAR); + HandleZeroColumnSizeAtFetch(columnSize); + columnBytes = CheckedMultiplySize( + CheckedAddSize(columnSize, 1, "Native fetch row size is too large"), + sizeof(SQLWCHAR), "Native fetch row size is too large"); break; case SQL_INTEGER: - rowSize += sizeof(SQLINTEGER); + columnBytes = sizeof(SQLINTEGER); break; case SQL_SMALLINT: - rowSize += sizeof(SQLSMALLINT); + columnBytes = sizeof(SQLSMALLINT); break; case SQL_REAL: - rowSize += sizeof(SQLREAL); + columnBytes = sizeof(SQLREAL); break; case SQL_FLOAT: - rowSize += sizeof(SQLFLOAT); + columnBytes = sizeof(SQLFLOAT); break; case SQL_DOUBLE: - rowSize += sizeof(SQLDOUBLE); + columnBytes = sizeof(SQLDOUBLE); break; case SQL_DECIMAL: case SQL_NUMERIC: - rowSize += MAX_DIGITS_IN_NUMERIC; + columnBytes = MAX_DIGITS_IN_NUMERIC; break; case SQL_TIMESTAMP: case SQL_TYPE_TIMESTAMP: case SQL_DATETIME: - rowSize += sizeof(SQL_TIMESTAMP_STRUCT); + columnBytes = sizeof(SQL_TIMESTAMP_STRUCT); break; case SQL_BIGINT: - rowSize += sizeof(SQLBIGINT); + columnBytes = sizeof(SQLBIGINT); break; case SQL_TYPE_DATE: - rowSize += sizeof(SQL_DATE_STRUCT); + columnBytes = sizeof(SQL_DATE_STRUCT); break; case SQL_SS_TIME2: - rowSize += sizeof(SQL_SS_TIME2_STRUCT); + columnBytes = sizeof(SQL_SS_TIME2_STRUCT); break; case SQL_GUID: - rowSize += sizeof(SQLGUID); + columnBytes = sizeof(SQLGUID); break; case SQL_TINYINT: case SQL_BIT: - rowSize += sizeof(SQLCHAR); + columnBytes = sizeof(SQLCHAR); break; case SQL_SS_UDT: - rowSize += (static_cast(columnSize) == SQL_NO_TOTAL || columnSize == 0) - ? SQL_MAX_LOB_SIZE - : columnSize; - break; case SQL_BINARY: case SQL_VARBINARY: case SQL_LONGVARBINARY: - rowSize += columnSize; + HandleZeroColumnSizeAtFetch(columnSize); + columnBytes = columnSize; break; case SQL_SS_TIMESTAMPOFFSET: - rowSize += sizeof(DateTimeOffset); + columnBytes = sizeof(DateTimeOffset); break; default: std::string columnName = columnMeta["ColumnName"].cast(); @@ -4817,6 +4837,7 @@ size_t calculateRowSize(py::list& columnNames, SQLUSMALLINT numCols) { ThrowStdException(errorString.str()); break; } + rowSize = CheckedAddSize(rowSize, columnBytes, "Native fetch row size is too large"); } return rowSize; } @@ -5075,8 +5096,8 @@ SQLRETURN FetchArrowBatch_wrap(SqlHandlePtr StatementHandle, py::list& capsules, // Fetch narrow char data as SQL_C_CHAR if on Linux/macOS and configured by the user charCtype = EffectiveCharCtypeForFetch(charCtype, "utf-8"); - // An overly large fetch size doesn't seem to help performance - int fetchSize = 64; + // An overly large fetch size doesn't seem to help performance. + int fetchSize = arrowBatchSize > 0 ? std::min(arrowBatchSize, 64) : 1; SQLRETURN ret; SQLHSTMT hStmt = StatementHandle->get(); @@ -5291,15 +5312,28 @@ SQLRETURN FetchArrowBatch_wrap(SqlHandlePtr StatementHandle, py::list& capsules, std::memset(arrowColumnProducer->valid.get(), 0xFF, bitmapSize); } - // Initialize column buffers - ReserveNativeFetchBytes( - reservedBytes, - CheckedMultiplySize(static_cast(numCols), static_cast(fetchSize), - "Native fetch indicator buffer is too large"), - sizeof(SQLLEN)); - ColumnBuffers buffers(numCols, fetchSize); + if (arrowBatchSize > 0 && !hasLobColumns) { + const size_t rowSize = calculateRowSize(columnNames, numCols, charCtype); + const size_t remainingBytes = MAX_NATIVE_FETCH_BYTES - reservedBytes; + const size_t rowsWithinBudget = remainingBytes / rowSize; + if (rowsWithinBudget == 0) { + ThrowStdException("Native fetch buffers exceed the 256 MiB allocation limit"); + } + fetchSize = static_cast(std::min(fetchSize, rowsWithinBudget)); + } + + const int bufferFetchSize = arrowBatchSize > 0 ? fetchSize : 0; + if (bufferFetchSize > 0) { + ReserveNativeFetchBytes( + reservedBytes, + CheckedMultiplySize(static_cast(numCols), + static_cast(bufferFetchSize), + "Native fetch indicator buffer is too large"), + sizeof(SQLLEN)); + } + ColumnBuffers buffers(numCols, bufferFetchSize); - if (!hasLobColumns && fetchSize > 0) { + if (!hasLobColumns && bufferFetchSize > 0) { ret = SQLBindColums(hStmt, buffers, columnNames, numCols, fetchSize, reservedBytes, charCtype); if (!SQL_SUCCEEDED(ret)) { @@ -6090,14 +6124,12 @@ SQLRETURN FetchAll_wrap(SqlHandlePtr StatementHandle, py::list& rows, } // No LOBs detected - use binding path with batch fetching - // Define a memory limit (1 GB) - const size_t memoryLimit = 1ULL * 1024 * 1024 * 1024; - size_t totalRowSize = calculateRowSize(columnNames, numCols); + size_t totalRowSize = calculateRowSize(columnNames, numCols, charCtype); - // Calculate fetch size based on the total row size and memory limit + // Calculate fetch size from the same storage budget enforced during binding. size_t numRowsInMemLimit; if (totalRowSize > 0) { - numRowsInMemLimit = static_cast(memoryLimit / totalRowSize); + numRowsInMemLimit = MAX_NATIVE_FETCH_BYTES / totalRowSize; } else { // Handle case where totalRowSize is 0 to avoid division by zero. // This can happen for NVARCHAR(MAX) cols. SQLDescribeCol returns 0 @@ -6128,6 +6160,8 @@ SQLRETURN FetchAll_wrap(SqlHandlePtr StatementHandle, py::list& rows, } else { fetchSize = 1000; } + fetchSize = static_cast( + std::min(fetchSize, std::max(numRowsInMemLimit, 1))); LOG("FetchAll_wrap: Fetching data in batch sizes of %d", fetchSize); size_t reservedBytes = 0; diff --git a/tests/test_004_cursor.py b/tests/test_004_cursor.py index 4441308b9..c6154fe32 100644 --- a/tests/test_004_cursor.py +++ b/tests/test_004_cursor.py @@ -1767,7 +1767,7 @@ def test_setinputsizes_rejects_excessive_column_size(): def test_executemany_rejects_excessive_cumulative_parameter_buffers(cursor): cursor.setinputsizes([(mssql_python.SQL_WVARCHAR, 50_000_000, 0)] * 3) with pytest.raises(RuntimeError, match="Parameter buffers exceed the 256 MiB allocation limit"): - cursor.executemany("SELECT ?, ?, ?", [("", "", "")]) + cursor.executemany("SELECT ?, ?, ?", [(None, None, None)]) def test_executemany_rejects_text_for_binary_parameter(cursor): @@ -1783,6 +1783,11 @@ def test_fetchmany_rejects_excessive_native_buffer(cursor): assert cursor.fetchone()[0] == "x" +def test_fetchall_clamps_wide_result_batch_to_native_buffer_budget(cursor): + cursor.execute("SELECT " + ", ".join("CAST(N'x' AS NVARCHAR(4000))" for _ in range(34))) + assert cursor.fetchall() == [("x",) * 34] + + def test_description(cursor): """Test description""" cursor.execute("SELECT * FROM #pytest_all_data_types WHERE id = 1") diff --git a/tests/test_004_cursor_arrow.py b/tests/test_004_cursor_arrow.py index b815e524b..b6e4bfb80 100644 --- a/tests/test_004_cursor_arrow.py +++ b/tests/test_004_cursor_arrow.py @@ -225,6 +225,21 @@ def test_arrow_empty_fetch(cursor: mssql_python.Cursor): _test_arrow_test_data(cursor, [col_data], fetch_length=0) +def test_arrow_wide_schema_only_and_single_row_fit_native_buffer_budget( + cursor: mssql_python.Cursor, +): + columns = ", ".join(f"CAST(N'x' AS NVARCHAR(4000)) AS col_{index}" for index in range(600)) + cursor.execute(f"SELECT {columns}") + + schema_batch = cursor.arrow_batch(0) + assert schema_batch.num_rows == 0 + assert schema_batch.num_columns == 600 + + data_batch = cursor.arrow_batch(1) + assert data_batch.num_rows == 1 + assert data_batch.num_columns == 600 + + def test_arrow_table_batchsize_negative(cursor: mssql_python.Cursor): cursor.execute("select 1 a") with pytest.raises(ValueError, match="batch_size"): diff --git a/tests/test_024_bulkcopy_arrow.py b/tests/test_024_bulkcopy_arrow.py index ee9858262..c8718284b 100644 --- a/tests/test_024_bulkcopy_arrow.py +++ b/tests/test_024_bulkcopy_arrow.py @@ -202,6 +202,10 @@ def test_batch_size_negative(self): with pytest.raises(ValueError, match="batch_size"): _bare_cursor().bulkcopy_arrow("t", pa.table({"a": [1]}), batch_size=-1) + def test_batch_size_exceeds_native_row_limit(self): + with pytest.raises(ValueError, match="batch_size"): + _bare_cursor().bulkcopy_arrow("t", pa.table({"a": [1]}), batch_size=1_000_001) + def test_batch_size_bool_rejected(self): for flag in (False, True): with pytest.raises(TypeError, match="batch_size"): From a0966897fbd55c77e2726671e30cbd6cb8484bfc Mon Sep 17 00:00:00 2001 From: gargsaumya Date: Wed, 7 Oct 2026 11:33:04 +0530 Subject: [PATCH 06/31] FIX: Close remaining native size validation gaps Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- mssql_python/pybind/ddbc_bindings.cpp | 49 +++++++++++++++++++++------ mssql_python/pybind/param_detect.hpp | 14 +++++--- tests/test_023_execute_path_parity.py | 12 +++++++ 3 files changed, 61 insertions(+), 14 deletions(-) diff --git a/mssql_python/pybind/ddbc_bindings.cpp b/mssql_python/pybind/ddbc_bindings.cpp index 4b580df3a..fc4aa8658 100644 --- a/mssql_python/pybind/ddbc_bindings.cpp +++ b/mssql_python/pybind/ddbc_bindings.cpp @@ -349,6 +349,23 @@ size_t CheckedMultiplySize(size_t left, size_t right, const char* errorMessage) return left * right; } +size_t CheckedFetchColumnSize(SQLULEN columnSize) { + const size_t result = static_cast(columnSize); + if (static_cast(result) != columnSize) { + ThrowStdException("Column size is too large"); + } + return result; +} + +SQLLEN CheckedFetchBufferLength(size_t elementCount, size_t elementSize) { + const size_t byteCount = + CheckedMultiplySize(elementCount, elementSize, "Column fetch stride is too large"); + if (byteCount > static_cast(std::numeric_limits::max())) { + ThrowStdException("Column fetch stride is too large"); + } + return static_cast(byteCount); +} + constexpr int MAX_NATIVE_ROW_COUNT = 1000000; constexpr size_t MAX_NATIVE_FETCH_BYTES = 256ULL * 1024 * 1024; constexpr size_t MAX_NATIVE_PARAMETER_BYTES = 256ULL * 1024 * 1024; @@ -4449,23 +4466,30 @@ SQLRETURN SQLBindColums(SQLHSTMT hStmt, ColumnBuffers& buffers, const Metadata& case SQL_VARCHAR: case SQL_LONGVARCHAR: { HandleZeroColumnSizeAtFetch(columnSize); + const size_t baseColumnSize = CheckedFetchColumnSize(columnSize); if (useWideChar) { // Bind VARCHAR columns as SQL_C_WCHAR so the ODBC driver // returns UTF-16 data, avoiding code-page decode issues. - uint64_t fetchBufferSize = columnSize + 1 /*null-terminator*/; + const size_t fetchBufferSize = CheckedAddSize( + baseColumnSize, 1, "Column fetch stride is too large"); ResizeNativeFetchBuffer(buffers.wcharBuffers[col - 1], CheckedMultiplySize(fetchSize, fetchBufferSize, "Native fetch buffer is too large"), reservedBytes); ret = SQLBindCol_ptr( hStmt, col, SQL_C_WCHAR, buffers.wcharBuffers[col - 1].data(), - fetchBufferSize * sizeof(SQLWCHAR), buffers.indicators[col - 1].data()); + CheckedFetchBufferLength(fetchBufferSize, sizeof(SQLWCHAR)), + buffers.indicators[col - 1].data()); } else { // Original narrow-char path #if defined(__APPLE__) || defined(__linux__) - uint64_t fetchBufferSize = columnSize * 4 + 1 /*null-terminator*/; + const size_t fetchBufferSize = CheckedAddSize( + CheckedMultiplySize(baseColumnSize, 4, + "Column fetch stride is too large"), + 1, "Column fetch stride is too large"); #else - uint64_t fetchBufferSize = columnSize + 1 /*null-terminator*/; + const size_t fetchBufferSize = CheckedAddSize( + baseColumnSize, 1, "Column fetch stride is too large"); #endif ResizeNativeFetchBuffer(buffers.charBuffers[col - 1], CheckedMultiplySize(fetchSize, fetchBufferSize, @@ -4473,7 +4497,8 @@ SQLRETURN SQLBindColums(SQLHSTMT hStmt, ColumnBuffers& buffers, const Metadata& reservedBytes); ret = SQLBindCol_ptr( hStmt, col, SQL_C_CHAR, buffers.charBuffers[col - 1].data(), - fetchBufferSize * sizeof(SQLCHAR), buffers.indicators[col - 1].data()); + CheckedFetchBufferLength(fetchBufferSize, sizeof(SQLCHAR)), + buffers.indicators[col - 1].data()); } break; } @@ -4483,13 +4508,14 @@ SQLRETURN SQLBindColums(SQLHSTMT hStmt, ColumnBuffers& buffers, const Metadata& // TODO: handle variable length data correctly. This logic wont // suffice HandleZeroColumnSizeAtFetch(columnSize); - uint64_t fetchBufferSize = columnSize + 1 /*null-terminator*/; + const size_t fetchBufferSize = CheckedAddSize( + CheckedFetchColumnSize(columnSize), 1, "Column fetch stride is too large"); ResizeNativeFetchBuffer(buffers.wcharBuffers[col - 1], CheckedMultiplySize(fetchSize, fetchBufferSize, "Native fetch buffer is too large"), reservedBytes); ret = SQLBindCol_ptr(hStmt, col, SQL_C_WCHAR, buffers.wcharBuffers[col - 1].data(), - fetchBufferSize * sizeof(SQLWCHAR), + CheckedFetchBufferLength(fetchBufferSize, sizeof(SQLWCHAR)), buffers.indicators[col - 1].data()); break; } @@ -4572,17 +4598,20 @@ SQLRETURN SQLBindColums(SQLHSTMT hStmt, ColumnBuffers& buffers, const Metadata& case SQL_SS_UDT: case SQL_BINARY: case SQL_VARBINARY: - case SQL_LONGVARBINARY: + case SQL_LONGVARBINARY: { // TODO: handle variable length data correctly. This logic wont // suffice HandleZeroColumnSizeAtFetch(columnSize); + const size_t fetchBufferSize = CheckedFetchColumnSize(columnSize); ResizeNativeFetchBuffer(buffers.charBuffers[col - 1], - CheckedMultiplySize(fetchSize, columnSize, + CheckedMultiplySize(fetchSize, fetchBufferSize, "Native fetch buffer is too large"), reservedBytes); ret = SQLBindCol_ptr(hStmt, col, SQL_C_BINARY, buffers.charBuffers[col - 1].data(), - columnSize, buffers.indicators[col - 1].data()); + CheckedFetchBufferLength(fetchBufferSize, 1), + buffers.indicators[col - 1].data()); break; + } case SQL_SS_TIMESTAMPOFFSET: ResizeNativeFetchBuffer(buffers.datetimeoffsetBuffers[col - 1], fetchSize, reservedBytes); diff --git a/mssql_python/pybind/param_detect.hpp b/mssql_python/pybind/param_detect.hpp index 172a98b51..fe5cdeaa2 100644 --- a/mssql_python/pybind/param_detect.hpp +++ b/mssql_python/pybind/param_detect.hpp @@ -257,10 +257,16 @@ inline void ApplyInputSizeOverride(PyObject* params, PyObject* inputSize, Py_ssi } } - info.isDAE = - (PyUnicode_Check(obj) && PyLongGreaterThan(columnSize, MAX_INLINE_CHAR)) || - ((PyBytes_Check(obj) || PyByteArray_Check(obj)) && - PyLongGreaterThan(columnSize, MAX_INLINE_BINARY)); + const bool textNeedsDAE = + PyUnicode_Check(obj) && + (PyLongGreaterThan(columnSize, MAX_INLINE_CHAR) || + PyUnicode_GET_LENGTH(obj) > MAX_INLINE_CHAR); + const bool binaryNeedsDAE = + (PyBytes_Check(obj) || PyByteArray_Check(obj)) && + (PyLongGreaterThan(columnSize, MAX_INLINE_BINARY) || + (PyBytes_Check(obj) ? PyBytes_GET_SIZE(obj) : PyByteArray_GET_SIZE(obj)) > + MAX_INLINE_BINARY); + info.isDAE = textNeedsDAE || binaryNeedsDAE; if (PyTime_Check(obj) && info.paramCType == PARAM_C_TYPE_TEXT) { NormalizeTimeParam(params, index, info.columnSize); diff --git a/tests/test_023_execute_path_parity.py b/tests/test_023_execute_path_parity.py index d9bada323..9453cb85f 100644 --- a/tests/test_023_execute_path_parity.py +++ b/tests/test_023_execute_path_parity.py @@ -296,6 +296,18 @@ def test_setinputsizes_text_binding_normalizes_time(cursor): ) +@pytest.mark.parametrize( + ("value", "sql_type"), + [ + ("x" * 4001, ddbc_sql_const.SQL_VARCHAR.value), + (b"x" * 8001, ddbc_sql_const.SQL_VARBINARY.value), + ], +) +def test_setinputsizes_uses_actual_value_size_for_dae(cursor, value, sql_type): + """A too-small declared size cannot bypass streaming for a large value.""" + assert _override_roundtrip(cursor, value, sql_type, 1) == value + + @pytest.mark.parametrize("sql_type", [None, ddbc_sql_const.SQL_VARCHAR.value]) def test_time_isoformat_must_return_string(cursor, sql_type): """Native time normalization rejects a broken subclass contract on either path.""" From 231ae90a543c4ef2637990ed1d27568d942537d3 Mon Sep 17 00:00:00 2001 From: gargsaumya Date: Wed, 7 Oct 2026 11:43:24 +0530 Subject: [PATCH 07/31] FIX: Measure override payloads in native units Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- mssql_python/pybind/ddbc_bindings.cpp | 3 +- mssql_python/pybind/param_detect.hpp | 52 +++++++++++++++++++-------- tests/test_023_execute_path_parity.py | 1 + 3 files changed, 41 insertions(+), 15 deletions(-) diff --git a/mssql_python/pybind/ddbc_bindings.cpp b/mssql_python/pybind/ddbc_bindings.cpp index fc4aa8658..d19b4013a 100644 --- a/mssql_python/pybind/ddbc_bindings.cpp +++ b/mssql_python/pybind/ddbc_bindings.cpp @@ -2290,7 +2290,8 @@ SQLRETURN SQLExecute_wrap(const SqlHandlePtr statementHandle, // Run DetectParamTypes BEFORE SQLPrepare so that type-detection errors // (unsupported type, NaN Decimal, precision overflow) don't leave the // cursor in a half-prepared state. - std::vector paramInfos = DetectParamTypes(params.ptr(), input_sizes.ptr()); + std::vector paramInfos = + DetectParamTypes(params.ptr(), input_sizes.ptr(), charEncoding); RETCODE rc; bool already_prepared = is_stmt_prepared[0].cast(); diff --git a/mssql_python/pybind/param_detect.hpp b/mssql_python/pybind/param_detect.hpp index fe5cdeaa2..9d28c63a5 100644 --- a/mssql_python/pybind/param_detect.hpp +++ b/mssql_python/pybind/param_detect.hpp @@ -190,6 +190,29 @@ inline bool PyLongGreaterThan(PyObject* value, long long threshold) { return overflow > 0 || (overflow == 0 && result > threshold); } +inline Py_ssize_t UnicodeUtf16Length(PyObject* value) { + const Py_ssize_t length = PyUnicode_GET_LENGTH(value); + if (PyUnicode_KIND(value) <= PyUnicode_2BYTE_KIND) { + return length; + } + + Py_ssize_t utf16Length = 0; + const Py_UCS4* data = PyUnicode_4BYTE_DATA(value); + for (Py_ssize_t index = 0; index < length; ++index) { + utf16Length += data[index] > 0xFFFF ? 2 : 1; + } + return utf16Length; +} + +inline Py_ssize_t EncodedUnicodeLength(PyObject* value, const std::string& encoding) { + py::object encoded = + steal(PyUnicode_AsEncodedString(value, encoding.c_str(), "strict")); + if (!encoded) { + throw py::error_already_set(); + } + return PyBytes_GET_SIZE(encoded.ptr()); +} + inline PyObject* FormatDecimalParam(PyObject* params, Py_ssize_t index, PyObject* value) { py::object formatted = steal(PyObject_CallMethod(value, "__format__", "s", "f")); if (!formatted) throw py::error_already_set(); @@ -216,7 +239,7 @@ inline void NormalizeTimeParam(PyObject* params, Py_ssize_t index, SQLULEN& colu } inline void ApplyInputSizeOverride(PyObject* params, PyObject* inputSize, Py_ssize_t index, - ParamInfo& info) { + ParamInfo& info, const std::string& charEncoding) { py::tuple values = borrow(inputSize); info.paramSQLType = values[0].cast(); info.paramCType = values[1].cast(); @@ -257,16 +280,25 @@ inline void ApplyInputSizeOverride(PyObject* params, PyObject* inputSize, Py_ssi } } + Py_ssize_t actualTextLength = 0; + if (PyUnicode_Check(obj)) { + actualTextLength = info.paramCType == SQL_C_CHAR + ? EncodedUnicodeLength(obj, charEncoding) + : UnicodeUtf16Length(obj); + } const bool textNeedsDAE = PyUnicode_Check(obj) && (PyLongGreaterThan(columnSize, MAX_INLINE_CHAR) || - PyUnicode_GET_LENGTH(obj) > MAX_INLINE_CHAR); + actualTextLength > MAX_INLINE_CHAR); const bool binaryNeedsDAE = (PyBytes_Check(obj) || PyByteArray_Check(obj)) && (PyLongGreaterThan(columnSize, MAX_INLINE_BINARY) || (PyBytes_Check(obj) ? PyBytes_GET_SIZE(obj) : PyByteArray_GET_SIZE(obj)) > MAX_INLINE_BINARY); info.isDAE = textNeedsDAE || binaryNeedsDAE; + if (info.isDAE && !PyLongGreaterThan(columnSize, MAX_INLINE_BINARY)) { + info.columnSize = 0; + } if (PyTime_Check(obj) && info.paramCType == PARAM_C_TYPE_TEXT) { NormalizeTimeParam(params, index, info.columnSize); @@ -302,7 +334,8 @@ inline void ApplyInputSizeOverride(PyObject* params, PyObject* inputSize, Py_ssi // // Takes raw PyObject* lists. Caller guarantees params is a fresh copy (cursor.py // does list(actual_params)), so in-place mutation via PyList_SetItem is safe. -inline std::vector DetectParamTypes(PyObject* params, PyObject* inputSizes) { +inline std::vector DetectParamTypes(PyObject* params, PyObject* inputSizes, + const std::string& charEncoding = "utf-8") { PyTypeCache::initialize(); const Py_ssize_t n = PyList_GET_SIZE(params); @@ -318,7 +351,7 @@ inline std::vector DetectParamTypes(PyObject* params, PyObject* input info.isDAE = false; if (i < inputSizeCount) { - ApplyInputSizeOverride(params, PyList_GET_ITEM(inputSizes, i), i, info); + ApplyInputSizeOverride(params, PyList_GET_ITEM(inputSizes, i), i, info, charEncoding); continue; } @@ -404,16 +437,7 @@ inline std::vector DetectParamTypes(PyObject* params, PyObject* input unsigned int kind = PyUnicode_KIND(obj); const void* udata = PyUnicode_DATA(obj); - Py_ssize_t utf16_len; - if (kind <= PyUnicode_2BYTE_KIND) { - utf16_len = length; - } else { - utf16_len = 0; - const Py_UCS4* data = PyUnicode_4BYTE_DATA(obj); - for (Py_ssize_t j = 0; j < length; ++j) { - utf16_len += (data[j] > 0xFFFF) ? 2 : 1; - } - } + const Py_ssize_t utf16_len = UnicodeUtf16Length(obj); // Detect whether the string needs wide-char (NVARCHAR) or narrow (VARCHAR) binding. // PyUnicode_IS_COMPACT_ASCII is a struct field check (O(1)), not a content scan. diff --git a/tests/test_023_execute_path_parity.py b/tests/test_023_execute_path_parity.py index 9453cb85f..cc39db777 100644 --- a/tests/test_023_execute_path_parity.py +++ b/tests/test_023_execute_path_parity.py @@ -300,6 +300,7 @@ def test_setinputsizes_text_binding_normalizes_time(cursor): ("value", "sql_type"), [ ("x" * 4001, ddbc_sql_const.SQL_VARCHAR.value), + ("\U0001f600" * 3000, ddbc_sql_const.SQL_WVARCHAR.value), (b"x" * 8001, ddbc_sql_const.SQL_VARBINARY.value), ], ) From 444fb7f17c5eff596598592d166727abc0428ea2 Mon Sep 17 00:00:00 2001 From: gargsaumya Date: Wed, 7 Oct 2026 11:53:14 +0530 Subject: [PATCH 08/31] FIX: Enforce peak native fetch memory limit Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- mssql_python/pybind/ddbc_bindings.cpp | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/mssql_python/pybind/ddbc_bindings.cpp b/mssql_python/pybind/ddbc_bindings.cpp index 48963d516..a634144a7 100644 --- a/mssql_python/pybind/ddbc_bindings.cpp +++ b/mssql_python/pybind/ddbc_bindings.cpp @@ -474,8 +474,19 @@ void ReserveNativeFetchBytes(size_t& reservedBytes, size_t count, size_t element template void ResizeNativeFetchBuffer(std::vector& buffer, size_t count, size_t& reservedBytes) { + if (buffer.size() >= count) { + return; + } + + const size_t oldBytes = CheckedMultiplySize( + buffer.size(), sizeof(ElementType), "Native fetch buffer size is too large"); ReserveNativeFetchBytes(reservedBytes, count, sizeof(ElementType)); - buffer.resize(count); + { + std::vector resizedBuffer(count); + std::copy(buffer.begin(), buffer.end(), resizedBuffer.begin()); + buffer.swap(resizedBuffer); + } + reservedBytes -= oldBytes; } template @@ -488,8 +499,7 @@ void EnsureNativeFetchBufferSize(std::vector& buffer, size_t requir while (newSize < requiredSize) { newSize = CheckedMultiplySize(newSize, 2, "Native fetch buffer size is too large"); } - ReserveNativeFetchBytes(reservedBytes, newSize - buffer.size(), sizeof(ElementType)); - buffer.resize(newSize); + ResizeNativeFetchBuffer(buffer, newSize, reservedBytes); } template @@ -5442,8 +5452,7 @@ SQLRETURN GetDataVar(SQLHSTMT hStmt, SQLUSMALLINT colNumber, SQLSMALLINT cType, start = dataVec.size() - sizeNullTerminator; // Resize buffer for next iteration - ReserveNativeFetchBytes(reservedBytes, end - dataVec.size(), sizeof(T)); - dataVec.resize(end); + ResizeNativeFetchBuffer(dataVec, end, reservedBytes); } else { // Unexpected return code return ret; From d5b44313d98889cc062fca73f155c083edb45bc0 Mon Sep 17 00:00:00 2001 From: gargsaumya Date: Wed, 7 Oct 2026 11:59:37 +0530 Subject: [PATCH 09/31] FIX: Bound streamed LOB native memory Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- mssql_python/pybind/ddbc_bindings.cpp | 9 ++++++++- mssql_python/pybind/param_detect.hpp | 2 +- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/mssql_python/pybind/ddbc_bindings.cpp b/mssql_python/pybind/ddbc_bindings.cpp index a634144a7..2b995328b 100644 --- a/mssql_python/pybind/ddbc_bindings.cpp +++ b/mssql_python/pybind/ddbc_bindings.cpp @@ -3471,6 +3471,8 @@ py::object FetchLobColumnData(SQLHSTMT hStmt, SQLUSMALLINT colIndex, SQLSMALLINT py::handle messages) { PERF_TIMER("FetchLobColumnData"); std::vector buffer; + size_t reservedBytes = 0; + ReserveNativeFetchBytes(reservedBytes, DAE_CHUNK_SIZE, sizeof(char)); SQLRETURN ret = SQL_SUCCESS_WITH_INFO; int loopCount = 0; @@ -3540,7 +3542,11 @@ py::object FetchLobColumnData(SQLHSTMT hStmt, SQLUSMALLINT colIndex, SQLSMALLINT } } if (bytesRead > 0) { - buffer.insert(buffer.end(), chunk.begin(), chunk.begin() + bytesRead); + const size_t previousSize = buffer.size(); + const size_t requiredSize = + CheckedAddSize(previousSize, bytesRead, "LOB fetch buffer is too large"); + ResizeNativeFetchBuffer(buffer, requiredSize, reservedBytes); + std::memcpy(buffer.data() + previousSize, chunk.data(), bytesRead); LOG("FetchLobColumnData: Appended %zu bytes at loop %d", bytesRead, loopCount); } if (ret == SQL_SUCCESS) { @@ -3561,6 +3567,7 @@ py::object FetchLobColumnData(SQLHSTMT hStmt, SQLUSMALLINT colIndex, SQLSMALLINT ThrowStdException("Wide-character LOB data has an invalid byte length"); } size_t wcharCount = buffer.size() / sizeof(SQLWCHAR); + ReserveNativeFetchBytes(reservedBytes, wcharCount, sizeof(SQLWCHAR)); std::vector alignedBuf(wcharCount); std::memcpy(alignedBuf.data(), buffer.data(), buffer.size()); return py::cast(dupeSqlWCharAsUtf16Le(alignedBuf.data(), wcharCount)); diff --git a/mssql_python/pybind/param_detect.hpp b/mssql_python/pybind/param_detect.hpp index 9d28c63a5..ae0cf7bf8 100644 --- a/mssql_python/pybind/param_detect.hpp +++ b/mssql_python/pybind/param_detect.hpp @@ -296,7 +296,7 @@ inline void ApplyInputSizeOverride(PyObject* params, PyObject* inputSize, Py_ssi (PyBytes_Check(obj) ? PyBytes_GET_SIZE(obj) : PyByteArray_GET_SIZE(obj)) > MAX_INLINE_BINARY); info.isDAE = textNeedsDAE || binaryNeedsDAE; - if (info.isDAE && !PyLongGreaterThan(columnSize, MAX_INLINE_BINARY)) { + if (info.isDAE) { info.columnSize = 0; } From 3b60b8cd37389caf0b5ee6635cbd9a2bb795c489 Mon Sep 17 00:00:00 2001 From: gargsaumya Date: Wed, 7 Oct 2026 12:08:14 +0530 Subject: [PATCH 10/31] FIX: Validate executemany payload sizes Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- mssql_python/cursor.py | 25 ++++++--- mssql_python/pybind/ddbc_bindings.cpp | 77 ++++++++++++++++++++------- tests/test_023_execute_path_parity.py | 20 +++++++ 3 files changed, 97 insertions(+), 25 deletions(-) diff --git a/mssql_python/cursor.py b/mssql_python/cursor.py index 7ebc0c35b..39c2f6267 100644 --- a/mssql_python/cursor.py +++ b/mssql_python/cursor.py @@ -2580,6 +2580,7 @@ def executemany( # pylint: disable=too-many-locals,too-many-branches,too-many-s param_info = ddbc_bindings.ParamInfo parameters_type = [] any_dae = False + encoding_settings = self._get_encoding_settings() # Check if we have explicit input sizes set if self._inputsizes: @@ -2615,10 +2616,22 @@ def executemany( # pylint: disable=too-many-locals,too-many-branches,too-many-s ): c_type = ddbc_sql_const.SQL_C_CHAR.value - # Check if this should be a DAE (data at execution) parameter based on column size + # Check if this should be a DAE (data at execution) parameter. if sample_value is not None: - if isinstance(sample_value, str) and column_size > MAX_INLINE_CHAR: - is_dae = True + if isinstance(sample_value, str): + if c_type == ddbc_sql_const.SQL_C_CHAR.value: + text_size = max( + len(value.encode(encoding_settings["encoding"])) + for value in column + if isinstance(value, str) + ) + else: + text_size = max( + sum(2 if ord(char) > 0xFFFF else 1 for char in value) + for value in column + if isinstance(value, str) + ) + is_dae = column_size > MAX_INLINE_CHAR or text_size > MAX_INLINE_CHAR elif isinstance(sample_value, (bytes, bytearray)) and column_size > 8000: is_dae = True @@ -2652,6 +2665,9 @@ def executemany( # pylint: disable=too-many-locals,too-many-branches,too-many-s # Always ensure at least a minimum size of 1 for empty strings column_size = max(max_binary_size, 1) + if is_dae: + column_size = 0 + paraminfo = param_info() paraminfo.paramCType = c_type paraminfo.paramSQLType = sql_type @@ -2816,9 +2832,6 @@ def executemany( # pylint: disable=too-many-locals,too-many-branches,too-many-s processed_parameters ) - # Get encoding settings - encoding_settings = self._get_encoding_settings() - # Debug logging: emit batch metadata only. Never log parameter values or # row representations here -- rows may contain PII (SSNs, emails, # balances) that would leak into log files and APM/log shippers even diff --git a/mssql_python/pybind/ddbc_bindings.cpp b/mssql_python/pybind/ddbc_bindings.cpp index 2b995328b..d2a1f9d02 100644 --- a/mssql_python/pybind/ddbc_bindings.cpp +++ b/mssql_python/pybind/ddbc_bindings.cpp @@ -2569,16 +2569,41 @@ SQLRETURN BindParameterArray(SqlHandle& handle, SQLHANDLE hStmt, const py::list& std::memset(wcharArray + i * (info.columnSize + 1), 0, (info.columnSize + 1) * sizeof(SQLWCHAR)); } else { - std::u16string wstr = columnValues[i].cast(); - // u16string is already UTF-16, so the - // original check is sufficient - if (wstr.length() > info.columnSize) { + if (!py::isinstance(columnValues[i])) { + ThrowStdException( + MakeParamMismatchErrorStr(info.paramCType, paramIndex)); + } + if (static_cast( + PyUnicode_GET_LENGTH(columnValues[i].ptr())) > + info.columnSize) { ThrowStdException("Input string exceeds allowed column size " "at parameter index " + std::to_string(paramIndex)); } - std::memcpy(wcharArray + i * (info.columnSize + 1), wstr.c_str(), - (wstr.length() + 1) * sizeof(SQLWCHAR)); + py::object encoded = + columnValues[i].attr("encode")("utf-16-le", "strict"); + char* encodedData = nullptr; + Py_ssize_t encodedSize = 0; + if (PyBytes_AsStringAndSize(encoded.ptr(), &encodedData, + &encodedSize) != 0) { + throw py::error_already_set(); + } + if (encodedSize < 0 || + static_cast(encodedSize) % sizeof(SQLWCHAR) != 0) { + ThrowStdException( + "Wide-character parameter has an invalid byte length"); + } + const size_t wcharLength = + static_cast(encodedSize) / sizeof(SQLWCHAR); + if (wcharLength > info.columnSize) { + ThrowStdException("Input string exceeds allowed column size " + "at parameter index " + + std::to_string(paramIndex)); + } + SQLWCHAR* destination = wcharArray + i * elementWidth; + std::copy_n(reinterpret_cast(encodedData), + wcharLength, destination); + destination[wcharLength] = 0; strLenOrIndArray[i] = SQL_NTS; } } @@ -2679,19 +2704,24 @@ SQLRETURN BindParameterArray(SqlHandle& handle, SQLHANDLE hStmt, const py::list& ThrowStdException(MakeParamMismatchErrorStr(info.paramCType, paramIndex)); } - std::string encodedStr; - + py::object encoded; + char* encodedData = nullptr; + Py_ssize_t encodedSize = 0; if (py::isinstance(columnValues[i])) { // Use Python's codec system to encode the string with specified // encoding try { - py::object encoded = + encoded = columnValues[i].attr("encode")(charEncoding, "strict"); - encodedStr = encoded.cast(); + if (PyBytes_AsStringAndSize(encoded.ptr(), &encodedData, + &encodedSize) != 0) { + throw py::error_already_set(); + } LOG("BindParameterArray: param[%d] row[%zu] SQL_C_CHAR - " "Encoded with '%s', " "size=%zu bytes", - paramIndex, i, charEncoding.c_str(), encodedStr.size()); + paramIndex, i, charEncoding.c_str(), + static_cast(encodedSize)); } catch (const py::error_already_set& e) { LOG_ERROR("BindParameterArray: param[%d] row[%zu] SQL_C_CHAR - " "Failed to encode " @@ -2702,22 +2732,31 @@ SQLRETURN BindParameterArray(SqlHandle& handle, SQLHANDLE hStmt, const py::list& std::to_string(paramIndex) + " row " + std::to_string(i) + " with encoding '" + charEncoding + "': " + e.what()); } + } else if (py::isinstance(columnValues[i])) { + encoded = py::reinterpret_borrow(columnValues[i]); + if (PyBytes_AsStringAndSize(encoded.ptr(), &encodedData, + &encodedSize) != 0) { + throw py::error_already_set(); + } } else { - // bytes/bytearray - use as-is (already encoded) - encodedStr = columnValues[i].cast(); + encodedData = PyByteArray_AsString(columnValues[i].ptr()); + encodedSize = PyByteArray_Size(columnValues[i].ptr()); + if (encodedData == nullptr || encodedSize < 0) { + throw py::error_already_set(); + } } - if (encodedStr.size() > info.columnSize) { + const size_t dataSize = static_cast(encodedSize); + if (dataSize > info.columnSize) { LOG("BindParameterArray: String/binary too " "long - param_index=%d, row=%zu, size=%zu, " "max=%zu", - paramIndex, i, encodedStr.size(), info.columnSize); + paramIndex, i, dataSize, info.columnSize); ThrowStdException("Input exceeds column size at index " + std::to_string(i)); } - std::memcpy(charArray + i * (info.columnSize + 1), encodedStr.c_str(), - encodedStr.size()); - strLenOrIndArray[i] = static_cast(encodedStr.size()); + std::copy_n(encodedData, dataSize, charArray + i * elementWidth); + strLenOrIndArray[i] = static_cast(dataSize); } } LOG("BindParameterArray: SQL_C_CHAR/BINARY bound - " @@ -3546,7 +3585,7 @@ py::object FetchLobColumnData(SQLHSTMT hStmt, SQLUSMALLINT colIndex, SQLSMALLINT const size_t requiredSize = CheckedAddSize(previousSize, bytesRead, "LOB fetch buffer is too large"); ResizeNativeFetchBuffer(buffer, requiredSize, reservedBytes); - std::memcpy(buffer.data() + previousSize, chunk.data(), bytesRead); + std::copy_n(chunk.data(), bytesRead, buffer.data() + previousSize); LOG("FetchLobColumnData: Appended %zu bytes at loop %d", bytesRead, loopCount); } if (ret == SQL_SUCCESS) { diff --git a/tests/test_023_execute_path_parity.py b/tests/test_023_execute_path_parity.py index cc39db777..595ae9adc 100644 --- a/tests/test_023_execute_path_parity.py +++ b/tests/test_023_execute_path_parity.py @@ -309,6 +309,26 @@ def test_setinputsizes_uses_actual_value_size_for_dae(cursor, value, sql_type): assert _override_roundtrip(cursor, value, sql_type, 1) == value +@pytest.mark.parametrize( + ("value", "sql_type"), + [ + pytest.param("x" * 4001, ddbc_sql_const.SQL_VARCHAR.value, id="varchar-encoded-bytes"), + pytest.param( + "\U0001f600" * 2001, + ddbc_sql_const.SQL_WVARCHAR.value, + id="nvarchar-utf16-units", + ), + ], +) +def test_executemany_setinputsizes_uses_actual_text_size_for_dae(cursor, value, sql_type): + """A too-small declared size cannot bypass streaming for an array-bound text value.""" + cursor.execute("CREATE TABLE #dae_batch (value NVARCHAR(MAX))") + cursor.setinputsizes([(sql_type, 1, 0)]) + cursor.executemany("INSERT INTO #dae_batch (value) VALUES (?)", [(value,), ("short",)]) + cursor.execute("SELECT value FROM #dae_batch ORDER BY LEN(value) DESC") + assert cursor.fetchall() == [(value,), ("short",)] + + @pytest.mark.parametrize("sql_type", [None, ddbc_sql_const.SQL_VARCHAR.value]) def test_time_isoformat_must_return_string(cursor, sql_type): """Native time normalization rejects a broken subclass contract on either path.""" From 9f5c11af1563feda8f6724fa0f375936073cb421 Mon Sep 17 00:00:00 2001 From: gargsaumya Date: Wed, 7 Oct 2026 12:18:06 +0530 Subject: [PATCH 11/31] FIX: Copy encoded wide text as bytes Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- mssql_python/pybind/ddbc_bindings.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mssql_python/pybind/ddbc_bindings.cpp b/mssql_python/pybind/ddbc_bindings.cpp index d2a1f9d02..422e6fc8b 100644 --- a/mssql_python/pybind/ddbc_bindings.cpp +++ b/mssql_python/pybind/ddbc_bindings.cpp @@ -2601,8 +2601,8 @@ SQLRETURN BindParameterArray(SqlHandle& handle, SQLHANDLE hStmt, const py::list& std::to_string(paramIndex)); } SQLWCHAR* destination = wcharArray + i * elementWidth; - std::copy_n(reinterpret_cast(encodedData), - wcharLength, destination); + std::copy_n(encodedData, static_cast(encodedSize), + reinterpret_cast(destination)); destination[wcharLength] = 0; strLenOrIndArray[i] = SQL_NTS; } From 5b26c36bf155a64c23e7287d6559697a5b64522f Mon Sep 17 00:00:00 2001 From: gargsaumya Date: Wed, 7 Oct 2026 12:31:31 +0530 Subject: [PATCH 12/31] FIX: Stream DAE payloads in bounded chunks Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- mssql_python/cursor.py | 29 +++- mssql_python/pybind/ddbc_bindings.cpp | 196 +++++++++++++------------- mssql_python/pybind/param_detect.hpp | 26 +++- 3 files changed, 144 insertions(+), 107 deletions(-) diff --git a/mssql_python/cursor.py b/mssql_python/cursor.py index 39c2f6267..f80fdbeb4 100644 --- a/mssql_python/cursor.py +++ b/mssql_python/cursor.py @@ -14,6 +14,7 @@ import decimal import logging import uuid +import codecs import datetime import warnings from typing import List, Mapping, Union, Any, Optional, Tuple, Sequence, TYPE_CHECKING, Iterable @@ -54,6 +55,22 @@ # Bound each native fetch allocation; Arrow initially reserves 42 bytes per variable-width row. MAX_NATIVE_ROW_COUNT: int = 1_000_000 MAX_NATIVE_PARAMETER_SIZE: int = 256 * 1024 * 1024 + + +def _encoded_length_exceeds(value: str, encoding: str, limit: int) -> bool: + encoder = codecs.getincrementalencoder(encoding)(errors="strict") + total = 0 + chunk_size = 4096 + for offset in range(0, len(value), chunk_size): + end = min(offset + chunk_size, len(value)) + total += len(encoder.encode(value[offset:end], final=end == len(value))) + if total > limit: + return True + if not value: + total += len(encoder.encode("", final=True)) + return total > limit + + # SQL BIGINT is a signed 64-bit integer. Ints outside this range have no BIGINT # encoding and must be rejected at detect time on both paths (see _map_sql_type). BIGINT_MIN: int = -(2**63) @@ -2620,8 +2637,12 @@ def executemany( # pylint: disable=too-many-locals,too-many-branches,too-many-s if sample_value is not None: if isinstance(sample_value, str): if c_type == ddbc_sql_const.SQL_C_CHAR.value: - text_size = max( - len(value.encode(encoding_settings["encoding"])) + is_dae = column_size > MAX_INLINE_CHAR or any( + _encoded_length_exceeds( + value, + encoding_settings["encoding"], + MAX_INLINE_CHAR, + ) for value in column if isinstance(value, str) ) @@ -2631,7 +2652,9 @@ def executemany( # pylint: disable=too-many-locals,too-many-branches,too-many-s for value in column if isinstance(value, str) ) - is_dae = column_size > MAX_INLINE_CHAR or text_size > MAX_INLINE_CHAR + is_dae = ( + column_size > MAX_INLINE_CHAR or text_size > MAX_INLINE_CHAR + ) elif isinstance(sample_value, (bytes, bytearray)) and column_size > 8000: is_dae = True diff --git a/mssql_python/pybind/ddbc_bindings.cpp b/mssql_python/pybind/ddbc_bindings.cpp index 422e6fc8b..66fea9790 100644 --- a/mssql_python/pybind/ddbc_bindings.cpp +++ b/mssql_python/pybind/ddbc_bindings.cpp @@ -535,6 +535,78 @@ static SQLRETURN stream_dae_chunks(const void* data, size_t total_bytes, PutData return SQL_SUCCESS; } +template +static SQLRETURN stream_unicode_dae_chunks(PyObject* value, const std::string& encoding, + PutDataFn put_data_fn) { + py::object encoderFactory = + py::module_::import("codecs").attr("getincrementalencoder")(encoding); + py::object encoder = encoderFactory("strict"); + const Py_ssize_t length = PyUnicode_GET_LENGTH(value); + constexpr Py_ssize_t chunkSize = 4096; + for (Py_ssize_t offset = 0; offset < length; offset += chunkSize) { + const Py_ssize_t end = std::min(offset + chunkSize, length); + py::object chunk = steal(PyUnicode_Substring(value, offset, end)); + if (!chunk) throw py::error_already_set(); + py::object encoded = encoder.attr("encode")(chunk, end == length); + char* data = nullptr; + Py_ssize_t size = 0; + if (PyBytes_AsStringAndSize(encoded.ptr(), &data, &size) != 0) { + throw py::error_already_set(); + } + if (size > 0) { + SQLRETURN rc = put_data_fn(data, static_cast(size)); + if (!SQL_SUCCEEDED(rc)) return rc; + } + } + if (length == 0) { + py::object encoded = encoder.attr("encode")(py::str(), true); + char* data = nullptr; + Py_ssize_t size = 0; + if (PyBytes_AsStringAndSize(encoded.ptr(), &data, &size) != 0) { + throw py::error_already_set(); + } + if (size > 0) return put_data_fn(data, static_cast(size)); + } + return SQL_SUCCESS; +} + +static SQLRETURN StreamDAEParameter(SQLHSTMT hStmt, const ParamInfo& info, + const std::string& charEncoding) { + PyObject* value = info.dataPtr.ptr(); + if (!value || value == Py_None) { + py::gil_scoped_release release; + return SQLPutData_ptr(hStmt, nullptr, 0); + } + + auto putImmutableData = [&](SQLPOINTER data, SQLLEN length) { + py::gil_scoped_release release; + return SQLPutData_ptr(hStmt, data, length); + }; + if (PyUnicode_Check(value)) { + if (info.paramCType == SQL_C_WCHAR) { + return stream_unicode_dae_chunks(value, "utf-16-le", putImmutableData); + } + if (info.paramCType == SQL_C_CHAR) { + return stream_unicode_dae_chunks(value, charEncoding, putImmutableData); + } + ThrowStdException("DAE only supports text C types for str values"); + } + if (PyBytes_Check(value)) { + return stream_dae_chunks(PyBytes_AS_STRING(value), + static_cast(PyBytes_GET_SIZE(value)), + putImmutableData); + } + if (PyByteArray_Check(value)) { + auto putMutableData = [&](SQLPOINTER data, SQLLEN length) { + return SQLPutData_ptr(hStmt, data, length); + }; + return stream_dae_chunks(PyByteArray_AS_STRING(value), + static_cast(PyByteArray_GET_SIZE(value)), + putMutableData); + } + ThrowStdException("DAE only supports str, bytes, or bytearray values"); +} + // GH-610: Resolve SQL type for a NULL parameter using per-handle cache. // On cache miss, calls SQLDescribeParam and stores the result. static DescribedParamInfo ResolveNullParamType(SqlHandle& handle, SQLHANDLE hStmt, int paramIndex) { @@ -2377,10 +2449,6 @@ SQLRETURN SQLExecute_wrap(const SqlHandlePtr statementHandle, // GIL is released around each ODBC call to match slow-path concurrency. if (rc == SQL_NEED_DATA) { SQLPOINTER paramToken = nullptr; - auto putData = [&](SQLPOINTER data, SQLLEN len) { - py::gil_scoped_release release; - return SQLPutData_ptr(hStmt, data, len); - }; while (true) { { py::gil_scoped_release release; @@ -2399,58 +2467,8 @@ SQLRETURN SQLExecute_wrap(const SqlHandlePtr statementHandle, if (matchedInfo < first || matchedInfo >= last) { ThrowStdException("SQLExecute: unrecognized paramToken from SQLParamData"); } - PyObject* pyObj = matchedInfo->dataPtr.ptr(); - if (!pyObj || pyObj == Py_None) { - py::gil_scoped_release release; - SQLPutData_ptr(hStmt, nullptr, 0); - continue; - } - - if (PyUnicode_Check(pyObj)) { - if (matchedInfo->paramCType == SQL_C_WCHAR) { - std::u16string u16 = - borrow(pyObj).cast(); - rc = stream_dae_chunks( - reinterpretU16stringAsSqlWChar(u16), - u16.size() * sizeof(SQLWCHAR), - putData); - if (!SQL_SUCCEEDED(rc)) return rc; - } else if (matchedInfo->paramCType == SQL_C_CHAR) { - std::string encodedStr; - py::object encoded = borrow(pyObj) - .attr("encode")(charEncoding, "strict"); - encodedStr = encoded.cast(); - rc = stream_dae_chunks(encodedStr.data(), encodedStr.size(), putData); - if (!SQL_SUCCEEDED(rc)) return rc; - } else { - ThrowStdException("SQLExecute: unsupported C type for str in DAE"); - } - } else if (PyBytes_Check(pyObj) || PyByteArray_Check(pyObj)) { - // matchedInfo->dataPtr holds a strong ref to pyObj for the whole loop. - const char* dataPtr = nullptr; - size_t totalBytes = 0; - std::string bytesStorage; // only used for the bytearray copy below - - if (PyBytes_Check(pyObj)) { - // bytes is immutable and kept alive by the strong ref above, so stream - // straight from its internal buffer with no copy. This is the large-blob - // DAE path, so skipping a full payload copy is the whole point. - dataPtr = PyBytes_AS_STRING(pyObj); - totalBytes = static_cast(PyBytes_GET_SIZE(pyObj)); - } else { - // bytearray is mutable and the GIL is released mid-stream, so copy to a - // stable buffer before streaming. - bytesStorage.assign(PyByteArray_AS_STRING(pyObj), - static_cast(PyByteArray_GET_SIZE(pyObj))); - dataPtr = bytesStorage.data(); - totalBytes = bytesStorage.size(); - } - - rc = stream_dae_chunks(dataPtr, totalBytes, putData); - if (!SQL_SUCCEEDED(rc)) return rc; - } else { - ThrowStdException("SQLExecute: DAE only supported for str or bytes"); - } + rc = StreamDAEParameter(hStmt, *matchedInfo, charEncoding); + if (!SQL_SUCCEEDED(rc)) return rc; } if (!SQL_SUCCEEDED(rc) && rc != SQL_NO_DATA) return rc; } @@ -3245,11 +3263,23 @@ SQLRETURN SQLExecuteMany_wrap(const SqlHandlePtr statementHandle, const std::u16 return rc; } else { LOG("SQLExecuteMany: Using DAE (data-at-execution) - row_count=%zu", - columnwise_params.size()); - size_t rowCount = columnwise_params.size(); + paramSetSize); + if (columnwise_params.size() != paramInfos.size()) { + ThrowStdException("Parameter count does not match parameter metadata count"); + } + rc = SQLSetStmtAttr_ptr(hStmt, SQL_ATTR_PARAMSET_SIZE, (SQLPOINTER)1, 0); + if (!SQL_SUCCEEDED(rc)) return rc; + size_t rowCount = paramSetSize; for (size_t rowIndex = 0; rowIndex < rowCount; ++rowIndex) { LOG("SQLExecuteMany: Processing DAE row %zu of %zu", rowIndex + 1, rowCount); - py::list rowParams = columnwise_params[rowIndex]; + py::list rowParams; + for (py::handle column : columnwise_params) { + py::sequence values = py::reinterpret_borrow(column); + if (static_cast(py::len(values)) != paramSetSize) { + ThrowStdException("Parameter column length does not match parameter set size"); + } + rowParams.append(values[rowIndex]); + } std::vector> paramBuffers; rc = BindParameters(*statementHandle, hStmt, rowParams, paramInfos, @@ -3284,47 +3314,15 @@ SQLRETURN SQLExecuteMany_wrap(const SqlHandlePtr statementHandle, const std::u16 return rc; } - py::object* py_obj_ptr = reinterpret_cast(token); - if (!py_obj_ptr) { - LOG("SQLExecuteMany: NULL token pointer in DAE - chunk=%zu", dae_chunk_count); - return SQL_ERROR; - } - - if (py::isinstance(*py_obj_ptr)) { - std::string data = py_obj_ptr->cast(); - SQLLEN data_len = static_cast(data.size()); - LOG("SQLExecuteMany: Sending string DAE data - chunk=%zu, " - "length=%lld", - dae_chunk_count, static_cast(data_len)); - rc = [&] { - py::gil_scoped_release release; - return SQLPutData_ptr(hStmt, (SQLPOINTER)data.c_str(), data_len); - }(); - if (!SQL_SUCCEEDED(rc) && rc != SQL_NEED_DATA) { - LOG("SQLExecuteMany: SQLPutData(string) failed - " - "chunk=%zu, rc=%d", - dae_chunk_count, rc); - } - } else if (py::isinstance(*py_obj_ptr) || - py::isinstance(*py_obj_ptr)) { - std::string data = py_obj_ptr->cast(); - SQLLEN data_len = static_cast(data.size()); - LOG("SQLExecuteMany: Sending bytes/bytearray DAE data - " - "chunk=%zu, length=%lld", - dae_chunk_count, static_cast(data_len)); - rc = [&] { - py::gil_scoped_release release; - return SQLPutData_ptr(hStmt, (SQLPOINTER)data.c_str(), data_len); - }(); - if (!SQL_SUCCEEDED(rc) && rc != SQL_NEED_DATA) { - LOG("SQLExecuteMany: SQLPutData(bytes) failed - " - "chunk=%zu, rc=%d", - dae_chunk_count, rc); - } - } else { - LOG("SQLExecuteMany: Unsupported DAE data type - chunk=%zu", dae_chunk_count); - return SQL_ERROR; + const ParamInfo* matchedInfo = reinterpret_cast(token); + const ParamInfo* first = paramInfos.data(); + const ParamInfo* last = first + paramInfos.size(); + if (matchedInfo < first || matchedInfo >= last) { + ThrowStdException( + "SQLExecuteMany: unrecognized paramToken from SQLParamData"); } + rc = StreamDAEParameter(hStmt, *matchedInfo, charEncoding); + if (!SQL_SUCCEEDED(rc)) return rc; dae_chunk_count++; } LOG("SQLExecuteMany: DAE completed for row %zu - total_chunks=%zu, " diff --git a/mssql_python/pybind/param_detect.hpp b/mssql_python/pybind/param_detect.hpp index ae0cf7bf8..f8a3e7d4f 100644 --- a/mssql_python/pybind/param_detect.hpp +++ b/mssql_python/pybind/param_detect.hpp @@ -205,12 +205,28 @@ inline Py_ssize_t UnicodeUtf16Length(PyObject* value) { } inline Py_ssize_t EncodedUnicodeLength(PyObject* value, const std::string& encoding) { - py::object encoded = - steal(PyUnicode_AsEncodedString(value, encoding.c_str(), "strict")); - if (!encoded) { - throw py::error_already_set(); + py::object encoderFactory = + py::module_::import("codecs").attr("getincrementalencoder")(encoding); + py::object encoder = encoderFactory("strict"); + const Py_ssize_t length = PyUnicode_GET_LENGTH(value); + constexpr Py_ssize_t chunkSize = 4096; + Py_ssize_t total = 0; + for (Py_ssize_t offset = 0; offset < length; offset += chunkSize) { + const Py_ssize_t end = std::min(offset + chunkSize, length); + py::object chunk = steal(PyUnicode_Substring(value, offset, end)); + if (!chunk) throw py::error_already_set(); + py::object encoded = encoder.attr("encode")(chunk, end == length); + const Py_ssize_t encodedSize = PyBytes_GET_SIZE(encoded.ptr()); + if (encodedSize > MAX_INLINE_CHAR - total) { + return MAX_INLINE_CHAR + 1; + } + total += encodedSize; + } + if (length == 0) { + py::object encoded = encoder.attr("encode")(py::str(), true); + total = PyBytes_GET_SIZE(encoded.ptr()); } - return PyBytes_GET_SIZE(encoded.ptr()); + return total; } inline PyObject* FormatDecimalParam(PyObject* params, Py_ssize_t index, PyObject* value) { From 3a57124f6337af1ab338d1de67690ce37e295381 Mon Sep 17 00:00:00 2001 From: gargsaumya Date: Wed, 7 Oct 2026 12:42:46 +0530 Subject: [PATCH 13/31] FIX: Complete bounded DAE streaming Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- mssql_python/pybind/ddbc_bindings.cpp | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/mssql_python/pybind/ddbc_bindings.cpp b/mssql_python/pybind/ddbc_bindings.cpp index 66fea9790..15808e442 100644 --- a/mssql_python/pybind/ddbc_bindings.cpp +++ b/mssql_python/pybind/ddbc_bindings.cpp @@ -525,6 +525,7 @@ std::string DescribeChar(unsigned char ch) { template // The callable hides whether the caller wraps SQLPutData with GIL management; chunk sizing stays shared. static SQLRETURN stream_dae_chunks(const void* data, size_t total_bytes, PutDataFn put_data_fn) { + if (total_bytes == 0) return put_data_fn(nullptr, 0); const char* bytes = static_cast(data); for (size_t offset = 0; offset < total_bytes; offset += DAE_CHUNK_SIZE) { size_t len = std::min(static_cast(DAE_CHUNK_SIZE), total_bytes - offset); @@ -566,6 +567,7 @@ static SQLRETURN stream_unicode_dae_chunks(PyObject* value, const std::string& e throw py::error_already_set(); } if (size > 0) return put_data_fn(data, static_cast(size)); + return put_data_fn(nullptr, 0); } return SQL_SUCCESS; } @@ -597,12 +599,20 @@ static SQLRETURN StreamDAEParameter(SQLHSTMT hStmt, const ParamInfo& info, putImmutableData); } if (PyByteArray_Check(value)) { - auto putMutableData = [&](SQLPOINTER data, SQLLEN length) { - return SQLPutData_ptr(hStmt, data, length); - }; - return stream_dae_chunks(PyByteArray_AS_STRING(value), - static_cast(PyByteArray_GET_SIZE(value)), - putMutableData); + const size_t totalBytes = static_cast(PyByteArray_GET_SIZE(value)); + if (totalBytes == 0) return putImmutableData(nullptr, 0); + std::vector chunk(std::min(static_cast(DAE_CHUNK_SIZE), totalBytes)); + for (size_t offset = 0; offset < totalBytes; offset += chunk.size()) { + const size_t currentSize = static_cast(PyByteArray_GET_SIZE(value)); + const size_t length = std::min(chunk.size(), totalBytes - offset); + if (currentSize < offset + length) { + ThrowStdException("bytearray changed size during DAE streaming"); + } + std::copy_n(PyByteArray_AS_STRING(value) + offset, length, chunk.data()); + SQLRETURN rc = putImmutableData(chunk.data(), static_cast(length)); + if (!SQL_SUCCEEDED(rc)) return rc; + } + return SQL_SUCCESS; } ThrowStdException("DAE only supports str, bytes, or bytearray values"); } @@ -3313,6 +3323,7 @@ SQLRETURN SQLExecuteMany_wrap(const SqlHandlePtr statementHandle, const std::u16 dae_chunk_count, rc); return rc; } + if (rc != SQL_NEED_DATA) break; const ParamInfo* matchedInfo = reinterpret_cast(token); const ParamInfo* first = paramInfos.data(); From c64fbddbc96558f335a795122eae27a00f78c30e Mon Sep 17 00:00:00 2001 From: gargsaumya Date: Wed, 7 Oct 2026 12:53:04 +0530 Subject: [PATCH 14/31] FIX: Align DAE and array metadata Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- mssql_python/cursor.py | 51 +++++++++++++++------------ mssql_python/pybind/ddbc_bindings.cpp | 7 ++-- tests/test_023_execute_path_parity.py | 10 ++++++ 3 files changed, 43 insertions(+), 25 deletions(-) diff --git a/mssql_python/cursor.py b/mssql_python/cursor.py index f80fdbeb4..b6cd81cf2 100644 --- a/mssql_python/cursor.py +++ b/mssql_python/cursor.py @@ -2633,30 +2633,35 @@ def executemany( # pylint: disable=too-many-locals,too-many-branches,too-many-s ): c_type = ddbc_sql_const.SQL_C_CHAR.value - # Check if this should be a DAE (data at execution) parameter. - if sample_value is not None: - if isinstance(sample_value, str): - if c_type == ddbc_sql_const.SQL_C_CHAR.value: - is_dae = column_size > MAX_INLINE_CHAR or any( - _encoded_length_exceeds( - value, - encoding_settings["encoding"], - MAX_INLINE_CHAR, - ) - for value in column - if isinstance(value, str) - ) - else: - text_size = max( - sum(2 if ord(char) > 0xFFFF else 1 for char in value) - for value in column - if isinstance(value, str) - ) - is_dae = ( - column_size > MAX_INLINE_CHAR or text_size > MAX_INLINE_CHAR + # Check every compatible value before selecting array binding. Mixed + # str/bytes columns cannot rely on the sample value's type. + if c_type in ( + ddbc_sql_const.SQL_C_CHAR.value, + ddbc_sql_const.SQL_C_WCHAR.value, + ): + text_values = [value for value in column if isinstance(value, str)] + if c_type == ddbc_sql_const.SQL_C_CHAR.value: + text_is_large = any( + _encoded_length_exceeds( + value, + encoding_settings["encoding"], + MAX_INLINE_CHAR, ) - elif isinstance(sample_value, (bytes, bytearray)) and column_size > 8000: - is_dae = True + for value in text_values + ) + binary_is_large = any( + len(value) > MAX_INLINE_CHAR + for value in column + if isinstance(value, (bytes, bytearray)) + ) + else: + text_is_large = any( + sum(2 if ord(char) > 0xFFFF else 1 for char in value) + > MAX_INLINE_CHAR + for value in text_values + ) + binary_is_large = False + is_dae = column_size > MAX_INLINE_CHAR or text_is_large or binary_is_large # Sanitize precision/scale for numeric types if sql_type in ( diff --git a/mssql_python/pybind/ddbc_bindings.cpp b/mssql_python/pybind/ddbc_bindings.cpp index 15808e442..6dd960106 100644 --- a/mssql_python/pybind/ddbc_bindings.cpp +++ b/mssql_python/pybind/ddbc_bindings.cpp @@ -391,7 +391,7 @@ void ReserveNativeParameterBytes(size_t& reservedBytes, size_t count, size_t ele size_t ParameterArrayElementSize(const ParamInfo& info) { switch (info.paramCType) { case SQL_C_LONG: - return sizeof(int); + return sizeof(int64_t); case SQL_C_DOUBLE: return sizeof(double); case SQL_C_WCHAR: @@ -773,6 +773,9 @@ SQLRETURN BindParameters(SqlHandle& handle, SQLHANDLE hStmt, const py::list& par for (int paramIndex = 0; paramIndex < params.size(); paramIndex++) { const auto& param = params[paramIndex]; ParamInfo& paramInfo = paramInfos[paramIndex]; + if (paramInfo.isDAE) { + paramInfo.dataPtr = py::reinterpret_borrow(param); + } LOG("BindParameters: Processing param[%d] - C_Type=%d, SQL_Type=%d, " "ColumnSize=%lu, DecimalDigits=%d, InputOutputType=%d", paramIndex, paramInfo.paramCType, paramInfo.paramSQLType, @@ -4725,7 +4728,7 @@ SQLRETURN SQLBindColums(SQLHSTMT hStmt, ColumnBuffers& buffers, const Metadata& reservedBytes); ret = SQLBindCol_ptr(hStmt, col, SQL_C_SS_TIMESTAMPOFFSET, buffers.datetimeoffsetBuffers[col - 1].data(), - sizeof(DateTimeOffset) * fetchSize, + sizeof(DateTimeOffset), buffers.indicators[col - 1].data()); break; default: diff --git a/tests/test_023_execute_path_parity.py b/tests/test_023_execute_path_parity.py index 595ae9adc..3a3c38935 100644 --- a/tests/test_023_execute_path_parity.py +++ b/tests/test_023_execute_path_parity.py @@ -329,6 +329,16 @@ def test_executemany_setinputsizes_uses_actual_text_size_for_dae(cursor, value, assert cursor.fetchall() == [(value,), ("short",)] +def test_executemany_setinputsizes_scans_mixed_text_values_for_dae(cursor): + """A bytes sample cannot hide a later oversized string from DAE selection.""" + value = "x" * 4001 + cursor.execute("CREATE TABLE #mixed_dae_batch (value VARCHAR(MAX))") + cursor.setinputsizes([(ddbc_sql_const.SQL_VARCHAR.value, 1, 0)]) + cursor.executemany("INSERT INTO #mixed_dae_batch (value) VALUES (?)", [(b"x",), (value,)]) + cursor.execute("SELECT value FROM #mixed_dae_batch ORDER BY LEN(value) DESC") + assert cursor.fetchall() == [(value,), ("x",)] + + @pytest.mark.parametrize("sql_type", [None, ddbc_sql_const.SQL_VARCHAR.value]) def test_time_isoformat_must_return_string(cursor, sql_type): """Native time normalization rejects a broken subclass contract on either path.""" From b8a8873f37839f1f4591e28cea07289f24b3fc4c Mon Sep 17 00:00:00 2001 From: gargsaumya Date: Wed, 7 Oct 2026 13:14:41 +0530 Subject: [PATCH 15/31] FIX: Satisfy strict DAE return analysis Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- mssql_python/pybind/ddbc_bindings.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/mssql_python/pybind/ddbc_bindings.cpp b/mssql_python/pybind/ddbc_bindings.cpp index 6dd960106..f20549905 100644 --- a/mssql_python/pybind/ddbc_bindings.cpp +++ b/mssql_python/pybind/ddbc_bindings.cpp @@ -615,6 +615,7 @@ static SQLRETURN StreamDAEParameter(SQLHSTMT hStmt, const ParamInfo& info, return SQL_SUCCESS; } ThrowStdException("DAE only supports str, bytes, or bytearray values"); + return SQL_ERROR; } // GH-610: Resolve SQL type for a NULL parameter using per-handle cache. From 9a55372eb6b8f5611713b4727b1e4f669b7b09a8 Mon Sep 17 00:00:00 2001 From: gargsaumya Date: Wed, 7 Oct 2026 13:36:12 +0530 Subject: [PATCH 16/31] FIX: Close remaining native size gaps Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- mssql_python/pybind/ddbc_bindings.cpp | 33 +++++++++++++-------------- mssql_python/pybind/param_detect.hpp | 5 ++++ tests/test_023_execute_path_parity.py | 7 ++++++ 3 files changed, 28 insertions(+), 17 deletions(-) diff --git a/mssql_python/pybind/ddbc_bindings.cpp b/mssql_python/pybind/ddbc_bindings.cpp index f20549905..20987ed1c 100644 --- a/mssql_python/pybind/ddbc_bindings.cpp +++ b/mssql_python/pybind/ddbc_bindings.cpp @@ -391,7 +391,7 @@ void ReserveNativeParameterBytes(size_t& reservedBytes, size_t count, size_t ele size_t ParameterArrayElementSize(const ParamInfo& info) { switch (info.paramCType) { case SQL_C_LONG: - return sizeof(int64_t); + return sizeof(int); case SQL_C_DOUBLE: return sizeof(double); case SQL_C_WCHAR: @@ -883,8 +883,7 @@ SQLRETURN BindParameters(SqlHandle& handle, SQLHANDLE hStmt, const py::list& par break; } case SQL_C_WCHAR: { - if (!py::isinstance(param) && !py::isinstance(param) && - !py::isinstance(param)) { + if (!py::isinstance(param)) { ThrowStdException(MakeParamMismatchErrorStr(paramInfo.paramCType, paramIndex)); } if (paramInfo.isDAE) { @@ -5568,7 +5567,6 @@ SQLRETURN FetchArrowBatch_wrap(SqlHandlePtr StatementHandle, py::list& capsules, bool hasLobColumns = false; std::vector dataTypes(numCols); - std::vector columnSizes(numCols); std::vector columnNullable(numCols); std::vector columnVarLen(numCols, false); std::vector nullCounts(numCols, 0); @@ -5587,7 +5585,6 @@ SQLRETURN FetchArrowBatch_wrap(SqlHandlePtr StatementHandle, py::list& capsules, SQLSMALLINT nullable = colMeta["Nullable"].cast(); dataTypes[i] = dataType; - columnSizes[i] = columnSize; columnNullable[i] = (nullable != SQL_NO_NULLS); if ((dataType == SQL_WVARCHAR || dataType == SQL_WLONGVARCHAR || dataType == SQL_VARCHAR || @@ -6104,19 +6101,21 @@ SQLRETURN FetchArrowBatch_wrap(SqlHandlePtr StatementHandle, py::list& capsules, case SQL_LONGVARBINARY: { auto target_vec = &arrowColumnProducer->varData; auto start = arrowColumnProducer->varVal[idxRowArrow]; - EnsureNativeFetchBufferSize( - *target_vec, - CheckedAddSize(start, dataLen, "Arrow value buffer is too large"), - reservedBytes); - const size_t sourceStride = hasLobColumns - ? buffers.charBuffers[idxCol].size() - : buffers.charBuffers[idxCol].size() / - static_cast(fetchSize); - const size_t sourceOffset = CheckedArrowSourceOffset( - buffers.charBuffers[idxCol], idxRowSql, sourceStride, dataLen); + if (dataLen > 0) { + EnsureNativeFetchBufferSize( + *target_vec, + CheckedAddSize(start, dataLen, "Arrow value buffer is too large"), + reservedBytes); + const size_t sourceStride = hasLobColumns + ? buffers.charBuffers[idxCol].size() + : buffers.charBuffers[idxCol].size() / + static_cast(fetchSize); + const size_t sourceOffset = CheckedArrowSourceOffset( + buffers.charBuffers[idxCol], idxRowSql, sourceStride, dataLen); - std::memcpy(&(*target_vec)[start], - &buffers.charBuffers[idxCol][sourceOffset], dataLen); + std::memcpy(&(*target_vec)[start], + &buffers.charBuffers[idxCol][sourceOffset], dataLen); + } arrowColumnProducer->varVal[idxRowArrow + 1] = start + dataLen; break; } diff --git a/mssql_python/pybind/param_detect.hpp b/mssql_python/pybind/param_detect.hpp index f8a3e7d4f..32bff929f 100644 --- a/mssql_python/pybind/param_detect.hpp +++ b/mssql_python/pybind/param_detect.hpp @@ -296,6 +296,11 @@ inline void ApplyInputSizeOverride(PyObject* params, PyObject* inputSize, Py_ssi } } + if (info.paramCType == SQL_C_WCHAR && + (PyBytes_Check(obj) || PyByteArray_Check(obj))) { + throw py::type_error("bytes values cannot be bound as SQL_C_WCHAR"); + } + Py_ssize_t actualTextLength = 0; if (PyUnicode_Check(obj)) { actualTextLength = info.paramCType == SQL_C_CHAR diff --git a/tests/test_023_execute_path_parity.py b/tests/test_023_execute_path_parity.py index 3a3c38935..b7550131d 100644 --- a/tests/test_023_execute_path_parity.py +++ b/tests/test_023_execute_path_parity.py @@ -339,6 +339,13 @@ def test_executemany_setinputsizes_scans_mixed_text_values_for_dae(cursor): assert cursor.fetchall() == [(value,), ("x",)] +def test_setinputsizes_rejects_bytes_for_wide_character_binding(cursor): + """Wide-character overrides require text rather than raw encoded bytes.""" + cursor.setinputsizes([(ddbc_sql_const.SQL_WVARCHAR.value, 1, 0)]) + with pytest.raises(TypeError, match="SQL_C_WCHAR"): + cursor.execute("SELECT ?", [b"x" * 8001]) + + @pytest.mark.parametrize("sql_type", [None, ddbc_sql_const.SQL_VARCHAR.value]) def test_time_isoformat_must_return_string(cursor, sql_type): """Native time normalization rejects a broken subclass contract on either path.""" From 7bb3a1afc4c799a33842c185cfd933bb8eb8f6fa Mon Sep 17 00:00:00 2001 From: gargsaumya Date: Wed, 7 Oct 2026 13:58:54 +0530 Subject: [PATCH 17/31] FIX: Restore driver-compatible DAE binding Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- mssql_python/cursor.py | 68 +++++++++++++++------------ mssql_python/pybind/ddbc_bindings.cpp | 1 + mssql_python/pybind/param_detect.hpp | 28 ++++++----- 3 files changed, 54 insertions(+), 43 deletions(-) diff --git a/mssql_python/cursor.py b/mssql_python/cursor.py index b6cd81cf2..934087a4e 100644 --- a/mssql_python/cursor.py +++ b/mssql_python/cursor.py @@ -1347,9 +1347,9 @@ def _reset_inputsizes(self) -> None: def _get_sql_to_c_type_map(cls): if cls._SQL_TO_C_TYPE is None: cls._SQL_TO_C_TYPE = { - ddbc_sql_const.SQL_CHAR.value: ddbc_sql_const.SQL_C_CHAR.value, - ddbc_sql_const.SQL_VARCHAR.value: ddbc_sql_const.SQL_C_CHAR.value, - ddbc_sql_const.SQL_LONGVARCHAR.value: ddbc_sql_const.SQL_C_CHAR.value, + ddbc_sql_const.SQL_CHAR.value: ddbc_sql_const.SQL_CHAR.value, + ddbc_sql_const.SQL_VARCHAR.value: ddbc_sql_const.SQL_CHAR.value, + ddbc_sql_const.SQL_LONGVARCHAR.value: ddbc_sql_const.SQL_CHAR.value, ddbc_sql_const.SQL_WCHAR.value: ddbc_sql_const.SQL_C_WCHAR.value, ddbc_sql_const.SQL_WVARCHAR.value: ddbc_sql_const.SQL_C_WCHAR.value, ddbc_sql_const.SQL_WLONGVARCHAR.value: ddbc_sql_const.SQL_C_WCHAR.value, @@ -2633,35 +2633,39 @@ def executemany( # pylint: disable=too-many-locals,too-many-branches,too-many-s ): c_type = ddbc_sql_const.SQL_C_CHAR.value - # Check every compatible value before selecting array binding. Mixed - # str/bytes columns cannot rely on the sample value's type. - if c_type in ( - ddbc_sql_const.SQL_C_CHAR.value, - ddbc_sql_const.SQL_C_WCHAR.value, + # Check every compatible value before selecting array binding. Use + # the SQL type because the public SQL_C_CHAR enum aliases SQL_C_WCHAR. + if sql_type in ( + ddbc_sql_const.SQL_CHAR.value, + ddbc_sql_const.SQL_VARCHAR.value, + ddbc_sql_const.SQL_LONGVARCHAR.value, ): text_values = [value for value in column if isinstance(value, str)] - if c_type == ddbc_sql_const.SQL_C_CHAR.value: - text_is_large = any( - _encoded_length_exceeds( - value, - encoding_settings["encoding"], - MAX_INLINE_CHAR, - ) - for value in text_values - ) - binary_is_large = any( - len(value) > MAX_INLINE_CHAR - for value in column - if isinstance(value, (bytes, bytearray)) + text_is_large = any( + _encoded_length_exceeds( + value, + encoding_settings["encoding"], + MAX_INLINE_CHAR, ) - else: - text_is_large = any( - sum(2 if ord(char) > 0xFFFF else 1 for char in value) - > MAX_INLINE_CHAR - for value in text_values - ) - binary_is_large = False - is_dae = column_size > MAX_INLINE_CHAR or text_is_large or binary_is_large + for value in text_values + ) + binary_is_large = any( + len(value) > MAX_INLINE_CHAR + for value in column + if isinstance(value, (bytes, bytearray)) + ) + is_dae = text_is_large or binary_is_large + elif sql_type in ( + ddbc_sql_const.SQL_WCHAR.value, + ddbc_sql_const.SQL_WVARCHAR.value, + ddbc_sql_const.SQL_WLONGVARCHAR.value, + ddbc_sql_const.SQL_SS_XML.value, + ): + is_dae = any( + sum(2 if ord(char) > 0xFFFF else 1 for char in value) > MAX_INLINE_CHAR + for value in column + if isinstance(value, str) + ) # Sanitize precision/scale for numeric types if sql_type in ( @@ -2676,6 +2680,7 @@ def executemany( # pylint: disable=too-many-locals,too-many-branches,too-many-s ddbc_sql_const.SQL_BINARY.value, ddbc_sql_const.SQL_VARBINARY.value, ddbc_sql_const.SQL_LONGVARBINARY.value, + ddbc_sql_const.SQL_SS_UDT.value, ): # Find the maximum size needed for any row's binary data max_binary_size = 0 @@ -2685,8 +2690,9 @@ def executemany( # pylint: disable=too-many-locals,too-many-branches,too-many-s max_binary_size = max(max_binary_size, len(value)) # For SQL Server VARBINARY(MAX), we need to use large object binding - if column_size > 8000 or max_binary_size > 8000: - sql_type = ddbc_sql_const.SQL_LONGVARBINARY.value + if max_binary_size > 8000: + if sql_type != ddbc_sql_const.SQL_SS_UDT.value: + sql_type = ddbc_sql_const.SQL_LONGVARBINARY.value is_dae = True # Update column_size to actual maximum size if it's larger diff --git a/mssql_python/pybind/ddbc_bindings.cpp b/mssql_python/pybind/ddbc_bindings.cpp index 20987ed1c..1a90c0ea9 100644 --- a/mssql_python/pybind/ddbc_bindings.cpp +++ b/mssql_python/pybind/ddbc_bindings.cpp @@ -4767,6 +4767,7 @@ SQLRETURN FetchBatchData(SQLHSTMT hStmt, ColumnBuffers& buffers, const Metadata& LOG("FetchBatchData: Fetching data in batches"); SQLRETURN ret; { + numRowsFetched = 0; // Release the GIL during the blocking ODBC fetch py::gil_scoped_release release; PERF_TIMER("FetchBatchData::SQLFetchScroll_call"); diff --git a/mssql_python/pybind/param_detect.hpp b/mssql_python/pybind/param_detect.hpp index 32bff929f..46adcb08c 100644 --- a/mssql_python/pybind/param_detect.hpp +++ b/mssql_python/pybind/param_detect.hpp @@ -308,17 +308,23 @@ inline void ApplyInputSizeOverride(PyObject* params, PyObject* inputSize, Py_ssi : UnicodeUtf16Length(obj); } const bool textNeedsDAE = - PyUnicode_Check(obj) && - (PyLongGreaterThan(columnSize, MAX_INLINE_CHAR) || - actualTextLength > MAX_INLINE_CHAR); + PyUnicode_Check(obj) && actualTextLength > MAX_INLINE_CHAR; const bool binaryNeedsDAE = (PyBytes_Check(obj) || PyByteArray_Check(obj)) && - (PyLongGreaterThan(columnSize, MAX_INLINE_BINARY) || - (PyBytes_Check(obj) ? PyBytes_GET_SIZE(obj) : PyByteArray_GET_SIZE(obj)) > - MAX_INLINE_BINARY); + (PyBytes_Check(obj) ? PyBytes_GET_SIZE(obj) : PyByteArray_GET_SIZE(obj)) > + MAX_INLINE_BINARY; info.isDAE = textNeedsDAE || binaryNeedsDAE; if (info.isDAE) { info.columnSize = 0; + if (textNeedsDAE) { + if (info.paramSQLType == SQL_CHAR || info.paramSQLType == SQL_VARCHAR) { + info.paramSQLType = SQL_LONGVARCHAR; + } else if (info.paramSQLType == SQL_WCHAR || info.paramSQLType == SQL_WVARCHAR) { + info.paramSQLType = SQL_WLONGVARCHAR; + } + } else if (info.paramSQLType == SQL_BINARY || info.paramSQLType == SQL_VARBINARY) { + info.paramSQLType = SQL_LONGVARBINARY; + } } if (PyTime_Check(obj) && info.paramCType == PARAM_C_TYPE_TEXT) { @@ -490,16 +496,13 @@ inline std::vector DetectParamTypes(PyObject* params, PyObject* input // Strings > 4000 UTF-16 code units exceed SQL Server's inline NVARCHAR(MAX) // threshold. Switch to data-at-execution (DAE) streaming: ODBC driver pulls // data in chunks via SQLPutData, avoiding a single massive buffer allocation. - // DAE path: match slow-path types exactly. - // Non-unicode (ASCII) → SQL_VARCHAR + PARAM_C_TYPE_TEXT, which is - // SQL_C_WCHAR and matches the slow path's SQL_C_CHAR (numerically - // -8 == SQL_C_WCHAR — a long-standing alias in the Python layer). - // Unicode → SQL_WVARCHAR + SQL_C_WCHAR (wide-char streaming) + // Long SQL types prevent a zero DAE ColumnSize from being interpreted + // as a fixed-width VARCHAR/NVARCHAR precision by the driver. info.isDAE = true; info.columnSize = 0; info.utf16Len = utf16_len; info.dataPtr = borrow(obj); - info.paramSQLType = is_unicode ? SQL_WVARCHAR : SQL_VARCHAR; + info.paramSQLType = is_unicode ? SQL_WLONGVARCHAR : SQL_LONGVARCHAR; info.paramCType = is_unicode ? SQL_C_WCHAR : PARAM_C_TYPE_TEXT; } else { info.columnSize = is_unicode ? utf16_len : length; @@ -520,6 +523,7 @@ inline std::vector DetectParamTypes(PyObject* params, PyObject* input info.isDAE = true; info.columnSize = 0; info.dataPtr = borrow(obj); + info.paramSQLType = SQL_LONGVARBINARY; } else { info.columnSize = std::max(length, 1); } From 37ec527db5b003d7bfd0ae1ec4983d27035c1db1 Mon Sep 17 00:00:00 2001 From: gargsaumya Date: Wed, 7 Oct 2026 14:10:37 +0530 Subject: [PATCH 18/31] FIX: Bound wide LOB conversion memory Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- mssql_python/cursor.py | 9 +++++---- mssql_python/pybind/ddbc_bindings.cpp | 13 ++++++++----- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/mssql_python/cursor.py b/mssql_python/cursor.py index 934087a4e..85621d5b8 100644 --- a/mssql_python/cursor.py +++ b/mssql_python/cursor.py @@ -48,6 +48,7 @@ MAX_INLINE_CHAR: int = ( 4000 # NVARCHAR/VARCHAR inline limit; this triggers NVARCHAR(MAX)/VARCHAR(MAX) + DAE ) +MAX_INLINE_BINARY: int = 8000 SMALLMONEY_MIN: decimal.Decimal = decimal.Decimal("-214748.3648") SMALLMONEY_MAX: decimal.Decimal = decimal.Decimal("214748.3647") MONEY_MIN: decimal.Decimal = decimal.Decimal("-922337203685477.5808") @@ -980,7 +981,7 @@ def _map_sql_type( # pylint: disable=too-many-arguments,too-many-positional-arg if isinstance(param, (bytes, bytearray)): length = len(param) - if length > 8000: # Use VARBINARY(MAX) for large blobs + if length > MAX_INLINE_BINARY: # Use VARBINARY(MAX) for large blobs return ( ddbc_sql_const.SQL_VARBINARY.value, ddbc_sql_const.SQL_C_BINARY.value, @@ -2650,7 +2651,7 @@ def executemany( # pylint: disable=too-many-locals,too-many-branches,too-many-s for value in text_values ) binary_is_large = any( - len(value) > MAX_INLINE_CHAR + len(value) > MAX_INLINE_BINARY for value in column if isinstance(value, (bytes, bytearray)) ) @@ -2690,7 +2691,7 @@ def executemany( # pylint: disable=too-many-locals,too-many-branches,too-many-s max_binary_size = max(max_binary_size, len(value)) # For SQL Server VARBINARY(MAX), we need to use large object binding - if max_binary_size > 8000: + if max_binary_size > MAX_INLINE_BINARY: if sql_type != ddbc_sql_const.SQL_SS_UDT.value: sql_type = ddbc_sql_const.SQL_LONGVARBINARY.value is_dae = True @@ -2780,7 +2781,7 @@ def executemany( # pylint: disable=too-many-locals,too-many-branches,too-many-s max_binary_size = max(max_binary_size, len(value)) # For SQL Server VARBINARY(MAX), we need to use large object binding - if max_binary_size > 8000: + if max_binary_size > MAX_INLINE_BINARY: paraminfo.paramSQLType = ddbc_sql_const.SQL_LONGVARBINARY.value paraminfo.isDAE = True diff --git a/mssql_python/pybind/ddbc_bindings.cpp b/mssql_python/pybind/ddbc_bindings.cpp index 1a90c0ea9..8a48c7d09 100644 --- a/mssql_python/pybind/ddbc_bindings.cpp +++ b/mssql_python/pybind/ddbc_bindings.cpp @@ -3617,11 +3617,14 @@ py::object FetchLobColumnData(SQLHSTMT hStmt, SQLUSMALLINT colIndex, SQLSMALLINT if (buffer.size() % sizeof(SQLWCHAR) != 0) { ThrowStdException("Wide-character LOB data has an invalid byte length"); } - size_t wcharCount = buffer.size() / sizeof(SQLWCHAR); - ReserveNativeFetchBytes(reservedBytes, wcharCount, sizeof(SQLWCHAR)); - std::vector alignedBuf(wcharCount); - std::memcpy(alignedBuf.data(), buffer.data(), buffer.size()); - return py::cast(dupeSqlWCharAsUtf16Le(alignedBuf.data(), wcharCount)); + int byteOrder = -1; + PyObject* decoded = + PyUnicode_DecodeUTF16(buffer.data(), static_cast(buffer.size()), + "strict", &byteOrder); + if (decoded == nullptr) { + throw py::error_already_set(); + } + return py::reinterpret_steal(decoded); } if (isBinary) { LOG("FetchLobColumnData: Returning binary data - %zu bytes for column " From 7b5a3b5ca6fa4854a771786d300a0bab18e61a3a Mon Sep 17 00:00:00 2001 From: gargsaumya Date: Wed, 7 Oct 2026 14:25:14 +0530 Subject: [PATCH 19/31] FIX: Align streamed parameter metadata Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- mssql_python/cursor.py | 55 ++++++++++++++-------------- mssql_python/pybind/param_detect.hpp | 27 +++++++++----- 2 files changed, 45 insertions(+), 37 deletions(-) diff --git a/mssql_python/cursor.py b/mssql_python/cursor.py index 85621d5b8..1bdf51ce2 100644 --- a/mssql_python/cursor.py +++ b/mssql_python/cursor.py @@ -1348,9 +1348,9 @@ def _reset_inputsizes(self) -> None: def _get_sql_to_c_type_map(cls): if cls._SQL_TO_C_TYPE is None: cls._SQL_TO_C_TYPE = { - ddbc_sql_const.SQL_CHAR.value: ddbc_sql_const.SQL_CHAR.value, - ddbc_sql_const.SQL_VARCHAR.value: ddbc_sql_const.SQL_CHAR.value, - ddbc_sql_const.SQL_LONGVARCHAR.value: ddbc_sql_const.SQL_CHAR.value, + ddbc_sql_const.SQL_CHAR.value: ddbc_sql_const.SQL_C_CHAR.value, + ddbc_sql_const.SQL_VARCHAR.value: ddbc_sql_const.SQL_C_CHAR.value, + ddbc_sql_const.SQL_LONGVARCHAR.value: ddbc_sql_const.SQL_C_CHAR.value, ddbc_sql_const.SQL_WCHAR.value: ddbc_sql_const.SQL_C_WCHAR.value, ddbc_sql_const.SQL_WVARCHAR.value: ddbc_sql_const.SQL_C_WCHAR.value, ddbc_sql_const.SQL_WLONGVARCHAR.value: ddbc_sql_const.SQL_C_WCHAR.value, @@ -2613,6 +2613,7 @@ def executemany( # pylint: disable=too-many-locals,too-many-branches,too-many-s # Prepare parameter type information with perf_phase("py::executemany::param_type_detection"): for col_index in range(param_count): + requires_row_fallback = False column = ( [row[col_index] for row in seq_of_parameters] if hasattr(seq_of_parameters, "__getitem__") @@ -2634,39 +2635,37 @@ def executemany( # pylint: disable=too-many-locals,too-many-branches,too-many-s ): c_type = ddbc_sql_const.SQL_C_CHAR.value - # Check every compatible value before selecting array binding. Use - # the SQL type because the public SQL_C_CHAR enum aliases SQL_C_WCHAR. - if sql_type in ( + # Text values stay on the driver's wide binding path. Bytes supplied + # for narrow SQL text types use real SQL_C_CHAR in scalar execution. + narrow_text_type = sql_type in ( ddbc_sql_const.SQL_CHAR.value, ddbc_sql_const.SQL_VARCHAR.value, ddbc_sql_const.SQL_LONGVARCHAR.value, - ): - text_values = [value for value in column if isinstance(value, str)] - text_is_large = any( - _encoded_length_exceeds( - value, - encoding_settings["encoding"], - MAX_INLINE_CHAR, - ) - for value in text_values - ) - binary_is_large = any( - len(value) > MAX_INLINE_BINARY - for value in column - if isinstance(value, (bytes, bytearray)) - ) - is_dae = text_is_large or binary_is_large - elif sql_type in ( + ) + wide_text_type = sql_type in ( ddbc_sql_const.SQL_WCHAR.value, ddbc_sql_const.SQL_WVARCHAR.value, ddbc_sql_const.SQL_WLONGVARCHAR.value, ddbc_sql_const.SQL_SS_XML.value, - ): - is_dae = any( + ) + if narrow_text_type or wide_text_type: + text_values = [value for value in column if isinstance(value, str)] + binary_values = [ + value for value in column if isinstance(value, (bytes, bytearray)) + ] + text_is_large = any( sum(2 if ord(char) > 0xFFFF else 1 for char in value) > MAX_INLINE_CHAR - for value in column - if isinstance(value, str) + for value in text_values + ) + binary_is_large = narrow_text_type and any( + len(value) > MAX_INLINE_BINARY for value in binary_values ) + requires_row_fallback = bool( + narrow_text_type and text_values and binary_values + ) + if narrow_text_type and binary_values and not text_values: + c_type = ddbc_sql_const.SQL_CHAR.value + is_dae = text_is_large or binary_is_large # Sanitize precision/scale for numeric types if sql_type in ( @@ -2790,7 +2789,7 @@ def executemany( # pylint: disable=too-many-locals,too-many-branches,too-many-s paraminfo.columnSize = max(max_binary_size, 1) parameters_type.append(paraminfo) - if paraminfo.isDAE: + if paraminfo.isDAE or requires_row_fallback: any_dae = True if any_dae: diff --git a/mssql_python/pybind/param_detect.hpp b/mssql_python/pybind/param_detect.hpp index 46adcb08c..96b53246f 100644 --- a/mssql_python/pybind/param_detect.hpp +++ b/mssql_python/pybind/param_detect.hpp @@ -296,6 +296,12 @@ inline void ApplyInputSizeOverride(PyObject* params, PyObject* inputSize, Py_ssi } } + if ((PyBytes_Check(obj) || PyByteArray_Check(obj)) && + (info.paramSQLType == SQL_CHAR || info.paramSQLType == SQL_VARCHAR || + info.paramSQLType == SQL_LONGVARCHAR)) { + info.paramCType = SQL_C_CHAR; + } + if (info.paramCType == SQL_C_WCHAR && (PyBytes_Check(obj) || PyByteArray_Check(obj))) { throw py::type_error("bytes values cannot be bound as SQL_C_WCHAR"); @@ -315,13 +321,16 @@ inline void ApplyInputSizeOverride(PyObject* params, PyObject* inputSize, Py_ssi MAX_INLINE_BINARY; info.isDAE = textNeedsDAE || binaryNeedsDAE; if (info.isDAE) { - info.columnSize = 0; - if (textNeedsDAE) { - if (info.paramSQLType == SQL_CHAR || info.paramSQLType == SQL_VARCHAR) { - info.paramSQLType = SQL_LONGVARCHAR; - } else if (info.paramSQLType == SQL_WCHAR || info.paramSQLType == SQL_WVARCHAR) { - info.paramSQLType = SQL_WLONGVARCHAR; - } + info.columnSize = + textNeedsDAE + ? static_cast(actualTextLength) + : static_cast(PyBytes_Check(obj) ? PyBytes_GET_SIZE(obj) + : PyByteArray_GET_SIZE(obj)); + if (info.paramSQLType == SQL_CHAR || info.paramSQLType == SQL_VARCHAR) { + info.paramSQLType = SQL_LONGVARCHAR; + } else if (textNeedsDAE && + (info.paramSQLType == SQL_WCHAR || info.paramSQLType == SQL_WVARCHAR)) { + info.paramSQLType = SQL_WLONGVARCHAR; } else if (info.paramSQLType == SQL_BINARY || info.paramSQLType == SQL_VARBINARY) { info.paramSQLType = SQL_LONGVARBINARY; } @@ -499,7 +508,7 @@ inline std::vector DetectParamTypes(PyObject* params, PyObject* input // Long SQL types prevent a zero DAE ColumnSize from being interpreted // as a fixed-width VARCHAR/NVARCHAR precision by the driver. info.isDAE = true; - info.columnSize = 0; + info.columnSize = utf16_len; info.utf16Len = utf16_len; info.dataPtr = borrow(obj); info.paramSQLType = is_unicode ? SQL_WLONGVARCHAR : SQL_LONGVARCHAR; @@ -521,7 +530,7 @@ inline std::vector DetectParamTypes(PyObject* params, PyObject* input info.decimalDigits = 0; if (length > MAX_INLINE_BINARY) { info.isDAE = true; - info.columnSize = 0; + info.columnSize = static_cast(length); info.dataPtr = borrow(obj); info.paramSQLType = SQL_LONGVARBINARY; } else { From 190760f29090e6b0b1bfa0189388ddb34093dd7e Mon Sep 17 00:00:00 2001 From: gargsaumya Date: Wed, 7 Oct 2026 14:38:21 +0530 Subject: [PATCH 20/31] FIX: Preserve MAX parameter type semantics Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- mssql_python/pybind/param_detect.hpp | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/mssql_python/pybind/param_detect.hpp b/mssql_python/pybind/param_detect.hpp index 96b53246f..288c16ddf 100644 --- a/mssql_python/pybind/param_detect.hpp +++ b/mssql_python/pybind/param_detect.hpp @@ -326,14 +326,6 @@ inline void ApplyInputSizeOverride(PyObject* params, PyObject* inputSize, Py_ssi ? static_cast(actualTextLength) : static_cast(PyBytes_Check(obj) ? PyBytes_GET_SIZE(obj) : PyByteArray_GET_SIZE(obj)); - if (info.paramSQLType == SQL_CHAR || info.paramSQLType == SQL_VARCHAR) { - info.paramSQLType = SQL_LONGVARCHAR; - } else if (textNeedsDAE && - (info.paramSQLType == SQL_WCHAR || info.paramSQLType == SQL_WVARCHAR)) { - info.paramSQLType = SQL_WLONGVARCHAR; - } else if (info.paramSQLType == SQL_BINARY || info.paramSQLType == SQL_VARBINARY) { - info.paramSQLType = SQL_LONGVARBINARY; - } } if (PyTime_Check(obj) && info.paramCType == PARAM_C_TYPE_TEXT) { @@ -505,13 +497,13 @@ inline std::vector DetectParamTypes(PyObject* params, PyObject* input // Strings > 4000 UTF-16 code units exceed SQL Server's inline NVARCHAR(MAX) // threshold. Switch to data-at-execution (DAE) streaming: ODBC driver pulls // data in chunks via SQLPutData, avoiding a single massive buffer allocation. - // Long SQL types prevent a zero DAE ColumnSize from being interpreted - // as a fixed-width VARCHAR/NVARCHAR precision by the driver. + // Advertise the validated payload size rather than zero or the caller's + // declared size so the driver selects the corresponding MAX representation. info.isDAE = true; info.columnSize = utf16_len; info.utf16Len = utf16_len; info.dataPtr = borrow(obj); - info.paramSQLType = is_unicode ? SQL_WLONGVARCHAR : SQL_LONGVARCHAR; + info.paramSQLType = is_unicode ? SQL_WVARCHAR : SQL_VARCHAR; info.paramCType = is_unicode ? SQL_C_WCHAR : PARAM_C_TYPE_TEXT; } else { info.columnSize = is_unicode ? utf16_len : length; @@ -532,7 +524,6 @@ inline std::vector DetectParamTypes(PyObject* params, PyObject* input info.isDAE = true; info.columnSize = static_cast(length); info.dataPtr = borrow(obj); - info.paramSQLType = SQL_LONGVARBINARY; } else { info.columnSize = std::max(length, 1); } From 5d0a2e323ff3d5cc69e036bae25be18f3ed032d9 Mon Sep 17 00:00:00 2001 From: gargsaumya Date: Wed, 7 Oct 2026 14:42:03 +0530 Subject: [PATCH 21/31] FIX: Size text by active C binding Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- mssql_python/cursor.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/mssql_python/cursor.py b/mssql_python/cursor.py index 1bdf51ce2..bafc19de6 100644 --- a/mssql_python/cursor.py +++ b/mssql_python/cursor.py @@ -2653,10 +2653,21 @@ def executemany( # pylint: disable=too-many-locals,too-many-branches,too-many-s binary_values = [ value for value in column if isinstance(value, (bytes, bytearray)) ] - text_is_large = any( - sum(2 if ord(char) > 0xFFFF else 1 for char in value) > MAX_INLINE_CHAR - for value in text_values - ) + if c_type == ddbc_sql_const.SQL_CHAR.value: + text_is_large = any( + _encoded_length_exceeds( + value, + encoding_settings["encoding"], + MAX_INLINE_CHAR, + ) + for value in text_values + ) + else: + text_is_large = any( + sum(2 if ord(char) > 0xFFFF else 1 for char in value) + > MAX_INLINE_CHAR + for value in text_values + ) binary_is_large = narrow_text_type and any( len(value) > MAX_INLINE_BINARY for value in binary_values ) From e98d558391d3e7513acef9c97608e18c318a795c Mon Sep 17 00:00:00 2001 From: gargsaumya Date: Wed, 7 Oct 2026 14:54:31 +0530 Subject: [PATCH 22/31] FIX: Bind NULL rows independently in DAE batches Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- mssql_python/pybind/ddbc_bindings.cpp | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/mssql_python/pybind/ddbc_bindings.cpp b/mssql_python/pybind/ddbc_bindings.cpp index 8a48c7d09..5f73fa572 100644 --- a/mssql_python/pybind/ddbc_bindings.cpp +++ b/mssql_python/pybind/ddbc_bindings.cpp @@ -3294,8 +3294,16 @@ SQLRETURN SQLExecuteMany_wrap(const SqlHandlePtr statementHandle, const std::u16 rowParams.append(values[rowIndex]); } + std::vector rowParamInfos = paramInfos; + for (size_t paramIndex = 0; paramIndex < rowParamInfos.size(); ++paramIndex) { + if (rowParams[paramIndex].is_none()) { + rowParamInfos[paramIndex].paramCType = SQL_C_DEFAULT; + rowParamInfos[paramIndex].isDAE = false; + rowParamInfos[paramIndex].dataPtr = py::none(); + } + } std::vector> paramBuffers; - rc = BindParameters(*statementHandle, hStmt, rowParams, paramInfos, + rc = BindParameters(*statementHandle, hStmt, rowParams, rowParamInfos, paramBuffers, charEncoding); if (!SQL_SUCCEEDED(rc)) { LOG("SQLExecuteMany: BindParameters failed for row %zu - rc=%d", rowIndex, rc); From 99e837cae8db6857cb28e1de68749777bf5cacdc Mon Sep 17 00:00:00 2001 From: gargsaumya Date: Wed, 7 Oct 2026 15:12:11 +0530 Subject: [PATCH 23/31] FIX: Validate row-local DAE tokens Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- mssql_python/pybind/ddbc_bindings.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mssql_python/pybind/ddbc_bindings.cpp b/mssql_python/pybind/ddbc_bindings.cpp index 5f73fa572..aaaf7d0a1 100644 --- a/mssql_python/pybind/ddbc_bindings.cpp +++ b/mssql_python/pybind/ddbc_bindings.cpp @@ -3337,8 +3337,8 @@ SQLRETURN SQLExecuteMany_wrap(const SqlHandlePtr statementHandle, const std::u16 if (rc != SQL_NEED_DATA) break; const ParamInfo* matchedInfo = reinterpret_cast(token); - const ParamInfo* first = paramInfos.data(); - const ParamInfo* last = first + paramInfos.size(); + const ParamInfo* first = rowParamInfos.data(); + const ParamInfo* last = first + rowParamInfos.size(); if (matchedInfo < first || matchedInfo >= last) { ThrowStdException( "SQLExecuteMany: unrecognized paramToken from SQLParamData"); From a72841c8d08a8bfa2283ec45604d61f8c35d66bd Mon Sep 17 00:00:00 2001 From: gargsaumya Date: Wed, 7 Oct 2026 15:25:44 +0530 Subject: [PATCH 24/31] FIX: Use MAX sentinel for oversized DAE Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- mssql_python/pybind/param_detect.hpp | 35 +++++++++++++++++++++++----- 1 file changed, 29 insertions(+), 6 deletions(-) diff --git a/mssql_python/pybind/param_detect.hpp b/mssql_python/pybind/param_detect.hpp index 288c16ddf..5a3e98d4f 100644 --- a/mssql_python/pybind/param_detect.hpp +++ b/mssql_python/pybind/param_detect.hpp @@ -140,6 +140,26 @@ inline constexpr int MAX_INLINE_CHAR = 4000; // Binary data longer than this uses DAE streaming (SQL Server max for non-MAX types) inline constexpr int MAX_INLINE_BINARY = 8000; +inline SQLULEN DAEColumnSize(SQLSMALLINT sqlType, SQLULEN actualSize) { + switch (sqlType) { + case SQL_CHAR: + case SQL_VARCHAR: + return actualSize > MAX_INLINE_BINARY ? 0 : actualSize; + case SQL_WCHAR: + case SQL_WVARCHAR: + return actualSize > MAX_INLINE_CHAR ? 0 : actualSize; + case SQL_BINARY: + case SQL_VARBINARY: + return actualSize > MAX_INLINE_BINARY ? 0 : actualSize; + case SQL_LONGVARCHAR: + case SQL_WLONGVARCHAR: + case SQL_LONGVARBINARY: + return 0; + default: + return actualSize; + } +} + // SQL Server maximum numeric precision inline constexpr int MAX_NUMERIC_PRECISION = 38; @@ -321,11 +341,12 @@ inline void ApplyInputSizeOverride(PyObject* params, PyObject* inputSize, Py_ssi MAX_INLINE_BINARY; info.isDAE = textNeedsDAE || binaryNeedsDAE; if (info.isDAE) { - info.columnSize = + const SQLULEN actualSize = textNeedsDAE ? static_cast(actualTextLength) : static_cast(PyBytes_Check(obj) ? PyBytes_GET_SIZE(obj) : PyByteArray_GET_SIZE(obj)); + info.columnSize = DAEColumnSize(info.paramSQLType, actualSize); } if (PyTime_Check(obj) && info.paramCType == PARAM_C_TYPE_TEXT) { @@ -497,13 +518,14 @@ inline std::vector DetectParamTypes(PyObject* params, PyObject* input // Strings > 4000 UTF-16 code units exceed SQL Server's inline NVARCHAR(MAX) // threshold. Switch to data-at-execution (DAE) streaming: ODBC driver pulls // data in chunks via SQLPutData, avoiding a single massive buffer allocation. - // Advertise the validated payload size rather than zero or the caller's - // declared size so the driver selects the corresponding MAX representation. + // Use the validated payload size when it is legal fixed-width metadata; + // larger values use the driver's MAX-length sentinel. info.isDAE = true; - info.columnSize = utf16_len; + const SQLSMALLINT sqlType = is_unicode ? SQL_WVARCHAR : SQL_VARCHAR; + info.columnSize = DAEColumnSize(sqlType, utf16_len); info.utf16Len = utf16_len; info.dataPtr = borrow(obj); - info.paramSQLType = is_unicode ? SQL_WVARCHAR : SQL_VARCHAR; + info.paramSQLType = sqlType; info.paramCType = is_unicode ? SQL_C_WCHAR : PARAM_C_TYPE_TEXT; } else { info.columnSize = is_unicode ? utf16_len : length; @@ -522,7 +544,8 @@ inline std::vector DetectParamTypes(PyObject* params, PyObject* input info.decimalDigits = 0; if (length > MAX_INLINE_BINARY) { info.isDAE = true; - info.columnSize = static_cast(length); + info.columnSize = + DAEColumnSize(SQL_VARBINARY, static_cast(length)); info.dataPtr = borrow(obj); } else { info.columnSize = std::max(length, 1); From 5919670f18d23b5511be0412d7a906ac65f9d6ec Mon Sep 17 00:00:00 2001 From: gargsaumya Date: Wed, 7 Oct 2026 15:38:46 +0530 Subject: [PATCH 25/31] FIX: Size DAE metadata through MAX boundary Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- mssql_python/pybind/param_detect.hpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mssql_python/pybind/param_detect.hpp b/mssql_python/pybind/param_detect.hpp index 5a3e98d4f..fa2717038 100644 --- a/mssql_python/pybind/param_detect.hpp +++ b/mssql_python/pybind/param_detect.hpp @@ -154,7 +154,7 @@ inline SQLULEN DAEColumnSize(SQLSMALLINT sqlType, SQLULEN actualSize) { case SQL_LONGVARCHAR: case SQL_WLONGVARCHAR: case SQL_LONGVARBINARY: - return 0; + return actualSize; default: return actualSize; } @@ -237,8 +237,8 @@ inline Py_ssize_t EncodedUnicodeLength(PyObject* value, const std::string& encod if (!chunk) throw py::error_already_set(); py::object encoded = encoder.attr("encode")(chunk, end == length); const Py_ssize_t encodedSize = PyBytes_GET_SIZE(encoded.ptr()); - if (encodedSize > MAX_INLINE_CHAR - total) { - return MAX_INLINE_CHAR + 1; + if (encodedSize > MAX_INLINE_BINARY - total) { + return MAX_INLINE_BINARY + 1; } total += encodedSize; } From db18317db25563051a3780a09af0de6440385706 Mon Sep 17 00:00:00 2001 From: gargsaumya Date: Wed, 7 Oct 2026 15:53:03 +0530 Subject: [PATCH 26/31] FIX: align DAE metadata with wide bindings Use the Unicode-safe MAX boundary for VARCHAR data-at-execution metadata and compare the wide fetch regression by row values rather than generated Row identity. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- mssql_python/pybind/param_detect.hpp | 2 +- tests/test_004_cursor.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mssql_python/pybind/param_detect.hpp b/mssql_python/pybind/param_detect.hpp index fa2717038..d586daac3 100644 --- a/mssql_python/pybind/param_detect.hpp +++ b/mssql_python/pybind/param_detect.hpp @@ -144,7 +144,7 @@ inline SQLULEN DAEColumnSize(SQLSMALLINT sqlType, SQLULEN actualSize) { switch (sqlType) { case SQL_CHAR: case SQL_VARCHAR: - return actualSize > MAX_INLINE_BINARY ? 0 : actualSize; + return actualSize > MAX_INLINE_CHAR ? 0 : actualSize; case SQL_WCHAR: case SQL_WVARCHAR: return actualSize > MAX_INLINE_CHAR ? 0 : actualSize; diff --git a/tests/test_004_cursor.py b/tests/test_004_cursor.py index 518dd15b4..02dec2e9f 100644 --- a/tests/test_004_cursor.py +++ b/tests/test_004_cursor.py @@ -1748,7 +1748,7 @@ def test_fetchmany_rejects_excessive_native_buffer(cursor): def test_fetchall_clamps_wide_result_batch_to_native_buffer_budget(cursor): cursor.execute("SELECT " + ", ".join("CAST(N'x' AS NVARCHAR(4000))" for _ in range(34))) - assert cursor.fetchall() == [("x",) * 34] + assert [tuple(row) for row in cursor.fetchall()] == [("x",) * 34] def test_description(cursor): From 400bd0f2191ac6ff3d2da45855a296e2f8ef16bc Mon Sep 17 00:00:00 2001 From: gargsaumya Date: Wed, 7 Oct 2026 16:11:21 +0530 Subject: [PATCH 27/31] FIX: preserve actual parameter metadata size Prevent a too-small setinputsizes declaration from truncating later non-DAE text or binary rows after executemany falls back to row execution. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- mssql_python/pybind/param_detect.hpp | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/mssql_python/pybind/param_detect.hpp b/mssql_python/pybind/param_detect.hpp index d586daac3..b7ec6a189 100644 --- a/mssql_python/pybind/param_detect.hpp +++ b/mssql_python/pybind/param_detect.hpp @@ -334,19 +334,23 @@ inline void ApplyInputSizeOverride(PyObject* params, PyObject* inputSize, Py_ssi : UnicodeUtf16Length(obj); } const bool textNeedsDAE = - PyUnicode_Check(obj) && actualTextLength > MAX_INLINE_CHAR; + !isNumeric && PyUnicode_Check(obj) && actualTextLength > MAX_INLINE_CHAR; const bool binaryNeedsDAE = (PyBytes_Check(obj) || PyByteArray_Check(obj)) && (PyBytes_Check(obj) ? PyBytes_GET_SIZE(obj) : PyByteArray_GET_SIZE(obj)) > MAX_INLINE_BINARY; info.isDAE = textNeedsDAE || binaryNeedsDAE; - if (info.isDAE) { - const SQLULEN actualSize = - textNeedsDAE - ? static_cast(actualTextLength) - : static_cast(PyBytes_Check(obj) ? PyBytes_GET_SIZE(obj) - : PyByteArray_GET_SIZE(obj)); - info.columnSize = DAEColumnSize(info.paramSQLType, actualSize); + if (!isNumeric && PyUnicode_Check(obj)) { + const SQLULEN actualSize = static_cast(actualTextLength); + info.columnSize = + info.isDAE ? DAEColumnSize(info.paramSQLType, actualSize) + : std::max(info.columnSize, actualSize); + } else if (PyBytes_Check(obj) || PyByteArray_Check(obj)) { + const SQLULEN actualSize = static_cast( + PyBytes_Check(obj) ? PyBytes_GET_SIZE(obj) : PyByteArray_GET_SIZE(obj)); + info.columnSize = + info.isDAE ? DAEColumnSize(info.paramSQLType, actualSize) + : std::max(info.columnSize, actualSize); } if (PyTime_Check(obj) && info.paramCType == PARAM_C_TYPE_TEXT) { From 26c4763d712585870f3b7fb9b1aa16c00607dec6 Mon Sep 17 00:00:00 2001 From: gargsaumya Date: Wed, 7 Oct 2026 16:31:29 +0530 Subject: [PATCH 28/31] FIX: address native validation review findings Reject boolean row counts before conversion, guard empty Arrow character copies, validate bytearray inputs before C API access, and make the DAE regressions compare row values reliably. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- mssql_python/pybind/ddbc_bindings.cpp | 57 ++++++++++++++++++--------- tests/test_010_pybind_functions.py | 4 +- tests/test_023_execute_path_parity.py | 15 ++++--- 3 files changed, 50 insertions(+), 26 deletions(-) diff --git a/mssql_python/pybind/ddbc_bindings.cpp b/mssql_python/pybind/ddbc_bindings.cpp index aaaf7d0a1..c9b2c0dce 100644 --- a/mssql_python/pybind/ddbc_bindings.cpp +++ b/mssql_python/pybind/ddbc_bindings.cpp @@ -378,6 +378,15 @@ void ValidateNativeRowCount(int value, const char* name, bool allowZero) { } } +int ValidateNativeRowCountArgument(py::handle value, const char* name, bool allowZero) { + if (PyBool_Check(value.ptr())) { + ThrowStdException(std::string(name) + " must be an integer, not bool"); + } + const int converted = value.cast(); + ValidateNativeRowCount(converted, name, allowZero); + return converted; +} + void ReserveNativeParameterBytes(size_t& reservedBytes, size_t count, size_t elementSize) { const size_t allocationBytes = CheckedMultiplySize(count, elementSize, "Parameter buffer size is too large"); @@ -2769,12 +2778,15 @@ SQLRETURN BindParameterArray(SqlHandle& handle, SQLHANDLE hStmt, const py::list& &encodedSize) != 0) { throw py::error_already_set(); } - } else { + } else if (py::isinstance(columnValues[i])) { encodedData = PyByteArray_AsString(columnValues[i].ptr()); encodedSize = PyByteArray_Size(columnValues[i].ptr()); if (encodedData == nullptr || encodedSize < 0) { throw py::error_already_set(); } + } else { + ThrowStdException( + MakeParamMismatchErrorStr(info.paramCType, paramIndex)); } const size_t dataSize = static_cast(encodedSize); @@ -5312,12 +5324,13 @@ struct FetchStateGuard { // the result set and populates the provided Python list with the row data. If // there are no more rows to fetch, it returns SQL_NO_DATA. If an error occurs // during fetching, it throws a runtime error. -SQLRETURN FetchMany_wrap(SqlHandlePtr StatementHandle, py::list& rows, int fetchSize, +SQLRETURN FetchMany_wrap(SqlHandlePtr StatementHandle, py::list& rows, py::handle fetchSizeArg, const std::string& charEncoding = "utf-16le", const std::string& wcharEncoding = "utf-16le", int charCtype = SQL_C_WCHAR, py::handle messages = {}) { PERF_TIMER("FetchMany_wrap"); - ValidateNativeRowCount(fetchSize, "Fetch size", false); + const int fetchSize = + ValidateNativeRowCountArgument(fetchSizeArg, "Fetch size", false); // Issue #531: upgrade SQL_C_CHAR + utf-8 to SQL_C_WCHAR on Windows so the // driver does lossless UTF-16 conversion instead of returning ACP bytes. charCtype = EffectiveCharCtypeForFetch(charCtype, charEncoding); @@ -5543,10 +5556,12 @@ int32_t days_from_civil(int y, int m, int d) { return era * 146097 + static_cast(doe) - 719468; } -SQLRETURN FetchArrowBatch_wrap(SqlHandlePtr StatementHandle, py::list& capsules, int arrowBatchSize, - int charCtype, py::handle messages = {}) { +SQLRETURN FetchArrowBatch_wrap(SqlHandlePtr StatementHandle, py::list& capsules, + py::handle arrowBatchSizeArg, int charCtype, + py::handle messages = {}) { PERF_TIMER("FetchArrowBatch_wrap"); - ValidateNativeRowCount(arrowBatchSize, "Arrow batch size", true); + const int arrowBatchSize = + ValidateNativeRowCountArgument(arrowBatchSizeArg, "Arrow batch size", true); const size_t batchSize = static_cast(arrowBatchSize); const size_t offsetCount = CheckedAddSize(batchSize, 1, "Arrow batch size is too large"); const size_t initialVarDataSize = @@ -6137,19 +6152,23 @@ SQLRETURN FetchArrowBatch_wrap(SqlHandlePtr StatementHandle, py::list& capsules, if (charCtype == SQL_C_CHAR) { auto target_vec = &arrowColumnProducer->varData; auto start = arrowColumnProducer->varVal[idxRowArrow]; - EnsureNativeFetchBufferSize( - *target_vec, - CheckedAddSize(start, dataLen, "Arrow value buffer is too large"), - reservedBytes); - const size_t sourceStride = hasLobColumns - ? buffers.charBuffers[idxCol].size() - : buffers.charBuffers[idxCol].size() / - static_cast(fetchSize); - const size_t sourceOffset = CheckedArrowSourceOffset( - buffers.charBuffers[idxCol], idxRowSql, sourceStride, dataLen); - - std::memcpy(&(*target_vec)[start], - &buffers.charBuffers[idxCol][sourceOffset], dataLen); + if (dataLen > 0) { + EnsureNativeFetchBufferSize( + *target_vec, + CheckedAddSize(start, dataLen, + "Arrow value buffer is too large"), + reservedBytes); + const size_t sourceStride = + hasLobColumns + ? buffers.charBuffers[idxCol].size() + : buffers.charBuffers[idxCol].size() / + static_cast(fetchSize); + const size_t sourceOffset = CheckedArrowSourceOffset( + buffers.charBuffers[idxCol], idxRowSql, sourceStride, dataLen); + + std::memcpy(&(*target_vec)[start], + &buffers.charBuffers[idxCol][sourceOffset], dataLen); + } arrowColumnProducer->varVal[idxRowArrow + 1] = start + dataLen; break; } diff --git a/tests/test_010_pybind_functions.py b/tests/test_010_pybind_functions.py index afed788e4..ec85f14ed 100644 --- a/tests/test_010_pybind_functions.py +++ b/tests/test_010_pybind_functions.py @@ -38,14 +38,14 @@ @pytest.mark.skipif(not DDBC_AVAILABLE, reason="ddbc_bindings not available") -@pytest.mark.parametrize("fetch_size", [-1, 1_000_001]) +@pytest.mark.parametrize("fetch_size", [-1, 1_000_001, True]) def test_fetchmany_rejects_unsafe_size_before_handle_access(fetch_size): with pytest.raises(RuntimeError, match="Fetch size"): ddbc.DDBCSQLFetchMany(None, [], fetch_size) @pytest.mark.skipif(not DDBC_AVAILABLE, reason="ddbc_bindings not available") -@pytest.mark.parametrize("batch_size", [-1, 1_000_001]) +@pytest.mark.parametrize("batch_size", [-1, 1_000_001, True]) def test_arrow_batch_rejects_unsafe_size_before_handle_access(batch_size): with pytest.raises(RuntimeError, match="Arrow batch size"): ddbc.DDBCSQLFetchArrowBatch(None, [], batch_size, 0) diff --git a/tests/test_023_execute_path_parity.py b/tests/test_023_execute_path_parity.py index b7550131d..daeb42514 100644 --- a/tests/test_023_execute_path_parity.py +++ b/tests/test_023_execute_path_parity.py @@ -322,11 +322,16 @@ def test_setinputsizes_uses_actual_value_size_for_dae(cursor, value, sql_type): ) def test_executemany_setinputsizes_uses_actual_text_size_for_dae(cursor, value, sql_type): """A too-small declared size cannot bypass streaming for an array-bound text value.""" - cursor.execute("CREATE TABLE #dae_batch (value NVARCHAR(MAX))") + table_name = ( + "#dae_batch_varchar" + if sql_type == ddbc_sql_const.SQL_VARCHAR.value + else "#dae_batch_nvarchar" + ) + cursor.execute(f"CREATE TABLE {table_name} (value NVARCHAR(MAX))") cursor.setinputsizes([(sql_type, 1, 0)]) - cursor.executemany("INSERT INTO #dae_batch (value) VALUES (?)", [(value,), ("short",)]) - cursor.execute("SELECT value FROM #dae_batch ORDER BY LEN(value) DESC") - assert cursor.fetchall() == [(value,), ("short",)] + cursor.executemany(f"INSERT INTO {table_name} (value) VALUES (?)", [(value,), ("short",)]) + cursor.execute(f"SELECT value FROM {table_name} ORDER BY LEN(value) DESC") + assert [tuple(row) for row in cursor.fetchall()] == [(value,), ("short",)] def test_executemany_setinputsizes_scans_mixed_text_values_for_dae(cursor): @@ -336,7 +341,7 @@ def test_executemany_setinputsizes_scans_mixed_text_values_for_dae(cursor): cursor.setinputsizes([(ddbc_sql_const.SQL_VARCHAR.value, 1, 0)]) cursor.executemany("INSERT INTO #mixed_dae_batch (value) VALUES (?)", [(b"x",), (value,)]) cursor.execute("SELECT value FROM #mixed_dae_batch ORDER BY LEN(value) DESC") - assert cursor.fetchall() == [(value,), ("x",)] + assert [tuple(row) for row in cursor.fetchall()] == [(value,), ("x",)] def test_setinputsizes_rejects_bytes_for_wide_character_binding(cursor): From f53323085fcf86a70bc68804b17e8b031a5818b5 Mon Sep 17 00:00:00 2001 From: gargsaumya Date: Wed, 7 Oct 2026 16:46:04 +0530 Subject: [PATCH 29/31] FIX: reset DAE bindings after each row Release ODBC parameter bindings before row-local buffers and SQLParamData tokens leave scope, and surface reset failures without disturbing execution diagnostics on error paths. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- mssql_python/pybind/ddbc_bindings.cpp | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/mssql_python/pybind/ddbc_bindings.cpp b/mssql_python/pybind/ddbc_bindings.cpp index c9b2c0dce..4190b27e2 100644 --- a/mssql_python/pybind/ddbc_bindings.cpp +++ b/mssql_python/pybind/ddbc_bindings.cpp @@ -3367,6 +3367,16 @@ SQLRETURN SQLExecuteMany_wrap(const SqlHandlePtr statementHandle, const std::u16 LOG("SQLExecuteMany: DAE row %zu failed - rc=%d", rowIndex, rc); return rc; } + + const SQLRETURN rowRc = rc; + rc = SQLFreeStmt_ptr(hStmt, SQL_RESET_PARAMS); + if (!SQL_SUCCEEDED(rc)) { + LOG("SQLExecuteMany: SQL_RESET_PARAMS failed for row %zu - " + "rc=%d", + rowIndex, rc); + return rc; + } + rc = rowRc; } LOG("SQLExecuteMany: All DAE rows processed successfully - " "total_rows=%zu", From 48a2f23af05e1f2205c0a38f54e0c95758e43bc7 Mon Sep 17 00:00:00 2001 From: gargsaumya Date: Thu, 8 Oct 2026 11:51:57 +0530 Subject: [PATCH 30/31] FIX: restore geometric LOB buffer growth Track LOB payload length separately from geometric allocation size, validate native executemany counts and input-size metadata before handle mutation, and add direct boundary and growth-count regressions. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- mssql_python/pybind/ddbc_bindings.cpp | 56 +++++++++++++---- mssql_python/pybind/param_detect.hpp | 88 +++++++++++++++++++++++++++ tests/test_010_pybind_functions.py | 36 +++++++++++ 3 files changed, 167 insertions(+), 13 deletions(-) diff --git a/mssql_python/pybind/ddbc_bindings.cpp b/mssql_python/pybind/ddbc_bindings.cpp index 4190b27e2..21ab00f33 100644 --- a/mssql_python/pybind/ddbc_bindings.cpp +++ b/mssql_python/pybind/ddbc_bindings.cpp @@ -511,6 +511,29 @@ void EnsureNativeFetchBufferSize(std::vector& buffer, size_t requir ResizeNativeFetchBuffer(buffer, newSize, reservedBytes); } +size_t ExtendNativeFetchBuffer(std::vector& buffer, size_t currentSize, size_t appendSize, + size_t& reservedBytes) { + const size_t requiredSize = + CheckedAddSize(currentSize, appendSize, "Native fetch buffer size is too large"); + EnsureNativeFetchBufferSize(buffer, requiredSize, reservedBytes); + return requiredSize; +} + +py::tuple TestLobBufferGrowth(size_t chunkCount, size_t chunkSize) { + std::vector buffer; + size_t dataSize = 0; + size_t reservedBytes = 0; + size_t growthCount = 0; + for (size_t chunk = 0; chunk < chunkCount; ++chunk) { + const size_t previousAllocation = buffer.size(); + dataSize = ExtendNativeFetchBuffer(buffer, dataSize, chunkSize, reservedBytes); + if (buffer.size() != previousAllocation) { + ++growthCount; + } + } + return py::make_tuple(growthCount, dataSize, buffer.size(), reservedBytes); +} + template std::unique_ptr AllocateArrowArray(size_t count, size_t& reservedBytes, const char* errorMessage) { @@ -2396,6 +2419,7 @@ SQLRETURN SQLExecute_wrap(const SqlHandlePtr statementHandle, bool use_prepare, const py::dict& encoding_settings) { PERF_TIMER("SQLExecute_wrap"); + ValidateInputSizes(input_sizes.ptr()); if (!statementHandle || !statementHandle->get()) { return SQL_INVALID_HANDLE; } @@ -3218,9 +3242,14 @@ SQLRETURN BindParameterArray(SqlHandle& handle, SQLHANDLE hStmt, const py::list& SQLRETURN SQLExecuteMany_wrap(const SqlHandlePtr statementHandle, const std::u16string& query, const py::list& columnwise_params, - std::vector& paramInfos, size_t paramSetSize, + std::vector& paramInfos, py::handle paramSetSizeArg, const py::dict& encodingSettings) { PERF_TIMER("SQLExecuteMany_wrap"); + const size_t paramSetSize = static_cast( + ValidateNativeRowCountArgument(paramSetSizeArg, "Parameter set size", false)); + if (!statementHandle || !statementHandle->get()) { + return SQL_INVALID_HANDLE; + } statementHandle->resultMetadata.clear(); LOG("SQLExecuteMany: Starting batch execution - param_count=%zu, " "param_set_size=%zu", @@ -3552,6 +3581,7 @@ py::object FetchLobColumnData(SQLHSTMT hStmt, SQLUSMALLINT colIndex, SQLSMALLINT py::handle messages) { PERF_TIMER("FetchLobColumnData"); std::vector buffer; + size_t dataSize = 0; size_t reservedBytes = 0; ReserveNativeFetchBytes(reservedBytes, DAE_CHUNK_SIZE, sizeof(char)); SQLRETURN ret = SQL_SUCCESS_WITH_INFO; @@ -3623,10 +3653,8 @@ py::object FetchLobColumnData(SQLHSTMT hStmt, SQLUSMALLINT colIndex, SQLSMALLINT } } if (bytesRead > 0) { - const size_t previousSize = buffer.size(); - const size_t requiredSize = - CheckedAddSize(previousSize, bytesRead, "LOB fetch buffer is too large"); - ResizeNativeFetchBuffer(buffer, requiredSize, reservedBytes); + const size_t previousSize = dataSize; + dataSize = ExtendNativeFetchBuffer(buffer, dataSize, bytesRead, reservedBytes); std::copy_n(chunk.data(), bytesRead, buffer.data() + previousSize); LOG("FetchLobColumnData: Appended %zu bytes at loop %d", bytesRead, loopCount); } @@ -3635,21 +3663,21 @@ py::object FetchLobColumnData(SQLHSTMT hStmt, SQLUSMALLINT colIndex, SQLSMALLINT break; } } - LOG("FetchLobColumnData: Total bytes collected=%zu for column %d", buffer.size(), colIndex); + LOG("FetchLobColumnData: Total bytes collected=%zu for column %d", dataSize, colIndex); - if (buffer.empty()) { + if (dataSize == 0) { if (isBinary) { return py::bytes(""); } return py::str(""); } if (isWideChar) { - if (buffer.size() % sizeof(SQLWCHAR) != 0) { + if (dataSize % sizeof(SQLWCHAR) != 0) { ThrowStdException("Wide-character LOB data has an invalid byte length"); } int byteOrder = -1; PyObject* decoded = - PyUnicode_DecodeUTF16(buffer.data(), static_cast(buffer.size()), + PyUnicode_DecodeUTF16(buffer.data(), static_cast(dataSize), "strict", &byteOrder); if (decoded == nullptr) { throw py::error_already_set(); @@ -3659,18 +3687,18 @@ py::object FetchLobColumnData(SQLHSTMT hStmt, SQLUSMALLINT colIndex, SQLSMALLINT if (isBinary) { LOG("FetchLobColumnData: Returning binary data - %zu bytes for column " "%d", - buffer.size(), colIndex); - return py::bytes(buffer.data(), buffer.size()); + dataSize, colIndex); + return py::bytes(buffer.data(), dataSize); } // For SQL_C_CHAR data, decode using the appropriate encoding. const std::string effectiveCharEncoding = GetEffectiveCharDecoding(charEncoding); - py::bytes raw_bytes(buffer.data(), buffer.size()); + py::bytes raw_bytes(buffer.data(), dataSize); try { py::object decoded = raw_bytes.attr("decode")(effectiveCharEncoding, "strict"); LOG("FetchLobColumnData: Decoded narrow string with '%s' - %zu bytes -> %zu chars for " "column %d", - effectiveCharEncoding.c_str(), buffer.size(), py::len(decoded), colIndex); + effectiveCharEncoding.c_str(), dataSize, py::len(decoded), colIndex); return decoded; } catch (const py::error_already_set& e) { LOG_ERROR("FetchLobColumnData: Failed to decode with '%s' for column %d: %s", @@ -7000,6 +7028,8 @@ PYBIND11_MODULE(ddbc_bindings, m) { "Fetch an arrow batch of given length from the result set", py::arg("StatementHandle"), py::arg("capsules"), py::arg("arrowBatchSize"), py::arg("charCtype"), py::arg("messages") = py::none()); + m.def("_test_lob_buffer_growth", &TestLobBufferGrowth, py::arg("chunk_count"), + py::arg("chunk_size")); m.def("DDBCSQLFreeHandle", &SQLFreeHandle_wrap, "Free a handle"); m.def("DDBCSQLResetStmt", &SQLResetStmt_wrap, "Close cursor and unbind params without freeing HSTMT"); diff --git a/mssql_python/pybind/param_detect.hpp b/mssql_python/pybind/param_detect.hpp index b7ec6a189..6443d3d23 100644 --- a/mssql_python/pybind/param_detect.hpp +++ b/mssql_python/pybind/param_detect.hpp @@ -274,6 +274,89 @@ inline void NormalizeTimeParam(PyObject* params, Py_ssize_t index, SQLULEN& colu } } +inline long long ValidatedInputSizeInteger(PyObject* value, const char* fieldName) { + if (!PyLong_Check(value) || PyBool_Check(value)) { + throw py::type_error(std::string(fieldName) + " must be an integer"); + } + int overflow = 0; + const long long result = PyLong_AsLongLongAndOverflow(value, &overflow); + if ((result == -1 && PyErr_Occurred()) || overflow != 0) { + PyErr_Clear(); + throw py::value_error(std::string(fieldName) + " is out of range"); + } + return result; +} + +inline void ValidateInputSizes(PyObject* inputSizes) { + if (inputSizes == Py_None) { + return; + } + if (!PyList_Check(inputSizes)) { + throw py::type_error("inputSizes must be None or a list"); + } + + const Py_ssize_t count = PyList_GET_SIZE(inputSizes); + for (Py_ssize_t index = 0; index < count; ++index) { + PyObject* entry = PyList_GET_ITEM(inputSizes, index); + if (!PyTuple_Check(entry) || PyTuple_GET_SIZE(entry) != 4) { + throw py::type_error("each inputSizes entry must be a four-item tuple"); + } + + const long long sqlType = + ValidatedInputSizeInteger(PyTuple_GET_ITEM(entry, 0), "SQL type"); + const long long cType = + ValidatedInputSizeInteger(PyTuple_GET_ITEM(entry, 1), "C type"); + PyObject* columnSize = PyTuple_GET_ITEM(entry, 2); + PyObject* decimalDigits = PyTuple_GET_ITEM(entry, 3); + if (!PyLong_Check(columnSize) || PyBool_Check(columnSize) || + !PyLong_Check(decimalDigits) || PyBool_Check(decimalDigits)) { + throw py::type_error("column size and decimal digits must be integers"); + } + + if (sqlType < std::numeric_limits::min() || + sqlType > std::numeric_limits::max() || + cType < std::numeric_limits::min() || + cType > std::numeric_limits::max()) { + throw py::value_error("SQL and C types must fit in SQLSMALLINT"); + } + const bool isNumeric = sqlType == SQL_DECIMAL || sqlType == SQL_NUMERIC; + py::int_ zero(0); + const int negativeColumnSize = + PyObject_RichCompareBool(columnSize, zero.ptr(), Py_LT); + const int negativeDecimalDigits = + PyObject_RichCompareBool(decimalDigits, zero.ptr(), Py_LT); + if (negativeColumnSize == -1 || negativeDecimalDigits == -1) { + throw py::error_already_set(); + } + if (negativeColumnSize == 1) { + throw py::value_error("column size must be non-negative"); + } + if (negativeDecimalDigits == 1) { + throw py::value_error("decimal digits must be non-negative"); + } + if (!isNumeric) { + const unsigned long long requestedSize = PyLong_AsUnsignedLongLong(columnSize); + if (requestedSize == static_cast(-1) && PyErr_Occurred()) { + PyErr_Clear(); + throw py::value_error("column size is out of range"); + } + if (requestedSize > std::numeric_limits::max()) { + throw py::value_error("column size is out of range"); + } + const unsigned long long requestedDigits = + PyLong_AsUnsignedLongLong(decimalDigits); + if (requestedDigits == static_cast(-1) && PyErr_Occurred()) { + PyErr_Clear(); + throw py::value_error("decimal digits are out of range"); + } + if (requestedDigits > + static_cast(std::numeric_limits::max())) { + throw py::value_error("decimal digits are out of range"); + } + } + } +} + inline void ApplyInputSizeOverride(PyObject* params, PyObject* inputSize, Py_ssize_t index, ParamInfo& info, const std::string& charEncoding) { py::tuple values = borrow(inputSize); @@ -391,6 +474,11 @@ inline std::vector DetectParamTypes(PyObject* params, PyObject* input const std::string& charEncoding = "utf-8") { PyTypeCache::initialize(); + if (!PyList_Check(params)) { + throw py::type_error("params must be a list"); + } + ValidateInputSizes(inputSizes); + const Py_ssize_t n = PyList_GET_SIZE(params); const Py_ssize_t inputSizeCount = inputSizes == Py_None ? 0 : PyList_GET_SIZE(inputSizes); std::vector infos(n); diff --git a/tests/test_010_pybind_functions.py b/tests/test_010_pybind_functions.py index ec85f14ed..ce4ec61db 100644 --- a/tests/test_010_pybind_functions.py +++ b/tests/test_010_pybind_functions.py @@ -51,6 +51,42 @@ def test_arrow_batch_rejects_unsafe_size_before_handle_access(batch_size): ddbc.DDBCSQLFetchArrowBatch(None, [], batch_size, 0) +@pytest.mark.skipif(not DDBC_AVAILABLE, reason="ddbc_bindings not available") +@pytest.mark.parametrize("param_set_size", [-1, 0, 1_000_001, True]) +def test_executemany_rejects_unsafe_size_before_handle_access(param_set_size): + with pytest.raises(RuntimeError, match="Parameter set size"): + ddbc.SQLExecuteMany(None, "", [], [], param_set_size, {}) + + +@pytest.mark.skipif(not DDBC_AVAILABLE, reason="ddbc_bindings not available") +@pytest.mark.parametrize( + ("input_sizes", "message"), + [ + ((12, -8, 1, 0), "inputSizes must be None or a list"), + ([None], "four-item tuple"), + ([(12, -8, 1)], "four-item tuple"), + ([(12, -8, True, 0)], "must be integers"), + ([(12, -8, -1, 0)], "column size must be non-negative"), + ([(12, -8, 2**128, 0)], "column size is out of range"), + ([(12, -8, 1, 2**128)], "decimal digits are out of range"), + ], +) +def test_execute_rejects_malformed_input_sizes_before_handle_access(input_sizes, message): + with pytest.raises((TypeError, ValueError), match=message): + ddbc.DDBCSQLExecute(None, "", [], input_sizes, [False], True, {}) + + +@pytest.mark.skipif(not DDBC_AVAILABLE, reason="ddbc_bindings not available") +def test_lob_buffer_growth_is_geometric(): + growth_count, data_size, allocation_size, reserved_bytes = ddbc._test_lob_buffer_growth( + 64, 64 * 1024 + ) + assert data_size == 4 * 1024 * 1024 + assert allocation_size == data_size + assert reserved_bytes == allocation_size + assert growth_count == 7 + + @pytest.mark.skipif(not DDBC_AVAILABLE, reason="ddbc_bindings not available") class TestPybindModuleInfo: """Test module information and architecture detection.""" From d0912f643faf4b0e789cb7b1a4c8e5b38929387b Mon Sep 17 00:00:00 2001 From: gargsaumya Date: Thu, 8 Oct 2026 12:35:44 +0530 Subject: [PATCH 31/31] FIX: bound LOB growth test helper Reject boolean, zero, oversized chunk counts and invalid chunk sizes before the internal production test binding can enter its simulation loop. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- mssql_python/pybind/ddbc_bindings.cpp | 11 ++++++++++- tests/test_010_pybind_functions.py | 15 +++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/mssql_python/pybind/ddbc_bindings.cpp b/mssql_python/pybind/ddbc_bindings.cpp index 21ab00f33..b89ca350c 100644 --- a/mssql_python/pybind/ddbc_bindings.cpp +++ b/mssql_python/pybind/ddbc_bindings.cpp @@ -519,7 +519,16 @@ size_t ExtendNativeFetchBuffer(std::vector& buffer, size_t currentSize, si return requiredSize; } -py::tuple TestLobBufferGrowth(size_t chunkCount, size_t chunkSize) { +py::tuple TestLobBufferGrowth(py::handle chunkCountArg, py::handle chunkSizeArg) { + const size_t chunkCount = static_cast( + ValidateNativeRowCountArgument(chunkCountArg, "LOB growth chunk count", false)); + if (!PyLong_Check(chunkSizeArg.ptr()) || PyBool_Check(chunkSizeArg.ptr())) { + ThrowStdException("LOB growth chunk size must be a positive integer"); + } + const size_t chunkSize = chunkSizeArg.cast(); + if (chunkSize == 0 || chunkSize > MAX_NATIVE_FETCH_BYTES) { + ThrowStdException("LOB growth chunk size must be between 1 and 256 MiB"); + } std::vector buffer; size_t dataSize = 0; size_t reservedBytes = 0; diff --git a/tests/test_010_pybind_functions.py b/tests/test_010_pybind_functions.py index ce4ec61db..41e175af4 100644 --- a/tests/test_010_pybind_functions.py +++ b/tests/test_010_pybind_functions.py @@ -87,6 +87,21 @@ def test_lob_buffer_growth_is_geometric(): assert growth_count == 7 +@pytest.mark.skipif(not DDBC_AVAILABLE, reason="ddbc_bindings not available") +@pytest.mark.parametrize( + ("chunk_count", "chunk_size", "message"), + [ + (1_000_001, 1, "chunk count"), + (True, 1, "chunk count"), + (1, 0, "chunk size"), + (1, True, "chunk size"), + ], +) +def test_lob_buffer_growth_helper_rejects_unsafe_sizes(chunk_count, chunk_size, message): + with pytest.raises((RuntimeError, TypeError), match=message): + ddbc._test_lob_buffer_growth(chunk_count, chunk_size) + + @pytest.mark.skipif(not DDBC_AVAILABLE, reason="ddbc_bindings not available") class TestPybindModuleInfo: """Test module information and architecture detection."""