Skip to content

Commit b06ac2c

Browse files
subrata-mssubrata-ms
authored
Merge branch 'main' into subrata-ms/cp1252_encoding
2 parents d7e894b + 590af57 commit b06ac2c

5 files changed

Lines changed: 797 additions & 9 deletions

File tree

‎mssql_python/constants.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -118,6 +118,7 @@ class ConstantsDDBC(Enum):
118118
SQL_DATETIMEOFFSET = -155
119119
SQL_SS_TIME2 = -154
120120
SQL_SS_XML = -152
121+
SQL_SS_VARIANT = -150
121122
SQL_C_SS_TIMESTAMPOFFSET = 0x4001
122123
SQL_SCOPE_CURROW = 0
123124
SQL_BEST_ROWID = 1
@@ -376,6 +377,7 @@ def get_valid_types(cls) -> set:
376377
ConstantsDDBC.SQL_SS_XML.value,
377378
ConstantsDDBC.SQL_GUID.value,
378379
ConstantsDDBC.SQL_SS_UDT.value,
380+
ConstantsDDBC.SQL_SS_VARIANT.value,
379381
}
380382

381383
# Could also add category methods for convenience

‎mssql_python/cursor.py‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -396,7 +396,7 @@ def _map_sql_type( # pylint: disable=too-many-arguments,too-many-positional-arg
396396
if param is None:
397397
logger.debug("_map_sql_type: NULL parameter - index=%d", i)
398398
return (
399-
ddbc_sql_const.SQL_VARCHAR.value,
399+
ddbc_sql_const.SQL_UNKNOWN_TYPE.value,
400400
ddbc_sql_const.SQL_C_DEFAULT.value,
401401
1,
402402
0,
@@ -883,6 +883,7 @@ def _get_c_type_for_sql_type(self, sql_type: int) -> int:
883883
# Other types
884884
ddbc_sql_const.SQL_GUID.value: ddbc_sql_const.SQL_C_GUID.value,
885885
ddbc_sql_const.SQL_SS_XML.value: ddbc_sql_const.SQL_C_WCHAR.value,
886+
ddbc_sql_const.SQL_SS_VARIANT.value: ddbc_sql_const.SQL_C_BINARY.value,
886887
}
887888
return sql_to_c_type.get(sql_type, ddbc_sql_const.SQL_C_DEFAULT.value)
888889

@@ -2208,6 +2209,16 @@ def executemany( # pylint: disable=too-many-locals,too-many-branches,too-many-s
22082209
min_val=min_val,
22092210
max_val=max_val,
22102211
)
2212+
2213+
# For executemany with all-NULL columns, SQL_UNKNOWN_TYPE doesn't work
2214+
# with array binding. Fall back to SQL_VARCHAR as a safe default.
2215+
if (
2216+
sample_value is None
2217+
and paraminfo.paramSQLType == ddbc_sql_const.SQL_UNKNOWN_TYPE.value
2218+
):
2219+
paraminfo.paramSQLType = ddbc_sql_const.SQL_VARCHAR.value
2220+
paraminfo.columnSize = 1
2221+
22112222
# Special handling for binary data in auto-detected types
22122223
if paraminfo.paramSQLType in (
22132224
ddbc_sql_const.SQL_BINARY.value,

‎mssql_python/pybind/ddbc_bindings.cpp‎

Lines changed: 159 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,19 @@
2828
#define SQL_MAX_NUMERIC_LEN 16
2929
#define SQL_SS_XML (-152)
3030
#define SQL_SS_UDT (-151)
31+
#define SQL_SS_VARIANT (-150)
32+
#define SQL_CA_SS_VARIANT_TYPE (1215)
33+
#ifndef SQL_C_DATE
34+
#define SQL_C_DATE (9)
35+
#endif
36+
#ifndef SQL_C_TIME
37+
#define SQL_C_TIME (10)
38+
#endif
39+
#ifndef SQL_C_TIMESTAMP
40+
#define SQL_C_TIMESTAMP (11)
41+
#endif
42+
// SQL Server-specific variant TIME type code
43+
#define SQL_SS_VARIANT_TIME (16384)
3144

3245
#define STRINGIFY_FOR_CASE(x) \
3346
case x: \
@@ -471,14 +484,21 @@ SQLRETURN BindParameters(SQLHANDLE hStmt, const py::list& params,
471484
hStmt, static_cast<SQLUSMALLINT>(paramIndex + 1), &describedType,
472485
&describedSize, &describedDigits, &nullable);
473486
if (!SQL_SUCCEEDED(rc)) {
474-
LOG("BindParameters: SQLDescribeParam failed for "
475-
"param[%d] (NULL parameter) - SQLRETURN=%d",
476-
paramIndex, rc);
477-
return rc;
487+
// SQLDescribeParam can fail for generic SELECT statements where
488+
// no table column is referenced. Fall back to SQL_VARCHAR as a safe
489+
// default.
490+
LOG_WARNING("BindParameters: SQLDescribeParam failed for "
491+
"param[%d] (NULL parameter) - SQLRETURN=%d, falling back to "
492+
"SQL_VARCHAR",
493+
paramIndex, rc);
494+
sqlType = SQL_VARCHAR;
495+
columnSize = 1;
496+
decimalDigits = 0;
497+
} else {
498+
sqlType = describedType;
499+
columnSize = describedSize;
500+
decimalDigits = describedDigits;
478501
}
479-
sqlType = describedType;
480-
columnSize = describedSize;
481-
decimalDigits = describedDigits;
482502
}
483503
dataPtr = nullptr;
484504
strLenOrIndPtr = AllocateParamBuffer<SQLLEN>(paramBuffers);
@@ -2907,6 +2927,67 @@ py::object FetchLobColumnData(SQLHSTMT hStmt, SQLUSMALLINT colIndex, SQLSMALLINT
29072927
}
29082928
}
29092929

2930+
// Helper function to map sql_variant's underlying C type to SQL data type
2931+
// This allows sql_variant to reuse existing fetch logic for each data type
2932+
SQLSMALLINT MapVariantCTypeToSQLType(SQLLEN variantCType) {
2933+
switch (variantCType) {
2934+
case SQL_C_SLONG:
2935+
case SQL_C_LONG:
2936+
return SQL_INTEGER;
2937+
case SQL_C_SSHORT:
2938+
case SQL_C_SHORT:
2939+
return SQL_SMALLINT;
2940+
case SQL_C_SBIGINT:
2941+
return SQL_BIGINT;
2942+
case SQL_C_FLOAT:
2943+
return SQL_REAL;
2944+
case SQL_C_DOUBLE:
2945+
return SQL_DOUBLE;
2946+
case SQL_C_BIT:
2947+
return SQL_BIT;
2948+
case SQL_C_CHAR:
2949+
return SQL_VARCHAR;
2950+
case SQL_C_WCHAR:
2951+
return SQL_WVARCHAR;
2952+
case SQL_C_DATE:
2953+
case SQL_C_TYPE_DATE:
2954+
return SQL_TYPE_DATE;
2955+
case SQL_C_TIME:
2956+
case SQL_C_TYPE_TIME:
2957+
case SQL_SS_VARIANT_TIME:
2958+
return SQL_TYPE_TIME;
2959+
case SQL_C_TIMESTAMP:
2960+
case SQL_C_TYPE_TIMESTAMP:
2961+
return SQL_TYPE_TIMESTAMP;
2962+
case SQL_C_BINARY:
2963+
return SQL_VARBINARY;
2964+
case SQL_C_GUID:
2965+
return SQL_GUID;
2966+
case SQL_C_NUMERIC:
2967+
return SQL_NUMERIC;
2968+
case SQL_C_TINYINT:
2969+
case SQL_C_UTINYINT:
2970+
case SQL_C_STINYINT:
2971+
return SQL_TINYINT;
2972+
default:
2973+
// Unknown C type code - fallback to WVARCHAR for string conversion
2974+
// Note: SQL Server enforces sql_variant restrictions at INSERT time, preventing
2975+
// invalid types (text, ntext, image, timestamp, xml, MAX types, nested variants,
2976+
// spatial types, hierarchyid, UDTs) from being stored. By the time we fetch data,
2977+
// only valid base types exist. This default handles unmapped/future type codes.
2978+
return SQL_WVARCHAR;
2979+
}
2980+
}
2981+
2982+
// Helper function to check if a column requires SQLGetData streaming (LOB or sql_variant)
2983+
static inline bool IsLobOrVariantColumn(SQLSMALLINT dataType, SQLULEN columnSize) {
2984+
return dataType == SQL_SS_VARIANT ||
2985+
((dataType == SQL_WVARCHAR || dataType == SQL_WLONGVARCHAR || dataType == SQL_VARCHAR ||
2986+
dataType == SQL_LONGVARCHAR || dataType == SQL_VARBINARY ||
2987+
dataType == SQL_LONGVARBINARY || dataType == SQL_SS_XML || dataType == SQL_SS_UDT) &&
2988+
(columnSize == 0 || columnSize == SQL_NO_TOTAL || columnSize > SQL_MAX_LOB_SIZE));
2989+
}
2990+
29102991
// Helper function to retrieve column data
29112992
SQLRETURN SQLGetData_wrap(SqlHandlePtr StatementHandle, SQLUSMALLINT colCount, py::list& row,
29122993
const std::string& charEncoding = "utf-8",
@@ -2949,7 +3030,42 @@ SQLRETURN SQLGetData_wrap(SqlHandlePtr StatementHandle, SQLUSMALLINT colCount, p
29493030
continue;
29503031
}
29513032

2952-
switch (dataType) {
3033+
// Preprocess sql_variant: detect underlying type to route to correct conversion logic
3034+
SQLSMALLINT effectiveDataType = dataType;
3035+
if (dataType == SQL_SS_VARIANT) {
3036+
// For sql_variant, we MUST call SQLGetData with SQL_C_BINARY (NULL buffer, len=0)
3037+
// first. This serves two purposes:
3038+
// 1. Detects NULL values via the indicator parameter
3039+
// 2. Initializes the variant metadata in the ODBC driver, which is required for
3040+
// SQLColAttribute(SQL_CA_SS_VARIANT_TYPE) to return the correct underlying C type.
3041+
// Without this probe call, SQLColAttribute returns incorrect type codes.
3042+
SQLLEN indicator;
3043+
ret = SQLGetData_ptr(hStmt, i, SQL_C_BINARY, NULL, 0, &indicator);
3044+
if (!SQL_SUCCEEDED(ret)) {
3045+
LOG_ERROR("SQLGetData: Failed to probe sql_variant column %d - SQLRETURN=%d", i,
3046+
ret);
3047+
row.append(py::none());
3048+
continue;
3049+
}
3050+
if (indicator == SQL_NULL_DATA) {
3051+
row.append(py::none());
3052+
continue;
3053+
}
3054+
// Now retrieve the underlying C type
3055+
SQLLEN variantCType = 0;
3056+
ret =
3057+
SQLColAttribute_ptr(hStmt, i, SQL_CA_SS_VARIANT_TYPE, NULL, 0, NULL, &variantCType);
3058+
if (!SQL_SUCCEEDED(ret)) {
3059+
LOG_ERROR("SQLGetData: Failed to get sql_variant underlying type for column %d", i);
3060+
row.append(py::none());
3061+
continue;
3062+
}
3063+
effectiveDataType = MapVariantCTypeToSQLType(variantCType);
3064+
LOG("SQLGetData: sql_variant column %d has variantCType=%ld, mapped to SQL type %d", i,
3065+
(long)variantCType, effectiveDataType);
3066+
}
3067+
3068+
switch (effectiveDataType) {
29533069
case SQL_CHAR:
29543070
case SQL_VARCHAR:
29553071
case SQL_LONGVARCHAR: {
@@ -4169,6 +4285,7 @@ SQLRETURN FetchMany_wrap(SqlHandlePtr StatementHandle, py::list& rows, int fetch
41694285
dataType == SQL_LONGVARCHAR || dataType == SQL_VARBINARY ||
41704286
dataType == SQL_LONGVARBINARY || dataType == SQL_SS_XML || dataType == SQL_SS_UDT) &&
41714287
(columnSize == 0 || columnSize == SQL_NO_TOTAL || columnSize > SQL_MAX_LOB_SIZE)) {
4288+
if (IsLobOrVariantColumn(dataType, columnSize)) {
41724289
lobColumns.push_back(i + 1); // 1-based
41734290
}
41744291
}
@@ -4258,6 +4375,40 @@ SQLRETURN FetchAll_wrap(SqlHandlePtr StatementHandle, py::list& rows,
42584375
return ret;
42594376
}
42604377

4378+
std::vector<SQLUSMALLINT> lobColumns;
4379+
for (SQLSMALLINT i = 0; i < numCols; i++) {
4380+
auto colMeta = columnNames[i].cast<py::dict>();
4381+
SQLSMALLINT dataType = colMeta["DataType"].cast<SQLSMALLINT>();
4382+
SQLULEN columnSize = colMeta["ColumnSize"].cast<SQLULEN>();
4383+
4384+
// Detect LOB columns that need SQLGetData streaming
4385+
// sql_variant always uses SQLGetData for native type preservation
4386+
if (IsLobOrVariantColumn(dataType, columnSize)) {
4387+
lobColumns.push_back(i + 1); // 1-based
4388+
}
4389+
}
4390+
4391+
// If we have LOBs → fall back to row-by-row fetch + SQLGetData_wrap
4392+
if (!lobColumns.empty()) {
4393+
LOG("FetchAll_wrap: LOB columns detected (%zu columns), using per-row "
4394+
"SQLGetData path",
4395+
lobColumns.size());
4396+
while (true) {
4397+
ret = SQLFetch_ptr(hStmt);
4398+
if (ret == SQL_NO_DATA)
4399+
break;
4400+
if (!SQL_SUCCEEDED(ret))
4401+
return ret;
4402+
4403+
py::list row;
4404+
SQLGetData_wrap(StatementHandle, numCols, row, charEncoding,
4405+
wcharEncoding); // <-- streams LOBs correctly
4406+
rows.append(row);
4407+
}
4408+
return SQL_SUCCESS;
4409+
}
4410+
4411+
// No LOBs detected - use binding path with batch fetching
42614412
// Define a memory limit (1 GB)
42624413
const size_t memoryLimit = 1ULL * 1024 * 1024 * 1024;
42634414
size_t totalRowSize = calculateRowSize(columnNames, numCols);

‎tests/test_004_cursor.py‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -658,6 +658,23 @@ def test_varbinary_full_capacity(cursor, db_connection):
658658
db_connection.commit()
659659

660660

661+
def test_execute_none_into_varbinary_column(cursor, db_connection):
662+
from mssql_python.constants import ConstantsDDBC
663+
664+
drop_table_if_exists(cursor, "#test_varbinary_null")
665+
try:
666+
cursor.execute("CREATE TABLE #test_varbinary_null (data VARBINARY(100))")
667+
db_connection.commit()
668+
cursor.setinputsizes([(ConstantsDDBC.SQL_VARBINARY.value, 100, 0)])
669+
cursor.execute("INSERT INTO #test_varbinary_null (data) VALUES (?)", None)
670+
db_connection.commit()
671+
cursor.execute("SELECT data FROM #test_varbinary_null")
672+
row = cursor.fetchone()
673+
assert row[0] is None
674+
finally:
675+
drop_table_if_exists(cursor, "#test_varbinary_null")
676+
677+
661678
def test_varbinary_max(cursor, db_connection):
662679
"""Test SQL_VARBINARY with MAX length"""
663680
try:

0 commit comments

Comments
 (0)