Skip to content

Commit b96534b

Browse files
gargsaumyagargsaumya
committed
Add SQL_VARIANT data type support with native Python type preservation
- Add SQL_VARIANT constant (-150) to constants.py - Implement preprocessing approach in ddbc_bindings.cpp: * MapVariantCTypeToSQLType helper maps C types to SQL types * SQLGetData_wrap detects sql_variant and maps to base type * Handles old-style date/time C codes (9, 10, 11) * Handles SQL Server TIME type (code 16384) * Routes to existing type conversion logic (no duplication) - Move LOB detection before calculateRowSize in FetchAll_wrap - Add comprehensive test suite (25 tests): * Tests all SQL base types: INT, BIGINT, SMALLINT, TINYINT, REAL, FLOAT, DECIMAL, NUMERIC, BIT, VARCHAR, NVARCHAR, DATE, TIME, DATETIME, DATETIME2, VARBINARY, UNIQUEIDENTIFIER, NULL * Tests all fetch methods: fetchone(), fetchmany(), fetchall() * Tests implicit vs explicit type casting * All tests passing Type mappings: - Integer types → Python int - Float types → Python float - Exact numeric → Python Decimal - Character types → Python str - Date/time types → Python date/time/datetime objects - Binary → Python bytes - GUID → Python str/UUID - NULL → Python None
1 parent 7388593 commit b96534b

4 files changed

Lines changed: 656 additions & 9 deletions

File tree

‎mssql_python/constants.py‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -118,6 +118,7 @@ class ConstantsDDBC(Enum):
118118
SQL_DATETIMEOFFSET = -155
119119
SQL_SS_TIME2 = -154
120120
SQL_SS_XML = -152
121+
SQL_SS_VARIANT = -150
121122
SQL_C_SS_TIMESTAMPOFFSET = 0x4001
122123
SQL_SCOPE_CURROW = 0
123124
SQL_BEST_ROWID = 1
@@ -375,7 +376,11 @@ def get_valid_types(cls) -> set:
375376
ConstantsDDBC.SQL_DATETIMEOFFSET.value,
376377
ConstantsDDBC.SQL_SS_XML.value,
377378
ConstantsDDBC.SQL_GUID.value,
379+
<<<<<<< HEAD
378380
ConstantsDDBC.SQL_SS_UDT.value,
381+
=======
382+
ConstantsDDBC.SQL_SS_VARIANT.value,
383+
>>>>>>> a7cc4c0d (Add SQL_VARIANT data type support with native Python type preservation)
379384
}
380385

381386
# Could also add category methods for convenience

‎mssql_python/cursor.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -879,6 +879,7 @@ def _get_c_type_for_sql_type(self, sql_type: int) -> int:
879879
# Other types
880880
ddbc_sql_const.SQL_GUID.value: ddbc_sql_const.SQL_C_GUID.value,
881881
ddbc_sql_const.SQL_SS_XML.value: ddbc_sql_const.SQL_C_WCHAR.value,
882+
ddbc_sql_const.SQL_SS_VARIANT.value: ddbc_sql_const.SQL_C_BINARY.value,
882883
}
883884
return sql_to_c_type.get(sql_type, ddbc_sql_const.SQL_C_DEFAULT.value)
884885

‎mssql_python/pybind/ddbc_bindings.cpp‎

Lines changed: 150 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,11 @@
2828
#define SQL_MAX_NUMERIC_LEN 16
2929
#define SQL_SS_XML (-152)
3030
#define SQL_SS_UDT (-151)
31+
#define SQL_SS_VARIANT (-150)
32+
#define SQL_CA_SS_BASE 1200
33+
#define SQL_CA_SS_VARIANT_TYPE (SQL_CA_SS_BASE + 15)
34+
#define SQL_CA_SS_VARIANT_SQL_TYPE (SQL_CA_SS_BASE + 16)
35+
#define SQL_CA_SS_VARIANT_SERVER_TYPE (SQL_CA_SS_BASE + 17)
3136

3237
#define STRINGIFY_FOR_CASE(x) \
3338
case x: \
@@ -2907,6 +2912,55 @@ py::object FetchLobColumnData(SQLHSTMT hStmt, SQLUSMALLINT colIndex, SQLSMALLINT
29072912
}
29082913
}
29092914

2915+
// Helper function to map sql_variant's underlying C type to SQL data type
2916+
// This allows sql_variant to reuse existing fetch logic for each data type
2917+
SQLSMALLINT MapVariantCTypeToSQLType(SQLLEN variantCType) {
2918+
switch (variantCType) {
2919+
case SQL_C_SLONG:
2920+
case SQL_C_LONG:
2921+
return SQL_INTEGER;
2922+
case SQL_C_SSHORT:
2923+
case SQL_C_SHORT:
2924+
return SQL_SMALLINT;
2925+
case SQL_C_SBIGINT:
2926+
return SQL_BIGINT;
2927+
case SQL_C_FLOAT:
2928+
return SQL_REAL;
2929+
case SQL_C_DOUBLE:
2930+
return SQL_DOUBLE;
2931+
case SQL_C_BIT:
2932+
return SQL_BIT;
2933+
case SQL_C_CHAR:
2934+
return SQL_VARCHAR;
2935+
case SQL_C_WCHAR:
2936+
return SQL_WVARCHAR;
2937+
// Date/time types - handle both old-style (9, 10, 11) and new-style (91, 92, 93) codes
2938+
case 9: // SQL_C_DATE (old style)
2939+
case SQL_C_TYPE_DATE: // 91 (new style)
2940+
return SQL_TYPE_DATE;
2941+
case 10: // SQL_C_TIME (old style)
2942+
case SQL_C_TYPE_TIME: // 92 (new style)
2943+
case 16384: // SQL Server variant TIME type (observed value)
2944+
return SQL_TYPE_TIME;
2945+
case 11: // SQL_C_TIMESTAMP (old style)
2946+
case SQL_C_TYPE_TIMESTAMP: // 93 (new style)
2947+
return SQL_TYPE_TIMESTAMP;
2948+
case SQL_C_BINARY:
2949+
return SQL_VARBINARY;
2950+
case SQL_C_GUID:
2951+
return SQL_GUID;
2952+
case SQL_C_NUMERIC:
2953+
return SQL_NUMERIC;
2954+
case SQL_C_TINYINT:
2955+
case SQL_C_UTINYINT:
2956+
case SQL_C_STINYINT:
2957+
return SQL_TINYINT;
2958+
default:
2959+
// Unknown type, fallback to WVARCHAR for string conversion
2960+
return SQL_WVARCHAR;
2961+
}
2962+
}
2963+
29102964
// Helper function to retrieve column data
29112965
SQLRETURN SQLGetData_wrap(SqlHandlePtr StatementHandle, SQLUSMALLINT colCount, py::list& row,
29122966
const std::string& charEncoding = "utf-8",
@@ -2945,7 +2999,40 @@ SQLRETURN SQLGetData_wrap(SqlHandlePtr StatementHandle, SQLUSMALLINT colCount, p
29452999
continue;
29463000
}
29473001

2948-
switch (dataType) {
3002+
printf("[DEBUG] SQLGetData_wrap: Column %d - dataType=%d, columnSize=%lu\n", i, dataType,
3003+
(unsigned long)columnSize);
3004+
3005+
// Preprocess sql_variant: detect underlying type and handle NULL
3006+
// This allows reuse of existing fetch logic instead of duplicating code
3007+
SQLSMALLINT effectiveDataType = dataType;
3008+
if (dataType == SQL_SS_VARIANT) {
3009+
// Step 1: Check for NULL using header read
3010+
SQLLEN indicator;
3011+
ret = SQLGetData_ptr(hStmt, i, SQL_C_BINARY, NULL, 0, &indicator);
3012+
if (indicator == SQL_NULL_DATA) {
3013+
row.append(py::none());
3014+
continue; // Skip to next column
3015+
}
3016+
3017+
// Step 2: Get the variant's underlying C data type
3018+
SQLLEN variantCType = 0;
3019+
ret =
3020+
SQLColAttribute_ptr(hStmt, i, SQL_CA_SS_VARIANT_TYPE, NULL, 0, NULL, &variantCType);
3021+
if (!SQL_SUCCEEDED(ret)) {
3022+
LOG("SQLGetData: Failed to get sql_variant underlying type for column %d", i);
3023+
row.append(py::none());
3024+
continue; // Skip to next column
3025+
}
3026+
3027+
printf("[DEBUG] SQLGetData_wrap: sql_variant column %d has variantCType=%ld\n", i,
3028+
(long)variantCType);
3029+
3030+
// Step 3: Map C type to SQL type so existing code can handle it
3031+
effectiveDataType = MapVariantCTypeToSQLType(variantCType);
3032+
printf("[DEBUG] SQLGetData_wrap: Mapped to effectiveDataType=%d\n", effectiveDataType);
3033+
}
3034+
3035+
switch (effectiveDataType) {
29493036
case SQL_CHAR:
29503037
case SQL_VARCHAR:
29513038
case SQL_LONGVARCHAR: {
@@ -4048,7 +4135,8 @@ size_t calculateRowSize(py::list& columnNames, SQLUSMALLINT numCols) {
40484135
break;
40494136
case SQL_SS_UDT:
40504137
rowSize += (static_cast<SQLLEN>(columnSize) == SQL_NO_TOTAL || columnSize == 0)
4051-
? SQL_MAX_LOB_SIZE : columnSize;
4138+
? SQL_MAX_LOB_SIZE
4139+
: columnSize;
40524140
break;
40534141
case SQL_BINARY:
40544142
case SQL_VARBINARY:
@@ -4110,11 +4198,19 @@ SQLRETURN FetchMany_wrap(SqlHandlePtr StatementHandle, py::list& rows, int fetch
41104198
SQLSMALLINT dataType = colMeta["DataType"].cast<SQLSMALLINT>();
41114199
SQLULEN columnSize = colMeta["ColumnSize"].cast<SQLULEN>();
41124200

4113-
if ((dataType == SQL_WVARCHAR || dataType == SQL_WLONGVARCHAR || dataType == SQL_VARCHAR ||
4114-
dataType == SQL_LONGVARCHAR || dataType == SQL_VARBINARY ||
4115-
dataType == SQL_LONGVARBINARY || dataType == SQL_SS_XML ||
4116-
dataType == SQL_SS_UDT) &&
4117-
(columnSize == 0 || columnSize == SQL_NO_TOTAL || columnSize > SQL_MAX_LOB_SIZE)) {
4201+
printf("[DEBUG] FetchMany_wrap: Column %d - dataType=%d, columnSize=%lu\n", i + 1, dataType,
4202+
(unsigned long)columnSize);
4203+
4204+
// Detect LOB columns that need SQLGetData streaming
4205+
// sql_variant always uses SQLGetData for native type preservation
4206+
if (dataType == SQL_SS_VARIANT) {
4207+
lobColumns.push_back(i + 1); // 1-based
4208+
} else if ((dataType == SQL_WVARCHAR || dataType == SQL_WLONGVARCHAR ||
4209+
dataType == SQL_VARCHAR || dataType == SQL_LONGVARCHAR ||
4210+
dataType == SQL_VARBINARY || dataType == SQL_LONGVARBINARY ||
4211+
dataType == SQL_SS_XML || dataType == SQL_SS_UDT) &&
4212+
(columnSize == 0 || columnSize == SQL_NO_TOTAL ||
4213+
columnSize > SQL_MAX_LOB_SIZE)) {
41184214
lobColumns.push_back(i + 1); // 1-based
41194215
}
41204216
}
@@ -4204,6 +4300,52 @@ SQLRETURN FetchAll_wrap(SqlHandlePtr StatementHandle, py::list& rows,
42044300
return ret;
42054301
}
42064302

4303+
// Detect LOB columns FIRST (before calculateRowSize)
4304+
// This allows sql_variant to skip the binding path entirely
4305+
std::vector<SQLUSMALLINT> lobColumns;
4306+
for (SQLSMALLINT i = 0; i < numCols; i++) {
4307+
auto colMeta = columnNames[i].cast<py::dict>();
4308+
SQLSMALLINT dataType = colMeta["DataType"].cast<SQLSMALLINT>();
4309+
SQLULEN columnSize = colMeta["ColumnSize"].cast<SQLULEN>();
4310+
4311+
printf("[DEBUG] FetchAll_wrap: Column %d - dataType=%d, columnSize=%lu\n", i + 1, dataType,
4312+
(unsigned long)columnSize);
4313+
4314+
// Detect LOB columns that need SQLGetData streaming
4315+
// sql_variant always uses SQLGetData for native type preservation
4316+
if (dataType == SQL_SS_VARIANT) {
4317+
lobColumns.push_back(i + 1); // 1-based
4318+
} else if ((dataType == SQL_WVARCHAR || dataType == SQL_WLONGVARCHAR ||
4319+
dataType == SQL_VARCHAR || dataType == SQL_LONGVARCHAR ||
4320+
dataType == SQL_VARBINARY || dataType == SQL_LONGVARBINARY ||
4321+
dataType == SQL_SS_XML) &&
4322+
(columnSize == 0 || columnSize == SQL_NO_TOTAL ||
4323+
columnSize > SQL_MAX_LOB_SIZE)) {
4324+
lobColumns.push_back(i + 1); // 1-based
4325+
}
4326+
}
4327+
4328+
// If we have LOBs → fall back to row-by-row fetch + SQLGetData_wrap
4329+
if (!lobColumns.empty()) {
4330+
LOG("FetchAll_wrap: LOB columns detected (%zu columns), using per-row "
4331+
"SQLGetData path",
4332+
lobColumns.size());
4333+
while (true) {
4334+
ret = SQLFetch_ptr(hStmt);
4335+
if (ret == SQL_NO_DATA)
4336+
break;
4337+
if (!SQL_SUCCEEDED(ret))
4338+
return ret;
4339+
4340+
py::list row;
4341+
SQLGetData_wrap(StatementHandle, numCols, row, charEncoding,
4342+
wcharEncoding); // <-- streams LOBs correctly
4343+
rows.append(row);
4344+
}
4345+
return SQL_SUCCESS;
4346+
}
4347+
4348+
// No LOBs detected - use binding path with batch fetching
42074349
// Define a memory limit (1 GB)
42084350
const size_t memoryLimit = 1ULL * 1024 * 1024 * 1024;
42094351
size_t totalRowSize = calculateRowSize(columnNames, numCols);
@@ -4252,8 +4394,7 @@ SQLRETURN FetchAll_wrap(SqlHandlePtr StatementHandle, py::list& rows,
42524394

42534395
if ((dataType == SQL_WVARCHAR || dataType == SQL_WLONGVARCHAR || dataType == SQL_VARCHAR ||
42544396
dataType == SQL_LONGVARCHAR || dataType == SQL_VARBINARY ||
4255-
dataType == SQL_LONGVARBINARY || dataType == SQL_SS_XML ||
4256-
dataType == SQL_SS_UDT) &&
4397+
dataType == SQL_LONGVARBINARY || dataType == SQL_SS_XML || dataType == SQL_SS_UDT) &&
42574398
(columnSize == 0 || columnSize == SQL_NO_TOTAL || columnSize > SQL_MAX_LOB_SIZE)) {
42584399
lobColumns.push_back(i + 1); // 1-based
42594400
}

0 commit comments

Comments
 (0)