diff --git a/docs/changeset-format.md b/docs/changeset-format.md index 4ab8dc50..b05a1a1c 100644 --- a/docs/changeset-format.md +++ b/docs/changeset-format.md @@ -1,25 +1,38 @@ # Changeset Format -The format for changesets is borrowed from SQLite3 session extension's internal format -and it is currently 100% compatible with it. Below are details of the format, extracted -from SQLite3 source code. +The format for changesets is based on the SQLite3 session extension's internal +format. Below are details of the format: ## Summary -A changeset is a collection of DELETE, UPDATE and INSERT operations on -one or more tables. Operations on a single table are grouped together, -but may occur in any order (i.e. deletes, updates and inserts are all -mixed together). +A changeset is a linear list of operations of various types, identified by a +one-byte tag: -Each group of changes begins with a table header: +- Table record (`'T'`) +- Data entry (`18`, `23`, `9`) +- Create table entry (`'a'`) +- Drop table entry (`'A'`) +- Add column entry (`'c'`) +- Drop column entry (`'C'`) + +Data operations on a single table are grouped together, preceded by a single +table record. The operations are processed as if they were executed +sequentially. + +## Table record + +The table record identifies the table and its columns: - 1 byte: Constant 0x54 (capital 'T') - Varint: Number of columns in the table. - nCol bytes: 0x01 for PK columns, 0x00 otherwise. -- N bytes: Unqualified table name (encoded using UTF-8). Nul-terminated. +- N bytes: Unqualified table name (encoded using UTF-8). Null-terminated. -Followed by one or more changes to the table. +## Data entry + +A data entry is a DELETE, UPDATE or INSERT operation on one table (identified +by last table record): - 1 byte: Either SQLITE_INSERT (0x12), UPDATE (0x17) or DELETE (0x09). - 1 byte: The "indirect-change" flag. @@ -48,6 +61,44 @@ with table columns modified by the UPDATE change contain the new values. Fields associated with table columns that are not modified are set to "undefined". +## Create table entry + +This entry creates a new empty table: + +- 1 byte: Constant 0x61 (lowercase 'a') +- Null-terminated string: Table name +- Varint: Number of columns in the table. +- nCol entries: Table column info. + +## Drop table entry + +This entry deletes an existing table by name. The table must be empty. Column +information is kept for the purpose of rebasing and inverting the changeset. + +- 1 byte: Constant 0x41 (uppercase 'A') +- Null-terminated string: Table name +- Varint: Number of columns in the table. +- nCol entries: Table column info. + +## Add column entry + +This entry adds a new column to an existing table. All existing rows will have +`NULL` filled in. + +- 1 byte: Constant 0x63 (lowercase 'c') +- Null-terminated string: Table name +- Table column info. + +## Drop column entry + +This entry deletes an existing column from a table. All existing rows must have +`NULL` values in this column. Column information is kept for the purpose of +rebasing and inverting the changeset. + +- 1 byte: Constant 0x43 (uppercase 'C') +- Null-terminated string: Table name +- Table column info. + # Record Format Unlike the SQLite database record format, each field is self-contained - @@ -69,7 +120,7 @@ is followed by: - Text values: A varint containing the number of bytes in the value (encoded using UTF-8). Followed by a buffer containing the UTF-8 representation - of the text value. There is no nul terminator. + of the text value. There is no null terminator. - Blob values: A varint containing the number of bytes in the value, followed by @@ -82,6 +133,19 @@ is followed by: An 8-byte big-endian IEEE 754-2008 real value. +# Table column info + +- Null-terminated string: column name +- 1 byte: Column type (same as record) +- 1 byte: Flags packed as bits. From LSb: + - is primary key + - is autoincrement + - is geometry column + - geometry has Z coordinate + - geometry has M coordinate +- Null-terminated string: geometry type (`POINT`, `LINE`, ...) +- Varint: SRS ID for geometry + # Varint Format Varint values are encoded in the same way as varints in the SQLite diff --git a/docs/schema-changes.md b/docs/schema-changes.md new file mode 100644 index 00000000..803bff73 --- /dev/null +++ b/docs/schema-changes.md @@ -0,0 +1,26 @@ +# Schema changes + +Geodiff supports diffing databases with different schemata. It identifies table +and column additions/deletions. + +Tables and columns are always created empty and any data present in the +database is recreated manually via `INSERT`/`UPDATE` entries, written after the +schema change entry. Likewise, deletion entries expect the table/column to be +empty, so `DELETE`/`UPDATE` entries clearing the data are written beforehand. +This simplifies inverting and rebasing, since the schema change entries work +separately from e.g. the ID renaming machinery. + +## Limitations and pitfalls + +Since we only look at the final state of the database, default values in +columns are not supported. Any default specified during creation of the column +will be simulated by an `UPDATE` for each row. This means that only the rows +present in the modified database will get the "default" value, and the default +won't be propagated when the diff is applied onto base. + +Renaming columns is supported only as a deletion & addition. This has similar +pitfalls to the default values - on rebase, values in the second database won't +be moved. Same with renaming tables. + +The intermediate states created by applying the resulting diff (e.g. "nulling +out" column before dropping it) may conflict with database constraints. diff --git a/geodiff/CMakeLists.txt b/geodiff/CMakeLists.txt index 5ec5382a..147f2d87 100644 --- a/geodiff/CMakeLists.txt +++ b/geodiff/CMakeLists.txt @@ -157,6 +157,8 @@ SET(geodiff_src src/driver.h src/tableschema.cpp src/tableschema.h + src/tableschemadiff.cpp + src/tableschemadiff.hpp src/drivers/sqlitedriver.cpp src/drivers/sqlitedriver.h diff --git a/geodiff/src/changeset.h b/geodiff/src/changeset.h index 98b81340..f6b8a415 100644 --- a/geodiff/src/changeset.h +++ b/geodiff/src/changeset.h @@ -8,9 +8,13 @@ #include #include +#include #include +#include #include +#include "tableschema.h" + /** * Representation of a single value stored in a column. @@ -200,9 +204,23 @@ struct ChangesetTable size_t columnCount() const { return primaryKeys.size(); } }; +/** + * Types of supported changeset records. + */ +enum class ChangesetEntryType +{ + OpTableRecord = 'T', //!< corresponds to ChangesetTable + OpInsert = 18, //!< corresponds to ChangesetDataEntry + OpUpdate = 23, //!< corresponds to ChangesetDataEntry + OpDelete = 9, //!< corresponds to ChangesetDataEntry + OpCreateTable = 'a', //!< corresponds to ChangesetTable + OpDropTable = 'A', + OpAddColumn = 'c', + OpDropColumn = 'C', +}; /** - * Details of a single change within a changeset + * Details of a single data change within a changeset * * Contents of old/new values array based on operation type: * - INSERT - new values contain data of the row to be inserted, old values array is invalid @@ -212,10 +230,11 @@ struct ChangesetTable * columns of old value are always present (but new value of pkey columns is undefined * if the primary key is not being changed). */ -struct ChangesetEntry +struct ChangesetDataEntry { enum OperationType { + // The values here must be kept in sync with values in ChangesetEntryType! OpInsert = 18, //!< equal to SQLITE_INSERT OpUpdate = 23, //!< equal to SQLITE_UPDATE OpDelete = 9, //!< equal to SQLITE_DELETE @@ -231,17 +250,17 @@ struct ChangesetEntry * Optional pointer to the source table information as stored in changeset. * * When the changeset entry has been read by ChangesetReader, the table always will be set to a valid - * instance. Do not delete the instance - it is owned by ChangesetReader. + * instance. * * When the changeset entry is being passed to ChangesetWriter, the table pointer is ignored * and it does not need to be set (writer has an explicit beginTable() call to set table). */ - ChangesetTable *table = nullptr; + std::shared_ptr table; //! a quick way for tests to create a changeset entry - static ChangesetEntry make( ChangesetTable *t, OperationType o, const std::vector &oldV, const std::vector &newV ) + static ChangesetDataEntry make( std::shared_ptr t, OperationType o, const std::vector &oldV, const std::vector &newV ) { - ChangesetEntry e; + ChangesetDataEntry e; e.op = o; e.oldValues = oldV; e.newValues = newV; @@ -250,4 +269,57 @@ struct ChangesetEntry } }; +//! Entry for CREATE TABLE command +struct ChangesetCreateTableEntry +{ + std::string tableName; + std::vector columns; +}; + +//! Entry for DROP TABLE command +struct ChangesetDropTableEntry +{ + std::string tableName; + std::vector columns; +}; + +//! Entry for ALTER TABLE ... ADD COLUMN command +struct ChangesetAddColumnEntry +{ + std::string tableName; + TableColumnInfo column; +}; + +//! Entry for ALTER TABLE ... DROP COLUMN command +struct ChangesetDropColumnEntry +{ + std::string tableName; + TableColumnInfo column; +}; + +struct ChangesetEntry : public std::variant < + ChangesetDataEntry, + ChangesetCreateTableEntry, + ChangesetDropTableEntry, + ChangesetAddColumnEntry, + ChangesetDropColumnEntry > +{ + using variant::variant; // Use std::variant's constructor + + ChangesetEntryType operationType() const + { + if ( const ChangesetDataEntry *e = std::get_if( this ) ) + return static_cast( e->op ); + else if ( std::holds_alternative( *this ) ) + return ChangesetEntryType::OpCreateTable; + else if ( std::holds_alternative( *this ) ) + return ChangesetEntryType::OpDropTable; + else if ( std::holds_alternative( *this ) ) + return ChangesetEntryType::OpAddColumn; + else if ( std::holds_alternative( *this ) ) + return ChangesetEntryType::OpDropColumn; + throw std::invalid_argument( "Unreachable - operationType()" ); + } +}; + #endif // CHANGESET_H diff --git a/geodiff/src/changesetconcat.cpp b/geodiff/src/changesetconcat.cpp index f0f4c64e..080a29a7 100644 --- a/geodiff/src/changesetconcat.cpp +++ b/geodiff/src/changesetconcat.cpp @@ -3,12 +3,12 @@ Copyright (C) 2021 Martin Dobias */ -#include "sqlite3.h" +#include "changeset.h" +#include #include -#include #include -#include +#include #include "geodifflogger.hpp" #include "geodiffcontext.hpp" @@ -17,53 +17,50 @@ #include "changesetwriter.h" -//! Hash value generator based on primary keys to have ChangesetEntry used in std::unordered_set -struct HashChangesetEntryPkey +struct ValueVectorHash { - size_t operator()( const ChangesetEntry *pentry ) const + size_t operator()( const std::vector &values ) const { size_t h = 0; - const ChangesetEntry &entry = *pentry; - const std::vector &pkeys = entry.table->primaryKeys; - const std::vector &values = entry.op == ChangesetEntry::OpInsert ? entry.newValues : entry.oldValues; - for ( size_t i = 0; i < pkeys.size(); ++i ) - { - if ( pkeys[i] ) - h ^= std::hash {}( values[i] ); - } + for ( size_t i = 0; i < values.size(); ++i ) + h ^= std::hash {}( values[i] ); return h; } }; - -//! Exact equality check based on primary keys to have ChangesetEntry used in std::unordered_set -struct EqualToChangesetEntryPkey +static std::vector entryPkey( const ChangesetDataEntry &entry ) { - bool operator()( const ChangesetEntry *plhs, const ChangesetEntry *prhs ) const + const std::vector &pkeys = entry.table->primaryKeys; + const std::vector &values = entry.op == ChangesetDataEntry::OpInsert ? entry.newValues : entry.oldValues; + std::vector pkeyValues; + for ( size_t i = 0; i < values.size(); ++i ) { - const ChangesetEntry &lhs = *plhs; - const ChangesetEntry &rhs = *prhs; - const std::vector &pkeys = lhs.table->primaryKeys; - const std::vector &lhsValues = lhs.op == ChangesetEntry::OpInsert ? lhs.newValues : lhs.oldValues; - const std::vector &rhsValues = rhs.op == ChangesetEntry::OpInsert ? rhs.newValues : rhs.oldValues; - for ( size_t i = 0; i < pkeys.size(); ++i ) - { - if ( pkeys[i] && lhsValues[i] != rhsValues[i] ) - return false; - } - return true; + if ( pkeys[i] ) + pkeyValues.push_back( values[i] ); } -}; + return pkeyValues; +} -typedef std::unordered_set TableEntriesSet; + +// primary key values -> data entry index in entries list +typedef std::unordered_map, size_t, ValueVectorHash> TableEntriesMap; //! Struct to keep information about table and its changes while concatenating struct TableChanges { - std::unique_ptr table; - TableEntriesSet entries; + // List of entries affecting this table. Wrapped in optional so we can do + // in-place O(1) deletions. + std::vector> entries; + // Entries output at the start. Used for column additions. + std::vector prefixEntries; + TableEntriesMap dataEntries; }; +// Output of concatenation is divided into phases, where entries can be freely +// merged. +// Indexed by table name. +typedef std::unordered_map OutputPhase; + //! This is a helper function used by mergeUpdate(). static Value mergeValue( const Value &vOne, const Value &vTwo ) @@ -127,62 +124,62 @@ enum MergeEntriesResult //! Takes two changeset entries e1 and e2 and merges their changes to e1 if possible. //! It is also possible that merging results in no change at all, or the change is not allowed -static MergeEntriesResult mergeEntriesForRow( ChangesetEntry *e1, ChangesetEntry *e2 ) +static MergeEntriesResult mergeEntriesForRow( ChangesetDataEntry &e1, const ChangesetDataEntry &e2 ) { // all these changes make no sense really, if they happen most likely something got broken // (e.g. adding a row with the same pkey twice) - if ( ( e1->op == ChangesetEntry::OpInsert && e2->op == ChangesetEntry::OpInsert ) || - ( e1->op == ChangesetEntry::OpUpdate && e2->op == ChangesetEntry::OpInsert ) || - ( e1->op == ChangesetEntry::OpDelete && e2->op == ChangesetEntry::OpUpdate ) || - ( e1->op == ChangesetEntry::OpDelete && e2->op == ChangesetEntry::OpDelete ) ) + if ( ( e1.op == ChangesetDataEntry::OpInsert && e2.op == ChangesetDataEntry::OpInsert ) || + ( e1.op == ChangesetDataEntry::OpUpdate && e2.op == ChangesetDataEntry::OpInsert ) || + ( e1.op == ChangesetDataEntry::OpDelete && e2.op == ChangesetDataEntry::OpUpdate ) || + ( e1.op == ChangesetDataEntry::OpDelete && e2.op == ChangesetDataEntry::OpDelete ) ) return Unsupported; - if ( e1->op == ChangesetEntry::OpInsert && e2->op == ChangesetEntry::OpDelete ) + if ( e1.op == ChangesetDataEntry::OpInsert && e2.op == ChangesetDataEntry::OpDelete ) return EntryRemoved; - if ( e1->op == ChangesetEntry::OpInsert && e2->op == ChangesetEntry::OpUpdate ) + if ( e1.op == ChangesetDataEntry::OpInsert && e2.op == ChangesetDataEntry::OpUpdate ) { // modify INSERT - update its values wherever the update has a newer value - for ( size_t i = 0; i < e1->table->columnCount(); ++i ) + for ( size_t i = 0; i < e1.table->columnCount(); ++i ) { - if ( e2->newValues[i].type() != Value::TypeUndefined ) - e1->newValues[i] = e2->newValues[i]; + if ( e2.newValues[i].type() != Value::TypeUndefined ) + e1.newValues[i] = e2.newValues[i]; } return EntryModified; } - if ( e1->op == ChangesetEntry::OpUpdate && e2->op == ChangesetEntry::OpUpdate ) + if ( e1.op == ChangesetDataEntry::OpUpdate && e2.op == ChangesetDataEntry::OpUpdate ) { // modify UPDATE std::vector oldVals, newVals; - if ( !mergeUpdate( *e1->table, e2->oldValues, e1->oldValues, e1->newValues, e2->newValues, oldVals, newVals ) ) + if ( !mergeUpdate( *e1.table, e2.oldValues, e1.oldValues, e1.newValues, e2.newValues, oldVals, newVals ) ) return EntryRemoved; - e1->oldValues = oldVals; - e1->newValues = newVals; + e1.oldValues = oldVals; + e1.newValues = newVals; return EntryModified; } - if ( e1->op == ChangesetEntry::OpUpdate && e2->op == ChangesetEntry::OpDelete ) + if ( e1.op == ChangesetDataEntry::OpUpdate && e2.op == ChangesetDataEntry::OpDelete ) { // turn into DELETE, use old values from delete when update does not list them - e1->op = ChangesetEntry::OpDelete; - for ( size_t i = 0; i < e1->table->columnCount(); ++i ) + e1.op = ChangesetDataEntry::OpDelete; + for ( size_t i = 0; i < e1.table->columnCount(); ++i ) { - if ( e1->oldValues[i].type() == Value::TypeUndefined ) - e1->oldValues[i] = e2->oldValues[i]; + if ( e1.oldValues[i].type() == Value::TypeUndefined ) + e1.oldValues[i] = e2.oldValues[i]; } return EntryModified; } - if ( e1->op == ChangesetEntry::OpDelete && e2->op == ChangesetEntry::OpInsert ) + if ( e1.op == ChangesetDataEntry::OpDelete && e2.op == ChangesetDataEntry::OpInsert ) { // turn into UPDATE std::vector oldVals, newVals; - if ( !mergeUpdate( *e1->table, e1->oldValues, {}, e2->newValues, {}, oldVals, newVals ) ) + if ( !mergeUpdate( *e1.table, e1.oldValues, {}, e2.newValues, {}, oldVals, newVals ) ) return EntryRemoved; - e1->op = ChangesetEntry::OpUpdate; - e1->oldValues = oldVals; - e1->newValues = newVals; + e1.op = ChangesetDataEntry::OpUpdate; + e1.oldValues = oldVals; + e1.newValues = newVals; return EntryModified; } @@ -198,8 +195,7 @@ void concatChangesets( const std::vector &filenames, const std::string &outputChangeset ) { - // hashtable: table name -> ( fid -> changeset entry ) - std::unordered_map result; + std::vector outputPhases = {{}}; for ( const std::string &inputFilename : filenames ) { @@ -207,51 +203,84 @@ void concatChangesets( if ( !reader.open( inputFilename ) ) throw GeoDiffException( "concatChangesets: unable to open input file: " + inputFilename ); - ChangesetEntry entry; - while ( reader.nextEntry( entry ) ) + ChangesetEntry fullEntry; + while ( reader.nextEntry( fullEntry ) ) { - auto tableIt = result.find( entry.table->name ); - if ( tableIt == result.end() ) - { - TableChanges &t = result[ entry.table->name ]; // adds new entry - t.table.reset( new ChangesetTable( *entry.table ) ); - ChangesetEntry *e = new ChangesetEntry( entry ); - e->table = t.table.get(); - t.entries.insert( e ); - } - else + OutputPhase &phase = outputPhases.back(); + + if ( ChangesetDataEntry *dEntry = std::get_if( &fullEntry ) ) { - TableChanges &t = tableIt->second; - auto entriesIt = t.entries.find( &entry ); - if ( entriesIt == t.entries.end() ) + TableChanges &t = phase[dEntry->table->name]; + auto entriesIt = t.dataEntries.find( entryPkey( *dEntry ) ); + if ( entriesIt == t.dataEntries.end() ) { // row with this pkey is not in our list yet - ChangesetEntry *e = new ChangesetEntry( entry ); - e->table = t.table.get(); - t.entries.insert( e ); + t.entries.push_back( *dEntry ); + t.dataEntries[entryPkey( *dEntry )] = t.entries.size() - 1; } else { // we need to merge the recorded entry with the new one - ChangesetEntry *entry0 = *entriesIt; - MergeEntriesResult mergeRes = mergeEntriesForRow( entry0, &entry ); + ChangesetDataEntry &entry0 = std::get( *t.entries[entriesIt->second] ); + MergeEntriesResult mergeRes = mergeEntriesForRow( entry0, *dEntry ); switch ( mergeRes ) { case EntryModified: break; // nothing else to do - the original entry got updated in place case EntryRemoved: - t.entries.erase( entriesIt ); - delete entry0; + t.entries[ entriesIt->second ] = std::nullopt; + t.dataEntries.erase( entriesIt ); break; case Unsupported: // we are discarding the new entry (there's no sensible way to integrate it) context->logger().warn( "concatChangesets: unsupported sequence of entries for a single row - discarding newer entry" ); - t.entries.erase( entriesIt ); - delete entry0; + t.entries[ entriesIt->second ] = std::nullopt; + t.dataEntries.erase( entriesIt ); break; } } } + else if ( ChangesetDropTableEntry *dtEntry = std::get_if( &fullEntry ) ) + { + phase[dtEntry->tableName].entries.push_back( *dtEntry ); + } + else if ( ChangesetDropColumnEntry *dcEntry = std::get_if( &fullEntry ) ) + { + // This entry only contains the column's name, not its index, so we + // can't apply its effects to the existing entries. The best we can do + // is just forward this entry. + phase[dcEntry->tableName].entries.push_back( *dcEntry ); + // We also need to start a new phase, since we can't merge entries + // anymore. + outputPhases.push_back( {{}} ); + } + else if ( ChangesetCreateTableEntry *ctEntry = std::get_if( &fullEntry ) ) + { + phase[ctEntry->tableName].entries = { *ctEntry }; + } + else if ( ChangesetAddColumnEntry *acEntry = std::get_if( &fullEntry ) ) + { + phase[acEntry->tableName].prefixEntries.push_back( *acEntry ); + // Add the column to all existing entries, since we pushed the column + // addition in front of them. + size_t newColumnCount = SIZE_MAX; + for ( auto &existingEntry : phase[acEntry->tableName].entries ) + { + if ( !existingEntry ) continue; + ChangesetDataEntry *existingDEntry = std::get_if( &*existingEntry ); + if ( !existingDEntry ) continue; + if ( newColumnCount == SIZE_MAX ) + newColumnCount = existingDEntry->table->columnCount() + 1; + if ( existingDEntry->table->columnCount() != newColumnCount ) + existingDEntry->table->primaryKeys.push_back( false ); + if ( existingDEntry->oldValues.size() != newColumnCount ) + existingDEntry->oldValues.push_back( Value::makeNull() ); + if ( existingDEntry->newValues.size() != newColumnCount ) + existingDEntry->newValues.push_back( Value::makeNull() ); + } + } + else + throw GeoDiffException( "concatChanges: unhandled entry " + std::to_string( fullEntry.index() ) ); } } @@ -259,17 +288,33 @@ void concatChangesets( writer.open( outputChangeset ); // output all we have captured - for ( auto it = result.begin(); it != result.end(); ++it ) + for ( const OutputPhase &outPhase : outputPhases ) { - const TableChanges &t = it->second; - if ( t.entries.size() == 0 ) - continue; - - writer.beginTable( *t.table ); - for ( ChangesetEntry *e : t.entries ) + for ( auto it = outPhase.begin(); it != outPhase.end(); ++it ) { - writer.writeEntry( *e ); - delete e; + const TableChanges &t = it->second; + + for ( const ChangesetEntry &e : t.prefixEntries ) + { + writer.writeEntry( e ); + } + + std::shared_ptr writtenSchema; + for ( const std::optional &e : t.entries ) + { + if ( e ) + { + if ( const ChangesetDataEntry *dEntry = std::get_if( &*e ) ) + { + if ( dEntry->table != writtenSchema ) + { + writer.beginTable( *dEntry->table ); + writtenSchema = dEntry->table; + } + } + writer.writeEntry( *e ); + } + } } } } diff --git a/geodiff/src/changesetreader.cpp b/geodiff/src/changesetreader.cpp index 503cc404..615a93e1 100644 --- a/geodiff/src/changesetreader.cpp +++ b/geodiff/src/changesetreader.cpp @@ -5,9 +5,12 @@ #include "changesetreader.h" +#include "changeset.h" #include "geodiffutils.hpp" #include "changesetgetvarint.h" #include "portableendian.h" +#include "sqliteutils.h" +#include "tableschema.h" #include #include @@ -42,31 +45,40 @@ bool ChangesetReader::nextEntry( ChangesetEntry &entry ) if ( mOffset >= mBuffer->size() ) break; // EOF - int type = readByte(); - if ( type == 'T' ) + ChangesetEntryType type = static_cast( readByte() ); + if ( type == ChangesetEntryType::OpTableRecord ) { readTableRecord(); // and now continue reading, we want an entry } - else if ( type == ChangesetEntry::OpInsert || type == ChangesetEntry::OpUpdate || type == ChangesetEntry::OpDelete ) + else if ( type == ChangesetEntryType::OpInsert || type == ChangesetEntryType::OpUpdate || type == ChangesetEntryType::OpDelete ) { - readByte(); - if ( type != ChangesetEntry::OpInsert ) - readRowValues( entry.oldValues ); - else - entry.oldValues.erase( entry.oldValues.begin(), entry.oldValues.end() ); - if ( type != ChangesetEntry::OpDelete ) - readRowValues( entry.newValues ); - else - entry.newValues.erase( entry.newValues.begin(), entry.newValues.end() ); - - entry.op = static_cast( type ); - entry.table = &mCurrentTable; + entry = readDataEntry( type ); return true; // we're done! } + else if ( type == ChangesetEntryType::OpCreateTable ) + { + entry = readCreateTableEntry(); + return true; + } + else if ( type == ChangesetEntryType::OpDropTable ) + { + entry = readDropTableEntry(); + return true; + } + else if ( type == ChangesetEntryType::OpAddColumn ) + { + entry = readAddColumnEntry(); + return true; + } + else if ( type == ChangesetEntryType::OpDropColumn ) + { + entry = readDropColumnEntry(); + return true; + } else { - throwReaderError( "Unknown entry type " + std::to_string( type ) ); + throwReaderError( "Unknown entry type " + std::to_string( static_cast( type ) ) ); } } return false; @@ -80,7 +92,7 @@ bool ChangesetReader::isEmpty() const void ChangesetReader::rewind() { mOffset = 0; - mCurrentTable = ChangesetTable(); + mCurrentTable = {}; } char ChangesetReader::readByte() @@ -118,12 +130,14 @@ std::string ChangesetReader::readNullTerminatedString() void ChangesetReader::readRowValues( std::vector &values ) { // let's ensure we have the right size of array - if ( values.size() != mCurrentTable.columnCount() ) + if ( !mCurrentTable ) + throwReaderError( "Tried to read row without table" ); + if ( values.size() != mCurrentTable->columnCount() ) { - values.resize( mCurrentTable.columnCount() ); + values.resize( mCurrentTable->columnCount() ); } - for ( size_t i = 0; i < mCurrentTable.columnCount(); ++i ) + for ( size_t i = 0; i < mCurrentTable->columnCount(); ++i ) { int type = readByte(); if ( type == Value::TypeInt ) // 0x01 @@ -185,16 +199,98 @@ void ChangesetReader::readTableRecord() if ( nCol < 0 || nCol > 65536 ) throwReaderError( "readByte: unexpected number of columns" ); - mCurrentTable.primaryKeys.clear(); + mCurrentTable = std::make_shared(); + mCurrentTable->primaryKeys.clear(); for ( int i = 0; i < nCol; ++i ) { - mCurrentTable.primaryKeys.push_back( readByte() ); + mCurrentTable->primaryKeys.push_back( readByte() ); } - mCurrentTable.name = readNullTerminatedString(); + mCurrentTable->name = readNullTerminatedString(); +} + +ChangesetDataEntry ChangesetReader::readDataEntry( ChangesetEntryType type ) +{ + ChangesetDataEntry entry; + readByte(); + if ( type != ChangesetEntryType::OpInsert ) + readRowValues( entry.oldValues ); + else + entry.oldValues.erase( entry.oldValues.begin(), entry.oldValues.end() ); + if ( type != ChangesetEntryType::OpDelete ) + readRowValues( entry.newValues ); + else + entry.newValues.erase( entry.newValues.begin(), entry.newValues.end() ); + + entry.op = static_cast( type ); + entry.table = mCurrentTable; + return entry; +} + +TableColumnInfo ChangesetReader::readColumnInfo() +{ + TableColumnInfo column; + column.name = readNullTerminatedString(); + column.type.baseType = static_cast( readByte() ); + column.type.dbType = column.type.baseTypeToString( column.type.baseType ); + char flags = readByte(); + column.isPrimaryKey = flags & 1; + column.isNotNull = flags & ( 1 << 1 ); + column.isAutoIncrement = flags & ( 1 << 2 ); + column.isGeometry = flags & ( 1 << 3 ); + column.geomHasZ = flags & ( 1 << 4 ); + column.geomHasM = flags & ( 1 << 5 ); + column.geomType = readNullTerminatedString(); + column.geomSrsId = readVarint(); + return column; } +ChangesetCreateTableEntry ChangesetReader::readCreateTableEntry() +{ + ChangesetCreateTableEntry entry; + entry.tableName = readNullTerminatedString(); + int columnCount = readVarint(); + if ( columnCount < 0 || columnCount > 65536 ) + throwReaderError( "readCreateTableEntry: unexpected number of columns" ); + entry.columns.resize( columnCount ); + for ( size_t i = 0; i < entry.columns.size(); i++ ) + { + entry.columns[i] = readColumnInfo(); + } + return entry; +} + +ChangesetDropTableEntry ChangesetReader::readDropTableEntry() +{ + ChangesetDropTableEntry entry; + entry.tableName = readNullTerminatedString(); + int columnCount = readVarint(); + if ( columnCount < 0 || columnCount > 65536 ) + throwReaderError( "readDropTableEntry: unexpected number of columns" ); + entry.columns.resize( columnCount ); + for ( size_t i = 0; i < entry.columns.size(); i++ ) + { + entry.columns[i] = readColumnInfo(); + } + return entry; +} + +ChangesetAddColumnEntry ChangesetReader::readAddColumnEntry() +{ + ChangesetAddColumnEntry entry; + entry.tableName = readNullTerminatedString(); + entry.column = readColumnInfo(); + return entry; +} + +ChangesetDropColumnEntry ChangesetReader::readDropColumnEntry() +{ + ChangesetDropColumnEntry entry; + entry.tableName = readNullTerminatedString(); + entry.column = readColumnInfo(); + return entry; +} void ChangesetReader::throwReaderError( const std::string &message ) const { diff --git a/geodiff/src/changesetreader.h b/geodiff/src/changesetreader.h index 37254e60..6ae6de21 100644 --- a/geodiff/src/changesetreader.h +++ b/geodiff/src/changesetreader.h @@ -10,6 +10,7 @@ #include "geodiff.h" #include "changeset.h" +#include "tableschema.h" class Buffer; @@ -45,6 +46,12 @@ class ChangesetReader std::string readNullTerminatedString(); void readRowValues( std::vector &values ); void readTableRecord(); + ChangesetDataEntry readDataEntry( ChangesetEntryType type ); + TableColumnInfo readColumnInfo(); + ChangesetCreateTableEntry readCreateTableEntry(); + ChangesetDropTableEntry readDropTableEntry(); + ChangesetAddColumnEntry readAddColumnEntry(); + ChangesetDropColumnEntry readDropColumnEntry(); void throwReaderError( const std::string &message ) const; @@ -52,7 +59,7 @@ class ChangesetReader std::unique_ptr mBuffer; - ChangesetTable mCurrentTable; // currently processed table + std::shared_ptr mCurrentTable; // currently processed table }; diff --git a/geodiff/src/changesetutils.cpp b/geodiff/src/changesetutils.cpp index bde8148c..d32fe394 100644 --- a/geodiff/src/changesetutils.cpp +++ b/geodiff/src/changesetutils.cpp @@ -6,6 +6,7 @@ #include "changesetutils.h" #include "base64utils.h" +#include "changeset.h" #include "geodiffutils.hpp" #include "changesetreader.h" #include "changesetwriter.h" @@ -21,58 +22,109 @@ ChangesetTable schemaToChangesetTable( const std::string &tableName, const Table return chTable; } -void invertChangeset( ChangesetReader &reader, ChangesetWriter &writer ) +// Returns inverted changeset entries in reverse order +std::vector invertChangesetReverse( ChangesetReader &reader ) { - std::string currentTableName; - std::vector currentPkeys; + std::vector invertedEntries; ChangesetEntry entry; while ( reader.nextEntry( entry ) ) { - assert( entry.table ); - if ( entry.table->name != currentTableName ) + if ( ChangesetDataEntry *dataEntry = std::get_if( &entry ) ) + { + if ( dataEntry->op == ChangesetDataEntry::OpInsert ) + { + ChangesetDataEntry out; + out.op = ChangesetDataEntry::OpDelete; + out.table = dataEntry->table; + out.oldValues = dataEntry->newValues; + invertedEntries.push_back( out ); + } + else if ( dataEntry->op == ChangesetDataEntry::OpDelete ) + { + ChangesetDataEntry out; + out.op = ChangesetDataEntry::OpInsert; + out.table = dataEntry->table; + out.newValues = dataEntry->oldValues; + invertedEntries.push_back( out ); + } + else if ( dataEntry->op == ChangesetDataEntry::OpUpdate ) + { + ChangesetDataEntry out; + out.op = ChangesetDataEntry::OpUpdate; + out.table = dataEntry->table; + out.newValues = dataEntry->oldValues; + out.oldValues = dataEntry->newValues; + // if a column is a part of pkey and has not been changed, + // the original entry has "old" value the pkey value and "new" + // value is undefined - let's reverse "old" and "new" in that case. + for ( size_t i = 0; i < dataEntry->table->primaryKeys.size(); ++i ) + { + if ( dataEntry->table->primaryKeys[i] && out.oldValues[i].type() == Value::TypeUndefined ) + { + out.oldValues[i] = out.newValues[i]; + out.newValues[i].setUndefined(); + } + } + invertedEntries.push_back( out ); + } + else + { + throw GeoDiffException( "Unknown entry operation!" ); + } + } + else if ( const ChangesetCreateTableEntry *ctEntry = std::get_if( &entry ) ) { - writer.beginTable( *entry.table ); - currentTableName = entry.table->name; - currentPkeys = entry.table->primaryKeys; + ChangesetDropTableEntry out; + out.tableName = ctEntry->tableName; + out.columns = ctEntry->columns; + invertedEntries.push_back( out ); } - - if ( entry.op == ChangesetEntry::OpInsert ) + else if ( const ChangesetAddColumnEntry *acEntry = std::get_if( &entry ) ) { - ChangesetEntry out; - out.op = ChangesetEntry::OpDelete; - out.oldValues = entry.newValues; - writer.writeEntry( out ); + ChangesetDropColumnEntry out; + out.tableName = acEntry->tableName; + out.column = acEntry->column; + invertedEntries.push_back( out ); } - else if ( entry.op == ChangesetEntry::OpDelete ) + else if ( const ChangesetDropTableEntry *dtEntry = std::get_if( &entry ) ) { - ChangesetEntry out; - out.op = ChangesetEntry::OpInsert; - out.newValues = entry.oldValues; - writer.writeEntry( out ); + ChangesetCreateTableEntry out; + out.tableName = dtEntry->tableName; + out.columns = dtEntry->columns; + invertedEntries.push_back( out ); } - else if ( entry.op == ChangesetEntry::OpUpdate ) + else if ( const ChangesetDropColumnEntry *dcEntry = std::get_if( &entry ) ) { - ChangesetEntry out; - out.op = ChangesetEntry::OpUpdate; - out.newValues = entry.oldValues; - out.oldValues = entry.newValues; - // if a column is a part of pkey and has not been changed, - // the original entry has "old" value the pkey value and "new" - // value is undefined - let's reverse "old" and "new" in that case. - for ( size_t i = 0; i < currentPkeys.size(); ++i ) - { - if ( currentPkeys[i] && out.oldValues[i].type() == Value::TypeUndefined ) - { - out.oldValues[i] = out.newValues[i]; - out.newValues[i].setUndefined(); - } - } - writer.writeEntry( out ); + ChangesetAddColumnEntry out; + out.tableName = dcEntry->tableName; + out.column = dcEntry->column; + invertedEntries.push_back( out ); } else { - throw GeoDiffException( "Unknown entry operation!" ); + throw GeoDiffException( "Cannot invert changeset entry variant " + std::to_string( entry.index() ) ); + } + } + return invertedEntries; +} + +void invertChangeset( ChangesetReader &reader, ChangesetWriter &writer ) +{ + std::vector invertedReverse = invertChangesetReverse( reader ); + ChangesetTable *currentTable = nullptr; + for ( size_t i = 1; i <= invertedReverse.size(); i++ ) + { + const auto &entry = invertedReverse[invertedReverse.size() - i]; + if ( const ChangesetDataEntry *dataEntry = std::get_if( &entry ) ) + { + if ( dataEntry->table.get() != currentTable ) + { + writer.beginTable( *dataEntry->table ); + currentTable = dataEntry->table.get(); + } } + + writer.writeEntry( entry ); } } @@ -112,21 +164,21 @@ nlohmann::json valueToJSON( const Value &value ) } -nlohmann::json changesetEntryToJSON( const ChangesetEntry &entry ) +nlohmann::json changesetDataEntryToJSON( const ChangesetDataEntry &entry ) { std::string status; - if ( entry.op == ChangesetEntry::OpUpdate ) + if ( entry.op == ChangesetDataEntry::OpUpdate ) status = "update"; - else if ( entry.op == ChangesetEntry::OpInsert ) + else if ( entry.op == ChangesetDataEntry::OpInsert ) status = "insert"; - else if ( entry.op == ChangesetEntry::OpDelete ) + else if ( entry.op == ChangesetDataEntry::OpDelete ) status = "delete"; // Check that the table column count matches the vector sizes to prevent // out-of-bounds errors. - if ( ( ( entry.op == ChangesetEntry::OpUpdate || entry.op == ChangesetEntry::OpInsert ) + if ( ( ( entry.op == ChangesetDataEntry::OpUpdate || entry.op == ChangesetDataEntry::OpInsert ) && entry.table->columnCount() != entry.newValues.size() ) - || ( ( entry.op == ChangesetEntry::OpUpdate || entry.op == ChangesetEntry::OpDelete ) + || ( ( entry.op == ChangesetDataEntry::OpUpdate || entry.op == ChangesetDataEntry::OpDelete ) && entry.table->columnCount() != entry.oldValues.size() ) ) throw GeoDiffException( "Table column count doesn't match value list size" ); @@ -139,8 +191,8 @@ nlohmann::json changesetEntryToJSON( const ChangesetEntry &entry ) Value valueOld, valueNew; for ( size_t i = 0; i < entry.table->columnCount(); ++i ) { - valueNew = ( entry.op == ChangesetEntry::OpUpdate || entry.op == ChangesetEntry::OpInsert ) ? entry.newValues[i] : Value(); - valueOld = ( entry.op == ChangesetEntry::OpUpdate || entry.op == ChangesetEntry::OpDelete ) ? entry.oldValues[i] : Value(); + valueNew = ( entry.op == ChangesetDataEntry::OpUpdate || entry.op == ChangesetDataEntry::OpInsert ) ? entry.newValues[i] : Value(); + valueOld = ( entry.op == ChangesetDataEntry::OpUpdate || entry.op == ChangesetDataEntry::OpDelete ) ? entry.oldValues[i] : Value(); nlohmann::json change; @@ -174,6 +226,74 @@ nlohmann::json changesetEntryToJSON( const ChangesetEntry &entry ) return res; } +static nlohmann::json columnInfoToJSON( const TableColumnInfo &column ) +{ + nlohmann::json res; + res["name"] = column.name; + res["type"] = column.type.dbType; + res["isPrimaryKey"] = column.isPrimaryKey; + res["isNotNull"] = column.isNotNull; + res["isAutoIncrement"] = column.isAutoIncrement; + res["isGeometry"] = column.isGeometry; + res["geomType"] = column.geomType; + res["geomSrsId"] = column.geomSrsId; + res["geomHasZ"] = column.geomHasZ; + res["geomHasM"] = column.geomHasM; + return res; +} + +nlohmann::json changesetEntryToJSON( const ChangesetEntry &entry ) +{ + if ( const ChangesetDataEntry *dataEntry = std::get_if( &entry ) ) + { + return changesetDataEntryToJSON( *dataEntry ); + } + else if ( const ChangesetCreateTableEntry *ctEntry = std::get_if( &entry ) ) + { + nlohmann::json res; + res["type"] = "create_table"; + res["tableName"] = ctEntry->tableName; + res["columns"] = nlohmann::json::array(); + for ( const TableColumnInfo &column : ctEntry->columns ) + { + res["columns"].push_back( columnInfoToJSON( column ) ); + } + return res; + } + else if ( const ChangesetDropTableEntry *dtEntry = std::get_if( &entry ) ) + { + nlohmann::json res; + res["type"] = "drop_table"; + res["tableName"] = dtEntry->tableName; + res["columns"] = nlohmann::json::array(); + for ( const TableColumnInfo &column : dtEntry->columns ) + { + res["columns"].push_back( columnInfoToJSON( column ) ); + } + return res; + } + else if ( const ChangesetAddColumnEntry *acEntry = std::get_if( &entry ) ) + { + nlohmann::json res; + res["type"] = "add_column"; + res["tableName"] = acEntry->tableName; + res["column"] = columnInfoToJSON( acEntry->column ); + return res; + } + else if ( const ChangesetDropColumnEntry *dcEntry = std::get_if( &entry ) ) + { + nlohmann::json res; + res["type"] = "drop_column"; + res["tableName"] = dcEntry->tableName; + res["column"] = columnInfoToJSON( dcEntry->column ); + return res; + } + else + { + throw GeoDiffException( "Cannot convert entry variant " + std::to_string( entry.index() ) + " to JSON" ); + } +} + nlohmann::json changesetToJSON( ChangesetReader &reader ) { auto entries = nlohmann::json::array(); @@ -209,14 +329,17 @@ nlohmann::json changesetToJSONSummary( ChangesetReader &reader ) ChangesetEntry entry; while ( reader.nextEntry( entry ) ) { - std::string tableName = entry.table->name; + if ( !std::holds_alternative( entry ) ) + continue; + ChangesetDataEntry &dataEntry = std::get( entry ); + std::string tableName = dataEntry.table->name; TableSummary &tableSummary = summary[tableName]; - if ( entry.op == ChangesetEntry::OpUpdate ) + if ( dataEntry.op == ChangesetDataEntry::OpUpdate ) ++tableSummary.updates; - else if ( entry.op == ChangesetEntry::OpInsert ) + else if ( dataEntry.op == ChangesetDataEntry::OpInsert ) ++tableSummary.inserts; - else if ( entry.op == ChangesetEntry::OpDelete ) + else if ( dataEntry.op == ChangesetDataEntry::OpDelete ) ++tableSummary.deletes; } @@ -239,62 +362,73 @@ nlohmann::json changesetToJSONSummary( ChangesetReader &reader ) nlohmann::json conflictToJSON( const ConflictFeature &conflict ) { - nlohmann::json res; - res[ "table" ] = std::string( conflict.tableName() ); - res[ "type" ] = "conflict"; - res[ "fid" ] = std::to_string( conflict.pk() ); + if ( const DataConflictFeature *dcf = std::get_if( &conflict ) ) + { + nlohmann::json res; + res[ "table" ] = std::string( dcf->tableName() ); + res[ "type" ] = "conflict"; + res[ "fid" ] = std::to_string( dcf->pk() ); - auto entries = nlohmann::json::array(); + auto entries = nlohmann::json::array(); + for ( const DataConflictItem &item : dcf->items() ) + { + nlohmann::json change; + change[ "column" ] = item.column(); - const std::vector items = conflict.items(); - for ( const ConflictItem &item : items ) - { - nlohmann::json change; - change[ "column" ] = item.column(); + nlohmann::json valueBase = valueToJSON( item.base() ); + nlohmann::json valueOld = valueToJSON( item.theirs() ); + nlohmann::json valueNew = valueToJSON( item.ours() ); - nlohmann::json valueBase = valueToJSON( item.base() ); - nlohmann::json valueOld = valueToJSON( item.theirs() ); - nlohmann::json valueNew = valueToJSON( item.ours() ); + if ( !valueBase.empty() ) + { + if ( valueBase == "null" ) + change[ "base" ] = nullptr; + else + change[ "base" ] = valueBase; + } + if ( !valueOld.empty() ) + { + if ( valueOld == "null" ) + change[ "old" ] = nullptr; + else + change[ "old" ] = valueOld; + } + if ( !valueNew.empty() ) + { + if ( valueNew == "null" ) + change[ "new" ] = nullptr; + else + change[ "new" ] = valueNew; + } - if ( !valueBase.empty() ) - { - if ( valueBase == "null" ) - change[ "base" ] = nullptr; - else - change[ "base" ] = valueBase; - } - if ( !valueOld.empty() ) - { - if ( valueOld == "null" ) - change[ "old" ] = nullptr; - else - change[ "old" ] = valueOld; - } - if ( !valueNew.empty() ) - { - if ( valueNew == "null" ) - change[ "new" ] = nullptr; - else - change[ "new" ] = valueNew; + entries.push_back( change ); } - - entries.push_back( change ); + res[ "changes" ] = entries; + return res; } - res[ "changes" ] = entries; - return res; + else if ( const TableSchemaConflict *tsc = std::get_if( &conflict ) ) + { + nlohmann::json res; + res[ "type" ] = "schema_conflict_table"; + res[ "table" ] = tsc->tableName; + return res; + } + else if ( const ColumnSchemaConflict *csc = std::get_if( &conflict ) ) + { + nlohmann::json res; + res[ "type" ] = "schema_conflict_column"; + res[ "table" ] = csc->tableName; + res[ "column" ] = csc->columnName; + return res; + } + return {}; } nlohmann::json conflictsToJSON( const std::vector &conflicts ) { auto entries = nlohmann::json::array(); for ( const ConflictFeature &item : conflicts ) - { - nlohmann::json msg = conflictToJSON( item ); - if ( msg.empty() ) - continue; - - entries.push_back( msg ); - } + entries.push_back( conflictToJSON( item ) ); nlohmann::json res; res[ "geodiff" ] = entries; diff --git a/geodiff/src/changesetutils.h b/geodiff/src/changesetutils.h index e836436f..dbe84dac 100644 --- a/geodiff/src/changesetutils.h +++ b/geodiff/src/changesetutils.h @@ -7,15 +7,16 @@ #define CHANGESETUTILS_H #include "geodiff.h" +#include "changeset.h" #include #include #include "json.hpp" -class ConflictFeature; +struct ConflictFeature; class ChangesetReader; class ChangesetWriter; -struct ChangesetEntry; +struct ChangesetDataEntry; struct ChangesetTable; struct TableSchema; struct Value; @@ -27,6 +28,8 @@ void invertChangeset( ChangesetReader &reader, ChangesetWriter &writer ); void concatChangesets( const Context *context, const std::vector &filenames, const std::string &outputChangeset ); +nlohmann::json changesetDataEntryToJSON( const ChangesetDataEntry &entry ); + nlohmann::json changesetEntryToJSON( const ChangesetEntry &entry ); nlohmann::json changesetToJSON( ChangesetReader &reader ); diff --git a/geodiff/src/changesetwriter.cpp b/geodiff/src/changesetwriter.cpp index bb57afc3..47aa48ed 100644 --- a/geodiff/src/changesetwriter.cpp +++ b/geodiff/src/changesetwriter.cpp @@ -5,6 +5,7 @@ #include "changesetwriter.h" +#include "changeset.h" #include "geodiffutils.hpp" #include "changesetputvarint.h" #include "portableendian.h" @@ -13,6 +14,7 @@ #include #include +#include void ChangesetWriter::open( const std::string &filename ) { @@ -29,7 +31,7 @@ void ChangesetWriter::beginTable( const ChangesetTable &table ) { mCurrentTable = table; - writeByte( 'T' ); + writeByte( ( int ) ChangesetEntryType::OpTableRecord ); writeVarint( ( int ) table.columnCount() ); for ( size_t i = 0; i < table.columnCount(); ++i ) writeByte( table.primaryKeys[i] ); @@ -38,15 +40,19 @@ void ChangesetWriter::beginTable( const ChangesetTable &table ) void ChangesetWriter::writeEntry( const ChangesetEntry &entry ) { - if ( entry.op != ChangesetEntry::OpInsert && entry.op != ChangesetEntry::OpUpdate && entry.op != ChangesetEntry::OpDelete ) - throw GeoDiffException( "wrong op for changeset entry" ); - writeByte( ( char ) entry.op ); - writeByte( 0 ); // "indirect" always false - - if ( entry.op != ChangesetEntry::OpInsert ) - writeRowValues( entry.oldValues ); - if ( entry.op != ChangesetEntry::OpDelete ) - writeRowValues( entry.newValues ); + if ( const ChangesetDataEntry *dataEntry = std::get_if( &entry ) ) + writeDataEntry( *dataEntry ); + else if ( const ChangesetCreateTableEntry *ctEntry = std::get_if( &entry ) ) + writeCreateTableEntry( *ctEntry ); + else if ( const ChangesetDropTableEntry *dtEntry = std::get_if( &entry ) ) + writeDropTableEntry( *dtEntry ); + else if ( const ChangesetAddColumnEntry *acEntry = std::get_if( &entry ) ) + writeAddColumnEntry( *acEntry ); + else if ( const ChangesetDropColumnEntry *dcEntry = std::get_if( &entry ) ) + writeDropColumnEntry( *dcEntry ); + else + throw GeoDiffException( "Tried to write unhandled changeset entry type! " + + std::to_string( entry.index() ) ); } void ChangesetWriter::writeByte( char c ) @@ -113,3 +119,67 @@ void ChangesetWriter::writeRowValues( const std::vector &values ) } } } + +void ChangesetWriter::writeColumnInfo( const TableColumnInfo &column ) +{ + writeNullTerminatedString( column.name ); + writeByte( static_cast( column.type.baseType ) ); + writeByte( ( column.isPrimaryKey << 0 ) + | ( column.isNotNull << 1 ) + | ( column.isAutoIncrement << 2 ) + | ( column.isGeometry << 3 ) + | ( column.geomHasZ << 4 ) + | ( column.geomHasM << 5 ) ); + writeNullTerminatedString( column.geomType ); + writeVarint( column.geomSrsId ); +} + + +void ChangesetWriter::writeDataEntry( const ChangesetDataEntry &entry ) +{ + if ( entry.op != ChangesetDataEntry::OpInsert && entry.op != ChangesetDataEntry::OpUpdate && entry.op != ChangesetDataEntry::OpDelete ) + throw GeoDiffException( "wrong op for changeset entry" ); + writeByte( ( char ) entry.op ); + writeByte( 0 ); // "indirect" always false + + if ( entry.op != ( int ) ChangesetEntryType::OpInsert ) + writeRowValues( entry.oldValues ); + if ( entry.op != ( int ) ChangesetEntryType::OpDelete ) + writeRowValues( entry.newValues ); +} + +void ChangesetWriter::writeCreateTableEntry( const ChangesetCreateTableEntry &entry ) +{ + writeByte( static_cast( ChangesetEntryType::OpCreateTable ) ); + writeNullTerminatedString( entry.tableName ); + writeVarint( static_cast( entry.columns.size() ) ); + for ( const TableColumnInfo &column : entry.columns ) + { + writeColumnInfo( column ); + } +} + +void ChangesetWriter::writeDropTableEntry( const ChangesetDropTableEntry &entry ) +{ + writeByte( static_cast( ChangesetEntryType::OpDropTable ) ); + writeNullTerminatedString( entry.tableName ); + writeVarint( static_cast( entry.columns.size() ) ); + for ( const TableColumnInfo &column : entry.columns ) + { + writeColumnInfo( column ); + } +} + +void ChangesetWriter::writeAddColumnEntry( const ChangesetAddColumnEntry &entry ) +{ + writeByte( static_cast( ChangesetEntryType::OpAddColumn ) ); + writeNullTerminatedString( entry.tableName ); + writeColumnInfo( entry.column ); +} + +void ChangesetWriter::writeDropColumnEntry( const ChangesetDropColumnEntry &entry ) +{ + writeByte( static_cast( ChangesetEntryType::OpDropColumn ) ); + writeNullTerminatedString( entry.tableName ); + writeColumnInfo( entry.column ); +} diff --git a/geodiff/src/changesetwriter.h b/geodiff/src/changesetwriter.h index eed79a86..dfe7f908 100644 --- a/geodiff/src/changesetwriter.h +++ b/geodiff/src/changesetwriter.h @@ -43,6 +43,13 @@ class ChangesetWriter void writeNullTerminatedString( const std::string &str ); void writeRowValues( const std::vector &values ); + void writeColumnInfo( const TableColumnInfo &column ); + + void writeDataEntry( const ChangesetDataEntry &entry ); + void writeCreateTableEntry( const ChangesetCreateTableEntry &entry ); + void writeDropTableEntry( const ChangesetDropTableEntry &entry ); + void writeAddColumnEntry( const ChangesetAddColumnEntry &entry ); + void writeDropColumnEntry( const ChangesetDropColumnEntry &entry ); std::ofstream mFile; diff --git a/geodiff/src/driver.h b/geodiff/src/driver.h index fd1e1f20..92f8c9f0 100644 --- a/geodiff/src/driver.h +++ b/geodiff/src/driver.h @@ -11,6 +11,7 @@ #include #include +#include "changeset.h" #include "geodiff.h" #include "tableschema.h" @@ -126,6 +127,13 @@ class Driver */ virtual void dumpData( ChangesetWriter &writer, bool useModified = false ) = 0; + /** + * Executes SQL statement on 'base' database. Returns list of rows, which + * are lists of values in columns, in the database's native string + * representation. + */ + virtual std::vector> executeSql( std::string sql ) = 0; + static const std::string SQLITEDRIVERNAME; static const std::string POSTGRESDRIVERNAME; diff --git a/geodiff/src/drivers/postgresdriver.cpp b/geodiff/src/drivers/postgresdriver.cpp index 38deef5a..226d3768 100644 --- a/geodiff/src/drivers/postgresdriver.cpp +++ b/geodiff/src/drivers/postgresdriver.cpp @@ -61,9 +61,9 @@ class PostgresTransaction ///// -void PostgresDriver::logApplyConflict( const std::string &type, const ChangesetEntry &entry ) const +void PostgresDriver::logApplyConflict( const std::string &type, const ChangesetDataEntry &entry ) const { - context()->logger().warn( "CONFLICT: " + type + ":\n" + changesetEntryToJSON( entry ).dump( 2 ) ); + context()->logger().warn( "CONFLICT: " + type + ":\n" + changesetDataEntryToJSON( entry ).dump( 2 ) ); } PostgresDriver::PostgresDriver( const Context *context ) @@ -606,8 +606,8 @@ static void handleInserted( const std::string &schemaNameBase, const std::string first = false; } - ChangesetEntry e; - e.op = reverse ? ChangesetEntry::OpDelete : ChangesetEntry::OpInsert; + ChangesetDataEntry e; + e.op = reverse ? ChangesetDataEntry::OpDelete : ChangesetDataEntry::OpInsert; int numColumns = static_cast( tbl.columns.size() ); for ( int i = 0; i < numColumns; ++i ) @@ -650,8 +650,8 @@ static void handleUpdated( const std::string &schemaNameBase, const std::string ** are set to "undefined". */ - ChangesetEntry e; - e.op = ChangesetEntry::OpUpdate; + ChangesetDataEntry e; + e.op = ChangesetDataEntry::OpUpdate; int numColumns = static_cast( tbl.columns.size() ); for ( int i = 0; i < numColumns; ++i ) @@ -794,7 +794,7 @@ static std::string sqlForDelete( const std::string &schemaName, const std::strin return sql; } -ChangeApplyResult PostgresDriver::applyChange( PostgresChangeApplyState &state, const ChangesetEntry &entry ) +ChangeApplyResult PostgresDriver::applyChange( PostgresChangeApplyState &state, const ChangesetDataEntry &entry ) { std::string tableName = entry.table->name; @@ -836,7 +836,7 @@ ChangeApplyResult PostgresDriver::applyChange( PostgresChangeApplyState &state, try { PostgresChangeApplyState::TableState &tbl = state.tableState[tableName]; - if ( entry.op == ChangesetEntry::OpInsert ) + if ( entry.op == ChangesetDataEntry::OpInsert ) { std::string sql = sqlForInsert( mBaseSchema, tableName, tbl.schema, entry.newValues ); PostgresResult res = execSql( mConn, sql ); @@ -849,7 +849,7 @@ ChangeApplyResult PostgresDriver::applyChange( PostgresChangeApplyState &state, tbl.autoIncrementMax = std::max( tbl.autoIncrementMax, pkey ); } } - else if ( entry.op == ChangesetEntry::OpUpdate ) + else if ( entry.op == ChangesetDataEntry::OpUpdate ) { std::string sql = sqlForUpdate( mBaseSchema, tableName, tbl.schema, entry.oldValues, entry.newValues ); PostgresResult res = execSql( mConn, sql ); @@ -860,7 +860,7 @@ ChangeApplyResult PostgresDriver::applyChange( PostgresChangeApplyState &state, return ChangeApplyResult::NoChange; } } - else if ( entry.op == ChangesetEntry::OpDelete ) + else if ( entry.op == ChangesetDataEntry::OpDelete ) { std::string sql = sqlForDelete( mBaseSchema, tableName, tbl.schema, entry.oldValues ); PostgresResult res = execSql( mConn, sql ); @@ -901,35 +901,35 @@ void PostgresDriver::applyChangeset( ChangesetReader &reader ) // See sqlitedriver.cpp for why and how we're trying to apply changes // multiple times int unrecoverableConflictCount = 0; - std::vector conflictingEntries; + std::vector conflictingEntries; ChangesetEntry entry; PostgresChangeApplyState state; - std::unordered_map> tableCopies; while ( reader.nextEntry( entry ) ) { - ChangeApplyResult res = applyChange( state, entry ); - switch ( res ) + if ( ChangesetDataEntry *dataEntry = std::get_if( &entry ) ) { - case ChangeApplyResult::Applied: - case ChangeApplyResult::Skipped: - break; - case ChangeApplyResult::ConstraintConflict: - if ( tableCopies.count( entry.table->name ) == 0 ) - // cppcheck-suppress stlFindInsert - tableCopies[entry.table->name] = std::unique_ptr( new ChangesetTable( *entry.table ) ); - entry.table = tableCopies[entry.table->name].get(); - conflictingEntries.push_back( entry ); - break; - case ChangeApplyResult::NoChange: - unrecoverableConflictCount++; - break; + ChangeApplyResult res = applyChange( state, *dataEntry ); + switch ( res ) + { + case ChangeApplyResult::Applied: + case ChangeApplyResult::Skipped: + break; + case ChangeApplyResult::ConstraintConflict: + conflictingEntries.push_back( *dataEntry ); + break; + case ChangeApplyResult::NoChange: + unrecoverableConflictCount++; + break; + } } + else + throw GeoDiffException( "Unhandled changeset entry: " + std::to_string( entry.index() ) ); } - std::vector newConflictingEntries; + std::vector newConflictingEntries; while ( conflictingEntries.size() > 0 ) { - for ( const ChangesetEntry ¢ry : conflictingEntries ) + for ( const ChangesetDataEntry ¢ry : conflictingEntries ) { ChangeApplyResult res = applyChange( state, centry ); switch ( res ) @@ -948,7 +948,7 @@ void PostgresDriver::applyChangeset( ChangesetReader &reader ) if ( newConflictingEntries.size() == conflictingEntries.size() ) { - for ( const ChangesetEntry ¢ry : conflictingEntries ) + for ( const ChangesetDataEntry ¢ry : conflictingEntries ) logApplyConflict( "unresolvable_conflict", centry ); throw GeoDiffConflictsException( "Could not resolve dependencies in constraint conflicts." ); } @@ -1078,8 +1078,8 @@ void PostgresDriver::dumpData( ChangesetWriter &writer, bool useModified ) writer.beginTable( schemaToChangesetTable( tableName, tbl ) ); } - ChangesetEntry e; - e.op = ChangesetEntry::OpInsert; + ChangesetDataEntry e; + e.op = ChangesetDataEntry::OpInsert; int numColumns = static_cast( tbl.columns.size() ); for ( int i = 0; i < numColumns; ++i ) { @@ -1089,3 +1089,38 @@ void PostgresDriver::dumpData( ChangesetWriter &writer, bool useModified ) } } } + +class SearchPathScope +{ + public: + SearchPathScope( PGconn *conn, const std::string &prependSchema ) + : mConn( conn ) + { + PostgresResult res = execSql( mConn, "SHOW search_path" ); + mOldSearchPath = res.value( 0, 0 ); + std::string newSearchPath = prependSchema + "," + mOldSearchPath; + execSql( mConn, "SET search_path TO " + newSearchPath ); + } + ~SearchPathScope() + { + execSql( mConn, "SET search_path TO " + mOldSearchPath ); + } + private: + std::string mOldSearchPath; + PGconn *mConn; +}; + +std::vector> PostgresDriver::executeSql( std::string sql ) +{ + SearchPathScope spScope( mConn, mBaseSchema ); + PostgresResult res = execSql( mConn, sql ); + std::vector> rows; + rows.resize( res.rowCount() ); + for ( int r = 0; r < static_cast( rows.size() ); ++r ) + { + rows[r].resize( res.columnCount() ); + for ( int i = 0; i < static_cast( rows[r].size() ); ++i ) + rows[r][i] = res.value( r, i ); + } + return rows; +} diff --git a/geodiff/src/drivers/postgresdriver.h b/geodiff/src/drivers/postgresdriver.h index 83bb277a..ac5126e6 100644 --- a/geodiff/src/drivers/postgresdriver.h +++ b/geodiff/src/drivers/postgresdriver.h @@ -7,6 +7,7 @@ #define POSTGRESDRIVER_H #include "driver.h" +#include "changeset.h" extern "C" { @@ -43,14 +44,15 @@ class PostgresDriver : public Driver void applyChangeset( ChangesetReader &reader ) override; void createTables( const std::vector &tables ) override; void dumpData( ChangesetWriter &writer, bool useModified = false ) override; + std::vector> executeSql( std::string sql ) override; private: - void logApplyConflict( const std::string &type, const ChangesetEntry &entry ) const; + void logApplyConflict( const std::string &type, const ChangesetDataEntry &entry ) const; void openPrivate( const DriverParametersMap &conn ); void close(); std::string getSequenceObjectName( const TableSchema &tbl, int &autoIncrementPkeyIndex ); void updateSequenceObject( const std::string &seqName, int64_t maxValue ); - ChangeApplyResult applyChange( PostgresChangeApplyState &state, const ChangesetEntry &entry ); + ChangeApplyResult applyChange( PostgresChangeApplyState &state, const ChangesetDataEntry &entry ); PGconn *mConn = nullptr; std::string mBaseSchema; diff --git a/geodiff/src/drivers/postgresutils.h b/geodiff/src/drivers/postgresutils.h index 2d6d2b30..e44a7025 100644 --- a/geodiff/src/drivers/postgresutils.h +++ b/geodiff/src/drivers/postgresutils.h @@ -64,6 +64,12 @@ class PostgresResult return ::PQntuples( mResult ); } + int columnCount() const + { + assert( mResult ); + return ::PQnfields( mResult ); + } + std::string affectedRows() const { assert( mResult ); diff --git a/geodiff/src/drivers/sqlitedriver.cpp b/geodiff/src/drivers/sqlitedriver.cpp index 91206c9c..ddac3c40 100644 --- a/geodiff/src/drivers/sqlitedriver.cpp +++ b/geodiff/src/drivers/sqlitedriver.cpp @@ -5,16 +5,22 @@ #include "sqlitedriver.h" +#include "changeset.h" #include "changesetreader.h" #include "changesetwriter.h" #include "changesetutils.h" +#include "driver.h" #include "geodiffcontext.hpp" #include "geodifflogger.hpp" #include "geodiffutils.hpp" #include "sqliteutils.h" +#include "tableschema.h" +#include "tableschemadiff.hpp" #include #include +#include +#include void SqliteDriver::logApplyConflict( const std::string &type, const ChangesetEntry &entry, bool isDbErr ) const @@ -107,6 +113,8 @@ SqliteDriver::SqliteDriver( const Context *context ) { } +// Opens 'base' DB (with implicit schema called 'main') and optionally +// 'modified' DB (with explicit schema 'modified') void SqliteDriver::open( const DriverParametersMap &conn ) { DriverParametersMap::const_iterator connBaseIt = conn.find( "base" ); @@ -123,6 +131,8 @@ void SqliteDriver::open( const DriverParametersMap &conn ) } mDb = std::make_shared(); + mDb->open( base ); + if ( mHasModified ) { std::string modified = connModifiedIt->second; @@ -132,15 +142,11 @@ void SqliteDriver::open( const DriverParametersMap &conn ) throw GeoDiffException( "Missing 'modified' file when opening sqlite driver: " + modified ); } - mDb->open( modified ); - - Buffer sqlBuf; - sqlBuf.printf( "ATTACH '%q' AS aux", base.c_str() ); - mDb->exec( sqlBuf ); - } - else - { - mDb->open( base ); + { + Buffer sqlBuf; + sqlBuf.printf( "ATTACH '%q' AS modified", modified.c_str() ); + mDb->exec( sqlBuf ); + } } // GeoPackage triggers require few functions like ST_IsEmpty() to be registered @@ -180,7 +186,7 @@ std::string SqliteDriver::databaseName( bool useModified ) { if ( mHasModified ) { - return useModified ? "main" : "aux"; + return useModified ? "modified" : "main"; } else { @@ -369,6 +375,16 @@ TableSchema SqliteDriver::tableSchema( const std::string &tableName, return tbl; } +DatabaseSchema SqliteDriver::getSchema( bool useModified ) +{ + std::vector tables; + for ( const std::string &name : listTables( useModified ) ) + { + tables.push_back( tableSchema( name, useModified ) ); + } + return {tables}; +} + /** * printf() with sqlite extensions - see https://www.sqlite.org/printf.html * for extra format options like %q or %Q @@ -389,64 +405,124 @@ static std::string sqlitePrintf( const char *zFormat, ... ) return res; } +struct TableDiffContext +{ + std::shared_ptr db; + const TableSchema &schemaBase; + const TableSchema &schemaModified; + std::vector commonColumns; + std::vector newColumns; + ChangesetWriter &writer; + bool tableEntryWritten = false; +}; + +static std::string sqlColumnsStr( const TableDiffContext &diffContext, bool reverse ) +{ + const char *tableName = ( reverse ? diffContext.schemaBase.name : diffContext.schemaModified.name ).c_str(); + + std::string colsStr; // Column list equivalent to modified schema + for ( const TableColumnInfo &c : diffContext.schemaModified.columns ) + { + if ( !colsStr.empty() ) + colsStr += ", "; + if ( reverse ) + { + // Check if this column also exists in base and NULL it out if not + bool found = false; + for ( const auto &commonCol : diffContext.commonColumns ) + { + if ( commonCol.name == c.name ) + { + found = true; + break; + } + } + if ( !found ) + { + colsStr += sqlitePrintf( "NULL AS \"%w\"", c.name.c_str() ); + continue; + } + } + colsStr += sqlitePrintf( "\"%w\".\"%w\".\"%w\"", + reverse ? "main" : "modified", tableName, c.name.c_str() ); + } + return colsStr; +} + //! Constructs SQL query to get all rows that do not exist in the other table (used for insert and delete) -static std::string sqlFindInserted( const std::string &tableName, const TableSchema &tbl, bool reverse ) +static std::string sqlFindInserted( const TableDiffContext &diffContext, bool reverse ) { - std::string exprPk; - for ( const TableColumnInfo &c : tbl.columns ) + const char *baseTableName = diffContext.schemaBase.name.c_str(); + const char *modifiedTableName = diffContext.schemaModified.name.c_str(); + + std::string exprPk; // Filter expression checking primary key is equal + for ( const TableColumnInfo &c : diffContext.commonColumns ) { if ( c.isPrimaryKey ) { if ( !exprPk.empty() ) exprPk += " AND "; - exprPk += sqlitePrintf( "\"%w\".\"%w\".\"%w\"=\"%w\".\"%w\".\"%w\"", - "main", tableName.c_str(), c.name.c_str(), "aux", tableName.c_str(), c.name.c_str() ); + exprPk += sqlitePrintf( "\"modified\".\"%w\".\"%w\"=\"main\".\"%w\".\"%w\"", + modifiedTableName, c.name.c_str(), baseTableName, c.name.c_str() ); } } - std::string sql = sqlitePrintf( "SELECT * FROM \"%w\".\"%w\" WHERE NOT EXISTS ( SELECT 1 FROM \"%w\".\"%w\" WHERE %s)", - reverse ? "aux" : "main", tableName.c_str(), - reverse ? "main" : "aux", tableName.c_str(), exprPk.c_str() ); + std::string sql = sqlitePrintf( "SELECT %s FROM \"%w\".\"%w\" WHERE NOT EXISTS ( SELECT 1 FROM \"%w\".\"%w\" WHERE %s)", + sqlColumnsStr( diffContext, reverse ).c_str(), + reverse ? "main" : "modified", reverse ? baseTableName : modifiedTableName, + reverse ? "modified" : "main", reverse ? modifiedTableName : baseTableName, exprPk.c_str() ); return sql; } //! Constructs SQL query to get all modified rows for a single table -static std::string sqlFindModified( const std::string &tableName, const TableSchema &tbl ) +static std::string sqlFindModified( const TableDiffContext &diffContext ) { + const char *baseTableName = diffContext.schemaBase.name.c_str(); + const char *modifiedTableName = diffContext.schemaModified.name.c_str(); + std::string exprPk; std::string exprOther; - for ( const TableColumnInfo &c : tbl.columns ) + for ( const TableColumnInfo &c : diffContext.commonColumns ) { if ( c.isPrimaryKey ) { if ( !exprPk.empty() ) exprPk += " AND "; - exprPk += sqlitePrintf( "\"%w\".\"%w\".\"%w\"=\"%w\".\"%w\".\"%w\"", - "main", tableName.c_str(), c.name.c_str(), "aux", tableName.c_str(), c.name.c_str() ); + exprPk += sqlitePrintf( "\"modified\".\"%w\".\"%w\"=\"main\".\"%w\".\"%w\"", + modifiedTableName, c.name.c_str(), baseTableName, c.name.c_str() ); } else // not a primary key column { if ( !exprOther.empty() ) exprOther += " OR "; - exprOther += sqlitePrintf( "\"%w\".\"%w\".\"%w\" IS NOT \"%w\".\"%w\".\"%w\"", - "main", tableName.c_str(), c.name.c_str(), "aux", tableName.c_str(), c.name.c_str() ); + exprOther += sqlitePrintf( "\"modified\".\"%w\".\"%w\" IS NOT \"main\".\"%w\".\"%w\"", + modifiedTableName, c.name.c_str(), baseTableName, c.name.c_str() ); } } - std::string sql; + + // Check for non-NULL values in newly-added columns + for ( const TableColumnInfo &c : diffContext.newColumns ) + { + if ( !exprOther.empty() ) + exprOther += " OR "; + + exprOther += sqlitePrintf( "\"modified\".\"%w\".\"%w\" IS NOT NULL", + modifiedTableName, c.name.c_str() ); + } + + std::string colsStr = sqlColumnsStr( diffContext, false ) + ", " + sqlColumnsStr( diffContext, true ); if ( exprOther.empty() ) { - sql = sqlitePrintf( "SELECT * FROM \"%w\".\"%w\", \"%w\".\"%w\" WHERE %s", - "main", tableName.c_str(), "aux", tableName.c_str(), exprPk.c_str() ); + return sqlitePrintf( "SELECT %s FROM \"modified\".\"%w\", \"main\".\"%w\" WHERE %s", + colsStr.c_str(), modifiedTableName, baseTableName, exprPk.c_str() ); } else { - sql = sqlitePrintf( "SELECT * FROM \"%w\".\"%w\", \"%w\".\"%w\" WHERE %s AND (%s)", - "main", tableName.c_str(), "aux", tableName.c_str(), exprPk.c_str(), exprOther.c_str() ); + return sqlitePrintf( "SELECT %s FROM \"modified\".\"%w\", \"main\".\"%w\" WHERE %s AND (%s)", + colsStr.c_str(), modifiedTableName, baseTableName, exprPk.c_str(), exprOther.c_str() ); } - - return sql; } @@ -470,25 +546,25 @@ static Value changesetValue( sqlite3_value *v ) return x; } -static void handleInserted( const Context *context, const std::string &tableName, const TableSchema &tbl, bool reverse, std::shared_ptr db, ChangesetWriter &writer, bool &first ) +static void handleInserted( const Context *context, TableDiffContext &diffContext, bool reverse ) { - std::string sqlInserted = sqlFindInserted( tableName, tbl, reverse ); + std::string sqlInserted = sqlFindInserted( diffContext, reverse ); Sqlite3Stmt statementI; - statementI.prepare( db, "%s", sqlInserted.c_str() ); + statementI.prepare( diffContext.db, "%s", sqlInserted.c_str() ); int rc; while ( SQLITE_ROW == ( rc = sqlite3_step( statementI.get() ) ) ) { - if ( first ) + if ( !diffContext.tableEntryWritten ) { - ChangesetTable chTable = schemaToChangesetTable( tableName, tbl ); - writer.beginTable( chTable ); - first = false; + ChangesetTable chTable = schemaToChangesetTable( diffContext.schemaModified.name, diffContext.schemaModified ); + diffContext.writer.beginTable( chTable ); + diffContext.tableEntryWritten = true; } - ChangesetEntry e; - e.op = reverse ? ChangesetEntry::OpDelete : ChangesetEntry::OpInsert; + ChangesetDataEntry e; + e.op = reverse ? ChangesetDataEntry::OpDelete : ChangesetDataEntry::OpInsert; - size_t numColumns = tbl.columns.size(); + size_t numColumns = diffContext.schemaModified.columns.size(); for ( size_t i = 0; i < numColumns; ++i ) { Sqlite3Value v( sqlite3_column_value( statementI.get(), static_cast( i ) ) ); @@ -498,20 +574,20 @@ static void handleInserted( const Context *context, const std::string &tableName e.newValues.push_back( changesetValue( v.value() ) ); } - writer.writeEntry( e ); + diffContext.writer.writeEntry( e ); } if ( rc != SQLITE_DONE ) { - logSqliteError( context, db, "Failed to write information about inserted rows in table " + tableName ); + logSqliteError( context, diffContext.db, "Failed to write information about inserted rows in table " + diffContext.schemaModified.name ); } } -static void handleUpdated( const Context *context, const std::string &tableName, const TableSchema &tbl, std::shared_ptr db, ChangesetWriter &writer, bool &first ) +static void handleUpdated( const Context *context, TableDiffContext &diffContext ) { - std::string sqlModified = sqlFindModified( tableName, tbl ); + std::string sqlModified = sqlFindModified( diffContext ); Sqlite3Stmt statement; - statement.prepare( db, "%s", sqlModified.c_str() ); + statement.prepare( diffContext.db, "%s", sqlModified.c_str() ); int rc; while ( SQLITE_ROW == ( rc = sqlite3_step( statement.get() ) ) ) { @@ -526,16 +602,16 @@ static void handleUpdated( const Context *context, const std::string &tableName, ** are set to "undefined". */ - ChangesetEntry e; - e.op = ChangesetEntry::OpUpdate; + ChangesetDataEntry e; + e.op = ChangesetDataEntry::OpUpdate; bool hasUpdates = false; - size_t numColumns = tbl.columns.size(); + size_t numColumns = diffContext.schemaModified.columns.size(); for ( size_t i = 0; i < numColumns; ++i ) { Sqlite3Value v1( sqlite3_column_value( statement.get(), static_cast( i + numColumns ) ) ); Sqlite3Value v2( sqlite3_column_value( statement.get(), static_cast( i ) ) ); - bool pkey = tbl.columns[i].isPrimaryKey; + bool pkey = diffContext.schemaModified.columns[i].isPrimaryKey; bool updated = ( v1 != v2 ); if ( updated ) { @@ -543,10 +619,10 @@ static void handleUpdated( const Context *context, const std::string &tableName, // multiple different string representations could be used for a single datetime value, // see "Time Values" section in https://sqlite.org/lang_datefunc.html // Use strftime() to take into account fractional seconds - if ( tbl.columns[i].type == TableColumnType::DATETIME ) + if ( diffContext.schemaModified.columns[i].type == TableColumnType::DATETIME ) { Sqlite3Stmt stmtDatetime; - stmtDatetime.prepare( db, "SELECT STRFTIME('%%Y-%%m-%%d %%H:%%M:%%f', ?1) IS NOT STRFTIME('%%Y-%%m-%%d %%H:%%M:%%f', ?2)" ); + stmtDatetime.prepare( diffContext.db, "SELECT STRFTIME('%%Y-%%m-%%d %%H:%%M:%%f', ?1) IS NOT STRFTIME('%%Y-%%m-%%d %%H:%%M:%%f', ?2)" ); sqlite3_bind_value( stmtDatetime.get(), 1, v1.value() ); sqlite3_bind_value( stmtDatetime.get(), 2, v2.value() ); int res = sqlite3_step( stmtDatetime.get() ); @@ -556,7 +632,7 @@ static void handleUpdated( const Context *context, const std::string &tableName, } else if ( SQLITE_DONE != res ) { - logSqliteError( context, db, "Failed to write information about updated rows in table " + tableName ); + logSqliteError( context, diffContext.db, "Failed to write information about updated rows in table " + diffContext.schemaModified.name ); } } @@ -571,56 +647,198 @@ static void handleUpdated( const Context *context, const std::string &tableName, if ( hasUpdates ) { - if ( first ) + if ( !diffContext.tableEntryWritten ) { - ChangesetTable chTable = schemaToChangesetTable( tableName, tbl ); - writer.beginTable( chTable ); - first = false; + ChangesetTable chTable = schemaToChangesetTable( diffContext.schemaModified.name, diffContext.schemaModified ); + diffContext.writer.beginTable( chTable ); + diffContext.tableEntryWritten = true; } - writer.writeEntry( e ); + diffContext.writer.writeEntry( e ); } } if ( rc != SQLITE_DONE ) { - logSqliteError( context, db, "Failed to write information about inserted rows in table " + tableName ); + logSqliteError( context, diffContext.db, "Failed to write information about inserted rows in table " + diffContext.schemaModified.name ); } } -void SqliteDriver::createChangeset( ChangesetWriter &writer ) +// To allow diff inversion to work, we first delete all rows when dropping a +// table, and NULL out all rows when dropping a column. +static void writeDataChangesForSchemaChange( std::shared_ptr db, const std::unordered_map ¤tSchemata, ChangesetWriter &writer, const ChangesetEntry &entry ) { - std::vector tablesBase = listTables( false ); - std::vector tablesModified = listTables( true ); + if ( const ChangesetDropColumnEntry *dcEntry = std::get_if( &entry ) ) + { + auto it = currentSchemata.find( dcEntry->tableName ); + if ( it == currentSchemata.end() ) + throw GeoDiffException( "Missing schema for table " + dcEntry->tableName ); + const TableSchema &table = it->second; + + size_t droppedColIdx = table.columnFromName( dcEntry->column.name ); + if ( droppedColIdx == SIZE_MAX ) + throw GeoDiffException( "Could not find column " + dcEntry->column.name + " to delete" ); - if ( tablesBase != tablesModified ) + std::string pkeyColStr; + for ( const TableColumnInfo &c : table.columns ) + { + if ( c.isPrimaryKey ) + { + if ( !pkeyColStr.empty() ) + pkeyColStr += ", "; + pkeyColStr += sqlitePrintf( "\"%w\"", c.name.c_str() ); + } + } + if ( pkeyColStr.empty() ) + throw GeoDiffException( "Table " + table.name + " has no primary key" ); + + Sqlite3Stmt stmt; + stmt.prepare( db, "SELECT %s, \"%w\" FROM \"main\".\"%w\" WHERE \"%w\" IS NOT NULL", + pkeyColStr.c_str(), dcEntry->column.name.c_str(), dcEntry->tableName.c_str(), dcEntry->column.name.c_str() ); + + writer.beginTable( schemaToChangesetTable( table.name, table ) ); + int rc; + while ( SQLITE_ROW == ( rc = sqlite3_step( stmt.get() ) ) ) + { + ChangesetDataEntry e; + e.op = ChangesetDataEntry::OpUpdate; + + size_t idxInResult = 0; + for ( size_t i = 0; i < table.columns.size(); ++i ) + { + bool isPkey = table.columns[i].isPrimaryKey; + bool isDroppedCol = i == droppedColIdx; + + if ( isPkey || isDroppedCol ) + { + Sqlite3Value v( sqlite3_column_value( stmt.get(), static_cast( idxInResult ) ) ); + e.oldValues.push_back( changesetValue( v.value() ) ); + idxInResult++; + } + else + e.oldValues.push_back( Value() ); + + if ( isDroppedCol ) + { + Value nullVal; + nullVal.setNull(); + e.newValues.push_back( nullVal ); + } + else + e.newValues.push_back( Value() ); + } + + writer.writeEntry( e ); + } + } + else if ( const ChangesetDropTableEntry *dtEntry = std::get_if( &entry ) ) + { + auto it = currentSchemata.find( dtEntry->tableName ); + if ( it == currentSchemata.end() ) + throw GeoDiffException( "Missing schema for table " + dtEntry->tableName ); + const TableSchema &table = it->second; + + Sqlite3Stmt stmt; + stmt.prepare( db, "SELECT * FROM \"main\".\"%w\"", dtEntry->tableName.c_str() ); + + writer.beginTable( schemaToChangesetTable( table.name, table ) ); + int rc; + while ( SQLITE_ROW == ( rc = sqlite3_step( stmt.get() ) ) ) + { + ChangesetDataEntry e; + e.op = ChangesetDataEntry::OpDelete; + + size_t numColumns = table.columns.size(); + for ( size_t i = 0; i < numColumns; ++i ) + { + Sqlite3Value v( sqlite3_column_value( stmt.get(), static_cast( i ) ) ); + e.oldValues.push_back( changesetValue( v.value() ) ); + } + + writer.writeEntry( e ); + } + } +} + +void SqliteDriver::createChangeset( ChangesetWriter &writer ) +{ + DatabaseSchema schemaBase = getSchema( false ); + DatabaseSchema schemaModified = getSchema( true ); + + // We keep table schemata that have exactly the written out schema-change + // entries applied. They're necessary to know the intermediate database state + // for any data changes (e.g. row deletions before table drop). + std::unordered_map currentSchemata; + for ( const TableSchema &tbl : schemaBase.tables ) + currentSchemata[tbl.name] = tbl; + + auto schemaDiffEntries = diffDatabaseSchema( schemaBase, schemaModified ); + for ( const ChangesetEntry &entry : schemaDiffEntries ) { - throw GeoDiffException( "Table names are not matching between the input databases.\n" - "Base: " + concatNames( tablesBase ) + "\n" + - "Modified: " + concatNames( tablesModified ) ); + writeDataChangesForSchemaChange( mDb, currentSchemata, writer, entry ); + writer.writeEntry( entry ); + + if ( const ChangesetAddColumnEntry *acEntry = std::get_if( &entry ) ) + simulateColumnChange( currentSchemata[acEntry->tableName], entry ); + else if ( const ChangesetDropColumnEntry *dcEntry = std::get_if( &entry ) ) + simulateColumnChange( currentSchemata[dcEntry->tableName], entry ); } - for ( const std::string &tableName : tablesBase ) + for ( const TableSchema &tblModified : schemaModified.tables ) { - TableSchema tbl = tableSchema( tableName ); - TableSchema tblNew = tableSchema( tableName, true ); + if ( !tblModified.hasPrimaryKey() ) + continue; // ignore tables without primary key - they can't be compared properly - // test that table schema in the modified is the same - if ( tbl != tblNew ) + // Find corresponding table in base DB + const TableSchema *tblBase = nullptr; + for ( const TableSchema &tbl : schemaBase.tables ) { - if ( !tbl.compareWithBaseTypes( tblNew ) ) - throw GeoDiffException( "GeoPackage Table schemas are not the same for table: " + tableName ); + if ( tbl.name == tblModified.name ) + { + tblBase = &tbl; + break; + } } - if ( !tbl.hasPrimaryKey() ) - continue; // ignore tables without primary key - they can't be compared properly + if ( !tblBase ) + { + // Table was newly added, just dump data using INSERTs + dumpTableData( writer, tblModified, true ); + continue; + } - bool first = true; + TableDiffContext diffContext = { mDb, *tblBase, tblModified, {}, {}, writer }; - handleInserted( context(), tableName, tbl, false, mDb, writer, first ); // INSERT - handleInserted( context(), tableName, tbl, true, mDb, writer, first ); // DELETE - handleUpdated( context(), tableName, tbl, mDb, writer, first ); // UPDATE - } + for ( const TableColumnInfo &baseColumn : tblBase->columns ) + { + for ( const TableColumnInfo &modifiedColumn : tblModified.columns ) + { + if ( baseColumn.name == modifiedColumn.name ) + { + diffContext.commonColumns.push_back( modifiedColumn ); + break; + } + } + } + + for ( const TableColumnInfo &modifiedColumn : tblModified.columns ) + { + bool found = false; + for ( const TableColumnInfo &baseColumn : tblBase->columns ) + { + if ( baseColumn.name == modifiedColumn.name ) + { + found = true; + break; + } + } + if ( !found ) + diffContext.newColumns.push_back( modifiedColumn ); + } + handleInserted( context(), diffContext, false ); // INSERT + handleInserted( context(), diffContext, true ); // DELETE + handleUpdated( context(), diffContext ); // UPDATE + } } static std::string sqlForInsert( const std::string &tableName, const TableSchema &tbl ) @@ -751,7 +969,7 @@ static void bindValue( sqlite3_stmt *stmt, int index, const Value &v ) } -ChangeApplyResult SqliteDriver::applyChange( SqliteChangeApplyState &state, const ChangesetEntry &entry ) +ChangeApplyResult SqliteDriver::applyDataChange( SqliteChangeApplyState &state, const ChangesetDataEntry &entry ) { std::string tableName = entry.table->name; @@ -761,7 +979,7 @@ ChangeApplyResult SqliteDriver::applyChange( SqliteChangeApplyState &state, cons if ( context()->isTableSkipped( tableName ) ) // skip table if necessary return ChangeApplyResult::Skipped; - if ( state.tableState.count( tableName ) == 0 ) + if ( state.tableState.count( entry.table ) == 0 ) { TableSchema schema = tableSchema( tableName ); @@ -777,14 +995,14 @@ ChangeApplyResult SqliteDriver::applyChange( SqliteChangeApplyState &state, cons throw GeoDiffException( "Mismatch of primary keys in table: " + tableName ); } - SqliteChangeApplyState::TableState &tbl = state.tableState[tableName]; + SqliteChangeApplyState::TableState &tbl = state.tableState[entry.table]; tbl.schema = schema; tbl.stmtInsert.prepare( mDb, sqlForInsert( tableName, schema ) ); tbl.stmtUpdate.prepare( mDb, sqlForUpdate( tableName, schema ) ); tbl.stmtDelete.prepare( mDb, sqlForDelete( tableName, schema ) ); } - SqliteChangeApplyState::TableState &tbl = state.tableState[tableName]; + SqliteChangeApplyState::TableState &tbl = state.tableState[entry.table]; if ( entry.op == SQLITE_INSERT ) { @@ -862,131 +1080,6 @@ ChangeApplyResult SqliteDriver::applyChange( SqliteChangeApplyState &state, cons return ChangeApplyResult::Applied; } - -void SqliteDriver::applyChangeset( ChangesetReader &reader ) -{ - TableSchema tbl; - - // this will acquire DB mutex and release it when the function ends (or when an exception is thrown) - Sqlite3DbMutexLocker dbMutexLocker( mDb ); - - // start transaction! - Sqlite3SavepointTransaction savepointTransaction( context(), mDb ); - - // Defer verifying foreign key constraints until end of transaction. This - // only applies inside our transaction, so we don't need to reset it. - Sqlite3Stmt statement; - statement.prepare( mDb, "pragma defer_foreign_keys = 1" ); - int rc = sqlite3_step( statement.get() ); - if ( SQLITE_DONE != rc ) - logSqliteError( context(), mDb, "Failed to defer foreign key checks" ); - statement.close(); - - // get all triggers sql commands - // that we do not recognize (gpkg triggers are filtered) - std::vector triggerNames; - std::vector triggerCmds; - sqliteTriggers( context(), mDb, triggerNames, triggerCmds ); - - for ( const std::string &name : triggerNames ) - { - statement.prepare( mDb, "drop trigger '%q'", name.c_str() ); - rc = sqlite3_step( statement.get() ); - if ( SQLITE_DONE != rc ) - { - logSqliteError( context(), mDb, "Failed to drop trigger " + name ); - } - statement.close(); - } - - int unrecoverableConflictCount = 0; - std::vector conflictingEntries; - ChangesetEntry entry; - SqliteChangeApplyState state; - std::unordered_map> tableCopies; - while ( reader.nextEntry( entry ) ) - { - ChangeApplyResult res = applyChange( state, entry ); - switch ( res ) - { - case ChangeApplyResult::Applied: - case ChangeApplyResult::Skipped: - break; // Applied correctly, continue onward. - case ChangeApplyResult::ConstraintConflict: - // Ordering conflict found, handle later. - // Effectively copying the entry isn't simple, since ChangesetReader is - // happy to change entry.table under our feet. We need to copy the - // table object, ideally only keeping one per table. - if ( tableCopies.count( entry.table->name ) == 0 ) - // cppcheck-suppress stlFindInsert - tableCopies[entry.table->name] = std::unique_ptr( new ChangesetTable( *entry.table ) ); - entry.table = tableCopies[entry.table->name].get(); - conflictingEntries.push_back( entry ); - break; - case ChangeApplyResult::NoChange: - unrecoverableConflictCount++; // Other issue, will throw at the end. - break; - } - } - - // Applying some entries may fail due to constraints, since they require the - // entries to be in some specific, unknown order. To work around this, we - // retry applying the conflicting entries until either we apply them all or we - // get stuck. - std::vector newConflictingEntries; - while ( conflictingEntries.size() > 0 ) - { - for ( const ChangesetEntry ¢ry : conflictingEntries ) - { - ChangeApplyResult res = applyChange( state, centry ); - switch ( res ) - { - case ChangeApplyResult::Applied: - case ChangeApplyResult::Skipped: - break; // Applied correctly, don't put it in the new list. - case ChangeApplyResult::ConstraintConflict: - newConflictingEntries.push_back( centry ); // Still conflicting, keep in list. - break; - case ChangeApplyResult::NoChange: - unrecoverableConflictCount++; // Other issue, will throw at the end. - break; - } - } - - // If we haven't been able to apply any of the conflicting entries this - // loop, then these conflicts can't be resolved by reordering entries. - if ( newConflictingEntries.size() == conflictingEntries.size() ) - { - for ( const ChangesetEntry ¢ry : conflictingEntries ) - logApplyConflict( "unresolvable_conflict", centry ); - throw GeoDiffConflictsException( "Could not resolve dependencies in constraint conflicts." ); - } - conflictingEntries = newConflictingEntries; - newConflictingEntries.clear(); - } - - // recreate triggers - for ( const std::string &cmd : triggerCmds ) - { - statement.prepare( mDb, "%s", cmd.c_str() ); - if ( SQLITE_DONE != sqlite3_step( statement.get() ) ) - { - logSqliteError( context(), mDb, "Failed to recreate trigger using SQL \"" + cmd + "\"" ); - } - statement.close(); - } - - if ( !unrecoverableConflictCount ) - { - savepointTransaction.commitChanges(); - } - else - { - throw GeoDiffConflictsException( "Conflicts encountered while applying changes! Total " + std::to_string( unrecoverableConflictCount ) ); - } -} - - static void addGpkgCrsDefinition( std::shared_ptr db, const CrsDefinition &crs ) { // gpkg_spatial_ref_sys @@ -1005,6 +1098,9 @@ static void addGpkgCrsDefinition( std::shared_ptr db, const CrsDefini if ( sqlite3_column_int( stmtCheck.get(), 0 ) ) return; // already there + if ( crs.wkt.size() == 0 ) + throw GeoDiffException( "Tried to add new CRS without WKT definition" ); + Sqlite3Stmt stmt; stmt.prepare( db, "INSERT INTO gpkg_spatial_ref_sys VALUES ('%q:%d', %d, '%q', %d, '%q', '')", crs.authName.c_str(), crs.authCode, crs.srsId, crs.authName.c_str(), crs.authCode, @@ -1061,110 +1157,411 @@ static void addGpkgSpatialTable( std::shared_ptr db, const TableSchem } } -void SqliteDriver::createTables( const std::vector &tables ) +static void createTable( std::shared_ptr db, const TableSchema &tbl ) { - // currently we always create geopackage meta tables. Maybe in the future we can skip - // that if there is a reason, and have that optional if none of the tables are spatial. - Sqlite3Stmt stmt1; - stmt1.prepare( mDb, "SELECT InitSpatialMetadata('main');" ); - int res = sqlite3_step( stmt1.get() ); - if ( res != SQLITE_ROW ) + if ( tbl.geometryColumn() != SIZE_MAX ) { - throwSqliteError( mDb->get(), "Failure initializing spatial metadata" ); + addGpkgCrsDefinition( db, tbl.crs ); + addGpkgSpatialTable( db, tbl, Extent() ); // TODO: is it OK to set zeros? } - for ( const TableSchema &tbl : tables ) + std::string sql, pkeyCols, columns; + for ( const TableColumnInfo &c : tbl.columns ) { - if ( startsWith( tbl.name, "gpkg_" ) ) - continue; + if ( !columns.empty() ) + columns += ", "; + + columns += sqlitePrintf( "\"%w\" %s", c.name.c_str(), c.type.dbType.c_str() ); + + if ( c.isNotNull ) + columns += " NOT NULL"; + + // we have also c.isAutoIncrement, but the SQLite AUTOINCREMENT keyword only applies + // to primary keys, and according to the docs, ordinary tables with INTEGER PRIMARY KEY column + // (which becomes alias to ROWID) does auto-increment, and AUTOINCREMENT just prevents + // reuse of ROWIDs from previously deleted rows. + // See https://sqlite.org/autoinc.html - if ( tbl.geometryColumn() != SIZE_MAX ) + if ( c.isPrimaryKey ) { - addGpkgCrsDefinition( mDb, tbl.crs ); - addGpkgSpatialTable( mDb, tbl, Extent() ); // TODO: is it OK to set zeros? + if ( !pkeyCols.empty() ) + pkeyCols += ", "; + pkeyCols += sqlitePrintf( "\"%w\"", c.name.c_str() ); } + } - std::string sql, pkeyCols, columns; - for ( const TableColumnInfo &c : tbl.columns ) - { - if ( !columns.empty() ) - columns += ", "; + sql = sqlitePrintf( "CREATE TABLE \"%w\" (", tbl.name.c_str() ); + if ( !columns.empty() ) + { + sql += columns; + } + if ( !pkeyCols.empty() ) + { + sql += ", PRIMARY KEY (" + pkeyCols + ")"; + } + sql += ");"; - columns += sqlitePrintf( "\"%w\" %s", c.name.c_str(), c.type.dbType.c_str() ); + Sqlite3Stmt stmt; + stmt.prepare( db, sql ); + if ( sqlite3_step( stmt.get() ) != SQLITE_DONE ) + { + throwSqliteError( db->get(), "Failure creating table: " + tbl.name ); + } +} + +static void removeGpkgSpatialTable( std::shared_ptr db, const std::string &tableName ) +{ + { + Sqlite3Stmt stmt; + stmt.prepare( db, "DELETE FROM gpkg_contents WHERE table_name = '%q'", + tableName.c_str() ); + int res = sqlite3_step( stmt.get() ); + if ( res != SQLITE_DONE ) + throwSqliteError( db->get(), "Failed to delete table from gpkg_contents table" ); + } - if ( c.isNotNull ) - columns += " NOT NULL"; + { + Sqlite3Stmt stmt; + stmt.prepare( db, "DELETE FROM gpkg_geometry_columns WHERE table_name = '%q'", + tableName.c_str() ); + int res = sqlite3_step( stmt.get() ); + if ( res != SQLITE_DONE ) + throwSqliteError( db->get(), "Failed to delete table from gpkg_geometry_columns table" ); + } +} - // we have also c.isAutoIncrement, but the SQLite AUTOINCREMENT keyword only applies - // to primary keys, and according to the docs, ordinary tables with INTEGER PRIMARY KEY column - // (which becomes alias to ROWID) does auto-increment, and AUTOINCREMENT just prevents - // reuse of ROWIDs from previously deleted rows. - // See https://sqlite.org/autoinc.html +void SqliteDriver::applySchemaChange( const ChangesetEntry &entry ) +{ + if ( const ChangesetCreateTableEntry *ctEntry = std::get_if( &entry ) ) + { + // TODO: Also save full CRS definition inside diff? It's pretty large and + // we'd need it for all tables with geometry columns & geometry columns + // themselves. + CrsDefinition tableCrs; + for ( const TableColumnInfo &col : ctEntry->columns ) + { + if ( col.isGeometry ) + tableCrs.srsId = col.geomSrsId; + } - if ( c.isPrimaryKey ) + Sqlite3SavepointTransaction transaction( context(), mDb ); + try + { + createTable( mDb, { ctEntry->tableName, ctEntry->columns, tableCrs } ); + } + catch ( const GeoDiffException & ) + { + // TODO: Make sure this only catches sqlite errors on CREATE TABLE + logApplyConflict( "create_table_failed", entry, true ); + throw; + } + transaction.commitChanges(); + } + else if ( const ChangesetDropTableEntry *dtEntry = std::get_if( &entry ) ) + { + // Check there's no data in table (zero rows) + { + Sqlite3Stmt stmt; + stmt.prepare( mDb, "SELECT COUNT(*) FROM \"%w\"", dtEntry->tableName.c_str() ); + if ( sqlite3_step( stmt.get() ) != SQLITE_ROW ) + throwSqliteError( mDb->get(), "Getting row count in " + dtEntry->tableName ); + if ( sqlite3_column_int( stmt.get(), 0 ) != 0 ) { - if ( !pkeyCols.empty() ) - pkeyCols += ", "; - pkeyCols += sqlitePrintf( "\"%w\"", c.name.c_str() ); + logApplyConflict( "drop_table_not_empty", entry ); + throw GeoDiffException( "Tried to drop non-empty table " + dtEntry->tableName ); } } - sql = sqlitePrintf( "CREATE TABLE \"%w\".\"%w\" (", "main", tbl.name.c_str() ); - if ( !columns.empty() ) + Sqlite3Stmt stmt; + stmt.prepare( mDb, "DROP TABLE \"%w\"", dtEntry->tableName.c_str() ); + if ( sqlite3_step( stmt.get() ) != SQLITE_DONE ) { - sql += columns; + logApplyConflict( "drop_table_failed", entry, true ); + throwSqliteError( mDb->get(), "Failure deleting table: " + dtEntry->tableName ); } - if ( !pkeyCols.empty() ) + removeGpkgSpatialTable( mDb, dtEntry->tableName ); + } + else if ( const ChangesetAddColumnEntry *acEntry = std::get_if( &entry ) ) + { + if ( acEntry->column.isGeometry ) + // Would need changing gpkg metadata + throw GeoDiffException( "Adding geometry columns is not supported" ); + if ( acEntry->column.isPrimaryKey ) + throw GeoDiffException( "Adding column to primary key is not supported" ); + if ( acEntry->column.isNotNull ) + throw GeoDiffException( "Adding not-null column is not supported" ); + + std::string sql = sqlitePrintf( "ALTER TABLE \"%w\" ADD COLUMN \"%w\" %s", + acEntry->tableName.c_str(), acEntry->column.name.c_str(), acEntry->column.type.dbType.c_str() ); + + if ( acEntry->column.isNotNull ) + sql += " NOT NULL"; + Sqlite3Stmt stmt; + stmt.prepare( mDb, "%s", sql.c_str() ); + if ( sqlite3_step( stmt.get() ) != SQLITE_DONE ) + { + logApplyConflict( "drop_column_failed", entry, true ); + throwSqliteError( mDb->get(), "Failure adding column: " + acEntry->tableName + "." + acEntry->column.name ); + } + } + else if ( const ChangesetDropColumnEntry *dcEntry = std::get_if( &entry ) ) + { + if ( dcEntry->column.isGeometry ) + throw GeoDiffException( "Dropping geometry columns is not supported" ); + if ( dcEntry->column.isPrimaryKey ) + throw GeoDiffException( "Dropping column from primary key is not supported" ); + + // Check there's no data in the column (all NULLs) { - sql += ", PRIMARY KEY (" + pkeyCols + ")"; + Sqlite3Stmt stmt; + stmt.prepare( mDb, "SELECT COUNT(*) FROM \"%w\" WHERE \"%w\" IS NOT NULL", + dcEntry->tableName.c_str(), dcEntry->column.name.c_str() ); + if ( sqlite3_step( stmt.get() ) != SQLITE_ROW ) + throwSqliteError( mDb->get(), "Getting row count in " + dcEntry->tableName + "." + dcEntry->column.name ); + if ( sqlite3_column_int( stmt.get(), 0 ) != 0 ) + { + logApplyConflict( "drop_column_not_empty", entry ); + throw GeoDiffException( "Tried to drop non-empty column " + dcEntry->tableName + "." + dcEntry->column.name ); + } } - sql += ");"; Sqlite3Stmt stmt; - stmt.prepare( mDb, sql ); + stmt.prepare( mDb, "ALTER TABLE \"%w\" DROP COLUMN \"%w\"", + dcEntry->tableName.c_str(), dcEntry->column.name.c_str() ); if ( sqlite3_step( stmt.get() ) != SQLITE_DONE ) { - throwSqliteError( mDb->get(), "Failure creating table: " + tbl.name ); + logApplyConflict( "drop_column_failed", entry, true ); + throwSqliteError( mDb->get(), "Failure deleting column: " + dcEntry->tableName + "." + dcEntry->column.name ); } } + else + { + throw GeoDiffException( "Unhandled entry type (should have been schema change) " + + std::to_string( entry.index() ) ); + } } - -void SqliteDriver::dumpData( ChangesetWriter &writer, bool useModified ) +void SqliteDriver::applyChangeset( ChangesetReader &reader ) { - std::string dbName = databaseName( useModified ); - std::vector tables = listTables(); - for ( const std::string &tableName : tables ) + TableSchema tbl; + + // this will acquire DB mutex and release it when the function ends (or when an exception is thrown) + Sqlite3DbMutexLocker dbMutexLocker( mDb ); + + // start transaction! + Sqlite3SavepointTransaction savepointTransaction( context(), mDb ); + + // Defer verifying foreign key constraints until end of transaction. This + // only applies inside our transaction, so we don't need to reset it. + Sqlite3Stmt statement; + statement.prepare( mDb, "pragma defer_foreign_keys = 1" ); + int rc = sqlite3_step( statement.get() ); + if ( SQLITE_DONE != rc ) + logSqliteError( context(), mDb, "Failed to defer foreign key checks" ); + statement.close(); + + // get all triggers sql commands + // that we do not recognize (gpkg triggers are filtered) + std::vector triggerNames; + std::vector triggerCmds; + sqliteTriggers( context(), mDb, triggerNames, triggerCmds ); + + for ( const std::string &name : triggerNames ) { - TableSchema tbl = tableSchema( tableName, useModified ); - if ( !tbl.hasPrimaryKey() ) - continue; // ignore tables without primary key - they can't be compared properly + statement.prepare( mDb, "drop trigger '%q'", name.c_str() ); + rc = sqlite3_step( statement.get() ); + if ( SQLITE_DONE != rc ) + { + logSqliteError( context(), mDb, "Failed to drop trigger " + name ); + } + statement.close(); + } - bool first = true; - Sqlite3Stmt statementI; - statementI.prepare( mDb, "SELECT * FROM \"%w\".\"%w\"", dbName.c_str(), tableName.c_str() ); - int rc; - while ( SQLITE_ROW == ( rc = sqlite3_step( statementI.get() ) ) ) + // Applying some entries may fail due to constraints, since they require the + // entries to be in some specific, unknown order. To work around this, we + // retry applying the conflicting entries until either we apply them all or we + // get stuck. + // + // We can only reorder data entries, not schema-changing DDL entries, so we + // gather conflicting data entries in a list until either we run out of + // entries or read a schema-change entry. + + int unrecoverableConflictCount = 0; + std::vector conflictingEntries; + ChangesetEntry entry; + SqliteChangeApplyState state; + while ( true ) + { + bool haveEntry = reader.nextEntry( entry ); + if ( !haveEntry || !std::holds_alternative( entry ) ) { - if ( first ) + // We can't reorder entries beyond this point (see above), retry applying + // conflicting ones. + std::vector newConflictingEntries; + while ( conflictingEntries.size() > 0 ) { - writer.beginTable( schemaToChangesetTable( tableName, tbl ) ); - first = false; + for ( const ChangesetDataEntry ¢ry : conflictingEntries ) + { + ChangeApplyResult res = applyDataChange( state, centry ); + switch ( res ) + { + case ChangeApplyResult::Applied: + case ChangeApplyResult::Skipped: + break; // Applied correctly, don't put it in the new list. + case ChangeApplyResult::ConstraintConflict: + newConflictingEntries.push_back( centry ); // Still conflicting, keep in list. + break; + case ChangeApplyResult::NoChange: + unrecoverableConflictCount++; // Other issue, will throw at the end. + break; + } + } + + // If we haven't been able to apply any of the conflicting entries this + // loop, then these conflicts can't be resolved by reordering entries. + if ( newConflictingEntries.size() == conflictingEntries.size() ) + { + for ( const ChangesetDataEntry ¢ry : conflictingEntries ) + logApplyConflict( "unresolvable_conflict", centry ); + throw GeoDiffConflictsException( "Could not resolve dependencies in constraint conflicts." ); + } + conflictingEntries = newConflictingEntries; + newConflictingEntries.clear(); } + } + if ( !haveEntry ) + break; - ChangesetEntry e; - e.op = ChangesetEntry::OpInsert; - size_t numColumns = tbl.columns.size(); - for ( size_t i = 0; i < numColumns; ++i ) + if ( const ChangesetDataEntry *dataEntry = std::get_if( &entry ) ) + { + ChangeApplyResult res = applyDataChange( state, *dataEntry ); + switch ( res ) { - Sqlite3Value v( sqlite3_column_value( statementI.get(), static_cast( i ) ) ); - e.newValues.push_back( changesetValue( v.value() ) ); + case ChangeApplyResult::Applied: + case ChangeApplyResult::Skipped: + break; // Applied correctly, continue onward. + case ChangeApplyResult::ConstraintConflict: + // Ordering conflict found, handle later. + conflictingEntries.push_back( *dataEntry ); + break; + case ChangeApplyResult::NoChange: + unrecoverableConflictCount++; // Other issue, will throw at the end. + break; } - writer.writeEntry( e ); } - if ( rc != SQLITE_DONE ) + else + { + applySchemaChange( entry ); + } + } + + // recreate triggers + for ( const std::string &cmd : triggerCmds ) + { + statement.prepare( mDb, "%s", cmd.c_str() ); + if ( SQLITE_DONE != sqlite3_step( statement.get() ) ) + { + logSqliteError( context(), mDb, "Failed to recreate trigger using SQL \"" + cmd + "\"" ); + } + statement.close(); + } + + if ( !unrecoverableConflictCount ) + { + savepointTransaction.commitChanges(); + } + else + { + throw GeoDiffConflictsException( "Conflicts encountered while applying changes! Total " + std::to_string( unrecoverableConflictCount ) ); + } +} + +void SqliteDriver::createTables( const std::vector &tables ) +{ + // currently we always create geopackage meta tables. Maybe in the future we can skip + // that if there is a reason, and have that optional if none of the tables are spatial. + + Sqlite3Stmt stmt1; + stmt1.prepare( mDb, "SELECT InitSpatialMetadata('main');" ); + int res = sqlite3_step( stmt1.get() ); + if ( res != SQLITE_ROW ) + throwSqliteError( mDb->get(), "Failure initializing spatial metadata" ); + + for ( const TableSchema &tbl : tables ) + { + if ( startsWith( tbl.name, "gpkg_" ) ) + continue; + createTable( mDb, tbl ); + } +} + +void SqliteDriver::dumpTableData( ChangesetWriter &writer, TableSchema tbl, bool useModified ) +{ + std::string dbName = databaseName( useModified ); + if ( !tbl.hasPrimaryKey() ) + return; // ignore tables without primary key - they can't be compared properly + + bool first = true; + Sqlite3Stmt statementI; + statementI.prepare( mDb, "SELECT * FROM \"%w\".\"%w\"", dbName.c_str(), tbl.name.c_str() ); + int rc; + while ( SQLITE_ROW == ( rc = sqlite3_step( statementI.get() ) ) ) + { + if ( first ) + { + writer.beginTable( schemaToChangesetTable( tbl.name, tbl ) ); + first = false; + } + + ChangesetDataEntry e; + e.op = ChangesetDataEntry::OpInsert; + size_t numColumns = tbl.columns.size(); + for ( size_t i = 0; i < numColumns; ++i ) { - logSqliteError( context(), mDb, "Failure dumping changeset" ); + Sqlite3Value v( sqlite3_column_value( statementI.get(), static_cast( i ) ) ); + e.newValues.push_back( changesetValue( v.value() ) ); } + writer.writeEntry( e ); + } + if ( rc != SQLITE_DONE ) + { + logSqliteError( context(), mDb, "Failure dumping changeset" ); + } +} + +void SqliteDriver::dumpData( ChangesetWriter &writer, bool useModified ) +{ + std::vector tables = listTables(); + for ( const std::string &tableName : tables ) + { + TableSchema tbl = tableSchema( tableName, useModified ); + dumpTableData( writer, tbl, useModified ); + } +} + +std::vector> SqliteDriver::executeSql( std::string sql ) +{ + Sqlite3Stmt stmt; + stmt.prepare( mDb, "%s", sql.c_str() ); + std::vector> rows; + int rc; + while ( ( rc = sqlite3_step( stmt.get() ) ) == SQLITE_ROW ) + { + std::vector values; + values.resize( sqlite3_column_count( stmt.get() ) ); + for ( size_t i = 0; i < values.size(); ++i ) + { + const unsigned char *text = sqlite3_column_text( stmt.get(), static_cast( i ) ); + if ( text ) + values[i] = reinterpret_cast( text ); + else + values[i] = ""; + } + rows.push_back( values ); + } + if ( rc != SQLITE_DONE ) + { + logSqliteError( context(), mDb, "Failure executing SQL: " + sql ); } + return rows; } diff --git a/geodiff/src/drivers/sqlitedriver.h b/geodiff/src/drivers/sqlitedriver.h index 7bbecef1..202aa888 100644 --- a/geodiff/src/drivers/sqlitedriver.h +++ b/geodiff/src/drivers/sqlitedriver.h @@ -10,6 +10,8 @@ #include "driver.h" #include "sqliteutils.h" +#include "changeset.h" +#include "tableschema.h" /** * Holds state that is useful to keep between entries when applying changeset. @@ -25,7 +27,7 @@ class SqliteChangeApplyState Sqlite3Stmt stmtDelete; }; - std::unordered_map tableState; + std::unordered_map, TableState> tableState; }; @@ -50,15 +52,19 @@ class SqliteDriver : public Driver void create( const DriverParametersMap &conn, bool overwrite = false ) override; std::vector listTables( bool useModified = false ) override; TableSchema tableSchema( const std::string &tableName, bool useModified = false ) override; + DatabaseSchema getSchema( bool useModified = false ); void createChangeset( ChangesetWriter &writer ) override; void applyChangeset( ChangesetReader &reader ) override; void createTables( const std::vector &tables ) override; void dumpData( ChangesetWriter &writer, bool useModified = false ) override; + std::vector> executeSql( std::string sql ) override; private: void logApplyConflict( const std::string &type, const ChangesetEntry &entry, bool isDbErr = false ) const; - ChangeApplyResult applyChange( SqliteChangeApplyState &state, const ChangesetEntry &entry ); + ChangeApplyResult applyDataChange( SqliteChangeApplyState &state, const ChangesetDataEntry &entry ); + void applySchemaChange( const ChangesetEntry &entry ); std::string databaseName( bool useModified = false ); + void dumpTableData( ChangesetWriter &writer, TableSchema tbl, bool useModified ); std::shared_ptr mDb; bool mHasModified = false; // whether there is also a second file attached diff --git a/geodiff/src/drivers/sqliteutils.cpp b/geodiff/src/drivers/sqliteutils.cpp index ebfe9c0d..7dd7b0fa 100644 --- a/geodiff/src/drivers/sqliteutils.cpp +++ b/geodiff/src/drivers/sqliteutils.cpp @@ -434,7 +434,7 @@ void sqliteTables( const Context *context, std::vector sqliteColumnNames( const Context *context, std::shared_ptr db, - const std::string &zDb, /* Database ("main" or "aux") to query */ + const std::string &zDb, /* Database ("main" or "modified") to query */ const std::string &tableName /* Name of table to return details of */ ) { diff --git a/geodiff/src/geodiff.cpp b/geodiff/src/geodiff.cpp index 3981e2c3..6c037426 100644 --- a/geodiff/src/geodiff.cpp +++ b/geodiff/src/geodiff.cpp @@ -27,6 +27,7 @@ #include #include #include +#include #include #include "json.hpp" @@ -267,7 +268,7 @@ int GEODIFF_createChangesetEx( GEODIFF_ContextH contextHandle, const char *drive { createChangesetEx( context, driverName, driverExtraInfo, base, modified, changeset ); } - catch ( const GeoDiffException &exc ) + catch ( const GeoDiffException &exc ) { return handleException( context, exc ); } @@ -533,7 +534,7 @@ int GEODIFF_createRebasedChangeset( static void createRebasedChangesetEx( Context *context, const char *driverName, - const char * /* driverExtraInfo */, + const char *driverExtraInfo, const char *base, const char *base2modified, const char *base2their, @@ -545,11 +546,23 @@ static void createRebasedChangesetEx( throw GeoDiffException( "NULL arguments to GEODIFF_createRebasedChangesetEx" ); } - // TODO: use driverName + driverExtraInfo + base when creating rebased - // changeset (e.g. to check whether a newly created ID is actually free) + // Open the base DB to get its schema + DatabaseSchema baseSchema; + { + DriverParametersMap conn; + conn["base"] = std::string( base ); + if ( driverExtraInfo && *driverExtraInfo ) + conn["conninfo"] = std::string( driverExtraInfo ); + std::unique_ptr driver( Driver::createDriver( context, std::string( driverName ) ) ); + if ( !driver ) + throw GeoDiffException( "Unable to use driver: " + std::string( driverName ) ); + driver->open( conn ); + for ( const std::string &tableName : driver->listTables() ) + baseSchema.tables.push_back( driver->tableSchema( tableName ) ); + } std::vector conflicts; - rebase( context, base2their, rebased, base2modified, conflicts ); + rebase( context, baseSchema, base2their, rebased, base2modified, conflicts ); // output conflicts if ( conflicts.empty() ) @@ -643,7 +656,8 @@ int GEODIFF_changesCount( int changesCount = 0; ChangesetEntry entry; while ( reader.nextEntry( entry ) ) - ++changesCount; + if ( std::holds_alternative( entry ) ) + ++changesCount; return changesCount; } @@ -1303,32 +1317,56 @@ void GEODIFF_CR_destroy( GEODIFF_ContextH /*contextHandle*/, GEODIFF_ChangesetRe delete static_cast( readerHandle ); } +static ChangesetDataEntry *getDataEntry( Context *context, GEODIFF_ChangesetEntryH entryHandle ) +{ + ChangesetEntry *entry = static_cast( entryHandle ); + ChangesetDataEntry *dataEntry = std::get_if( entry ); + if ( !dataEntry && context ) + setAndLogError( context, "Entry is not a data entry" ); + return dataEntry; +} + int GEODIFF_CE_operation( GEODIFF_ContextH /*contextHandle*/, GEODIFF_ChangesetEntryH entryHandle ) { - return static_cast( entryHandle )->op; + const ChangesetEntry *entry = static_cast( entryHandle ); + return static_cast( entry->operationType() ); } -GEODIFF_ChangesetTableH GEODIFF_CE_table( GEODIFF_ContextH /*contextHandle*/, GEODIFF_ChangesetEntryH entryHandle ) +GEODIFF_ChangesetTableH GEODIFF_CE_table( GEODIFF_ContextH contextHandle, GEODIFF_ChangesetEntryH entryHandle ) { - ChangesetTable *table = static_cast( entryHandle )->table; - return table; + Context *context = static_cast( contextHandle ); + ChangesetDataEntry *dataEntry = getDataEntry( context, entryHandle ); + if ( !dataEntry ) + return nullptr; + return dataEntry->table.get(); } -int GEODIFF_CE_countValues( GEODIFF_ContextH /*contextHandle*/, GEODIFF_ChangesetEntryH entryHandle ) +int GEODIFF_CE_countValues( GEODIFF_ContextH contextHandle, GEODIFF_ChangesetEntryH entryHandle ) { - ChangesetEntry *entry = static_cast( entryHandle ); - size_t ret = entry->op == ChangesetEntry::OpDelete ? entry->oldValues.size() : entry->newValues.size(); + Context *context = static_cast( contextHandle ); + ChangesetDataEntry *dataEntry = getDataEntry( context, entryHandle ); + if ( !dataEntry ) + return GEODIFF_ERROR; + size_t ret = dataEntry->op == ChangesetDataEntry::OpDelete ? dataEntry->oldValues.size() : dataEntry->newValues.size(); return ( int ) ret; } -GEODIFF_ValueH GEODIFF_CE_oldValue( GEODIFF_ContextH /*contextHandle*/, GEODIFF_ChangesetEntryH entryHandle, int i ) +GEODIFF_ValueH GEODIFF_CE_oldValue( GEODIFF_ContextH contextHandle, GEODIFF_ChangesetEntryH entryHandle, int i ) { - return new Value( static_cast( entryHandle )->oldValues[i] ); + Context *context = static_cast( contextHandle ); + const ChangesetDataEntry *dataEntry = getDataEntry( context, entryHandle ); + if ( !dataEntry ) + return nullptr; + return new Value( dataEntry->oldValues[i] ); } -GEODIFF_ValueH GEODIFF_CE_newValue( GEODIFF_ContextH /*contextHandle*/, GEODIFF_ChangesetEntryH entryHandle, int i ) +GEODIFF_ValueH GEODIFF_CE_newValue( GEODIFF_ContextH contextHandle, GEODIFF_ChangesetEntryH entryHandle, int i ) { - return new Value( static_cast( entryHandle )->newValues[i] ); + Context *context = static_cast( contextHandle ); + const ChangesetDataEntry *dataEntry = getDataEntry( context, entryHandle ); + if ( !dataEntry ) + return nullptr; + return new Value( dataEntry->newValues[i] ); } void GEODIFF_CE_destroy( GEODIFF_ContextH /*contextHandle*/, GEODIFF_ChangesetEntryH entryHandle ) @@ -1406,3 +1444,33 @@ int GEODIFF_createWkbFromGpkgHeader( GEODIFF_ContextH contextHandle, const char return GEODIFF_SUCCESS; } + +int GEODIFF_changesetHasSchemaChangeEntries( GEODIFF_ContextH contextHandle, const char *changeset, bool *schemaChangePresent ) +{ + Context *context = static_cast( contextHandle ); + if ( !context || !changeset ) + { + return GEODIFF_ERROR; + } + + try + { + ChangesetReader reader; + reader.open( changeset ); + ChangesetEntry entry; + while ( reader.nextEntry( entry ) ) + { + if ( !std::holds_alternative( entry ) ) + { + *schemaChangePresent = true; + return GEODIFF_SUCCESS; + } + } + *schemaChangePresent = false; + return GEODIFF_SUCCESS; + } + catch ( const GeoDiffException &exc ) + { + return handleException( context, exc ); + } +} diff --git a/geodiff/src/geodiff.h b/geodiff/src/geodiff.h index 885c2bd8..1ebee05a 100644 --- a/geodiff/src/geodiff.h +++ b/geodiff/src/geodiff.h @@ -501,7 +501,7 @@ GEODIFF_EXPORT void GEODIFF_CR_destroy( // /** - * Reads entry's operation type - whether it is an insert, update or delete. + * Reads entry's operation type. See ChangesetEntryType for possible return values. */ GEODIFF_EXPORT int GEODIFF_CE_operation( GEODIFF_ContextH contextHandle, @@ -511,6 +511,8 @@ GEODIFF_EXPORT int GEODIFF_CE_operation( * Returns table-related information object of the entry. The returned object is owned * by geodiff and does not need to be deleted by the caller. It is only valid while * the changeset entry is not deleted. + * + * If the entry is not a data entry, returns NULL. */ GEODIFF_EXPORT GEODIFF_ChangesetTableH GEODIFF_CE_table( GEODIFF_ContextH contextHandle, @@ -518,6 +520,8 @@ GEODIFF_EXPORT GEODIFF_ChangesetTableH GEODIFF_CE_table( /** * Returns number of items in the list of old/new values. + * + * If the entry is not a data entry, returns NULL. */ GEODIFF_EXPORT int GEODIFF_CE_countValues( GEODIFF_ContextH contextHandle, @@ -527,6 +531,8 @@ GEODIFF_EXPORT int GEODIFF_CE_countValues( * Returns old value of an entry (only valid for UPDATE and DELETE). * The ownership of the value object is passed to the caller - GEODIFF_V_destroy() * should be called when the value object is not needed anymore. + * + * If the entry is not a data entry, returns NULL. */ GEODIFF_EXPORT GEODIFF_ValueH GEODIFF_CE_oldValue( GEODIFF_ContextH contextHandle, @@ -537,6 +543,8 @@ GEODIFF_EXPORT GEODIFF_ValueH GEODIFF_CE_oldValue( * Returns new value of an entry (only valid for UPDATE and INSERT). * The ownership of the value object is passed to the caller - GEODIFF_V_destroy() * should be called when the value object is not needed anymore. + * + * If the entry is not a data entry, returns NULL. */ GEODIFF_EXPORT GEODIFF_ValueH GEODIFF_CE_newValue( GEODIFF_ContextH contextHandle, @@ -645,6 +653,15 @@ GEODIFF_EXPORT int GEODIFF_createWkbFromGpkgHeader( const char **wkb, size_t *wkbLength ); +/** + * Sets schemaChangePresent to true if the changeset at the given path contains + * any schema-change entry (add/drop table/column). + */ +GEODIFF_EXPORT int GEODIFF_changesetHasSchemaChangeEntries( + GEODIFF_ContextH contextHandle, + const char *changeset, + bool *schemaChangePresent ); + #ifdef __cplusplus } diff --git a/geodiff/src/geodiffrebase.cpp b/geodiff/src/geodiffrebase.cpp index 03416726..9af82831 100644 --- a/geodiff/src/geodiffrebase.cpp +++ b/geodiff/src/geodiffrebase.cpp @@ -1,9 +1,10 @@ -/* +/* GEODIFF - MIT License Copyright (C) 2019 Peter Petrik */ #include "geodiffrebase.hpp" +#include "changeset.h" #include "geodiffutils.hpp" #include "geodiff.h" #include "geodifflogger.hpp" @@ -11,6 +12,9 @@ #include "changesetreader.h" #include "changesetwriter.h" +#include "changesetutils.h" +#include "tableschema.h" +#include "tableschemadiff.hpp" #include #include @@ -22,6 +26,7 @@ #include #include #include +#include #include #include #include @@ -50,7 +55,7 @@ struct TableRebaseInfo { std::set inserted; //!< pkeys that were inserted std::set deleted; //!< pkeys that were deleted - std::map > updated; //!< new column values for each recorded row (identified by pkey) + std::map> updated; //!< new column values (by name) for each updated row (identified by pkey) void dump( std::ostringstream &ret ) { @@ -73,6 +78,7 @@ struct TableRebaseInfo struct DatabaseRebaseInfo { std::map tables; //!< mapping for each table (key = table name) + DatabaseSchema theirSchema; void dump( const Context *context ) { @@ -92,7 +98,8 @@ struct DatabaseRebaseInfo }; -//! structure that keeps track of how we modify primary keys of the rebased changeset +//! structure that keeps track of how we modify primary keys and column indices +// of the rebased changeset. struct RebaseMapping { @@ -191,7 +198,7 @@ struct RebaseMapping /////////////////////////////////////// -int _get_primary_key( const ChangesetEntry &entry ) +int _get_primary_key( const ChangesetDataEntry &entry ) { int fid; int nFidColumn; @@ -202,35 +209,85 @@ int _get_primary_key( const ChangesetEntry &entry ) int _parse_old_changeset( const Context *context, + const DatabaseSchema &baseSchema, ChangesetReader &reader_BASE_THEIRS, DatabaseRebaseInfo &dbInfo ) { + dbInfo.theirSchema = baseSchema; + ChangesetEntry entry; while ( reader_BASE_THEIRS.nextEntry( entry ) ) { - std::string tableName = entry.table->name; - - // skip table if necessary - if ( context->isTableSkipped( tableName ) ) + if ( std::holds_alternative( entry ) ) { - continue; - } + ChangesetDataEntry &dataEntry = std::get( entry ); - int pk = _get_primary_key( entry ); + std::string tableName = dataEntry.table->name; - TableRebaseInfo &tableInfo = dbInfo.tables[tableName]; + // skip table if necessary + if ( context->isTableSkipped( tableName ) ) + { + continue; + } + + int pk = _get_primary_key( dataEntry ); + + TableRebaseInfo &tableInfo = dbInfo.tables[tableName]; - if ( entry.op == ChangesetEntry::OpInsert ) + if ( dataEntry.op == ChangesetDataEntry::OpInsert ) + { + tableInfo.inserted.insert( pk ); + } + if ( dataEntry.op == ChangesetDataEntry::OpDelete ) + { + tableInfo.deleted.insert( pk ); + } + if ( dataEntry.op == ChangesetDataEntry::OpUpdate ) + { + const TableSchema *ts = dbInfo.theirSchema.tableByName( tableName ); + if ( !ts ) + throw GeoDiffException( "Update entry for table not in schema: " + tableName ); + std::map namedVals; + for ( size_t i = 0; i < dataEntry.newValues.size() && i < ts->columns.size(); i++ ) + { + if ( dataEntry.newValues[i].type() != Value::TypeUndefined ) + namedVals[ts->columns[i].name] = dataEntry.newValues[i]; + } + tableInfo.updated[pk] = std::move( namedVals ); + } + } + else if ( const ChangesetCreateTableEntry *ctEntry = std::get_if( &entry ) ) { - tableInfo.inserted.insert( pk ); + if ( context->isTableSkipped( ctEntry->tableName ) ) + continue; + // Create dbInfo entry to signify table is changed + ( void ) dbInfo.tables[ctEntry->tableName]; + simulateSchemaChange( dbInfo.theirSchema, entry ); } - if ( entry.op == ChangesetEntry::OpDelete ) + else if ( const ChangesetDropTableEntry *dtEntry = std::get_if( &entry ) ) { - tableInfo.deleted.insert( pk ); + if ( context->isTableSkipped( dtEntry->tableName ) ) + continue; + ( void ) dbInfo.tables[dtEntry->tableName]; + simulateSchemaChange( dbInfo.theirSchema, entry ); } - if ( entry.op == ChangesetEntry::OpUpdate ) + else if ( const ChangesetAddColumnEntry *acEntry = std::get_if( &entry ) ) { - tableInfo.updated[pk] = entry.newValues; + if ( context->isTableSkipped( acEntry->tableName ) ) + continue; + ( void ) dbInfo.tables[acEntry->tableName]; + simulateSchemaChange( dbInfo.theirSchema, entry ); + } + else if ( const ChangesetDropColumnEntry *dcEntry = std::get_if( &entry ) ) + { + if ( context->isTableSkipped( dcEntry->tableName ) ) + continue; + ( void ) dbInfo.tables[dcEntry->tableName]; + simulateSchemaChange( dbInfo.theirSchema, entry ); + } + else + { + throw GeoDiffException( "Unhandled entry type in rebase: " + std::to_string( entry.index() ) ); } } @@ -260,7 +317,11 @@ int _find_mapping_for_new_changeset( ChangesetEntry entry; while ( reader.nextEntry( entry ) ) { - std::string tableName = entry.table->name; + if ( !std::holds_alternative( entry ) ) + continue; + const ChangesetDataEntry &dataEntry = std::get( entry ); + + std::string tableName = dataEntry.table->name; // skip table if necessary if ( context->isTableSkipped( tableName ) ) @@ -274,9 +335,9 @@ int _find_mapping_for_new_changeset( const TableRebaseInfo &tableInfo = tableIt->second; - if ( entry.op == ChangesetEntry::OpInsert ) + if ( dataEntry.op == ChangesetDataEntry::OpInsert ) { - int pk = _get_primary_key( entry ); + int pk = _get_primary_key( dataEntry ); if ( tableInfo.inserted.find( pk ) != tableInfo.inserted.end() ) { @@ -296,9 +357,9 @@ int _find_mapping_for_new_changeset( mapping.unmappedInsertIds[tableName].insert( pk ); } } - else if ( entry.op == ChangesetEntry::OpUpdate ) + else if ( dataEntry.op == ChangesetDataEntry::OpUpdate ) { - int pk = _get_primary_key( entry ); + int pk = _get_primary_key( dataEntry ); if ( tableInfo.deleted.find( pk ) != tableInfo.deleted.end() ) { @@ -306,9 +367,9 @@ int _find_mapping_for_new_changeset( mapping.addPkeyMapping( tableName, pk, RebaseMapping::INVALID_FID ); } } - else if ( entry.op == ChangesetEntry::OpDelete ) + else if ( dataEntry.op == ChangesetDataEntry::OpDelete ) { - int pk = _get_primary_key( entry ); + int pk = _get_primary_key( dataEntry ); if ( tableInfo.deleted.find( pk ) != tableInfo.deleted.end() ) { @@ -355,12 +416,12 @@ int _find_mapping_for_new_changeset( } -bool _handle_insert( const ChangesetEntry &entry, const RebaseMapping &mapping, ChangesetEntry &outEntry ) +bool _handle_insert( const ChangesetDataEntry &entry, const RebaseMapping &mapping, + const std::map &colMap, + ChangesetDataEntry &outEntry ) { - size_t numColumns = entry.table->columnCount(); - - outEntry.op = ChangesetEntry::OpInsert; - outEntry.newValues.resize( numColumns ); + outEntry.op = ChangesetDataEntry::OpInsert; + outEntry.newValues.resize( outEntry.table->columnCount() ); // resolve primary key and patched primary key int pk = _get_primary_key( entry ); @@ -368,31 +429,35 @@ bool _handle_insert( const ChangesetEntry &entry, const RebaseMapping &mapping, if ( mapping.hasOldPkey( entry.table->name, pk ) ) { - // conflict 2 concurrent updates... + // conflict 2 concurrent inserts... newPk = mapping.getNewPkey( entry.table->name, pk ); } - for ( size_t i = 0; i < numColumns; i++ ) + // NULL-out the new values vector - if a column is not present in the map, we + // set it to NULL. + for ( size_t i = 0; i < outEntry.newValues.size(); i++ ) + outEntry.newValues[i].setNull(); + + for ( const auto &[inIdx, outIdx] : colMap ) { - if ( entry.table->primaryKeys[i] ) - { - outEntry.newValues[i].setInt( newPk ); - } + if ( outEntry.table->primaryKeys[outIdx] ) + outEntry.newValues[outIdx].setInt( newPk ); else - { - outEntry.newValues[i] = entry.newValues[i]; - } + outEntry.newValues[outIdx] = entry.newValues[inIdx]; } return true; } -bool _handle_delete( const ChangesetEntry &entry, const RebaseMapping &mapping, - const TableRebaseInfo &tableInfo, ChangesetEntry &outEntry ) +bool _handle_delete( const ChangesetDataEntry &entry, const RebaseMapping &mapping, + const TableRebaseInfo &tableInfo, + const std::map &colMap, + const TableSchema &inTableSchema, + ChangesetDataEntry &outEntry ) { - size_t numColumns = entry.table->columnCount(); - - outEntry.op = ChangesetEntry::OpDelete; - outEntry.oldValues.resize( numColumns ); + outEntry.op = ChangesetDataEntry::OpDelete; + outEntry.oldValues.resize( outEntry.table->columnCount() ); + for ( Value &val : outEntry.oldValues ) + val.setNull(); // resolve primary key and patched primary key int pk = _get_primary_key( entry ); @@ -400,49 +465,45 @@ bool _handle_delete( const ChangesetEntry &entry, const RebaseMapping &mapping, if ( mapping.hasOldPkey( entry.table->name, pk ) ) { - // conflict 2 concurrent updates... - newPk = mapping.getNewPkey( entry.table->name, pk ); - // conflict 2 concurrent deletes... + newPk = mapping.getNewPkey( entry.table->name, pk ); if ( newPk == RebaseMapping::INVALID_FID ) return false; } // find the previously new values (will be used as the old values in the rebased version) - std::vector patchedVals; + const std::map *patchedMap = nullptr; auto a = tableInfo.updated.find( pk ); - if ( a == tableInfo.updated.end() ) - patchedVals.resize( static_cast( numColumns ) ); - else - patchedVals = a->second; + if ( a != tableInfo.updated.end() ) + patchedMap = &a->second; - for ( size_t i = 0; i < numColumns; i++ ) + for ( const auto &[inIdx, outIdx] : colMap ) { - if ( entry.table->primaryKeys[i] ) + if ( outEntry.table->primaryKeys[outIdx] ) { - outEntry.oldValues[i].setInt( newPk ); + outEntry.oldValues[outIdx].setInt( newPk ); } else { // if the value was patched in the previous commit, use that one as base - Value value; - const Value &patchedVal = patchedVals[i]; - if ( patchedVal.type() != Value::TypeUndefined ) + Value patchedVal; + if ( patchedMap ) { - value = patchedVal; + auto it = patchedMap->find( inTableSchema.columns[inIdx].name ); + if ( it != patchedMap->end() ) + patchedVal = it->second; } + if ( patchedVal.type() != Value::TypeUndefined ) + outEntry.oldValues[outIdx] = patchedVal; else - { // otherwise the value is same for both patched and this, so use base value - value = entry.oldValues[i]; - } - outEntry.oldValues[i] = value; + outEntry.oldValues[outIdx] = entry.oldValues[inIdx]; } } return true; } -void _addConflictItem( ConflictFeature &conflictFeature, int i, +void _addConflictItem( DataConflictFeature &conflictFeature, int i, const Value &base, const Value &theirs, const Value &ours ) { // 4th attribute in gpkg_contents is modified date @@ -451,19 +512,20 @@ void _addConflictItem( ConflictFeature &conflictFeature, int i, return; // ok safe to add it - ConflictItem item( i, base, theirs, ours ); + DataConflictItem item( i, base, theirs, ours ); conflictFeature.addItem( item ); } -bool _handle_update( const ChangesetEntry &entry, const RebaseMapping &mapping, - const TableRebaseInfo &tableInfo, ChangesetEntry &outEntry, +bool _handle_update( const ChangesetDataEntry &entry, const RebaseMapping &mapping, + const TableRebaseInfo &tableInfo, + const std::map &colMap, + const TableSchema &inTableSchema, + ChangesetDataEntry &outEntry, std::vector &conflicts ) { - size_t numColumns = entry.table->columnCount(); - - outEntry.op = ChangesetEntry::OpUpdate; - outEntry.oldValues.resize( numColumns ); - outEntry.newValues.resize( numColumns ); + outEntry.op = ChangesetDataEntry::OpUpdate; + outEntry.oldValues.resize( outEntry.table->columnCount() ); + outEntry.newValues.resize( outEntry.table->columnCount() ); // get values from patched (new) master int pk = _get_primary_key( entry ); @@ -473,12 +535,12 @@ bool _handle_update( const ChangesetEntry &entry, const RebaseMapping &mapping, if ( newPk == RebaseMapping::INVALID_FID ) { // our UPDATE conflicts with their DELETE: record as conflict, delete wins - ConflictFeature conflictFeature( pk, entry.table->name ); - for ( size_t i = 0; i < numColumns; i++ ) + DataConflictFeature conflictFeature( pk, entry.table->name ); + for ( const auto &[inIdx, outIdx] : colMap ) { - if ( entry.newValues[i].type() != Value::TypeUndefined ) + if ( entry.newValues[inIdx].type() != Value::TypeUndefined ) { - _addConflictItem( conflictFeature, ( int ) i, entry.oldValues[i], Value(), entry.newValues[i] ); + _addConflictItem( conflictFeature, outIdx, entry.oldValues[inIdx], Value(), entry.newValues[inIdx] ); } } if ( conflictFeature.isValid() ) @@ -488,128 +550,259 @@ bool _handle_update( const ChangesetEntry &entry, const RebaseMapping &mapping, } // find the previously new values (will be used as the old values in the rebased version) - std::vector patchedVals; + const std::map *patchedMap = nullptr; auto a = tableInfo.updated.find( pk ); - if ( a == tableInfo.updated.end() ) - patchedVals.resize( static_cast( numColumns ) ); - else - patchedVals = a->second; + if ( a != tableInfo.updated.end() ) + patchedMap = &a->second; - ConflictFeature conflictFeature( pk, entry.table->name ); + DataConflictFeature conflictFeature( pk, entry.table->name ); bool entryHasChanges = false; - for ( size_t i = 0; i < numColumns; i++ ) + for ( const auto &[inIdx, outIdx] : colMap ) { - Value patchedVal = patchedVals[i]; - if ( patchedVal.type() != Value::TypeUndefined && entry.newValues[i].type() != Value::TypeUndefined ) + Value patchedVal; + if ( patchedMap ) + { + auto it = patchedMap->find( inTableSchema.columns[inIdx].name ); + if ( it != patchedMap->end() ) + patchedVal = it->second; + } + + if ( patchedVal.type() != Value::TypeUndefined && entry.newValues[inIdx].type() != Value::TypeUndefined ) { - if ( patchedVal == entry.newValues[i] ) + if ( patchedVal == entry.newValues[inIdx] ) { // both "old" and "new" changeset modify the column's value to the same value - that // means that in our rebased changeset there's no further change and there's no conflict - outEntry.oldValues[i].setUndefined(); - outEntry.newValues[i].setUndefined(); + outEntry.oldValues[outIdx].setUndefined(); + outEntry.newValues[outIdx].setUndefined(); } else { // we have edit conflict here: both "old" changeset and the "new" changeset modify the same // column of the same row. Rebased changeset will get the "old" value updated to the new (patched) // value of the older changeset - outEntry.oldValues[i] = patchedVal; - outEntry.newValues[i] = entry.newValues[i]; + outEntry.oldValues[outIdx] = patchedVal; + outEntry.newValues[outIdx] = entry.newValues[inIdx]; entryHasChanges = true; - _addConflictItem( conflictFeature, ( int ) i, entry.oldValues[i], patchedVal, entry.newValues[i] ); + _addConflictItem( conflictFeature, outIdx, entry.oldValues[inIdx], patchedVal, entry.newValues[inIdx] ); } } else { // the "new" changeset stays as is without modifications - outEntry.oldValues[i] = entry.oldValues[i]; - outEntry.newValues[i] = entry.newValues[i]; + outEntry.oldValues[outIdx] = entry.oldValues[inIdx]; + outEntry.newValues[outIdx] = entry.newValues[inIdx]; // if a column is pkey, it would have "new" value undefined in the entry and that's not an actual change - if ( entry.newValues[i].type() != Value::TypeUndefined ) + if ( entry.newValues[inIdx].type() != Value::TypeUndefined ) entryHasChanges = true; } } if ( conflictFeature.isValid() ) - { conflicts.push_back( conflictFeature ); - } return entryHasChanges; } //! throws GeoDiffException on error void _prepare_new_changeset( const Context *context, ChangesetReader &reader, const std::string &changesetNew, - const RebaseMapping &mapping, const DatabaseRebaseInfo &dbInfo, + RebaseMapping &mapping, const DatabaseRebaseInfo &dbInfo, + const DatabaseSchema &baseSchema, std::vector &conflicts ) { - ChangesetEntry entry; - std::map tableDefinitions; + // The base DB schema with our changes from already processed entries applied + // on top. + DatabaseSchema currentSchema = baseSchema; + // The base DB schema with our their changes and then ourchanges from already + // processed entries applied on top. + DatabaseSchema outputSchema = dbInfo.theirSchema; + // table schema -> (old column index -> new column index) + // Column being absent means its index didn't change. + std::map> columnIndexMap; + // We record conflicting tables/columns and skip them when processing further + // changes + std::set conflictingTables; + std::map> conflictingColumns; + std::map > tableChanges; + // Cached output ChangesetTable for the current table. + std::shared_ptr outChangesetTable; + + ChangesetEntry entry; while ( reader.nextEntry( entry ) ) { - std::string tableName = entry.table->name; - - // skip table if necessary - if ( context->isTableSkipped( tableName ) ) + if ( std::holds_alternative( entry ) ) { - continue; - } + ChangesetDataEntry &dataEntry = std::get( entry ); + std::string tableName = dataEntry.table->name; - // Inserts table into the definitions, if it doesn't already contain it - tableDefinitions.insert( {tableName, *entry.table} ); + // skip table if necessary + if ( context->isTableSkipped( tableName ) || conflictingTables.count( tableName ) ) + continue; - auto tablesIt = dbInfo.tables.find( tableName ); - if ( tablesIt == dbInfo.tables.end() ) - { - // we have change in different table that was modified in theirs modifications - // just copy plain the change to the output buffer - tableChanges[tableName].push_back( entry ); - continue; - } + TableSchema *tableSchema = currentSchema.tableByName( tableName ); - bool writeEntry = false; - ChangesetEntry outEntry; + if ( !tableSchema ) + throw GeoDiffException( "Tried rebasing data entry for table not in schema: " + tableName ); - // commits to same table -> now save the change to changeset - switch ( entry.op ) - { - case ChangesetEntry::OpUpdate: - writeEntry = _handle_update( entry, mapping, tablesIt->second, outEntry, conflicts ); - break; + // Get the output table schema (theirs + our schema changes so far). + TableSchema *outTableSchema = outputSchema.tableByName( tableName ); + if ( !outTableSchema ) + // Table was dropped by theirs. + continue; - case ChangesetEntry::OpInsert: - writeEntry = _handle_insert( entry, mapping, outEntry ); - break; + // Compute column mapping (input index -> output index) on first encounter. + if ( columnIndexMap.find( dataEntry.table.get() ) == columnIndexMap.end() ) + { + std::map colMap; + auto columnsToSkip = conflictingColumns[tableName]; + for ( size_t i = 0; i < tableSchema->columns.size(); i++ ) + { + const std::string &colName = tableSchema->columns[i].name; + if ( columnsToSkip.count( colName ) ) + continue; + for ( size_t j = 0; j < outTableSchema->columns.size(); j++ ) + { + if ( outTableSchema->columns[j].name == colName ) + { + colMap[static_cast( i )] = static_cast( j ); + break; + } + } + } + columnIndexMap[dataEntry.table.get()] = std::move( colMap ); + } + + const std::map &colMap = columnIndexMap[dataEntry.table.get()]; + + // Rebuild cached output ChangesetTable when the table name changes. + if ( !outChangesetTable || outChangesetTable->name != tableName ) + outChangesetTable = std::make_shared( schemaToChangesetTable( tableName, *outTableSchema ) ); + + auto tablesIt = dbInfo.tables.find( tableName ); + if ( tablesIt == dbInfo.tables.end() ) + { + // Table not touched by theirs data-wise - copy through as-is. + tableChanges[tableName].push_back( entry ); + continue; + } + + bool writeEntry = false; + ChangesetDataEntry outEntry; + outEntry.table = outChangesetTable; + + // commits to same table -> now save the change to changeset + switch ( dataEntry.op ) + { + case ChangesetDataEntry::OpUpdate: + writeEntry = _handle_update( dataEntry, mapping, tablesIt->second, colMap, *tableSchema, outEntry, conflicts ); + break; + + case ChangesetDataEntry::OpInsert: + writeEntry = _handle_insert( dataEntry, mapping, colMap, outEntry ); + break; + + case ChangesetDataEntry::OpDelete: + writeEntry = _handle_delete( dataEntry, mapping, tablesIt->second, colMap, *tableSchema, outEntry ); + break; + } - case ChangesetEntry::OpDelete: - writeEntry = _handle_delete( entry, mapping, tablesIt->second, outEntry ); - break; + if ( writeEntry ) + tableChanges[tableName].push_back( outEntry ); } + else + { + simulateSchemaChange( currentSchema, entry ); + outChangesetTable = nullptr; // Invalidate cached schema, columns may change + + // Check whether the same change is already contained in theirs. If not, + // add it to the output. + bool isDuplicate = false; + std::string schemaEntryTableName; + if ( const ChangesetCreateTableEntry *ctEntry = std::get_if( &entry ) ) + { + schemaEntryTableName = ctEntry->tableName; + const TableSchema *existing = outputSchema.tableByName( ctEntry->tableName ); + if ( existing ) + { + if ( existing->columns != ctEntry->columns ) + { + conflicts.push_back( TableSchemaConflict { ctEntry->tableName } ); + conflictingTables.insert( ctEntry->tableName ); + } + isDuplicate = true; + } + } + else if ( const ChangesetDropTableEntry *dtEntry = std::get_if( &entry ) ) + { + schemaEntryTableName = dtEntry->tableName; + isDuplicate = outputSchema.tableByName( dtEntry->tableName ) == nullptr; + } + else if ( const ChangesetAddColumnEntry *acEntry = std::get_if( &entry ) ) + { + schemaEntryTableName = acEntry->tableName; + TableSchema *table = outputSchema.tableByName( acEntry->tableName ); + if ( table ) + { + auto it = std::find_if( table->columns.begin(), table->columns.end(), + [&]( const TableColumnInfo & c ) { return c.name == acEntry->column.name; } ); + if ( it != table->columns.end() ) + { + if ( *it != acEntry->column ) + { + conflicts.push_back( ColumnSchemaConflict { acEntry->tableName, acEntry->column.name } ); + conflictingColumns[acEntry->tableName].insert( acEntry->column.name ); + } + isDuplicate = true; + } + } + else + throw GeoDiffException( "During rebase tried to add column " + acEntry->tableName + "." + acEntry->column.name + " to non-existent table" ); + } + else if ( const ChangesetDropColumnEntry *dcEntry = std::get_if( &entry ) ) + { + schemaEntryTableName = dcEntry->tableName; + TableSchema *table = outputSchema.tableByName( dcEntry->tableName ); + if ( table ) + { + auto it = std::find_if( table->columns.begin(), table->columns.end(), + [&]( const TableColumnInfo & c ) { return c.name == dcEntry->column.name; } ); + isDuplicate = it == table->columns.end(); + } + else + throw GeoDiffException( "During rebase tried to drop column " + dcEntry->tableName + "." + dcEntry->column.name + " from non-existent table" ); + } - if ( writeEntry ) - tableChanges[tableName].push_back( outEntry ); + if ( !isDuplicate ) + { + simulateSchemaChange( outputSchema, entry ); + tableChanges[schemaEntryTableName].push_back( entry ); + } + } } ChangesetWriter writer; writer.open( changesetNew ); - for ( auto it : tableDefinitions ) + for ( const auto &it : tableChanges ) { - auto chit = tableChanges.find( it.first ); - if ( chit == tableChanges.end() ) - continue; - - const std::vector &changes = chit->second; + const std::vector &changes = it.second; if ( changes.empty() ) continue; - writer.beginTable( it.second ); + const ChangesetTable *defWritten = nullptr; for ( const ChangesetEntry &writeEntry : changes ) { + if ( auto dataEntry = std::get_if( &writeEntry ) ) + { + if ( defWritten != dataEntry->table.get() ) + { + writer.beginTable( *dataEntry->table ); + defWritten = dataEntry->table.get(); + } + } writer.writeEntry( writeEntry ); } } @@ -617,6 +810,7 @@ void _prepare_new_changeset( const Context *context, void rebase( const Context *context, + const DatabaseSchema &baseSchema, const std::string &changeset_BASE_THEIRS, const std::string &changeset_THEIRS_MODIFIED, const std::string &changeset_BASE_MODIFIED, @@ -651,7 +845,7 @@ void rebase( // 1. go through the original changeset and extract data that will be needed in the second step DatabaseRebaseInfo dbInfo; - int rc = _parse_old_changeset( context, reader_BASE_THEIRS, dbInfo ); + int rc = _parse_old_changeset( context, baseSchema, reader_BASE_THEIRS, dbInfo ); if ( rc != GEODIFF_SUCCESS ) throw GeoDiffException( "Could not parse changeset_BASE_THEIRS: " + changeset_BASE_THEIRS ); @@ -664,5 +858,5 @@ void rebase( reader_BASE_MODIFIED.rewind(); // 3. go through the changeset to be rebased again and write it with changes determined in step 2 - _prepare_new_changeset( context, reader_BASE_MODIFIED, changeset_THEIRS_MODIFIED, mapping, dbInfo, conflicts ); + _prepare_new_changeset( context, reader_BASE_MODIFIED, changeset_THEIRS_MODIFIED, mapping, dbInfo, baseSchema, conflicts ); } diff --git a/geodiff/src/geodiffrebase.hpp b/geodiff/src/geodiffrebase.hpp index 2c960c41..25a539ca 100644 --- a/geodiff/src/geodiffrebase.hpp +++ b/geodiff/src/geodiffrebase.hpp @@ -9,11 +9,13 @@ #include #include #include "geodiffutils.hpp" +#include "tableschema.h" class Logger; //! throws GeoDiffException on error void rebase( const Context *context, + const DatabaseSchema &baseSchema, //in const std::string &changeset_BASE_THEIRS, //in const std::string &changeset_THEIRS_MODIFIED, // out const std::string &changeset_BASE_MODIFIED, //in diff --git a/geodiff/src/geodiffutils.cpp b/geodiff/src/geodiffutils.cpp index 8281ac25..cd301936 100644 --- a/geodiff/src/geodiffutils.cpp +++ b/geodiff/src/geodiffutils.cpp @@ -346,7 +346,7 @@ bool isLayerTable( const std::string &tableName ) //// -void get_primary_key( const ChangesetEntry &entry, int &fid, int &nColumn ) +void get_primary_key( const ChangesetDataEntry &entry, int &fid, int &nColumn ) { const std::vector &tablePkeys = entry.table->primaryKeys; @@ -375,11 +375,11 @@ void get_primary_key( const ChangesetEntry &entry, int &fid, int &nColumn ) // now get the value Value pkeyValue; - if ( entry.op == ChangesetEntry::OpInsert ) + if ( entry.op == ChangesetDataEntry::OpInsert ) { pkeyValue = entry.newValues[pk_column_number]; } - else if ( entry.op == ChangesetEntry::OpDelete || entry.op == ChangesetEntry::OpUpdate ) + else if ( entry.op == ChangesetDataEntry::OpDelete || entry.op == ChangesetDataEntry::OpUpdate ) { pkeyValue = entry.oldValues[pk_column_number]; } @@ -556,40 +556,40 @@ void TmpFile::setPath( const std::string &path ) mPath = path; } -ConflictFeature::ConflictFeature( int pk, - const std::string &tableName ) +DataConflictFeature::DataConflictFeature( int pk, + const std::string &tableName ) : mPk( pk ) , mTableName( tableName ) { } -bool ConflictFeature::isValid() const +bool DataConflictFeature::isValid() const { return !mItems.empty(); } -void ConflictFeature::addItem( const ConflictItem &item ) +void DataConflictFeature::addItem( const DataConflictItem &item ) { mItems.push_back( item ); } -const std::string &ConflictFeature::tableName() const +const std::string &DataConflictFeature::tableName() const { return mTableName; } -int ConflictFeature::pk() const +int DataConflictFeature::pk() const { return mPk; } -const std::vector &ConflictFeature::items() const +const std::vector &DataConflictFeature::items() const { return mItems; } -ConflictItem::ConflictItem( int column, const Value &base, - const Value &theirs, const Value &ours ) +DataConflictItem::DataConflictItem( int column, const Value &base, + const Value &theirs, const Value &ours ) : mColumn( column ) , mBase( base ) , mTheirs( theirs ) @@ -598,22 +598,22 @@ ConflictItem::ConflictItem( int column, const Value &base, } -Value ConflictItem::base() const +Value DataConflictItem::base() const { return mBase; } -Value ConflictItem::theirs() const +Value DataConflictItem::theirs() const { return mTheirs; } -Value ConflictItem::ours() const +Value DataConflictItem::ours() const { return mOurs; } -int ConflictItem::column() const +int DataConflictItem::column() const { return mColumn; } diff --git a/geodiff/src/geodiffutils.hpp b/geodiff/src/geodiffutils.hpp index d4250981..bd311e59 100644 --- a/geodiff/src/geodiffutils.hpp +++ b/geodiff/src/geodiffutils.hpp @@ -12,6 +12,7 @@ #include #include #include +#include #include #include @@ -20,7 +21,7 @@ #include "geodiffcontext.hpp" class Buffer; -struct ChangesetEntry; +struct ChangesetDataEntry; class GeoDiffException: public std::exception { @@ -138,7 +139,7 @@ int indexOf( const std::vector &arr, const std::string &val ); std::string concatNames( const std::vector &names ); -void get_primary_key( const ChangesetEntry &entry, int &fid, int &nColumn ); +void get_primary_key( const ChangesetDataEntry &entry, int &fid, int &nColumn ); //! Returns value of an environment variable - or returns default value if it is not set @@ -201,10 +202,10 @@ class TmpFile }; -class ConflictItem +class DataConflictItem { public: - ConflictItem( + DataConflictItem( int column, const Value &base, const Value &theirs, @@ -222,19 +223,39 @@ class ConflictItem Value mOurs; }; -class ConflictFeature +class DataConflictFeature { public: - ConflictFeature( int pk, const std::string &tableName ); + DataConflictFeature( int pk, const std::string &tableName ); bool isValid() const; - void addItem( const ConflictItem &item ); + void addItem( const DataConflictItem &item ); const std::string &tableName() const; int pk() const; - const std::vector &items() const; + const std::vector &items() const; private: int mPk; std::string mTableName; - std::vector mItems; + std::vector mItems; +}; + +//! Schema conflict: two changesets created or modified the same table with +//different definitions +struct TableSchemaConflict +{ + std::string tableName; +}; + +//! Schema conflict: two changesets added the same column with different +//definitions +struct ColumnSchemaConflict +{ + std::string tableName; + std::string columnName; +}; + +struct ConflictFeature : public std::variant +{ + using variant::variant; }; diff --git a/geodiff/src/tableschema.cpp b/geodiff/src/tableschema.cpp index e67364be..4998f782 100644 --- a/geodiff/src/tableschema.cpp +++ b/geodiff/src/tableschema.cpp @@ -8,6 +8,7 @@ #include "geodiffcontext.hpp" #include "geodifflogger.hpp" +#include "geodiffutils.hpp" #include @@ -224,6 +225,13 @@ void baseToPostgres( TableSchema &tbl ) } } +TableSchema *DatabaseSchema::tableByName( const std::string &name ) +{ + auto it = std::find_if( tables.begin(), tables.end(), + [&name]( const TableSchema & t ) { return t.name == name; } ); + return it != tables.end() ? &*it : nullptr; +} + void tableSchemaConvert( const std::string &driverDstName, TableSchema &tbl ) { if ( driverDstName == Driver::SQLITEDRIVERNAME ) diff --git a/geodiff/src/tableschema.h b/geodiff/src/tableschema.h index 3cb1410a..4292aec1 100644 --- a/geodiff/src/tableschema.h +++ b/geodiff/src/tableschema.h @@ -10,7 +10,7 @@ #include #include -#include "geodiffutils.hpp" +#include "geodiffcontext.hpp" /* Information about column type, converted to base type */ struct TableColumnType @@ -225,6 +225,15 @@ struct TableSchema } }; +/** Information about all tables in the database. */ +struct DatabaseSchema +{ + std::vector tables; + + //! Returns pointer to the table with the given name, or nullptr if not found + TableSchema *tableByName( const std::string &name ); +}; + //! Converts column name to base type and returns struct with both names TableColumnType columnType( const Context *context, const std::string &columnType, const std::string &driverName, bool isGeometry = false ); diff --git a/geodiff/src/tableschemadiff.cpp b/geodiff/src/tableschemadiff.cpp new file mode 100644 index 00000000..1572d31d --- /dev/null +++ b/geodiff/src/tableschemadiff.cpp @@ -0,0 +1,183 @@ +/* + GEODIFF - MIT License + Copyright (C) 2026 David Koňařík +*/ + +#include "tableschemadiff.hpp" +#include "changeset.h" +#include "geodiffutils.hpp" +#include "tableschema.h" +#include +#include +#include + +template +static std::vector names( const std::vector &items ) +{ + std::vector names; + names.reserve( items.size() ); + for ( const auto &item : items ) + { + names.push_back( item.name ); + } + return names; +} + +template std::vector names( const std::vector &items ); + +template +static std::unordered_map byName( const std::vector &items ) +{ + std::unordered_map map; + for ( const T &item : items ) + { + map[item.name] = &item; + } + return map; +} + +void simulateColumnChange( TableSchema &schema, const ChangesetEntry &entry ) +{ + if ( const ChangesetAddColumnEntry *acEntry = std::get_if( &entry ) ) + { + auto it = std::find_if( schema.columns.begin(), schema.columns.end(), + [&]( const TableColumnInfo & c ) { return c.name == acEntry->column.name; } ); + if ( it != schema.columns.end() ) + throw GeoDiffException( "Tried simulating addition of already-existing column " + acEntry->column.name ); + schema.columns.push_back( acEntry->column ); + } + else if ( const ChangesetDropColumnEntry *dcEntry = std::get_if( &entry ) ) + { + auto it = std::find_if( schema.columns.begin(), schema.columns.end(), + [&]( const TableColumnInfo & c ) { return c.name == dcEntry->column.name; } ); + if ( it == schema.columns.end() ) + throw GeoDiffException( "Tried simulating deletion of non-existent column " + dcEntry->column.name ); + schema.columns.erase( it ); + } +} + +void simulateSchemaChange( DatabaseSchema &schema, const ChangesetEntry &entry ) +{ + if ( const ChangesetCreateTableEntry *ctEntry = std::get_if( &entry ) ) + { + if ( schema.tableByName( ctEntry->tableName ) ) + throw GeoDiffException( "Tried simulating creation of already-existing table " + ctEntry->tableName ); + TableSchema ts; + ts.name = ctEntry->tableName; + ts.columns = ctEntry->columns; + schema.tables.push_back( ts ); + } + else if ( const ChangesetDropTableEntry *dtEntry = std::get_if( &entry ) ) + { + auto it = std::find_if( schema.tables.begin(), schema.tables.end(), + [&]( const TableSchema & t ) { return t.name == dtEntry->tableName; } ); + if ( it == schema.tables.end() ) + throw GeoDiffException( "Tried simulating deletion of non-existent table " + dtEntry->tableName ); + schema.tables.erase( it ); + } + else if ( const ChangesetAddColumnEntry *acEntry = std::get_if( &entry ) ) + { + TableSchema *table = schema.tableByName( acEntry->tableName ); + if ( !table ) + throw GeoDiffException( "Tried to add column " + acEntry->column.name + " to non-existent table " + acEntry->tableName ); + simulateColumnChange( *table, entry ); + } + else if ( const ChangesetDropColumnEntry *dcEntry = std::get_if( &entry ) ) + { + TableSchema *table = schema.tableByName( dcEntry->tableName ); + if ( !table ) + throw GeoDiffException( "Tried to delete column " + dcEntry->column.name + " from non-existent table " + dcEntry->tableName ); + simulateColumnChange( *table, entry ); + } +} + +std::vector diffTableSchema( const TableSchema &base, const TableSchema &modified ) +{ + if ( base.crs != modified.crs ) + throw GeoDiffException( "Tried to compare tables with different CRSs (named" + + base.name + " and " + modified.name + ")" ); + + std::vector entries; + + const std::unordered_map baseColumns = byName( base.columns ); + const std::unordered_map modifiedColumns = byName( modified.columns ); + std::vector baseColNames = names( base.columns ); + std::vector modifiedColNames = names( modified.columns ); + std::sort( baseColNames.begin(), baseColNames.end() ); + std::sort( modifiedColNames.begin(), modifiedColNames.end() ); + + std::vector deletedColNames; + std::set_difference( baseColNames.begin(), baseColNames.end(), + modifiedColNames.begin(), modifiedColNames.end(), + std::back_inserter( deletedColNames ) ); + for ( const std::string &colName : deletedColNames ) + { + entries.push_back( ChangesetDropColumnEntry{base.name, *baseColumns.at( colName )} ); + } + + std::vector newColNames; + std::set_difference( modifiedColNames.begin(), modifiedColNames.end(), + baseColNames.begin(), baseColNames.end(), + std::back_inserter( newColNames ) ); + for ( const std::string &colName : newColNames ) + { + entries.push_back( ChangesetAddColumnEntry{base.name, *modifiedColumns.at( colName )} ); + } + + std::vector oldColNames; + std::set_intersection( modifiedColNames.begin(), modifiedColNames.end(), + baseColNames.begin(), baseColNames.end(), + std::back_inserter( oldColNames ) ); + for ( const std::string &colName : oldColNames ) + { + // Compare column type by base type enum rather than the exact db-specific + // string to avoid regression with DB pairs that use compatible types. + if ( !baseColumns.at( colName )->compareWithBaseTypes( *modifiedColumns.at( colName ) ) ) + throw GeoDiffException( "Columns differ: " + + base.name + "." + colName + " and " + modified.name + "." + colName ); + } + + return entries; +} + +std::vector diffDatabaseSchema( const DatabaseSchema &base, const DatabaseSchema &modified ) +{ + std::vector entries; + + const std::unordered_map baseTables = byName( base.tables ); + const std::unordered_map modifiedTables = byName( modified.tables ); + std::vector baseTableNames = names( base.tables ); + std::vector modifiedTableNames = names( modified.tables ); + std::sort( baseTableNames.begin(), baseTableNames.end() ); + std::sort( modifiedTableNames.begin(), modifiedTableNames.end() ); + + std::vector deletedTableNames; + std::set_difference( baseTableNames.begin(), baseTableNames.end(), + modifiedTableNames.begin(), modifiedTableNames.end(), + std::back_inserter( deletedTableNames ) ); + for ( const std::string &name : deletedTableNames ) + { + entries.push_back( ChangesetDropTableEntry{name, baseTables.at( name )->columns} ); + } + + std::vector newTableNames; + std::set_difference( modifiedTableNames.begin(), modifiedTableNames.end(), + baseTableNames.begin(), baseTableNames.end(), + std::back_inserter( newTableNames ) ); + for ( const std::string &name : newTableNames ) + { + entries.push_back( ChangesetCreateTableEntry{name, modifiedTables.at( name )->columns} ); + } + + std::vector oldTableNames; + std::set_intersection( modifiedTableNames.begin(), modifiedTableNames.end(), + baseTableNames.begin(), baseTableNames.end(), + std::back_inserter( oldTableNames ) ); + for ( const std::string &name : oldTableNames ) + { + std::vector tableEntries = diffTableSchema( *baseTables.at( name ), *modifiedTables.at( name ) ); + entries.insert( entries.end(), tableEntries.begin(), tableEntries.end() ); + } + + return entries; +} diff --git a/geodiff/src/tableschemadiff.hpp b/geodiff/src/tableschemadiff.hpp new file mode 100644 index 00000000..d5be967a --- /dev/null +++ b/geodiff/src/tableschemadiff.hpp @@ -0,0 +1,17 @@ +/* + GEODIFF - MIT License + Copyright (C) 2026 David Koňařík +*/ + +#ifndef TABLESCHEMADIFF_H +#define TABLESCHEMADIFF_H + +#include "changeset.h" +#include "tableschema.h" + +std::vector diffTableSchema( const TableSchema &base, const TableSchema &modified ); +std::vector diffDatabaseSchema( const DatabaseSchema &base, const DatabaseSchema &modified ); +void simulateColumnChange( TableSchema &schema, const ChangesetEntry &entry ); +void simulateSchemaChange( DatabaseSchema &schema, const ChangesetEntry &entry ); + +#endif // TABLESCHEMADIFF_H diff --git a/geodiff/tests/geodiff_testutils.cpp b/geodiff/tests/geodiff_testutils.cpp index 9913e71e..edc6fa6c 100644 --- a/geodiff/tests/geodiff_testutils.cpp +++ b/geodiff/tests/geodiff_testutils.cpp @@ -17,6 +17,7 @@ #include #include +#include "changeset.h" #include "changesetreader.h" #include "changesetwriter.h" #include "geodiffutils.hpp" @@ -63,6 +64,11 @@ std::string pathjoin( const std::string &dir, const std::string &dir2, const std return res; } +std::string pathjoin( const std::string &dir, const std::string &dir2, const std::string &dir3, const std::string &filename ) +{ + return pathjoin( pathjoin( dir, dir2, dir3 ), filename ); +} + std::string testdir() { return TEST_DATA_DIR; @@ -299,15 +305,15 @@ void writeSingleTableChangeset( std::string filename, const ChangesetTable &tabl } -static bool testAllEntriesInOtherVector( const std::vector &tableEntriesA, const std::vector &tableEntriesB ) +static bool testAllEntriesInOtherVector( const std::vector &tableEntriesA, const std::vector &tableEntriesB ) { for ( size_t i = 0; i < tableEntriesA.size(); ++i ) { - const ChangesetEntry &entryI = tableEntriesA[i]; + const ChangesetDataEntry &entryI = tableEntriesA[i]; bool found = false; for ( size_t j = 0; j < tableEntriesB.size(); ++j ) { - const ChangesetEntry &entryJ = tableEntriesB[j]; + const ChangesetDataEntry &entryJ = tableEntriesB[j]; if ( entryI.op == entryJ.op && entryI.oldValues == entryJ.oldValues && entryI.newValues == entryJ.newValues ) { found = true; @@ -331,20 +337,28 @@ bool compareDiffsByContent( std::string diffA, std::string diffB ) return false; std::unordered_map > tablesA, tablesB; - std::unordered_map > entriesA, entriesB; + std::unordered_map > dataEntriesA, dataEntriesB; ChangesetEntry entryA, entryB; while ( readerA.nextEntry( entryA ) ) { - if ( tablesA.find( entryA.table->name ) == tablesA.end() ) - tablesA[entryA.table->name] = entryA.table->primaryKeys; - entriesA[entryA.table->name].push_back( entryA ); + if ( ChangesetDataEntry *dataEntryA = std::get_if( &entryA ) ) + { + if ( tablesA.find( dataEntryA->table->name ) == tablesA.end() ) + tablesA[dataEntryA->table->name] = dataEntryA->table->primaryKeys; + dataEntriesA[dataEntryA->table->name].push_back( *dataEntryA ); + } + // TODO(dvdkon): Handle other entries? } while ( readerB.nextEntry( entryB ) ) { - if ( tablesB.find( entryB.table->name ) == tablesB.end() ) - tablesB[entryB.table->name] = entryB.table->primaryKeys; - entriesB[entryB.table->name].push_back( entryB ); + if ( ChangesetDataEntry *dataEntryB = std::get_if( &entryB ) ) + { + if ( tablesB.find( dataEntryB->table->name ) == tablesB.end() ) + tablesB[dataEntryB->table->name] = dataEntryB->table->primaryKeys; + dataEntriesB[dataEntryB->table->name].push_back( *dataEntryB ); + } + // TODO(dvdkon): Handle other entries? } if ( tablesA != tablesB ) @@ -353,11 +367,11 @@ bool compareDiffsByContent( std::string diffA, std::string diffB ) for ( auto tableIt = tablesA.begin(); tableIt != tablesA.end(); ++tableIt ) { std::string tableName = tableIt->first; - if ( entriesA[tableName].size() != entriesB[tableName].size() ) + if ( dataEntriesA[tableName].size() != dataEntriesB[tableName].size() ) return false; - if ( !testAllEntriesInOtherVector( entriesA[tableName], entriesB[tableName] ) ) + if ( !testAllEntriesInOtherVector( dataEntriesA[tableName], dataEntriesB[tableName] ) ) return false; - if ( !testAllEntriesInOtherVector( entriesB[tableName], entriesA[tableName] ) ) + if ( !testAllEntriesInOtherVector( dataEntriesB[tableName], dataEntriesA[tableName] ) ) return false; } return true; diff --git a/geodiff/tests/geodiff_testutils.hpp b/geodiff/tests/geodiff_testutils.hpp index d47ef034..54d14542 100644 --- a/geodiff/tests/geodiff_testutils.hpp +++ b/geodiff/tests/geodiff_testutils.hpp @@ -10,6 +10,7 @@ #include #include +#include "changeset.h" #include "geodiff.h" #include "geodiff_config.hpp" @@ -23,6 +24,7 @@ std::string testdir(); std::string pathjoin( const std::string &dir, const std::string &filename ); std::string pathjoin( const std::string &dir, const std::string &dir2, const std::string &filename ); +std::string pathjoin( const std::string &dir, const std::string &dir2, const std::string &dir3, const std::string &filename ); void makedir( const std::string &dir ); void init_test(); @@ -53,7 +55,7 @@ bool fileContentEquals( const std::string &file1, const std::string &file2 ); bool isFileEmpty( const std::string &filepath ); struct ChangesetTable; -struct ChangesetEntry; +struct ChangesetDataEntry; //! Helper function to write a diff file for a couple of tables void writeChangeset( std::string filename, const std::unordered_map &tables, diff --git a/geodiff/tests/test_changeset_reader.cpp b/geodiff/tests/test_changeset_reader.cpp index 0e8969a2..628320fe 100644 --- a/geodiff/tests/test_changeset_reader.cpp +++ b/geodiff/tests/test_changeset_reader.cpp @@ -4,6 +4,8 @@ */ #include "gtest/gtest.h" +#include +#include "changeset.h" #include "geodiff_testutils.hpp" #include "geodiff.h" @@ -26,17 +28,19 @@ TEST( ChangesetReaderTest, test_read_insert ) ChangesetEntry entry; EXPECT_TRUE( reader.nextEntry( entry ) ); - EXPECT_EQ( entry.op, ChangesetEntry::OpInsert ); - EXPECT_EQ( entry.table->name, "simple" ); - EXPECT_EQ( entry.table->primaryKeys.size(), 4 ); - EXPECT_EQ( entry.table->primaryKeys[0], true ); - EXPECT_EQ( entry.table->primaryKeys[1], false ); - EXPECT_EQ( entry.newValues.size(), 4 ); - EXPECT_EQ( entry.newValues[0].type(), Value::TypeInt ); - EXPECT_EQ( entry.newValues[0].getInt(), 4 ); - EXPECT_EQ( entry.newValues[1].type(), Value::TypeBlob ); - EXPECT_EQ( entry.newValues[2].type(), Value::TypeText ); - EXPECT_EQ( entry.newValues[2].getString(), "my new point A" ); + EXPECT_TRUE( std::holds_alternative( entry ) ); + ChangesetDataEntry &dataEntry = std::get( entry ); + EXPECT_EQ( dataEntry.op, ChangesetDataEntry::OpInsert ); + EXPECT_EQ( dataEntry.table->name, "simple" ); + EXPECT_EQ( dataEntry.table->primaryKeys.size(), 4 ); + EXPECT_EQ( dataEntry.table->primaryKeys[0], true ); + EXPECT_EQ( dataEntry.table->primaryKeys[1], false ); + EXPECT_EQ( dataEntry.newValues.size(), 4 ); + EXPECT_EQ( dataEntry.newValues[0].type(), Value::TypeInt ); + EXPECT_EQ( dataEntry.newValues[0].getInt(), 4 ); + EXPECT_EQ( dataEntry.newValues[1].type(), Value::TypeBlob ); + EXPECT_EQ( dataEntry.newValues[2].type(), Value::TypeText ); + EXPECT_EQ( dataEntry.newValues[2].getString(), "my new point A" ); EXPECT_FALSE( reader.nextEntry( entry ) ); EXPECT_FALSE( reader.nextEntry( entry ) ); @@ -51,26 +55,29 @@ TEST( ChangesetReaderTest, test_read_update ) ChangesetEntry entry; EXPECT_TRUE( reader.nextEntry( entry ) ); - EXPECT_EQ( entry.op, ChangesetEntry::OpUpdate ); - EXPECT_EQ( entry.table->name, "simple" ); + EXPECT_TRUE( std::holds_alternative( entry ) ); + ChangesetDataEntry &dataEntry = std::get( entry ); - EXPECT_EQ( entry.oldValues.size(), 4 ); - EXPECT_EQ( entry.newValues.size(), 4 ); + EXPECT_EQ( dataEntry.op, ChangesetDataEntry::OpUpdate ); + EXPECT_EQ( dataEntry.table->name, "simple" ); + + EXPECT_EQ( dataEntry.oldValues.size(), 4 ); + EXPECT_EQ( dataEntry.newValues.size(), 4 ); // pkey - unchanged - EXPECT_EQ( entry.oldValues[0].type(), Value::TypeInt ); - EXPECT_EQ( entry.oldValues[0].getInt(), 2 ); - EXPECT_EQ( entry.newValues[0].type(), Value::TypeUndefined ); + EXPECT_EQ( dataEntry.oldValues[0].type(), Value::TypeInt ); + EXPECT_EQ( dataEntry.oldValues[0].getInt(), 2 ); + EXPECT_EQ( dataEntry.newValues[0].type(), Value::TypeUndefined ); // geometry - changed - EXPECT_EQ( entry.oldValues[1].type(), Value::TypeBlob ); - EXPECT_EQ( entry.newValues[1].type(), Value::TypeBlob ); + EXPECT_EQ( dataEntry.oldValues[1].type(), Value::TypeBlob ); + EXPECT_EQ( dataEntry.newValues[1].type(), Value::TypeBlob ); // unchanged - EXPECT_EQ( entry.oldValues[2].type(), Value::TypeUndefined ); - EXPECT_EQ( entry.newValues[2].type(), Value::TypeUndefined ); + EXPECT_EQ( dataEntry.oldValues[2].type(), Value::TypeUndefined ); + EXPECT_EQ( dataEntry.newValues[2].type(), Value::TypeUndefined ); // changed - EXPECT_EQ( entry.oldValues[3].type(), Value::TypeInt ); - EXPECT_EQ( entry.oldValues[3].getInt(), 2 ); - EXPECT_EQ( entry.newValues[3].type(), Value::TypeInt ); - EXPECT_EQ( entry.newValues[3].getInt(), 9999 ); + EXPECT_EQ( dataEntry.oldValues[3].type(), Value::TypeInt ); + EXPECT_EQ( dataEntry.oldValues[3].getInt(), 2 ); + EXPECT_EQ( dataEntry.newValues[3].type(), Value::TypeInt ); + EXPECT_EQ( dataEntry.newValues[3].getInt(), 9999 ); EXPECT_FALSE( reader.nextEntry( entry ) ); } @@ -84,17 +91,20 @@ TEST( ChangesetReaderTest, test_read_delete ) ChangesetEntry entry; EXPECT_TRUE( reader.nextEntry( entry ) ); - EXPECT_EQ( entry.op, ChangesetEntry::OpDelete ); - EXPECT_EQ( entry.table->name, "simple" ); - - EXPECT_EQ( entry.oldValues.size(), 4 ); - EXPECT_EQ( entry.oldValues[0].type(), Value::TypeInt ); - EXPECT_EQ( entry.oldValues[0].getInt(), 2 ); - EXPECT_EQ( entry.oldValues[1].type(), Value::TypeBlob ); - EXPECT_EQ( entry.oldValues[2].type(), Value::TypeText ); - EXPECT_EQ( entry.oldValues[2].getString(), "feature2" ); - EXPECT_EQ( entry.oldValues[3].type(), Value::TypeInt ); - EXPECT_EQ( entry.oldValues[3].getInt(), 2 ); + EXPECT_TRUE( std::holds_alternative( entry ) ); + ChangesetDataEntry &dataEntry = std::get( entry ); + + EXPECT_EQ( dataEntry.op, ChangesetDataEntry::OpDelete ); + EXPECT_EQ( dataEntry.table->name, "simple" ); + + EXPECT_EQ( dataEntry.oldValues.size(), 4 ); + EXPECT_EQ( dataEntry.oldValues[0].type(), Value::TypeInt ); + EXPECT_EQ( dataEntry.oldValues[0].getInt(), 2 ); + EXPECT_EQ( dataEntry.oldValues[1].type(), Value::TypeBlob ); + EXPECT_EQ( dataEntry.oldValues[2].type(), Value::TypeText ); + EXPECT_EQ( dataEntry.oldValues[2].getString(), "feature2" ); + EXPECT_EQ( dataEntry.oldValues[3].type(), Value::TypeInt ); + EXPECT_EQ( dataEntry.oldValues[3].getInt(), 2 ); EXPECT_FALSE( reader.nextEntry( entry ) ); EXPECT_FALSE( reader.nextEntry( entry ) ); diff --git a/geodiff/tests/test_changeset_utils.cpp b/geodiff/tests/test_changeset_utils.cpp index 1fd95a14..10cb21df 100644 --- a/geodiff/tests/test_changeset_utils.cpp +++ b/geodiff/tests/test_changeset_utils.cpp @@ -4,6 +4,8 @@ */ #include "gtest/gtest.h" +#include +#include "changeset.h" #include "geodiff_testutils.hpp" #include "geodiff.h" @@ -12,6 +14,7 @@ #include "changesetwriter.h" #include "geodiffutils.hpp" +#include "tableschema.h" #include "json.hpp" @@ -48,12 +51,14 @@ TEST( ChangesetUtils, test_invert_insert ) ChangesetEntry entry; EXPECT_TRUE( readerInv.nextEntry( entry ) ); - EXPECT_EQ( entry.op, ChangesetEntry::OpDelete ); - EXPECT_EQ( entry.table->name, "simple" ); - EXPECT_EQ( entry.oldValues.size(), 4 ); - EXPECT_EQ( entry.oldValues[0].getInt(), 4 ); - EXPECT_EQ( entry.oldValues[2].getString(), "my new point A" ); - EXPECT_EQ( entry.oldValues[3].getInt(), 1 ); + EXPECT_TRUE( std::holds_alternative( entry ) ); + ChangesetDataEntry &dataEntry = std::get( entry ); + EXPECT_EQ( dataEntry.op, ChangesetDataEntry::OpDelete ); + EXPECT_EQ( dataEntry.table->name, "simple" ); + EXPECT_EQ( dataEntry.oldValues.size(), 4 ); + EXPECT_EQ( dataEntry.oldValues[0].getInt(), 4 ); + EXPECT_EQ( dataEntry.oldValues[2].getString(), "my new point A" ); + EXPECT_EQ( dataEntry.oldValues[3].getInt(), 1 ); EXPECT_FALSE( readerInv.nextEntry( entry ) ); } @@ -69,12 +74,14 @@ TEST( ChangesetUtils, test_invert_delete ) ChangesetEntry entry; EXPECT_TRUE( readerInv.nextEntry( entry ) ); - EXPECT_EQ( entry.op, ChangesetEntry::OpInsert ); - EXPECT_EQ( entry.table->name, "simple" ); - EXPECT_EQ( entry.newValues.size(), 4 ); - EXPECT_EQ( entry.newValues[0].getInt(), 2 ); - EXPECT_EQ( entry.newValues[2].getString(), "feature2" ); - EXPECT_EQ( entry.newValues[3].getInt(), 2 ); + EXPECT_TRUE( std::holds_alternative( entry ) ); + ChangesetDataEntry &dataEntry = std::get( entry ); + EXPECT_EQ( dataEntry.op, ChangesetDataEntry::OpInsert ); + EXPECT_EQ( dataEntry.table->name, "simple" ); + EXPECT_EQ( dataEntry.newValues.size(), 4 ); + EXPECT_EQ( dataEntry.newValues[0].getInt(), 2 ); + EXPECT_EQ( dataEntry.newValues[2].getString(), "feature2" ); + EXPECT_EQ( dataEntry.newValues[3].getInt(), 2 ); EXPECT_FALSE( readerInv.nextEntry( entry ) ); } @@ -90,17 +97,19 @@ TEST( ChangesetUtils, test_invert_update ) ChangesetEntry entry; EXPECT_TRUE( readerInv.nextEntry( entry ) ); - EXPECT_EQ( entry.op, ChangesetEntry::OpUpdate ); - EXPECT_EQ( entry.table->name, "simple" ); - EXPECT_EQ( entry.oldValues.size(), 4 ); - EXPECT_EQ( entry.oldValues[0].type(), Value::TypeInt ); - EXPECT_EQ( entry.oldValues[0].getInt(), 2 ); - EXPECT_EQ( entry.oldValues[2].type(), Value::TypeUndefined ); - EXPECT_EQ( entry.oldValues[3].getInt(), 9999 ); - EXPECT_EQ( entry.newValues.size(), 4 ); - EXPECT_EQ( entry.newValues[0].type(), Value::TypeUndefined ); - EXPECT_EQ( entry.newValues[2].type(), Value::TypeUndefined ); - EXPECT_EQ( entry.newValues[3].getInt(), 2 ); + EXPECT_TRUE( std::holds_alternative( entry ) ); + ChangesetDataEntry &dataEntry = std::get( entry ); + EXPECT_EQ( dataEntry.op, ChangesetDataEntry::OpUpdate ); + EXPECT_EQ( dataEntry.table->name, "simple" ); + EXPECT_EQ( dataEntry.oldValues.size(), 4 ); + EXPECT_EQ( dataEntry.oldValues[0].type(), Value::TypeInt ); + EXPECT_EQ( dataEntry.oldValues[0].getInt(), 2 ); + EXPECT_EQ( dataEntry.oldValues[2].type(), Value::TypeUndefined ); + EXPECT_EQ( dataEntry.oldValues[3].getInt(), 9999 ); + EXPECT_EQ( dataEntry.newValues.size(), 4 ); + EXPECT_EQ( dataEntry.newValues[0].type(), Value::TypeUndefined ); + EXPECT_EQ( dataEntry.newValues[2].type(), Value::TypeUndefined ); + EXPECT_EQ( dataEntry.newValues[3].getInt(), 2 ); EXPECT_FALSE( readerInv.nextEntry( entry ) ); } @@ -131,6 +140,18 @@ TEST( ChangesetUtils, test_export_json ) doExportAndCompare( pathjoin( testdir(), "2_deletes", "base-deleted_A" ), pathjoin( tmpdir(), "test_export_json", "delete-diff.json" ) ); + + doExportAndCompare( pathjoin( testdir(), "modified_scheme", "changesets", "added_attribute" ), + pathjoin( tmpdir(), "test_export_json", "added_attribute.json" ) ); + + doExportAndCompare( pathjoin( testdir(), "modified_scheme", "changesets", "added_table" ), + pathjoin( tmpdir(), "test_export_json", "added_table.json" ) ); + + doExportAndCompare( pathjoin( testdir(), "modified_scheme", "changesets", "delete_attribute" ), + pathjoin( tmpdir(), "test_export_json", "delete_attribute.json" ) ); + + doExportAndCompare( pathjoin( testdir(), "modified_scheme", "changesets", "delete_table" ), + pathjoin( tmpdir(), "test_export_json", "delete_table.json" ) ); } TEST( ChangesetUtils, test_export_json_summary ) @@ -175,62 +196,62 @@ void testConcat( std::string testName, void testConcatOneTable( std::string testName, - const ChangesetTable &table, + const std::shared_ptr table, std::vector entries1, std::vector entries2, std::vector entriesExpected ) { testConcat( testName, - { std::make_pair( table.name, table ) }, - { std::make_pair( table.name, entries1 ) }, - { std::make_pair( table.name, entries2 ) }, - { std::make_pair( table.name, entriesExpected ) } ); + { std::make_pair( table->name, *table ) }, + { std::make_pair( table->name, entries1 ) }, + { std::make_pair( table->name, entries2 ) }, + { std::make_pair( table->name, entriesExpected ) } ); } TEST( ChangesetUtils, test_concat_changesets_simple_table ) { // basic table with one pkey column - ChangesetTable tableFoo; - tableFoo.name = "foo"; - tableFoo.primaryKeys.push_back( true ); // fid (pkey) - tableFoo.primaryKeys.push_back( false ); // name - tableFoo.primaryKeys.push_back( false ); // rating - - ChangesetEntry fooInsert123 = ChangesetEntry::make( - &tableFoo, ChangesetEntry::OpInsert, {}, + std::shared_ptr tableFoo = std::make_shared(); + tableFoo->name = "foo"; + tableFoo->primaryKeys.push_back( true ); // fid (pkey) + tableFoo->primaryKeys.push_back( false ); // name + tableFoo->primaryKeys.push_back( false ); // rating + + ChangesetDataEntry fooInsert123 = ChangesetDataEntry::make( + tableFoo, ChangesetDataEntry::OpInsert, {}, { Value::makeInt( 123 ), Value::makeText( "hello" ), Value::makeInt( 5 ) } ); - ChangesetEntry fooDelete123 = ChangesetEntry::make( - &tableFoo, ChangesetEntry::OpDelete, + ChangesetDataEntry fooDelete123 = ChangesetDataEntry::make( + tableFoo, ChangesetDataEntry::OpDelete, { Value::makeInt( 123 ), Value::makeText( "hello" ), Value::makeInt( 5 ) }, {} ); - ChangesetEntry fooUpdate123 = ChangesetEntry::make( - &tableFoo, ChangesetEntry::OpUpdate, + ChangesetDataEntry fooUpdate123 = ChangesetDataEntry::make( + tableFoo, ChangesetDataEntry::OpUpdate, { Value::makeInt( 123 ), Value::makeText( "hello" ), Value::makeInt( 5 ) }, { Value(), Value::makeText( "world" ), Value::makeInt( 4 ) } ); - ChangesetEntry fooDelete123_2 = ChangesetEntry::make( - &tableFoo, ChangesetEntry::OpDelete, + ChangesetDataEntry fooDelete123_2 = ChangesetDataEntry::make( + tableFoo, ChangesetDataEntry::OpDelete, { Value::makeInt( 123 ), Value::makeText( "world" ), Value::makeInt( 4 ) }, {} ); - ChangesetEntry fooUpdate123_2 = ChangesetEntry::make( - &tableFoo, ChangesetEntry::OpUpdate, + ChangesetDataEntry fooUpdate123_2 = ChangesetDataEntry::make( + tableFoo, ChangesetDataEntry::OpUpdate, { Value::makeInt( 123 ), Value(), Value::makeInt( 4 ) }, { Value(), Value(), Value::makeInt( 1 ) } ); - ChangesetEntry fooUpdate123_inverse = ChangesetEntry::make( - &tableFoo, ChangesetEntry::OpUpdate, + ChangesetDataEntry fooUpdate123_inverse = ChangesetDataEntry::make( + tableFoo, ChangesetDataEntry::OpUpdate, { Value::makeInt( 123 ), Value::makeText( "world" ), Value::makeInt( 4 ) }, { Value(), Value::makeText( "hello" ), Value::makeInt( 5 ) } ); - ChangesetEntry fooUpdate123_pkey = ChangesetEntry::make( - &tableFoo, ChangesetEntry::OpUpdate, + ChangesetDataEntry fooUpdate123_pkey = ChangesetDataEntry::make( + tableFoo, ChangesetDataEntry::OpUpdate, { Value::makeInt( 123 ), Value(), Value() }, { Value::makeInt( 124 ), Value(), Value() } ); - ChangesetEntry fooUpdate456 = ChangesetEntry::make( - &tableFoo, ChangesetEntry::OpUpdate, + ChangesetDataEntry fooUpdate456 = ChangesetDataEntry::make( + tableFoo, ChangesetDataEntry::OpUpdate, { Value::makeInt( 456 ), Value(), Value::makeInt( 1 ) }, { Value(), Value(), Value::makeInt( 2 ) } ); @@ -238,37 +259,37 @@ TEST( ChangesetUtils, test_concat_changesets_simple_table ) testConcatOneTable( "foo-insert-update", tableFoo, { fooInsert123 }, { fooUpdate123 }, { - ChangesetEntry::make( &tableFoo, ChangesetEntry::OpInsert, {}, + ChangesetDataEntry::make( tableFoo, ChangesetDataEntry::OpInsert, {}, { Value::makeInt( 123 ), Value::makeText( "world" ), Value::makeInt( 4 ) } - ) + ) } ); testConcatOneTable( "foo-insert-delete", tableFoo, { fooInsert123 }, { fooDelete123 }, {} ); testConcatOneTable( "foo-update-update", tableFoo, { fooUpdate123 }, { fooUpdate123_2 }, { - ChangesetEntry::make( &tableFoo, ChangesetEntry::OpUpdate, + ChangesetDataEntry::make( tableFoo, ChangesetDataEntry::OpUpdate, { Value::makeInt( 123 ), Value::makeText( "hello" ), Value::makeInt( 5 ) }, { Value(), Value::makeText( "world" ), Value::makeInt( 1 ) } - ) + ) } ); testConcatOneTable( "foo-update-inv-update", tableFoo, { fooUpdate123 }, { fooUpdate123_inverse }, { } ); testConcatOneTable( "foo-update-delete", tableFoo, { fooUpdate123 }, { fooDelete123_2 }, { - ChangesetEntry::make( &tableFoo, ChangesetEntry::OpDelete, + ChangesetDataEntry::make( tableFoo, ChangesetDataEntry::OpDelete, { Value::makeInt( 123 ), Value::makeText( "hello" ), Value::makeInt( 5 ) }, {} - ) + ) } ); testConcatOneTable( "foo-delete-insert", tableFoo, { fooDelete123_2 }, { fooInsert123 }, { - ChangesetEntry::make( &tableFoo, ChangesetEntry::OpUpdate, + ChangesetDataEntry::make( tableFoo, ChangesetDataEntry::OpUpdate, { Value::makeInt( 123 ), Value::makeText( "world" ), Value::makeInt( 4 ) }, { Value(), Value::makeText( "hello" ), Value::makeInt( 5 ) } - ) + ) } ); testConcatOneTable( "foo-delete-inv-insert", tableFoo, { fooDelete123 }, { fooInsert123 }, { } ); @@ -289,17 +310,17 @@ TEST( ChangesetUtils, test_concat_changesets_simple_table ) TEST( ChangesetUtils, test_concat_changesets_no_pkey_table ) { // a table with no pkey - ChangesetTable tableNoPkey; - tableNoPkey.name = "table_no_pkey"; - tableNoPkey.primaryKeys.push_back( false ); - tableNoPkey.primaryKeys.push_back( false ); + std::shared_ptr tableNoPkey = std::make_shared(); + tableNoPkey->name = "table_no_pkey"; + tableNoPkey->primaryKeys.push_back( false ); + tableNoPkey->primaryKeys.push_back( false ); - ChangesetEntry noPkeyInsert1 = ChangesetEntry::make( - &tableNoPkey, ChangesetEntry::OpInsert, {}, + ChangesetDataEntry noPkeyInsert1 = ChangesetDataEntry::make( + tableNoPkey, ChangesetDataEntry::OpInsert, {}, { Value::makeInt( 1 ), Value::makeText( "hey" ) } ); - ChangesetEntry noPkeyUpdate2 = ChangesetEntry::make( - &tableNoPkey, ChangesetEntry::OpUpdate, + ChangesetDataEntry noPkeyUpdate2 = ChangesetDataEntry::make( + tableNoPkey, ChangesetDataEntry::OpUpdate, { Value::makeInt( 2 ), Value::makeText( "huh" ) }, { Value(), Value::makeText( "ho!" ) } ); @@ -310,32 +331,32 @@ TEST( ChangesetUtils, test_concat_changesets_no_pkey_table ) TEST( ChangesetUtils, test_concat_changesets_multiple_tables ) { - ChangesetTable tableFoo; - tableFoo.name = "foo"; - tableFoo.primaryKeys.push_back( true ); // fid (pkey) - tableFoo.primaryKeys.push_back( false ); // name - tableFoo.primaryKeys.push_back( false ); // rating - - ChangesetTable tableBar; - tableBar.name = "bar"; - tableBar.primaryKeys.push_back( true ); // fid (pkey) - tableBar.primaryKeys.push_back( false ); // name - - ChangesetEntry fooInsert123 = ChangesetEntry::make( - &tableFoo, ChangesetEntry::OpInsert, {}, + std::shared_ptr tableFoo = std::make_shared(); + tableFoo->name = "foo"; + tableFoo->primaryKeys.push_back( true ); // fid (pkey) + tableFoo->primaryKeys.push_back( false ); // name + tableFoo->primaryKeys.push_back( false ); // rating + + std::shared_ptr tableBar = std::make_shared(); + tableBar->name = "bar"; + tableBar->primaryKeys.push_back( true ); // fid (pkey) + tableBar->primaryKeys.push_back( false ); // name + + ChangesetDataEntry fooInsert123 = ChangesetDataEntry::make( + tableFoo, ChangesetDataEntry::OpInsert, {}, { Value::makeInt( 123 ), Value::makeText( "hello" ), Value::makeInt( 5 ) } ); - ChangesetEntry barInsert123 = ChangesetEntry::make( - &tableBar, ChangesetEntry::OpInsert, {}, + ChangesetDataEntry barInsert123 = ChangesetDataEntry::make( + tableBar, ChangesetDataEntry::OpInsert, {}, { Value::makeInt( 123 ), Value::makeText( "ha!" ) } ); - ChangesetEntry barUpdate123 = ChangesetEntry::make( - &tableFoo, ChangesetEntry::OpUpdate, + ChangesetDataEntry barUpdate123 = ChangesetDataEntry::make( + tableFoo, ChangesetDataEntry::OpUpdate, { Value::makeInt( 123 ), Value::makeText( "ha!" ) }, { Value(), Value::makeText( ":-)" ) } ); testConcat( "multi-related-insert-update", - { std::make_pair( "foo", tableFoo ), std::make_pair( "bar", tableBar ) }, + { std::make_pair( "foo", *tableFoo ), std::make_pair( "bar", *tableBar ) }, // changeset 1 { std::make_pair( "foo", std::vector( { fooInsert123 } ) ), @@ -346,17 +367,15 @@ TEST( ChangesetUtils, test_concat_changesets_multiple_tables ) // expected result { std::make_pair( "foo", std::vector( { - ChangesetEntry::make( &tableFoo, ChangesetEntry::OpInsert, {}, - { Value::makeInt( 123 ), Value::makeText( "hello" ), Value::makeInt( 5 ) } - ) } ) ), + ChangesetDataEntry::make( tableFoo, ChangesetDataEntry::OpInsert, {}, + { Value::makeInt( 123 ), Value::makeText( "hello" ), Value::makeInt( 5 ) } ) } ) ), std::make_pair( "bar", std::vector( { - ChangesetEntry::make( &tableBar, ChangesetEntry::OpInsert, {}, - { Value::makeInt( 123 ), Value::makeText( ":-)" ) } - ) } ) ) + ChangesetDataEntry::make( tableBar, ChangesetDataEntry::OpInsert, {}, + { Value::makeInt( 123 ), Value::makeText( ":-)" ) } ) } ) ) } ); testConcat( "multi-unrelated-insert-update", - { std::make_pair( "foo", tableFoo ), std::make_pair( "bar", tableBar ) }, + { std::make_pair( "foo", *tableFoo ), std::make_pair( "bar", *tableBar ) }, // changeset 1 { std::make_pair( "foo", std::vector( { fooInsert123 } ) ) }, // changeset 2 @@ -364,14 +383,12 @@ TEST( ChangesetUtils, test_concat_changesets_multiple_tables ) // expected result { std::make_pair( "foo", std::vector( { - ChangesetEntry::make( &tableFoo, ChangesetEntry::OpInsert, {}, - { Value::makeInt( 123 ), Value::makeText( "hello" ), Value::makeInt( 5 ) } - ) } ) ), + ChangesetDataEntry::make( tableFoo, ChangesetDataEntry::OpInsert, {}, + { Value::makeInt( 123 ), Value::makeText( "hello" ), Value::makeInt( 5 ) } ) } ) ), std::make_pair( "bar", std::vector( { - ChangesetEntry::make( &tableBar, ChangesetEntry::OpUpdate, + ChangesetDataEntry::make( tableBar, ChangesetDataEntry::OpUpdate, { Value::makeInt( 123 ), Value::makeText( "ha!" ) }, - { Value(), Value::makeText( ":-)" ) } - ) } ) ) + { Value(), Value::makeText( ":-)" ) } ) } ) ) } ); } diff --git a/geodiff/tests/test_driver_postgres.cpp b/geodiff/tests/test_driver_postgres.cpp index a5aa1f9e..9a464c22 100644 --- a/geodiff/tests/test_driver_postgres.cpp +++ b/geodiff/tests/test_driver_postgres.cpp @@ -1100,6 +1100,24 @@ TEST( PostgresDriverTest, test_timestamp_miliseconds ) PQfinish( c ); } +TEST( PostgresDriverTest, execute_sql ) +{ + std::string conninfo = pgTestConnInfo(); + execSqlCommands( conninfo, pathjoin( testdir(), "postgres", "base.sql" ) ); + + DriverParametersMap params; + params["conninfo"] = conninfo; + params["base"] = "gd_base"; + + std::unique_ptr driver( Driver::createDriver( static_cast( testContext() ), "postgres" ) ); + ASSERT_TRUE( driver ); + driver->open( params ); + + std::vector> result = driver->executeSql( "SELECT fid, name FROM simple LIMIT 2" ); + std::vector> expected = {{"1", "feature1"}, {"2", "feature2"}}; + ASSERT_EQ( result, expected ); +} + int main( int argc, char **argv ) { diff --git a/geodiff/tests/test_driver_sqlite.cpp b/geodiff/tests/test_driver_sqlite.cpp index 330cacb8..1cb096d6 100644 --- a/geodiff/tests/test_driver_sqlite.cpp +++ b/geodiff/tests/test_driver_sqlite.cpp @@ -501,6 +501,17 @@ TEST( SqliteDriverTest, make_copy_sqlite_concurrent ) ASSERT_EQ( sqlite3_column_int( stmtSafe.get(), 0 ), 2 ); } +TEST( SqliteDriverTest, execute_sql ) +{ + std::string fileBase = pathjoin( testdir(), "base.gpkg" ); + std::unique_ptr driver( Driver::createDriver( static_cast( testContext() ), "sqlite" ) ); + driver->open( Driver::sqliteParametersSingleSource( fileBase ) ); + + std::vector> result = driver->executeSql( "SELECT fid, name FROM simple LIMIT 2" ); + std::vector> expected = {{"1", "feature1"}, {"2", "feature2"}}; + ASSERT_EQ( result, expected ); +} + int main( int argc, char **argv ) { diff --git a/geodiff/tests/test_geometry_utils.cpp b/geodiff/tests/test_geometry_utils.cpp index 892e3a52..78239d03 100644 --- a/geodiff/tests/test_geometry_utils.cpp +++ b/geodiff/tests/test_geometry_utils.cpp @@ -4,6 +4,8 @@ */ #include "gtest/gtest.h" +#include +#include "changeset.h" #include "geodiff_testutils.hpp" #include "geodiff.h" @@ -20,12 +22,17 @@ TEST( GeometryUtilsTest, test_wkb_from_geometry ) ChangesetEntry entry; EXPECT_TRUE( reader.nextEntry( entry ) ); - EXPECT_EQ( entry.table->name, "gpkg_contents" ); + EXPECT_TRUE( std::holds_alternative( entry ) ); + ChangesetDataEntry &dataEntry = std::get( entry ); + EXPECT_EQ( dataEntry.table->name, "gpkg_contents" ); EXPECT_TRUE( reader.nextEntry( entry ) ); - EXPECT_EQ( entry.table->name, "simple" ); - EXPECT_EQ( entry.oldValues[1].type(), Value::TypeBlob ); - std::string gpkgWkb = entry.oldValues[1].getString(); + EXPECT_TRUE( std::holds_alternative( entry ) ); + dataEntry = std::get( entry ); + + EXPECT_EQ( dataEntry.table->name, "simple" ); + EXPECT_EQ( dataEntry.oldValues[1].type(), Value::TypeBlob ); + std::string gpkgWkb = dataEntry.oldValues[1].getString(); const char *c_gpkgWkb = gpkgWkb.c_str(); size_t length = gpkgWkb.length(); diff --git a/geodiff/tests/test_modified_scheme.cpp b/geodiff/tests/test_modified_scheme.cpp index e857e0d6..88049b48 100644 --- a/geodiff/tests/test_modified_scheme.cpp +++ b/geodiff/tests/test_modified_scheme.cpp @@ -3,10 +3,20 @@ Copyright (C) 2019 Peter Petrik */ +#include +#include + #include "gtest/gtest.h" -#include "geodiff_testutils.hpp" + +#include "changesetreader.h" +#include "changesetutils.h" +#include "changesetwriter.h" +#include "driver.h" #include "geodiff.h" +#include "geodiff_testutils.hpp" +#include "geodiffrebase.hpp" #include "geodiffutils.hpp" +#include "tableschema.h" TEST( ModifiedSchemeSqlite3Test, add_attribute ) { @@ -16,35 +26,41 @@ TEST( ModifiedSchemeSqlite3Test, add_attribute ) std::string base = pathjoin( testdir(), "base.gpkg" ); std::string modified = pathjoin( testdir(), "modified_scheme", "added_attribute.gpkg" ); - std::string changeset = pathjoin( tmpdir(), testname, "changeset.bin" ); + std::string changeset = pathjoin( tmpdir(), testname, "changeset.diff" ); + std::string expected = pathjoin( testdir(), "modified_scheme", "changesets", "added_attribute.diff" ); - ASSERT_EQ( GEODIFF_createChangeset( testContext(), base.c_str(), modified.c_str(), changeset.c_str() ), GEODIFF_ERROR ); + ASSERT_EQ( GEODIFF_createChangeset( testContext(), base.c_str(), modified.c_str(), changeset.c_str() ), GEODIFF_SUCCESS ); + EXPECT_TRUE( fileContentEquals( changeset, expected ) ); } TEST( ModifiedSchemeSqlite3Test, add_table ) { - std::cout << "geopackage add table to table" << std::endl; + std::cout << "geopackage add table to database" << std::endl; std::string testname = "add_table"; makedir( pathjoin( tmpdir(), testname ) ); std::string base = pathjoin( testdir(), "base.gpkg" ); std::string modified = pathjoin( testdir(), "modified_scheme", "added_table.gpkg" ); - std::string changeset = pathjoin( tmpdir(), testname, "changeset.bin" ); + std::string changeset = pathjoin( tmpdir(), testname, "changeset.diff" ); + std::string expected = pathjoin( testdir(), "modified_scheme", "changesets", "added_table.diff" ); - ASSERT_EQ( GEODIFF_createChangeset( testContext(), base.c_str(), modified.c_str(), changeset.c_str() ), GEODIFF_ERROR ); + ASSERT_EQ( GEODIFF_createChangeset( testContext(), base.c_str(), modified.c_str(), changeset.c_str() ), GEODIFF_SUCCESS ); + EXPECT_TRUE( fileContentEquals( changeset, expected ) ); } TEST( ModifiedSchemeSqlite3Test, delete_attribute ) { - std::cout << "geopackage add attribute to table" << std::endl; + std::cout << "geopackage delete attribute from table" << std::endl; std::string testname = "delete_attribute"; makedir( pathjoin( tmpdir(), testname ) ); std::string base = pathjoin( testdir(), "modified_scheme", "added_attribute.gpkg" ); std::string modified = pathjoin( testdir(), "base.gpkg" ); - std::string changeset = pathjoin( tmpdir(), testname, "changeset.bin" ); + std::string changeset = pathjoin( tmpdir(), testname, "changeset.diff" ); + std::string expected = pathjoin( testdir(), "modified_scheme", "changesets", "delete_attribute.diff" ); - ASSERT_EQ( GEODIFF_createChangeset( testContext(), base.c_str(), modified.c_str(), changeset.c_str() ), GEODIFF_ERROR ); + ASSERT_EQ( GEODIFF_createChangeset( testContext(), base.c_str(), modified.c_str(), changeset.c_str() ), GEODIFF_SUCCESS ); + EXPECT_TRUE( fileContentEquals( changeset, expected ) ); } TEST( ModifiedSchemeSqlite3Test, delete_table ) @@ -55,22 +71,26 @@ TEST( ModifiedSchemeSqlite3Test, delete_table ) std::string base = pathjoin( testdir(), "modified_scheme", "added_table.gpkg" ); std::string modified = pathjoin( testdir(), "base.gpkg" ); - std::string changeset = pathjoin( tmpdir(), testname, "changeset.bin" ); + std::string changeset = pathjoin( tmpdir(), testname, "changeset.diff" ); + std::string expected = pathjoin( testdir(), "modified_scheme", "changesets", "delete_table.diff" ); - ASSERT_EQ( GEODIFF_createChangeset( testContext(), base.c_str(), modified.c_str(), changeset.c_str() ), GEODIFF_ERROR ); + ASSERT_EQ( GEODIFF_createChangeset( testContext(), base.c_str(), modified.c_str(), changeset.c_str() ), GEODIFF_SUCCESS ); + EXPECT_TRUE( fileContentEquals( changeset, expected ) ); } TEST( ModifiedSchemeSqlite3Test, rename_table ) { std::cout << "geopackage table count is same, but tables have different name" << std::endl; - std::string testname = "delete_table"; + std::string testname = "rename_table"; makedir( pathjoin( tmpdir(), testname ) ); std::string base = pathjoin( testdir(), "modified_scheme", "added_table.gpkg" ); std::string modified = pathjoin( testdir(), "modified_scheme", "added_table2.gpkg" ); - std::string changeset = pathjoin( tmpdir(), testname, "changeset.bin" ); + std::string changeset = pathjoin( tmpdir(), testname, "changeset.diff" ); + std::string expected = pathjoin( testdir(), "modified_scheme", "changesets", "rename_table.diff" ); - ASSERT_EQ( GEODIFF_createChangeset( testContext(), base.c_str(), modified.c_str(), changeset.c_str() ), GEODIFF_ERROR ); + ASSERT_EQ( GEODIFF_createChangeset( testContext(), base.c_str(), modified.c_str(), changeset.c_str() ), GEODIFF_SUCCESS ); + EXPECT_TRUE( fileContentEquals( changeset, expected ) ); } TEST( ModifiedSchemeSqlite3Test, rename_attribute ) @@ -81,9 +101,733 @@ TEST( ModifiedSchemeSqlite3Test, rename_attribute ) std::string base = pathjoin( testdir(), "modified_scheme", "added_attribute.gpkg" ); std::string modified = pathjoin( testdir(), "modified_scheme", "added_attribute2.gpkg" ); - std::string changeset = pathjoin( tmpdir(), testname, "changeset.bin" ); + std::string changeset = pathjoin( tmpdir(), testname, "changeset.diff" ); + std::string expected = pathjoin( testdir(), "modified_scheme", "changesets", "rename_attribute.diff" ); + + ASSERT_EQ( GEODIFF_createChangeset( testContext(), base.c_str(), modified.c_str(), changeset.c_str() ), GEODIFF_SUCCESS ); + EXPECT_TRUE( fileContentEquals( changeset, expected ) ); +} + +// Create driver and fill DB with sample data +static std::unique_ptr createSampleDb( std::string driverName, std::string testname, std::string dbname ) +{ + DriverParametersMap params; + if ( driverName == "sqlite" ) + { + std::string dir = pathjoin( tmpdir(), testname ); + makedir( dir ); + params["base"] = pathjoin( dir, dbname + ".gpkg" ); + } +#ifdef HAVE_POSTGRES + else if ( driverName == "postgres" ) + { + params["base"] = testname + "_" + dbname; + params["conninfo"] = pgTestConnInfo(); + } +#endif + + std::unique_ptr driver( Driver::createDriver( static_cast( testContext() ), driverName ) ); + driver->create( params, true ); + + TableColumnInfo fidCol; + fidCol.name = "fid"; + fidCol.type = columnType( static_cast( testContext() ), "integer", driverName ); + fidCol.isPrimaryKey = true; + fidCol.isAutoIncrement = true; + + TableColumnInfo geometryCol; + geometryCol.name = "geometry"; + geometryCol.type = columnType( static_cast( testContext() ), "point", driverName, true ); + geometryCol.isGeometry = true; + + TableColumnInfo nameCol; + nameCol.name = "name"; + nameCol.type = columnType( static_cast( testContext() ), "text", driverName ); + + driver->createTables( + { + {"tram_stops", { fidCol, geometryCol, nameCol } }, + } ); + + driver->executeSql( "INSERT INTO tram_stops(fid, name) VALUES " + "(1, 'Ohrada'), " + "(2, 'Petřiny'), " + "(3, 'Park Maxe van der Stoela')" ); + + return driver; +} + +// Open driver with both base & modified as created by createBaseDb above +static std::unique_ptr openBaseModifiedDb( std::string driverName, std::string testname, std::string baseName, std::string modifiedName ) +{ + DriverParametersMap params; + if ( driverName == "sqlite" ) + { + std::string dir = pathjoin( tmpdir(), testname ); + makedir( dir ); + params["base"] = pathjoin( dir, baseName + ".gpkg" ); + if ( modifiedName.size() ) + params["modified"] = pathjoin( dir, modifiedName + ".gpkg" ); + } +#ifdef HAVE_POSTGRES + else if ( driverName == "postgres" ) + { + params["base"] = testname + "_" + baseName; + if ( modifiedName.size() ) + params["modified"] = testname + "_" + modifiedName; + params["conninfo"] = pgTestConnInfo(); + } +#endif + + std::unique_ptr driver( Driver::createDriver( static_cast( testContext() ), driverName ) ); + driver->open( params ); + return driver; +} + +static void testSchemaDiffWith( std::string driverName, std::string testname, std::function modification ) +{ + // Create base and modified DB + { + std::unique_ptr baseDb = createSampleDb( driverName, testname, "base" ); + std::unique_ptr modifiedDb = createSampleDb( driverName, testname, "modified" ); + modification( *modifiedDb ); + } + + // Create diff base->modified + std::string diffPath = pathjoin( tmpdir(), testname, "diff" ); + { + std::unique_ptr baseModifiedDriver = openBaseModifiedDb( driverName, testname, "base", "modified" ); + ChangesetWriter writer; + writer.open( diffPath ); + baseModifiedDriver->createChangeset( writer ); + } + + // Apply diff to base + { + ChangesetReader reader; + reader.open( diffPath ); + std::unique_ptr baseDb = openBaseModifiedDb( driverName, testname, "base", "" ); + baseDb->applyChangeset( reader ); + } + + // Check that base and modified are now equal + std::string diff2Path = pathjoin( tmpdir(), testname, "diff2" ); + { + std::unique_ptr baseModifiedDriver = openBaseModifiedDb( driverName, testname, "base", "modified" ); + ChangesetWriter writer; + writer.open( diff2Path ); + baseModifiedDriver->createChangeset( writer ); + } + uintmax_t diff2Size = std::filesystem::file_size( diff2Path ); + ASSERT_EQ( diff2Size, 0 ); + + // Invert diff + std::string invertedDiffPath = pathjoin( tmpdir(), testname, "diff-inv" ); + { + ChangesetReader reader; + reader.open( diffPath ); + ChangesetWriter writer; + writer.open( invertedDiffPath ); + invertChangeset( reader, writer ); + } + + // Apply inverted diff to base + { + ChangesetReader reader; + reader.open( invertedDiffPath ); + std::unique_ptr baseDb = openBaseModifiedDb( driverName, testname, "base", "" ); + baseDb->applyChangeset( reader ); + } + + // Check that base and original base are now equal + std::string diff3Path = pathjoin( tmpdir(), testname, "diff2" ); + { + createSampleDb( driverName, testname, "base2" ); + std::unique_ptr baseModifiedDriver = openBaseModifiedDb( driverName, testname, "base2", "base" ); + ChangesetWriter writer; + writer.open( diff3Path ); + baseModifiedDriver->createChangeset( writer ); + } + uintmax_t diff3Size = std::filesystem::file_size( diff3Path ); + ASSERT_EQ( diff3Size, 0 ); +} + +TEST( ModifiedSchemeTest, create_table ) +{ + // TODO: Postgres support + std::string driverName = "sqlite"; + + testSchemaDiffWith( driverName, "create_table", [ = ]( Driver & modifiedDb ) + { + TableColumnInfo fidCol; + fidCol.name = "fid"; + fidCol.type = columnType( static_cast( testContext() ), "integer", driverName ); + fidCol.isPrimaryKey = true; + fidCol.isAutoIncrement = true; + + TableColumnInfo geometryCol; + geometryCol.name = "geometry"; + geometryCol.type = columnType( static_cast( testContext() ), "point", driverName, true ); + geometryCol.isGeometry = true; + + TableColumnInfo materialCol; + materialCol.name = "material"; + materialCol.type = columnType( static_cast( testContext() ), "text", driverName ); + + TableSchema benchesTable{"benches", {fidCol, geometryCol, materialCol}}; + + modifiedDb.createTables( {benchesTable} ); + modifiedDb.executeSql( "INSERT INTO benches (fid, material) VALUES (1, 'wood'), (2, 'steel')" ); + } ); +} + +TEST( ModifiedSchemeTest, add_column ) +{ + // TODO: Postgres support + std::string driverName = "sqlite"; + + testSchemaDiffWith( driverName, "add_column", [ = ]( Driver & modifiedDb ) + { + modifiedDb.executeSql( "INSERT INTO tram_stops (fid, name) VALUES (4, 'Palmovka')" ); + modifiedDb.executeSql( "UPDATE tram_stops SET name = 'Pohořelec' WHERE fid = 1" ); + modifiedDb.executeSql( "ALTER TABLE tram_stops ADD COLUMN bench_count integer" ); + modifiedDb.executeSql( "UPDATE tram_stops SET bench_count = 1 WHERE fid = 1" ); + modifiedDb.executeSql( "UPDATE tram_stops SET bench_count = 4 WHERE fid = 2" ); + } ); +} + +TEST( ModifiedSchemeTest, drop_column ) +{ + // TODO: Postgres support + std::string driverName = "sqlite"; + + testSchemaDiffWith( driverName, "drop_column", [ = ]( Driver & modifiedDb ) + { + modifiedDb.executeSql( "INSERT INTO tram_stops (fid, name) VALUES (4, 'Palmovka')" ); + modifiedDb.executeSql( "UPDATE tram_stops SET name = 'Pohořelec' WHERE fid = 1" ); + modifiedDb.executeSql( "ALTER TABLE tram_stops DROP COLUMN name" ); + modifiedDb.executeSql( "INSERT INTO tram_stops (fid) VALUES (5)" ); + } ); +} + +TEST( ModifiedSchemeTest, drop_table ) +{ + // TODO: Postgres support + std::string driverName = "sqlite"; + + testSchemaDiffWith( driverName, "drop_table", [ = ]( Driver & modifiedDb ) + { + modifiedDb.executeSql( "INSERT INTO tram_stops (fid, name) VALUES (4, 'Palmovka')" ); + modifiedDb.executeSql( "UPDATE tram_stops SET name = 'Pohořelec' WHERE fid = 1" ); + modifiedDb.executeSql( "DROP TABLE tram_stops" ); + } ); +} + +static void testSchemaDiffRebaseWith( std::string driverName, std::string testname, int expectedConflicts, std::function theirs, std::function ours, std::function expected ) +{ + // Create base and modified Dbs + DatabaseSchema baseSchema; + { + std::unique_ptr baseDb = createSampleDb( driverName, testname, "base" ); + for ( const std::string &tableName : baseDb->listTables() ) + baseSchema.tables.push_back( baseDb->tableSchema( tableName ) ); + std::unique_ptr theirsDb = createSampleDb( driverName, testname, "theirs" ); + theirs( *theirsDb ); + std::unique_ptr oursDb = createSampleDb( driverName, testname, "ours" ); + ours( *oursDb ); + } + + // Create diff base->theirs + std::string base2TheirsPath = pathjoin( tmpdir(), testname, "base2theirs" ); + { + std::unique_ptr baseTheirsDriver = openBaseModifiedDb( driverName, testname, "base", "theirs" ); + ChangesetWriter writer; + writer.open( base2TheirsPath ); + baseTheirsDriver->createChangeset( writer ); + } + + // Create diff base->ours + std::string base2OursPath = pathjoin( tmpdir(), testname, "base2ours" ); + { + std::unique_ptr baseOursDriver = openBaseModifiedDb( driverName, testname, "base", "ours" ); + ChangesetWriter writer; + writer.open( base2OursPath ); + baseOursDriver->createChangeset( writer ); + } + + // Rebase base->ours diff to theirs->both + std::string theirs2bothPath = pathjoin( tmpdir(), testname, "theirs2both" ); + { + std::vector conflicts; + rebase( static_cast( testContext() ), baseSchema, base2TheirsPath, theirs2bothPath, base2OursPath, conflicts ); + ASSERT_EQ( static_cast( conflicts.size() ), expectedConflicts ) << conflicts.size() << " conflicts in rebase (expected " << expectedConflicts << "): " << conflictsToJSON( conflicts ).dump( 2 ); + } + + if ( expectedConflicts > 0 ) + return; + + // Apply both diffs to both + { + std::unique_ptr bothDb = createSampleDb( driverName, testname, "both" ); + + { + ChangesetReader reader; + reader.open( base2TheirsPath ); + bothDb->applyChangeset( reader ); + } + + { + ChangesetReader reader; + reader.open( theirs2bothPath ); + bothDb->applyChangeset( reader ); + } + } + + // Check that base and both are now equal + std::string expected2bothPath = pathjoin( tmpdir(), testname, "expected2both" ); + { + std::unique_ptr expectedDb = createSampleDb( driverName, testname, "expected" ); + expected( *expectedDb ); + + std::unique_ptr expectedBothDriver = openBaseModifiedDb( driverName, testname, "expected", "both" ); + ChangesetWriter writer; + writer.open( expected2bothPath ); + expectedBothDriver->createChangeset( writer ); + } + uintmax_t expected2bothSize = std::filesystem::file_size( expected2bothPath ); + ASSERT_EQ( expected2bothSize, 0 ); +} + +TEST( ModifiedSchemeTest, rebase_redundant_drop_column ) +{ + // TODO: Postgres support + std::string driverName = "sqlite"; + + testSchemaDiffRebaseWith( driverName, "rebase_redundant_drop_column", 0, + [ = ]( Driver & db ) + { + db.executeSql( "ALTER TABLE tram_stops DROP COLUMN name" ); + db.executeSql( "INSERT INTO tram_stops (fid) VALUES (4)" ); + }, + [ = ]( Driver & db ) + { + db.executeSql( "ALTER TABLE tram_stops DROP COLUMN name" ); + db.executeSql( "INSERT INTO tram_stops (fid) VALUES (4)" ); + }, + [ = ]( Driver & db ) + { + db.executeSql( "ALTER TABLE tram_stops DROP COLUMN name" ); + db.executeSql( "INSERT INTO tram_stops (fid) VALUES (4)" ); + db.executeSql( "INSERT INTO tram_stops (fid) VALUES (5)" ); + } ); +} + +TEST( ModifiedSchemeTest, rebase_redundant_drop_table ) +{ + // TODO: Postgres support + std::string driverName = "sqlite"; + + testSchemaDiffRebaseWith( driverName, "rebase_redundant_drop_table", 0, + [ = ]( Driver & db ) + { + db.executeSql( "DROP TABLE tram_stops" ); + }, + [ = ]( Driver & db ) + { + db.executeSql( "DROP TABLE tram_stops" ); + }, + [ = ]( Driver & db ) + { + db.executeSql( "DROP TABLE tram_stops" ); + } ); +} + +TEST( ModifiedSchemeTest, rebase_redundant_add_column ) +{ + // TODO: Postgres support + std::string driverName = "sqlite"; + + testSchemaDiffRebaseWith( driverName, "rebase_redundant_add_column", 0, + [ = ]( Driver & db ) + { + db.executeSql( "ALTER TABLE tram_stops ADD COLUMN bench_count integer" ); + db.executeSql( "INSERT INTO tram_stops (fid, name, bench_count) VALUES (4, 'Palmovka', 3)" ); + db.executeSql( "UPDATE tram_stops SET bench_count = 1 WHERE fid = 1" ); + }, + [ = ]( Driver & db ) + { + db.executeSql( "ALTER TABLE tram_stops ADD COLUMN bench_count integer" ); + db.executeSql( "UPDATE tram_stops SET bench_count = 1 WHERE fid = 1" ); + db.executeSql( "INSERT INTO tram_stops (fid, name, bench_count) VALUES (5, 'Drinopol', 2)" ); + }, + [ = ]( Driver & db ) + { + db.executeSql( "ALTER TABLE tram_stops ADD COLUMN bench_count integer" ); + db.executeSql( "INSERT INTO tram_stops (fid, name, bench_count) VALUES (4, 'Palmovka', 3)" ); + db.executeSql( "INSERT INTO tram_stops (fid, name, bench_count) VALUES (5, 'Drinopol', 2)" ); + db.executeSql( "UPDATE tram_stops SET bench_count = 1 WHERE fid = 1" ); + } ); +} + +TEST( ModifiedSchemeTest, rebase_redundant_create_table ) +{ + // TODO: Postgres support + std::string driverName = "sqlite"; + + testSchemaDiffRebaseWith( driverName, "rebase_redundant_create_table", 0, + [ = ]( Driver & db ) + { + db.executeSql( "CREATE TABLE vehicles (fid INTEGER, name TEXT, type TEXT)" ); + db.executeSql( "INSERT INTO vehicles VALUES (1, 'T3', 'tram')" ); + db.executeSql( "INSERT INTO vehicles VALUES (2, 'KT8D5', 'tram')" ); + }, + [ = ]( Driver & db ) + { + db.executeSql( "CREATE TABLE vehicles (fid INTEGER, name TEXT, type TEXT)" ); + db.executeSql( "INSERT INTO vehicles VALUES (1, 'T3', 'tram')" ); + db.executeSql( "INSERT INTO vehicles VALUES (2, '14T', 'tram')" ); + }, + [ = ]( Driver & db ) + { + db.executeSql( "CREATE TABLE vehicles (fid INTEGER, name TEXT, type TEXT)" ); + db.executeSql( "INSERT INTO vehicles VALUES (1, 'T3', 'tram')" ); + db.executeSql( "INSERT INTO vehicles VALUES (2, 'KT8D5', 'tram')" ); + db.executeSql( "INSERT INTO vehicles VALUES (3, '14T', 'tram')" ); + } ); +} + +TEST( ModifiedSchemeTest, rebase_conflict_create_table ) +{ + // TODO: Postgres support + std::string driverName = "sqlite"; + + testSchemaDiffRebaseWith( driverName, "rebase_conflict_create_table", 1, + [ = ]( Driver & db ) + { + db.executeSql( "CREATE TABLE vehicles (fid INTEGER, name TEXT, type TEXT)" ); + }, + [ = ]( Driver & db ) + { + db.executeSql( "CREATE TABLE vehicles (fid INTEGER, name TEXT, manufacturer TEXT)" ); + }, + [ = ]( Driver & db ) { } ); +} + +TEST( ModifiedSchemeTest, rebase_conflict_add_column ) +{ + // TODO: Postgres support + std::string driverName = "sqlite"; + + testSchemaDiffRebaseWith( driverName, "rebase_conflict_add_column", 1, + [ = ]( Driver & db ) + { + db.executeSql( "ALTER TABLE tram_stops ADD COLUMN bench_count INTEGER" ); + }, + [ = ]( Driver & db ) + { + db.executeSql( "ALTER TABLE tram_stops ADD COLUMN bench_count TEXT" ); + db.executeSql( "INSERT INTO tram_stops (fid, name, bench_count) VALUES (4, 'Palmovka', 'three')" ); + }, + [ = ]( Driver & db ) + { + db.executeSql( "INSERT INTO tram_stops (fid, name) VALUES (4, 'Palmovka')" ); + } ); +} + +TEST( ModifiedSchemeTest, rebase_add_column ) +{ + // TODO: Postgres support + std::string driverName = "sqlite"; + + testSchemaDiffRebaseWith( driverName, "rebase_add_column", 0, + [ = ]( Driver & db ) + { + db.executeSql( "ALTER TABLE tram_stops ADD COLUMN bench_count integer" ); + db.executeSql( "INSERT INTO tram_stops (fid, name, bench_count) VALUES (4, 'Palmovka', 3)" ); + db.executeSql( "UPDATE tram_stops SET bench_count = 1 WHERE fid = 1" ); + }, + [ = ]( Driver & db ) + { + db.executeSql( "INSERT INTO tram_stops (fid, name) VALUES (4, 'Drinopol')" ); + }, + [ = ]( Driver & db ) + { + db.executeSql( "ALTER TABLE tram_stops ADD COLUMN bench_count integer" ); + db.executeSql( "INSERT INTO tram_stops (fid, name, bench_count) VALUES (4, 'Palmovka', 3)" ); + db.executeSql( "INSERT INTO tram_stops (fid, name, bench_count) VALUES (5, 'Drinopol', NULL)" ); + db.executeSql( "UPDATE tram_stops SET bench_count = 1 WHERE fid = 1" ); + } ); +} + +TEST( ModifiedSchemeTest, rebase_add_empty_column ) +{ + // TODO: Postgres support + std::string driverName = "sqlite"; + + testSchemaDiffRebaseWith( driverName, "rebase_add_empty_column", 0, + [ = ]( Driver & db ) + { + db.executeSql( "ALTER TABLE tram_stops ADD COLUMN bench_count integer" ); + }, + [ = ]( Driver & db ) + { + db.executeSql( "INSERT INTO tram_stops (fid, name) VALUES (4, 'Drinopol')" ); + }, + [ = ]( Driver & db ) + { + db.executeSql( "ALTER TABLE tram_stops ADD COLUMN bench_count integer" ); + db.executeSql( "INSERT INTO tram_stops (fid, name, bench_count) VALUES (4, 'Drinopol', NULL)" ); + } ); +} + +TEST( ModifiedSchemeTest, rebase_add_empty_column_and_delete ) +{ + // TODO: Postgres support + std::string driverName = "sqlite"; + + testSchemaDiffRebaseWith( driverName, "rebase_add_empty_column_and_delete", 0, + [ = ]( Driver & db ) + { + db.executeSql( "ALTER TABLE tram_stops ADD COLUMN bench_count integer" ); + }, + [ = ]( Driver & db ) + { + db.executeSql( "DELETE FROM tram_stops WHERE fid = 1" ); + }, + [ = ]( Driver & db ) + { + db.executeSql( "ALTER TABLE tram_stops ADD COLUMN bench_count integer" ); + db.executeSql( "DELETE FROM tram_stops WHERE fid = 1" ); + } ); +} + +TEST( ModifiedSchemeTest, rebase_drop_column ) +{ + // TODO: Postgres support + std::string driverName = "sqlite"; + + testSchemaDiffRebaseWith( driverName, "rebase_drop_column", 0, + [ = ]( Driver & db ) + { + db.executeSql( "ALTER TABLE tram_stops DROP COLUMN name" ); + db.executeSql( "INSERT INTO tram_stops (fid) VALUES (4)" ); + }, + [ = ]( Driver & db ) + { + db.executeSql( "INSERT INTO tram_stops (fid, name) VALUES (4, 'Palmovka')" ); + }, + [ = ]( Driver & db ) + { + db.executeSql( "ALTER TABLE tram_stops DROP COLUMN name" ); + db.executeSql( "INSERT INTO tram_stops (fid) VALUES (4)" ); + db.executeSql( "INSERT INTO tram_stops (fid) VALUES (5)" ); + // TODO: Add conflict entry for deleted 'Palmovka' value + } ); +} + +TEST( ModifiedSchemeTest, rebase_rename_column ) +{ + // TODO: Postgres support + std::string driverName = "sqlite"; + + testSchemaDiffRebaseWith( driverName, "rebase_rename_column", 0, + [ = ]( Driver & db ) + { + db.executeSql( "ALTER TABLE tram_stops RENAME COLUMN name TO description" ); + db.executeSql( "INSERT INTO tram_stops (fid, description) VALUES (4, 'Palmovka')" ); + }, + [ = ]( Driver & db ) + { + db.executeSql( "INSERT INTO tram_stops (fid, name) VALUES (4, 'Drinopol')" ); + }, + [ = ]( Driver & db ) + { + db.executeSql( "ALTER TABLE tram_stops RENAME COLUMN name TO description" ); + db.executeSql( "INSERT INTO tram_stops (fid, description) VALUES (4, 'Palmovka')" ); + db.executeSql( "INSERT INTO tram_stops (fid, description) VALUES (5, NULL)" ); + } ); +} + +// Helper: create a second DB by copying an existing one (SQLite only) and applying changes +static void copySqliteDb( const std::string &srcPath, const std::string &dstPath ) +{ + std::filesystem::copy( srcPath, dstPath, std::filesystem::copy_options::overwrite_existing ); +} + +// Concatenation test: verify that concat(diff(base, step1), diff(step1, step2)) equals diff(base, step2). +static void testSchemaConcatWith( std::string driverName, std::string testname, + std::function step1, + std::function step2 ) +{ + std::string dir = pathjoin( tmpdir(), testname ); + makedir( dir ); + + // Create base, step1 (= base + step1), step2 (= step1 + step2) DBs + { + createSampleDb( driverName, testname, "base" ); + + createSampleDb( driverName, testname, "step1" ); + { + auto db = openBaseModifiedDb( driverName, testname, "step1", "" ); + step1( *db ); + } + + if ( driverName == "sqlite" ) + copySqliteDb( pathjoin( dir, "step1.gpkg" ), pathjoin( dir, "step2.gpkg" ) ); + { + auto db = openBaseModifiedDb( driverName, testname, "step2", "" ); + step2( *db ); + } + } + + // Generate diff1 (base → step1) + std::string diff1 = pathjoin( dir, "diff1" ); + { + auto driver = openBaseModifiedDb( driverName, testname, "base", "step1" ); + ChangesetWriter writer; + writer.open( diff1 ); + driver->createChangeset( writer ); + } + + // Generate diff2 (step1 → step2) + std::string diff2 = pathjoin( dir, "diff2" ); + { + auto driver = openBaseModifiedDb( driverName, testname, "step1", "step2" ); + ChangesetWriter writer; + writer.open( diff2 ); + driver->createChangeset( writer ); + } + + // Concat diff1 + diff2 + std::string concatDiff = pathjoin( dir, "concat" ); + const char *inputs[] = { diff1.c_str(), diff2.c_str() }; + ASSERT_EQ( GEODIFF_concatChanges( testContext(), 2, inputs, concatDiff.c_str() ), GEODIFF_SUCCESS ); + + // Apply concat to base + { + ChangesetReader reader; + reader.open( concatDiff ); + auto db = openBaseModifiedDb( driverName, testname, "base", "" ); + db->applyChangeset( reader ); + } + + // base should now equal step2 + std::string verifyDiff = pathjoin( dir, "verify" ); + { + auto driver = openBaseModifiedDb( driverName, testname, "base", "step2" ); + ChangesetWriter writer; + writer.open( verifyDiff ); + driver->createChangeset( writer ); + } + ASSERT_EQ( std::filesystem::file_size( verifyDiff ), 0u ); +} + +// --- Basic: one schema change per concat --- + +TEST( ModifiedSchemeTest, concat_add_column ) +{ + std::string driverName = "sqlite"; + + testSchemaConcatWith( driverName, "concat_add_column", + [ = ]( Driver & db ) + { + db.executeSql( "ALTER TABLE tram_stops ADD COLUMN bench_count integer" ); + db.executeSql( "UPDATE tram_stops SET bench_count = 2 WHERE fid = 1" ); + db.executeSql( "INSERT INTO tram_stops (fid, name, bench_count) VALUES (4, 'Palmovka', 3)" ); + }, + [ = ]( Driver & db ) + { + db.executeSql( "UPDATE tram_stops SET bench_count = 5 WHERE fid = 1" ); + db.executeSql( "UPDATE tram_stops SET name = 'Červený Vrch' WHERE fid = 2" ); + db.executeSql( "INSERT INTO tram_stops (fid, name, bench_count) VALUES (5, 'Drinopol', 1)" ); + } ); +} + +TEST( ModifiedSchemeTest, concat_drop_column ) +{ + std::string driverName = "sqlite"; + + testSchemaConcatWith( driverName, "concat_drop_column", + [ = ]( Driver & db ) + { + db.executeSql( "INSERT INTO tram_stops (fid, name) VALUES (4, 'Palmovka')" ); + db.executeSql( "UPDATE tram_stops SET name = 'Pohořelec' WHERE fid = 1" ); + }, + [ = ]( Driver & db ) + { + db.executeSql( "ALTER TABLE tram_stops DROP COLUMN name" ); + db.executeSql( "INSERT INTO tram_stops (fid) VALUES (5)" ); + } ); +} + +TEST( ModifiedSchemeTest, concat_create_table ) +{ + std::string driverName = "sqlite"; + + testSchemaConcatWith( driverName, "concat_create_table", + [ = ]( Driver & db ) + { + db.executeSql( "CREATE TABLE benches (fid INTEGER PRIMARY KEY, material TEXT)" ); + db.executeSql( "INSERT INTO benches VALUES (1, 'wood'), (2, 'steel')" ); + db.executeSql( "INSERT INTO tram_stops (fid, name) VALUES (4, 'Palmovka')" ); + }, + [ = ]( Driver & db ) + { + db.executeSql( "INSERT INTO benches VALUES (3, 'concrete')" ); + db.executeSql( "UPDATE tram_stops SET name = 'Pohořelec' WHERE fid = 1" ); + } ); +} + +TEST( ModifiedSchemeTest, concat_drop_table ) +{ + std::string driverName = "sqlite"; + + testSchemaConcatWith( driverName, "concat_drop_table", + [ = ]( Driver & db ) + { + db.executeSql( "INSERT INTO tram_stops (fid, name) VALUES (4, 'Palmovka')" ); + db.executeSql( "UPDATE tram_stops SET name = 'Pohořelec' WHERE fid = 1" ); + }, + [ = ]( Driver & db ) + { + db.executeSql( "DROP TABLE tram_stops" ); + } ); +} + +TEST( ModifiedSchemeTest, concat_add_two_columns ) +{ + std::string driverName = "sqlite"; + + testSchemaConcatWith( driverName, "concat_add_two_columns", + [ = ]( Driver & db ) + { + db.executeSql( "ALTER TABLE tram_stops ADD COLUMN bench_count integer" ); + db.executeSql( "UPDATE tram_stops SET bench_count = 1 WHERE fid = 1" ); + }, + [ = ]( Driver & db ) + { + db.executeSql( "ALTER TABLE tram_stops ADD COLUMN line_number text" ); + db.executeSql( "UPDATE tram_stops SET line_number = 'A' WHERE fid = 2" ); + db.executeSql( "INSERT INTO tram_stops (fid, name, bench_count, line_number) VALUES (4, 'Palmovka', 3, 'B')" ); + } ); +} + +TEST( ModifiedSchemeTest, concat_add_column_then_drop_it ) +{ + std::string driverName = "sqlite"; - ASSERT_EQ( GEODIFF_createChangeset( testContext(), base.c_str(), modified.c_str(), changeset.c_str() ), GEODIFF_ERROR ); + testSchemaConcatWith( driverName, "concat_add_column_then_drop_it", + [ = ]( Driver & db ) + { + db.executeSql( "ALTER TABLE tram_stops ADD COLUMN temp text" ); + db.executeSql( "UPDATE tram_stops SET temp = 'x' WHERE fid = 1" ); + db.executeSql( "INSERT INTO tram_stops (fid, name, temp) VALUES (4, 'Palmovka', 'y')" ); + }, + [ = ]( Driver & db ) + { + db.executeSql( "INSERT INTO tram_stops (fid, name) VALUES (5, 'Drinopol')" ); + db.executeSql( "ALTER TABLE tram_stops DROP COLUMN temp" ); + } ); } int main( int argc, char **argv ) diff --git a/geodiff/tests/testdata/modified_scheme/changed_attribute_type.gpkg b/geodiff/tests/testdata/modified_scheme/changed_attribute_type.gpkg new file mode 100644 index 00000000..7db48d73 Binary files /dev/null and b/geodiff/tests/testdata/modified_scheme/changed_attribute_type.gpkg differ diff --git a/geodiff/tests/testdata/modified_scheme/changesets/added_attribute.diff b/geodiff/tests/testdata/modified_scheme/changesets/added_attribute.diff new file mode 100644 index 00000000..d3ca0032 Binary files /dev/null and b/geodiff/tests/testdata/modified_scheme/changesets/added_attribute.diff differ diff --git a/geodiff/tests/testdata/modified_scheme/changesets/added_attribute.json b/geodiff/tests/testdata/modified_scheme/changesets/added_attribute.json new file mode 100644 index 00000000..ce7ca428 --- /dev/null +++ b/geodiff/tests/testdata/modified_scheme/changesets/added_attribute.json @@ -0,0 +1,65 @@ +{ + "geodiff": [ + { + "column": { + "geomHasM": false, + "geomHasZ": false, + "geomSrsId": -1, + "geomType": "", + "isAutoIncrement": false, + "isGeometry": false, + "isNotNull": false, + "isPrimaryKey": false, + "name": "added_field", + "type": "integer" + }, + "tableName": "simple", + "type": "add_column" + }, + { + "changes": [ + { + "column": 0, + "old": 1 + }, + { + "column": 4, + "new": 1, + "old": null + } + ], + "table": "simple", + "type": "update" + }, + { + "changes": [ + { + "column": 0, + "old": 2 + }, + { + "column": 4, + "new": 2, + "old": null + } + ], + "table": "simple", + "type": "update" + }, + { + "changes": [ + { + "column": 0, + "old": 3 + }, + { + "column": 4, + "new": 3, + "old": null + } + ], + "table": "simple", + "type": "update" + } + ] +} diff --git a/geodiff/tests/testdata/modified_scheme/changesets/added_table.diff b/geodiff/tests/testdata/modified_scheme/changesets/added_table.diff new file mode 100644 index 00000000..94093e43 Binary files /dev/null and b/geodiff/tests/testdata/modified_scheme/changesets/added_table.diff differ diff --git a/geodiff/tests/testdata/modified_scheme/changesets/added_table.json b/geodiff/tests/testdata/modified_scheme/changesets/added_table.json new file mode 100644 index 00000000..763e5852 --- /dev/null +++ b/geodiff/tests/testdata/modified_scheme/changesets/added_table.json @@ -0,0 +1,64 @@ +{ + "geodiff": [ + { + "columns": [ + { + "geomHasM": false, + "geomHasZ": false, + "geomSrsId": -1, + "geomType": "", + "isAutoIncrement": true, + "isGeometry": false, + "isNotNull": true, + "isPrimaryKey": true, + "name": "fid", + "type": "integer" + }, + { + "geomHasM": false, + "geomHasZ": false, + "geomSrsId": 4326, + "geomType": "POLYGON", + "isAutoIncrement": false, + "isGeometry": true, + "isNotNull": false, + "isPrimaryKey": false, + "name": "geometry", + "type": "geometry" + }, + { + "geomHasM": false, + "geomHasZ": false, + "geomSrsId": -1, + "geomType": "", + "isAutoIncrement": false, + "isGeometry": false, + "isNotNull": false, + "isPrimaryKey": false, + "name": "myfield", + "type": "text" + } + ], + "tableName": "added_table", + "type": "create_table" + }, + { + "changes": [ + { + "column": 0, + "new": 1 + }, + { + "column": 1, + "new": "R1AAA+YQAADwIZt/aSztvyhTjE5mN9u/MjKhWTE40z8KvmmFKQnlPwEDAAAAAQAAAAUAAACMGhYhZuPqv2eg8/JZ2OE/8CGbf2ks7b/WaZRh4P7TPyhTjE5mN9u/MjKhWTE40z+QaI2BBKXjvwq+aYUpCeU/jBoWIWbj6r9noPPyWdjhPw==" + }, + { + "column": 2, + "new": "hello" + } + ], + "table": "added_table", + "type": "insert" + } + ] +} diff --git a/geodiff/tests/testdata/modified_scheme/changesets/delete_attribute.diff b/geodiff/tests/testdata/modified_scheme/changesets/delete_attribute.diff new file mode 100644 index 00000000..f9e07b45 Binary files /dev/null and b/geodiff/tests/testdata/modified_scheme/changesets/delete_attribute.diff differ diff --git a/geodiff/tests/testdata/modified_scheme/changesets/delete_attribute.json b/geodiff/tests/testdata/modified_scheme/changesets/delete_attribute.json new file mode 100644 index 00000000..4cdcf771 --- /dev/null +++ b/geodiff/tests/testdata/modified_scheme/changesets/delete_attribute.json @@ -0,0 +1,65 @@ +{ + "geodiff": [ + { + "changes": [ + { + "column": 0, + "old": 1 + }, + { + "column": 4, + "new": null, + "old": 1 + } + ], + "table": "simple", + "type": "update" + }, + { + "changes": [ + { + "column": 0, + "old": 2 + }, + { + "column": 4, + "new": null, + "old": 2 + } + ], + "table": "simple", + "type": "update" + }, + { + "changes": [ + { + "column": 0, + "old": 3 + }, + { + "column": 4, + "new": null, + "old": 3 + } + ], + "table": "simple", + "type": "update" + }, + { + "column": { + "geomHasM": false, + "geomHasZ": false, + "geomSrsId": -1, + "geomType": "", + "isAutoIncrement": false, + "isGeometry": false, + "isNotNull": false, + "isPrimaryKey": false, + "name": "added_field", + "type": "integer" + }, + "tableName": "simple", + "type": "drop_column" + } + ] +} diff --git a/geodiff/tests/testdata/modified_scheme/changesets/delete_table.diff b/geodiff/tests/testdata/modified_scheme/changesets/delete_table.diff new file mode 100644 index 00000000..79e3557b Binary files /dev/null and b/geodiff/tests/testdata/modified_scheme/changesets/delete_table.diff differ diff --git a/geodiff/tests/testdata/modified_scheme/changesets/delete_table.json b/geodiff/tests/testdata/modified_scheme/changesets/delete_table.json new file mode 100644 index 00000000..6730d725 --- /dev/null +++ b/geodiff/tests/testdata/modified_scheme/changesets/delete_table.json @@ -0,0 +1,64 @@ +{ + "geodiff": [ + { + "changes": [ + { + "column": 0, + "old": 1 + }, + { + "column": 1, + "old": "R1AAA+YQAADwIZt/aSztvyhTjE5mN9u/MjKhWTE40z8KvmmFKQnlPwEDAAAAAQAAAAUAAACMGhYhZuPqv2eg8/JZ2OE/8CGbf2ks7b/WaZRh4P7TPyhTjE5mN9u/MjKhWTE40z+QaI2BBKXjvwq+aYUpCeU/jBoWIWbj6r9noPPyWdjhPw==" + }, + { + "column": 2, + "old": "hello" + } + ], + "table": "added_table", + "type": "delete" + }, + { + "columns": [ + { + "geomHasM": false, + "geomHasZ": false, + "geomSrsId": -1, + "geomType": "", + "isAutoIncrement": true, + "isGeometry": false, + "isNotNull": true, + "isPrimaryKey": true, + "name": "fid", + "type": "integer" + }, + { + "geomHasM": false, + "geomHasZ": false, + "geomSrsId": 4326, + "geomType": "POLYGON", + "isAutoIncrement": false, + "isGeometry": true, + "isNotNull": false, + "isPrimaryKey": false, + "name": "geometry", + "type": "geometry" + }, + { + "geomHasM": false, + "geomHasZ": false, + "geomSrsId": -1, + "geomType": "", + "isAutoIncrement": false, + "isGeometry": false, + "isNotNull": false, + "isPrimaryKey": false, + "name": "myfield", + "type": "text" + } + ], + "tableName": "added_table", + "type": "drop_table" + } + ] +} diff --git a/geodiff/tests/testdata/modified_scheme/changesets/rename_attribute.diff b/geodiff/tests/testdata/modified_scheme/changesets/rename_attribute.diff new file mode 100644 index 00000000..f864165b Binary files /dev/null and b/geodiff/tests/testdata/modified_scheme/changesets/rename_attribute.diff differ diff --git a/geodiff/tests/testdata/modified_scheme/changesets/rename_table.diff b/geodiff/tests/testdata/modified_scheme/changesets/rename_table.diff new file mode 100644 index 00000000..656e019a Binary files /dev/null and b/geodiff/tests/testdata/modified_scheme/changesets/rename_table.diff differ diff --git a/pygeodiff/geodifflib.py b/pygeodiff/geodifflib.py index 68fd7137..2a545d38 100644 --- a/pygeodiff/geodifflib.py +++ b/pygeodiff/geodifflib.py @@ -642,6 +642,22 @@ def create_wkb_from_gpkg_header(self, context, geometry): wkb = copy.deepcopy(out[: out_size.value]) return wkb + def has_schema_change_entries(self, context, changeset): + func = self.lib.GEODIFF_changesetHasSchemaChangeEntries + func.argtypes = [ + ctypes.c_void_p, + ctypes.c_char_p, + ctypes.POINTER(ctypes.c_bool), + ] + func.restype = ctypes.c_int + + out = ctypes.c_bool() + res = func( + context, ctypes.c_char_p(changeset.encode("utf-8")), ctypes.byref(out) + ) + self._parse_return_code(context, res, "has_schema_change_entries") + return out.value + class ChangesetReader(object): """Wrapper around GEODIFF_CR_* functions from C API""" @@ -682,10 +698,16 @@ def __next__(self): class ChangesetEntry(object): """Wrapper around GEODIFF_CE_* functions from C API""" - # constants as defined in ChangesetEntry::OperationType enum + # constants as defined in ChangesetEntryType enum OP_INSERT = 18 OP_UPDATE = 23 OP_DELETE = 9 + OP_CREATE_TABLE = ord("a") + OP_DROP_TABLE = ord("A") + OP_ADD_COLUMN = ord("c") + OP_DROP_COLUMN = ord("C") + + _DATA_OPS = (OP_INSERT, OP_UPDATE, OP_DELETE) def __init__(self, geodiff, context, entry_ptr): self.geodiff = geodiff @@ -693,6 +715,11 @@ def __init__(self, geodiff, context, entry_ptr): self.context = context self.operation = self.geodiff._CE_operation(self.context, self.entry_ptr) + + if self.operation not in self._DATA_OPS: + # Only data entries have associated table and values + return + self.values_count = self.geodiff._CE_count(self.context, self.entry_ptr) if self.operation == self.OP_DELETE or self.operation == self.OP_UPDATE: diff --git a/pygeodiff/main.py b/pygeodiff/main.py index 16bee442..9e91b8fd 100644 --- a/pygeodiff/main.py +++ b/pygeodiff/main.py @@ -255,6 +255,14 @@ def list_changes_summary(self, changeset, json): self._lazy_load() return self.clib.list_changes_summary(self.context, changeset, json) + def has_schema_change_entries(self, changeset): + """ + :returns: whether changeset contains at least one schema change entry + :raises GeoDiffLibError: raised on error + """ + self._lazy_load() + return self.clib.has_schema_change_entries(self.context, changeset) + def has_changes(self, changeset): """ :returns: whether changeset contains at least one change diff --git a/pygeodiff/tests/test_changeset_reader.py b/pygeodiff/tests/test_changeset_reader.py index 9ded1521..23dd8c70 100644 --- a/pygeodiff/tests/test_changeset_reader.py +++ b/pygeodiff/tests/test_changeset_reader.py @@ -91,3 +91,33 @@ def test_delete(self): ) # with DELETE the "new_values" attribute is not set i += 1 self.assertEqual(i, 1) + + def test_has_schema_change_entries(self): + # data-only changeset + changeset = os.path.join( + geodiff_test_dir(), "1_geopackage", "base-modified_1_geom.diff" + ) + self.assertFalse(self.geodiff.has_schema_change_entries(changeset)) + + # changeset with schema changes + changeset = os.path.join( + geodiff_test_dir(), "modified_scheme", "changesets", "added_table.diff" + ) + self.assertTrue(self.geodiff.has_schema_change_entries(changeset)) + + def test_schema_change_added_table(self): + changeset = os.path.join( + geodiff_test_dir(), "modified_scheme", "changesets", "added_table.diff" + ) + entries = list(self.geodiff.read_changeset(changeset)) + self.assertEqual(len(entries), 2) + + # first entry: CREATE TABLE (schema change, no table/values attributes) + self.assertEqual(entries[0].operation, ChangesetEntry.OP_CREATE_TABLE) + self.assertFalse(hasattr(entries[0], "table")) + self.assertFalse(hasattr(entries[0], "old_values")) + self.assertFalse(hasattr(entries[0], "new_values")) + + # second entry: INSERT (data entry for the new table) + self.assertEqual(entries[1].operation, ChangesetEntry.OP_INSERT) + self.assertEqual(entries[1].table.name, "added_table") diff --git a/pygeodiff/tests/test_errors.py b/pygeodiff/tests/test_errors.py index b3766859..e2e43b07 100644 --- a/pygeodiff/tests/test_errors.py +++ b/pygeodiff/tests/test_errors.py @@ -20,7 +20,7 @@ def test_unsupported_change_error(self): create_dir("unsupported_change") base = geodiff_test_dir() + "/base.gpkg" - modified = geodiff_test_dir() + "/modified_scheme/added_attribute.gpkg" + modified = geodiff_test_dir() + "/modified_scheme/changed_attribute_type.gpkg" changeset = tmpdir() + "/pyunsupported_change/changeset.bin" try: