v3.2.0

CHANGELOG.md

@@ -2,6 +2,10 @@
 
 Represents the **NuGet** versions.
 
+## v3.2.0
+- *Enhancement:* The `MigrationCommand.Execute` has been enhanced to support the execution of YAML/JSON data files. Raw SQL must _now_ be prefixed by `>` to explicitly differentiate from a file path.
+- *Enhancement:* The `MigrationCommand.Inspect` has been enhanced to also report whether each column has been identified as a JSON column (i.e. `DbColumnSchema.IsJson`).
+
 ## v3.1.2
 - *Fixed:* Where a `DbException` occurs executing a command, only the exception message is now logged; not the full stack trace as this is misleading (experience improvement).
 - *Fixed:* Prior to executing a command that requires the database, its existence will be checked first and an appropriate error emitted where not found (experience improvement).

Common.targets

@@ -1,6 +1,6 @@
 <Project>
   <PropertyGroup>
-    <Version>3.1.2</Version>
+    <Version>3.2.0</Version>
     <LangVersion>preview</LangVersion>
     <Authors>Avanade</Authors>
     <Company>Avanade</Company>

README.md

@@ -78,7 +78,7 @@ Command | Description
 `ResetAndAll` | Performs `Reset` and `All` (designed primarily for testing).
 `ResetAndData` | Performs `Reset` and `Data` (designed primarily for testing).
 `ResetAndDatabase` | Performs `Reset` and `Database` (designed primarily for testing).
-`Execute` | Executes the SQL statement(s) passed as additional arguments.
+`Execute` | Executes the SQL statement(s) passed as additional arguments (each being a file-path or raw SQL statement).
 `Script` | Creates a new [`migration`](#Migrate) script file using the defined naming convention.
 [`Inspect`](#Inspect) | Inspects one or more existing database tables and outputs the inferred schema (columns, types, nullability, defaults, primary key, identity, computed and unique flags) as markdown to the console.
 
@@ -210,8 +210,7 @@ Arguments:
   command                    Database migration command (see https://github.com/Avanade/dbex#commands-functions).
                              Allowed values are: None, Drop, Create, Migrate, CodeGen, Schema, Deploy, Reset, Data, DeployWithData, Database, DropAndDatabase, All, DropAndAll,
                              ResetAndData, ResetAndDatabase, ResetAndAll, Execute, Script, Inspect.
-  args                       Additional arguments; 'Script' arguments (first being the script name) -or- 'Execute' (each a SQL statement to invoke) -or- 'Inspect' (schema followed by one or more table names).
-
+  args                       Additional arguments; 'Script' arguments (first being the script name) -or- 'Execute' (each being a file-path or raw SQL statement) -or- 'Inspect' (schema plus one or more table names).
 Options:
   -?|-h|--help               Show help information.
   -cs|--connection-string    Database connection string.
@@ -285,13 +284,16 @@ dotnet run script cdc Foo Bar
 
 #### Execute command
 
-The execute command allows one or more SQL Statements, and/or Script files, to be executed directly against the database. This is intended for enabling commands to be executed only. No response other than success or failure will be acknowledged; as such this is not intended for performing queries.
+The execute command allows one or more SQL Statements, and/or Script files, to be executed directly against the database. This is intended for enabling commands to be executed only. No response other than success or failure will be acknowledged; as such this is not intended for performing queries. A raw SQL statement must be prefixed with a `>` character, otherwise it will be treated as a file path to a SQL script file. The SQL statement(s) or script file(s) must be specified in the order they are to be executed.
+
+Additionally, YAML and JSON data seeding files can be specified to be executed directly against the database.
 
 Examples as follows.
 
 ```
-dotnet run execute "create schema [Xyz] authorization [dbo]"
+dotnet run execute "> create schema [Xyz] authorization [dbo]"
 dotnet run execute ./schema/createscehma.sql
+dotnet run execute ./data/data.yaml ./data/other.json
 ```
 
 <br/>

src/DbEx.MySql/MySqlSchemaConfig.cs

@@ -84,7 +84,7 @@ public override DbColumnSchema CreateColumnFromInformationSchema(DbTableSchema t
             IsDotNetTimeOnly = RemovePrecisionFromDataType(dt).Equals("TIME", StringComparison.OrdinalIgnoreCase)
         };
 
-        c.IsJsonContent = c.Type.Equals("JSON", StringComparison.OrdinalIgnoreCase) || (c.DotNetName == "string" && c.Name.EndsWith(JsonColumnNameSuffix, StringComparison.Ordinal));
+        c.IsJsonContent = c.Type.Equals("JSON", StringComparison.OrdinalIgnoreCase) || (c.DotNetType == "string" && c.Name.EndsWith(JsonColumnNameSuffix, StringComparison.Ordinal));
         if (c.IsJsonContent && c.Name.EndsWith(JsonColumnNameSuffix, StringComparison.Ordinal))
             c.DotNetCleanedName = DbTableSchema.CreateDotNetName(c.Name[..^JsonColumnNameSuffix.Length]);
 

src/DbEx.Postgres/PostgresSchemaConfig.cs

@@ -83,7 +83,7 @@ public override DbColumnSchema CreateColumnFromInformationSchema(DbTableSchema t
             IsDotNetTimeOnly = RemovePrecisionFromDataType(dr.GetValue<string>("DATA_TYPE")!).Equals("TIME WITHOUT TIME ZONE", StringComparison.OrdinalIgnoreCase)
         };
 
-        c.IsJsonContent = c.Type.Equals("JSON", StringComparison.OrdinalIgnoreCase) || (c.DotNetName == "string" && c.Name.EndsWith(JsonColumnNameSuffix, StringComparison.Ordinal));
+        c.IsJsonContent = c.Type.Equals("JSON", StringComparison.OrdinalIgnoreCase) || (c.DotNetType == "string" && c.Name.EndsWith(JsonColumnNameSuffix, StringComparison.Ordinal));
         if (c.IsJsonContent && c.Name.EndsWith(JsonColumnNameSuffix, StringComparison.Ordinal))
             c.DotNetCleanedName = DbTableSchema.CreateDotNetName(c.Name[..^JsonColumnNameSuffix.Length]);
 

src/DbEx.SqlServer/SqlServerSchemaConfig.cs

@@ -80,7 +80,8 @@ public override DbColumnSchema CreateColumnFromInformationSchema(DbTableSchema t
             IsDotNetTimeOnly = RemovePrecisionFromDataType(dr.GetValue<string>("DATA_TYPE")!).Equals("TIME", StringComparison.OrdinalIgnoreCase),
         };
 
-        if (c.IsJsonContent = c.DotNetName == "string" && c.Name.EndsWith(JsonColumnNameSuffix, StringComparison.Ordinal))
+        c.IsJsonContent = c.Type.Equals("JSON", StringComparison.OrdinalIgnoreCase) || (c.DotNetType == "string" && c.Name.EndsWith(JsonColumnNameSuffix, StringComparison.Ordinal));
+        if (c.IsJsonContent && c.Name.EndsWith(JsonColumnNameSuffix, StringComparison.Ordinal))
             c.DotNetCleanedName = DbTableSchema.CreateDotNetName(c.Name[..^JsonColumnNameSuffix.Length]);
 
         return c;
@@ -196,7 +197,7 @@ public override string ToDotNetTypeName(DbColumnSchema schema)
 
         return dbType.ToUpperInvariant() switch
         {
-            "NCHAR" or "CHAR" or "NVARCHAR" or "VARCHAR" or "TEXT" or "NTEXT" => "string",
+            "NCHAR" or "CHAR" or "NVARCHAR" or "VARCHAR" or "TEXT" or "NTEXT" or "JSON" => "string",
             "DECIMAL" or "MONEY" or "NUMERIC" or "SMALLMONEY" => "decimal",
             "DATETIME" or "DATETIME2" or "SMALLDATETIME" => "DateTime",
             "DATETIMEOFFSET" => "DateTimeOffset",

src/DbEx/Console/MigrationConsoleBase.cs

@@ -107,7 +107,7 @@ public async Task<int> RunAsync(string[] args, CancellationToken cancellationTok
         ConsoleOptions.Add(DropSchemaObjectsName, app.Option("-dso|--drop-schema-objects", "Drop all known schema objects before applying; bypasses automatic skip where all scripts are replacements.", CommandOptionType.NoValue));
         ConsoleOptions.Add(AcceptPromptsOptionName, app.Option("--accept-prompts", "Accept prompts; command should _not_ stop and wait for user confirmation (DROP or RESET commands).", CommandOptionType.NoValue));
         ConsoleOptions.Add(ExpectNoChangesName, app.Option("--expect-no-changes", "Indicates to expect no changes during code-generation (i.e. result in error on change).", CommandOptionType.NoValue));
-        _additionalArgs = app.Argument("args", "Additional arguments; 'Script' arguments (first being the script name) -or- 'Execute' (each a SQL statement to invoke).", multipleValues: true);
+        _additionalArgs = app.Argument("args", "Additional arguments; 'Script' arguments (first being the script name) -or- 'Execute' (each being a file-path or raw SQL statement) -or- 'Inspect' (schema where applicable plus one or more table names).", multipleValues: true);
 
         OnBeforeExecute(app);
 

src/DbEx/Migration/DatabaseMigrationBase.cs

@@ -175,6 +175,9 @@ public virtual async Task<bool> MigrateAsync(CancellationToken cancellationToken
         if (!await CommandExecuteAsync(MigrationCommand.Drop, "DATABASE DROP: Checking database existence and dropping where found...", DatabaseDropAsync, null, cancellationToken).ConfigureAwait(false))
             return false;
 
+        if (Args.MigrationCommand == MigrationCommand.Drop)
+            return true;    // Where only dropping the database, then exit here to avoid errant database exists check coming up.
+
         // Database create.
         if (!await CommandExecuteAsync(MigrationCommand.Create, "DATABASE CREATE: Checking database existence and creating where not found...", DatabaseCreateAsync, null, cancellationToken).ConfigureAwait(false))
             return false;
@@ -333,7 +336,7 @@ protected async Task<bool> CommandExecuteAsync(string title, Func<CancellationTo
         }
         catch (Exception ex)
         {
-            Logger.LogError(ex, "{Content}", ex.Message);
+            Logger.LogCritical(ex, "{Content}", ex.Message);
             return false;
         }
     }
@@ -367,9 +370,14 @@ protected virtual async Task<bool> ExecuteScriptsAsync(IEnumerable<DatabaseMigra
             {
                 await ExecuteScriptAsync(script, cancellationToken).ConfigureAwait(false);
             }
+            catch (DbException dbex)
+            {
+                Logger.LogError("{Content}", $"A database error occurred: {dbex.Message}");
+                return false;
+            }
             catch (Exception ex)
             {
-                Logger.LogCritical(ex, "An error occurred executing the script: {Message}", ex.Message);
+                Logger.LogCritical(ex, "{Content}", ex.Message);
                 return false;
             }
 
@@ -1032,11 +1040,11 @@ private async Task<bool> CreateScriptInternalAsync(string? name, IDictionary<str
     public async Task<bool> ExecuteSqlStatementsAsync(string[]? statements, CancellationToken cancellationToken = default)
     {
         PreExecutionInitialization();
-        return await CommandExecuteAsync("DATABASE EXECUTE: Executes the SQL statement(s)...", async ct => await ExecuteSqlStatementsInternalAsync(statements, ct).ConfigureAwait(false), null, cancellationToken).ConfigureAwait(false);
+        return await CommandExecuteAsync($"DATABASE EXECUTE: Executing the {statements?.Length ?? 0} statement(s)...", async ct => await ExecuteSqlStatementsInternalAsync(statements, ct).ConfigureAwait(false), null, cancellationToken).ConfigureAwait(false);
     }
 
     /// <summary>
-    /// Executes the raw SQL statements.
+    /// Executes the SQL statements (files or raw SQL).
     /// </summary>
     private async Task<bool> ExecuteSqlStatementsInternalAsync(string[]? statements, CancellationToken cancellationToken)
     {
@@ -1049,18 +1057,82 @@ private async Task<bool> ExecuteSqlStatementsInternalAsync(string[]? statements,
         if (statements.Length >= 1000)
             throw new ArgumentException("A maximum of 999 SQL statements may be executed at one-time.", nameof(statements));
 
-        var sn = $"{DateTime.UtcNow.ToString("yyyyMMdd-HHmmss", System.Globalization.CultureInfo.InvariantCulture)}-console-execute-";
-
-        var scripts = new List<DatabaseMigrationScript>();
         for (int i = 0; i < statements.Length; i++)
         {
-            if (File.Exists(statements[i]))
-                scripts.Add(new DatabaseMigrationScript(this, new FileInfo(statements[i]), statements[i]));
+            Logger.LogInformation("{Content}", string.Empty);
+
+            if (i > 0)
+            {
+                Logger.LogInformation("{Content}", $"  {new string('-', 78)}");
+                Logger.LogInformation("{Content}", string.Empty);
+            }
+
+            if (!await ExecuteSqlStatementInternalAsync(statements[i], i, cancellationToken).ConfigureAwait(false))
+                return false;
+        }
+
+        return true;
+    }
+
+    /// <summary>
+    /// Executes an individual SQL statement (file or raw SQL).
+    /// </summary>
+    private async Task<bool> ExecuteSqlStatementInternalAsync(string statement, int index, CancellationToken cancellationToken)
+    {
+        // If the statement is not a file, assume it is a raw SQL statement and create a temporary script to execute it.
+        if (!File.Exists(statement))
+        {
+            if (string.IsNullOrWhiteSpace(statement))
+            {
+                Logger.LogWarning("{Content}", $"** Statement is empty; skipping... [#{index + 1}]");
+                return true;
+            }
+
+            if (statement.StartsWith('>'))
+                statement = statement[1..];
             else
-                scripts.Add(new DatabaseMigrationScript(this, statements[i], $"{sn}{i + 1:000}.{SchemaConfig.ScriptSuffix}"));
+            {
+                Logger.LogError("{Content}", $"Error: Statement is not a file (does not exist) or does not start with '>' (raw SQL). [#{index + 1}]");
+                return false;
+            }
+
+            if (string.IsNullOrWhiteSpace(statement))
+            {
+                Logger.LogWarning("{Content}", $"** Statement is empty; skipping... [#{index + 1}]");
+                return true;
+            }
+
+            Logger.LogInformation("{Content}", $"** Executing: Raw SQL [#{index + 1}]...");
+            var script = new DatabaseMigrationScript(this, statement, $"{DateTime.UtcNow.ToString("yyyyMMdd-HHmmss", System.Globalization.CultureInfo.InvariantCulture)}-console-execute.{index + 1:000}.{SchemaConfig.ScriptSuffix}");
+            await ExecuteScriptAsync(script, cancellationToken).ConfigureAwait(false);
+            return true;
+        }
+
+        var fi = new FileInfo(statement);
+
+        // If the statement is a YAML or JSON file, parse it and execute the data insert/merge.
+        if (statement.EndsWith(".yaml", StringComparison.InvariantCultureIgnoreCase) || statement.EndsWith(".yml", StringComparison.InvariantCultureIgnoreCase))
+        {
+            Logger.LogInformation("{Content}", $"** Parsing and executing: {fi.FullName} [#{index + 1}]...");
+            using var sr = new StreamReader(statement);
+            var schema = await Database.SelectSchemaAsync(this, cancellationToken).ConfigureAwait(false);
+            var tables = await new DataParser(this, schema).ParseYamlAsync(sr, cancellationToken).ConfigureAwait(false);
+            return await DatabaseDataAsync(tables, cancellationToken).ConfigureAwait(false);
         }
 
-        return await ExecuteScriptsAsync(scripts, false, cancellationToken).ConfigureAwait(false);
+        if (statement.EndsWith(".json", StringComparison.InvariantCultureIgnoreCase) || statement.EndsWith(".jsn", StringComparison.InvariantCultureIgnoreCase))
+        {
+            Logger.LogInformation("{Content}", $"** Parsing and executing: {fi.FullName} [#{index + 1}]...");
+            using var sr = new StreamReader(statement);
+            var tables = await new DataParser(this, await Database.SelectSchemaAsync(this, cancellationToken).ConfigureAwait(false)).ParseJsonAsync(sr, cancellationToken).ConfigureAwait(false);
+            return await DatabaseDataAsync(tables, cancellationToken).ConfigureAwait(false);
+        }
+
+        // Assume the statement is a SQL file, create a script to execute it.
+        Logger.LogInformation("{Content}", $"** Executing: {fi.FullName} [#{index + 1}] ...");
+        var sqlScript = new DatabaseMigrationScript(this, new FileInfo(statement), Path.GetFileName(statement));
+        await ExecuteScriptAsync(sqlScript, cancellationToken).ConfigureAwait(false);
+        return true;
     }
 
     /// <summary>
@@ -1148,7 +1220,8 @@ private void InspectTableMarkdown(string? schema, string name, DbSchema.DbTableS
                 c.IsPrimaryKey ? "Yes" : "No",
                 c.IsIdentity ? "Yes" : "No",
                 c.IsComputed ? "Yes" : "No",
-                c.IsUnique ? "Yes" : "No"
+                c.IsUnique ? "Yes" : "No",
+                c.IsJsonContent ? "Yes" : "No"
             ]);
         }
 
@@ -1160,12 +1233,13 @@ private void InspectTableMarkdown(string? schema, string name, DbSchema.DbTableS
         var identityMaxLength = Math.Max("Identity".Length, columns.Max(x => x[5].Length));
         var computedMaxLength = Math.Max("Computed".Length, columns.Max(x => x[6].Length));
         var uniqueMaxLength = Math.Max("Unique".Length, columns.Max(x => x[7].Length));
+        var jsonMaxLength = Math.Max("JSON".Length, columns.Max(x => x[8].Length));
 
-        Logger.LogInformation("{Content}", $"| Column{new string(' ', columnMaxLength - "Column".Length)} | Type{new string(' ', typeMaxLength - "Type".Length)} | Null{new string(' ', isNullMaxLength - "Null".Length)} | Default{new string(' ', defaultMaxLength - "Default".Length)} | PK{new string(' ', pkMaxLength - "PK".Length)} | Identity{new string(' ', identityMaxLength - "Identity".Length)} | Computed{new string(' ', computedMaxLength - "Computed".Length)} | Unique{new string(' ', uniqueMaxLength - "Unique".Length)} |");
-        Logger.LogInformation("{Content}", $"|-{"".PadRight(columnMaxLength, '-')}-|-{"".PadRight(typeMaxLength, '-')}-|-{"".PadRight(isNullMaxLength, '-')}-|-{"".PadRight(defaultMaxLength, '-')}-|-{"".PadRight(pkMaxLength, '-')}-|-{"".PadRight(identityMaxLength, '-')}-|-{"".PadRight(computedMaxLength, '-')}-|-{"".PadRight(uniqueMaxLength, '-')}-|");
+        Logger.LogInformation("{Content}", $"| Column{new string(' ', columnMaxLength - "Column".Length)} | Type{new string(' ', typeMaxLength - "Type".Length)} | Null{new string(' ', isNullMaxLength - "Null".Length)} | Default{new string(' ', defaultMaxLength - "Default".Length)} | PK{new string(' ', pkMaxLength - "PK".Length)} | Identity{new string(' ', identityMaxLength - "Identity".Length)} | Computed{new string(' ', computedMaxLength - "Computed".Length)} | Unique{new string(' ', uniqueMaxLength - "Unique".Length)} | JSON{new string(' ', jsonMaxLength - "JSON".Length)} |");
+        Logger.LogInformation("{Content}", $"|-{"".PadRight(columnMaxLength, '-')}-|-{"".PadRight(typeMaxLength, '-')}-|-{"".PadRight(isNullMaxLength, '-')}-|-{"".PadRight(defaultMaxLength, '-')}-|-{"".PadRight(pkMaxLength, '-')}-|-{"".PadRight(identityMaxLength, '-')}-|-{"".PadRight(computedMaxLength, '-')}-|-{"".PadRight(uniqueMaxLength, '-')}-|-{"".PadRight(jsonMaxLength, '-')}-|");
         foreach (var column in columns)
         {
-            Logger.LogInformation("{Content}", $"| {column[0].PadRight(columnMaxLength)} | {column[1].PadRight(typeMaxLength)} | {column[2].PadRight(isNullMaxLength)} | {column[3].PadRight(defaultMaxLength)} | {column[4].PadRight(pkMaxLength)} | {column[5].PadRight(identityMaxLength)} | {column[6].PadRight(computedMaxLength)} | {column[7].PadRight(uniqueMaxLength)} |");
+            Logger.LogInformation("{Content}", $"| {column[0].PadRight(columnMaxLength)} | {column[1].PadRight(typeMaxLength)} | {column[2].PadRight(isNullMaxLength)} | {column[3].PadRight(defaultMaxLength)} | {column[4].PadRight(pkMaxLength)} | {column[5].PadRight(identityMaxLength)} | {column[6].PadRight(computedMaxLength)} | {column[7].PadRight(uniqueMaxLength)} | {column[8].PadRight(jsonMaxLength)} |");
         }
 
         Logger.LogInformation("{Content}", string.Empty);

tests/DbEx.Test/Data.yaml

@@ -0,0 +1,4 @@
+Test:
+- Contact:
+  - { ContactId: 101, ContactType: E, Gender: M, Name: Bob, DateOfBirth: 2001-10-22, Addresses: [ { ContactAddressId: 1010, Street: "1 Main Street" } ] }
+  - { ContactId: 102, ContactType: I, Name: ^jane_name, Phone: 1234, Addresses: [ { ContactAddressId: 2020, ContactId: 102, Street: "1 Main Street" } ] }

tests/DbEx.Test/DbEx.Test.csproj

@@ -49,6 +49,9 @@
     <None Update="appsettings.json">
       <CopyToOutputDirectory>Always</CopyToOutputDirectory>
     </None>
+    <None Update="Data.yaml">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
   </ItemGroup>
 
 </Project>