diff --git a/BExIS++.sln b/BExIS++.sln index 0e2a080257..3d65216cd8 100644 --- a/BExIS++.sln +++ b/BExIS++.sln @@ -1,6 +1,6 @@ Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio Version 17 -VisualStudioVersion = 17.9.34723.18 +# Visual Studio Version 18 +VisualStudioVersion = 18.2.11415.280 MinimumVisualStudioVersion = 10.0.40219.1 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Console", "Console", "{A24A6801-2ECC-4F47-8284-8C93277D3030}" EndProject diff --git a/Components/DLM/BExIS.Dlm.Entities/BExIS.Dlm.Entities.csproj b/Components/DLM/BExIS.Dlm.Entities/BExIS.Dlm.Entities.csproj index 600c5452ba..9c5d322fed 100644 --- a/Components/DLM/BExIS.Dlm.Entities/BExIS.Dlm.Entities.csproj +++ b/Components/DLM/BExIS.Dlm.Entities/BExIS.Dlm.Entities.csproj @@ -124,6 +124,7 @@ + diff --git a/Components/DLM/BExIS.Dlm.Entities/SpeciesMatching/SpeciesMatchingResult.cs b/Components/DLM/BExIS.Dlm.Entities/SpeciesMatching/SpeciesMatchingResult.cs new file mode 100644 index 0000000000..9ef0c2c37c --- /dev/null +++ b/Components/DLM/BExIS.Dlm.Entities/SpeciesMatching/SpeciesMatchingResult.cs @@ -0,0 +1,78 @@ +using BExIS.Dlm.Entities.Data; +using BExIS.Security.Entities.Subjects; +using System; +using Vaiona.Entities.Common; + +namespace BExIS.Dlm.Entities.SpeciesMatching +{ + public class SpeciesMatchingResult : BaseEntity + { + + // original unchanged name (used for matching if EditedName is empty, and for display purposes) + public virtual string OriginalName { get; set; } + + // edited name after data cleaning + manual corrections (used for matching) + public virtual string EditedName { get; set; } + + // matched name from the external source (the result) + public virtual string MatchedName { get; set; } + + // taxonomic status of the matched name (e.g. accepted, synonym, etc.) + public virtual string Status { get; set; } + + // type of the match (e.g. exact, fuzzy, etc.) + public virtual string MatchType { get; set; } + + // taxonomic rank of the matched name (e.g. species, genus, etc.) + public virtual string MatchRank { get; set; } + + // unique identifier of the matched name in the external source (e.g. GBIF taxon ID) + public virtual string MatchId { get; set; } + + // authorship of the matched name + public virtual string MatchAuthorship { get; set; } + + // accepted name if (for example) the matched name is a synonym + public virtual string AcceptedScientificName { get; set; } + + // unique identifier of the accepted name in the external source (e.g. GBIF taxon ID) + public virtual string AcceptedId { get; set; } + + // authorship of the accepted name + public virtual string AcceptedAuthorship { get; set; } + + // higher classification of the matched name (e.g. kingdom, phylum, class, order, family, genus) + public virtual string TaxonKingdom { get; set; } + + public virtual string TaxonPhylum { get; set; } + + public virtual string TaxonClass { get; set; } + + public virtual string TaxonOrder { get; set; } + + public virtual string TaxonFamily { get; set; } + + public virtual string TaxonGenus { get; set; } + + // timestamp of the match (can vary by hours due to processing and queue times on different APIs) + public virtual DateTime TimestampMatch { get; set; } + + // source of the match (e.g. Catalogue of Life, GBIF, etc.) + public virtual string MatchSource { get; set; } + + // version of the source used for matching + public virtual string MatchSourceVersion { get; set; } + + // indicates whether the match has been confirmed by the user + public virtual bool ConfirmedByUser { get; set; } + + // reference to the dataset where the original name was taken from + public virtual Dataset Dataset { get; set; } + + // VersionId + Dataset make the unique key for the matching result + public virtual long DatasetVersionId { get; set; } + + // reference to the Matching step (or -1 if the result is not associated with a specific step) + public virtual long StepId { get; set; } + } +} \ No newline at end of file diff --git a/Components/DLM/BExIS.Dlm.Orm.NH/BExIS.Dlm.Orm.NH.csproj b/Components/DLM/BExIS.Dlm.Orm.NH/BExIS.Dlm.Orm.NH.csproj index dc82b66e33..642b8d5b8e 100644 --- a/Components/DLM/BExIS.Dlm.Orm.NH/BExIS.Dlm.Orm.NH.csproj +++ b/Components/DLM/BExIS.Dlm.Orm.NH/BExIS.Dlm.Orm.NH.csproj @@ -239,6 +239,7 @@ Designer + PreserveNewest Designer diff --git a/Components/DLM/BExIS.Dlm.Orm.NH/Mappings/Default/SpeciesMatching/SpeciesMatchingResult.hbm.xml b/Components/DLM/BExIS.Dlm.Orm.NH/Mappings/Default/SpeciesMatching/SpeciesMatchingResult.hbm.xml new file mode 100644 index 0000000000..0ef0bb127f --- /dev/null +++ b/Components/DLM/BExIS.Dlm.Orm.NH/Mappings/Default/SpeciesMatching/SpeciesMatchingResult.hbm.xml @@ -0,0 +1,68 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/Components/DLM/BExIS.Dlm.Services/BExIS.Dlm.Services.csproj b/Components/DLM/BExIS.Dlm.Services/BExIS.Dlm.Services.csproj index d131d355b9..10e5308551 100644 --- a/Components/DLM/BExIS.Dlm.Services/BExIS.Dlm.Services.csproj +++ b/Components/DLM/BExIS.Dlm.Services/BExIS.Dlm.Services.csproj @@ -166,6 +166,7 @@ + diff --git a/Components/DLM/BExIS.Dlm.Services/SpeciesMatching/SpeciesMatchingResultManager.cs b/Components/DLM/BExIS.Dlm.Services/SpeciesMatching/SpeciesMatchingResultManager.cs new file mode 100644 index 0000000000..f9c0d30372 --- /dev/null +++ b/Components/DLM/BExIS.Dlm.Services/SpeciesMatching/SpeciesMatchingResultManager.cs @@ -0,0 +1,142 @@ +using BExIS.Dlm.Entities.Data; +using BExIS.Dlm.Entities.SpeciesMatching; +using System; +using System.Collections.Generic; +using System.Diagnostics.Contracts; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Vaiona.Persistence.Api; + +namespace BExIS.Dlm.Services.SpeciesMatching +{ + public class SpeciesMatchingResultManager : IDisposable + { + + private IUnitOfWork guow = null; + + public SpeciesMatchingResultManager() + { + guow = this.GetIsolatedUnitOfWork(); + this.Repo = guow.GetReadOnlyRepository(); + } + + private bool isDisposed = false; + + ~SpeciesMatchingResultManager() + { + Dispose(true); + } + + public void Dispose() + { + Dispose(true); + } + + protected virtual void Dispose(bool disposing) + { + if (!isDisposed) + { + if (disposing) + { + if (guow != null) + guow.Dispose(); + isDisposed = true; + } + } + } + + public IReadOnlyRepository Repo { get; private set; } + + public SpeciesMatchingResult Create(SpeciesMatchingResult matchingResult) + { + if (matchingResult == null) throw new ArgumentNullException("Species matching result must not be null."); + if (matchingResult.Dataset == null) throw new ArgumentNullException("Dataset must not be null."); + if (matchingResult.OriginalName == null) throw new ArgumentNullException("Dataset must not be null."); + + using (IUnitOfWork uow = this.GetUnitOfWork()) + { + try + { + IRepository repo = uow.GetRepository(); + repo.Put(matchingResult); + uow.Commit(); + + return (matchingResult); + } + catch (Exception ex) + { + throw new Exception("SpeciesMatchingResult creation failed.", ex); + } + } + } + + public SpeciesMatchingResult Update(SpeciesMatchingResult matchingResult) + { + if (matchingResult == null) throw new ArgumentNullException("Species matching result must not be null."); + + Contract.Ensures(Contract.Result() != null && Contract.Result().Id >= 0); + + using (IUnitOfWork uow = this.GetUnitOfWork()) + { + try + { + IRepository repo = uow.GetRepository(); + repo.Merge(matchingResult); + var merged = repo.Get(matchingResult.Id); + repo.Put(merged); + uow.Commit(); + + return (merged); + } + catch (Exception ex) + { + throw new Exception("SpeciesMatchingResult creation failed.", ex); + } + } + } + public bool Delete(long id) + { + if (id == 0) throw new ArgumentException("Species matching result must not be null."); + + Contract.Ensures(Contract.Result() != null && Contract.Result().Id >= 0); + + using (IUnitOfWork uow = this.GetUnitOfWork()) + { + IRepository repo = uow.GetRepository(); + + var e = repo.Get(id); + + if (e != null) + { + repo.Delete(e); + uow.Commit(); + + return true; + } + else + { + throw new ArgumentException(string.Format("the species matching result with the id {0} does not exist", id)); + } + } + } + + public bool Delete(SpeciesMatchingResult matchingResult) + { + if (matchingResult == null) throw new ArgumentNullException("Entity template must not be null."); + + Contract.Ensures(Contract.Result() != null && Contract.Result().Id >= 0); + + using (IUnitOfWork uow = this.GetUnitOfWork()) + { + IRepository repo = uow.GetRepository(); + + repo.Delete(matchingResult); + uow.Commit(); + + return true; + } + } + + } +} diff --git a/Components/DLM/BExIS.Dlm.Tests/BExIS.Dlm.Tests.csproj b/Components/DLM/BExIS.Dlm.Tests/BExIS.Dlm.Tests.csproj index 752f098988..66b31b052d 100644 --- a/Components/DLM/BExIS.Dlm.Tests/BExIS.Dlm.Tests.csproj +++ b/Components/DLM/BExIS.Dlm.Tests/BExIS.Dlm.Tests.csproj @@ -231,6 +231,7 @@ + diff --git a/Components/DLM/BExIS.Dlm.Tests/Services/SpeciesMatching/SpeciesMatchingResultManagerTest.cs b/Components/DLM/BExIS.Dlm.Tests/Services/SpeciesMatching/SpeciesMatchingResultManagerTest.cs new file mode 100644 index 0000000000..dbbd6ed6ce --- /dev/null +++ b/Components/DLM/BExIS.Dlm.Tests/Services/SpeciesMatching/SpeciesMatchingResultManagerTest.cs @@ -0,0 +1,71 @@ +using BExIS.App.Testing; +using BExIS.Dlm.Entities.Data; +using BExIS.Dlm.Entities.SpeciesMatching; +using BExIS.Dlm.Services.Data; +using BExIS.Dlm.Services.MetadataStructure; +using BExIS.Dlm.Services.SpeciesMatching; +using BExIS.Security.Services.Objects; +using BExIS.Security.Services.Subjects; +using BExIS.Utils.Config; +using NUnit.Framework; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace BExIS.Dlm.Tests.Services.SpeciesMatching +{ + internal class SpeciesMatchingResultManagerTest + { + private TestSetupHelper helper = null; + + [OneTimeSetUp] + public void OneTimeSetUp() + { + helper = new TestSetupHelper(WebApiConfig.Register, false); + } + + [OneTimeTearDown] + public void OneTimeTearDown() + { + } + + [Test()] + public void Create_Valid_EntityTemplate() + { + using (var speciesMatchingResultManager = new SpeciesMatchingResultManager()) + using (var userManager = new UserManager()) + using (var datasetManager = new DatasetManager()) + { + //Arrange + SpeciesMatchingResult matchingResult = new SpeciesMatchingResult(); + + var user = userManager.Users.FirstOrDefault(); + var dataset = datasetManager.DatasetRepo.Get().FirstOrDefault(); + + matchingResult.OriginalName = "Sunflower"; + matchingResult.EditedName = ""; + matchingResult.MatchedName = ""; + matchingResult.Status = ""; + matchingResult.MatchType = ""; + matchingResult.TimestampMatch = DateTime.Now; + matchingResult.MatchSource = ""; + matchingResult.MatchSourceVersion = ""; + matchingResult.ConfirmedByUser = false; + matchingResult.Dataset = dataset; + matchingResult.DatasetVersionId = 1; + matchingResult.StepId = -1; + + //Act + var created = speciesMatchingResultManager.Create(matchingResult); + var fromdb = speciesMatchingResultManager.Repo.Get().LastOrDefault(); + + //Assert + Assert.IsNotNull(created); + Assert.IsNotNull(fromdb); + Assert.That(created.Id.Equals(fromdb.Id)); + } + } + } +} diff --git a/Console/BExIS.Web.Shell/Areas/DCM/BExIS.Modules.Dcm.UI.Svelte/BExIS.Modules.SMM.UI.csproj b/Console/BExIS.Web.Shell/Areas/DCM/BExIS.Modules.Dcm.UI.Svelte/BExIS.Modules.SMM.UI.csproj new file mode 100644 index 0000000000..f8d00ee09d --- /dev/null +++ b/Console/BExIS.Web.Shell/Areas/DCM/BExIS.Modules.Dcm.UI.Svelte/BExIS.Modules.SMM.UI.csproj @@ -0,0 +1,276 @@ + + + + + Debug + AnyCPU + + + 2.0 + {37402CAB-EB81-4D08-8791-8653949C0FEB} + {349c5851-65df-11da-9384-00065b846f21};{fae04ec0-301f-11d3-bf4b-00c04f79efbc} + Library + Properties + BExIS.Modules.Smm.UI + BExIS.Modules.Smm.UI + v4.8 + true + + + + + + + + + + + + true + full + false + false + bin\ + DEBUG;TRACE + prompt + 4 + + + pdbonly + true + true + bin\ + TRACE + prompt + 4 + + + + ..\..\..\..\packages\Microsoft.AspNet.Identity.Core.2.2.4\lib\net45\Microsoft.AspNet.Identity.Core.dll + + + + ..\..\..\..\..\packages\Microsoft.AspNet.WebHelpers.3.2.8\lib\net45\Microsoft.Web.Helpers.dll + + + ..\..\..\..\..\packages\Microsoft.Web.Infrastructure.2.0.0\lib\net40\Microsoft.Web.Infrastructure.dll + + + ..\..\..\..\packages\Newtonsoft.Json.13.0.3\lib\net45\Newtonsoft.Json.dll + + + + + + + ..\..\..\..\packages\Microsoft.AspNet.WebApi.Client.5.2.3\lib\net45\System.Net.Http.Formatting.dll + + + ..\..\..\..\..\packages\System.Runtime.CompilerServices.Unsafe.6.0.0\lib\net461\System.Runtime.CompilerServices.Unsafe.dll + + + + ..\..\..\..\..\packages\Microsoft.AspNet.WebPages.3.2.9\lib\net45\System.Web.Helpers.dll + + + ..\..\..\..\packages\Microsoft.AspNet.WebApi.Core.5.2.3\lib\net45\System.Web.Http.dll + + + ..\..\..\..\..\packages\Microsoft.AspNet.Mvc.5.2.8\lib\net45\System.Web.Mvc.dll + + + ..\..\..\..\..\packages\Microsoft.AspNet.Razor.3.2.9\lib\net45\System.Web.Razor.dll + + + + ..\..\..\..\..\packages\Microsoft.AspNet.WebPages.3.2.9\lib\net45\System.Web.WebPages.dll + + + ..\..\..\..\..\packages\Microsoft.AspNet.WebPages.3.2.9\lib\net45\System.Web.WebPages.Deployment.dll + + + ..\..\..\..\..\packages\Microsoft.AspNet.WebPages.3.2.9\lib\net45\System.Web.WebPages.Razor.dll + + + + + ..\..\..\..\..\packages\Microsoft.AspNet.WebPages.Data.3.2.9\lib\net45\WebMatrix.Data.dll + + + ..\..\..\..\..\packages\Microsoft.AspNet.WebPages.WebData.3.2.9\lib\net45\WebMatrix.WebData.dll + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Web.config + + + Web.config + + + Web.config + + + + +<<<<<<< Updated upstream + + +======= +>>>>>>> Stashed changes + + + + + {c230693b-d780-438b-b26c-82257642dd5c} + BExIS.Security.Entities + + + {681c8bc4-55f0-4f43-a685-90e246a88cb1} + BExIS.Security.Services + + + {7D7FBF8E-37D7-4A4C-B40E-3F267E9B9760} + BExIS.App.Bootstrap + + + {b4e7b1bf-01b4-40af-8d19-b8f362167261} + BExIS.Dlm.Entities + + + {c4ca0a99-0af3-4372-a9b7-b9073599bd8b} + BExIS.Dlm.Services + + + {c8a05313-b960-406e-92ec-c1e5b3f47fcd} + BExIS.IO.Transform.Validation + + + {DE0AD99C-C559-422F-8132-CC4D7C46FF83} + BExIS.UI + + + {0815d220-3625-4e23-bbbc-8152345637fe} + Vaiona.Entities + + + {e8b37581-1cac-463d-903b-b4bee8b2b0e3} + Vaiona.Logging + + + {640bf81d-354a-4bf0-85fc-f0ad587cf8a2} + Vaiona.Persistence.Api + + + {63fcacaa-9534-4fdd-a082-78dcc06baf28} + Vaiona.Utils + + + {705f8751-e58a-453e-a7fd-0c310fd3cae8} + Vaiona.Web.Mvc.Modularity + + + {5f5d22e8-8c05-49cd-854e-8fe8eff1aa6c} + Vaiona.Web.Mvc + + + {782B71C1-707F-4AB1-80E9-90D2880635B4} + BExIS.Utils + + + {252F7872-A69C-43A6-84B4-4D2ABDBDD9AB} + BExIS.Xml.Helpers + + + + + + + + 10.0 + $(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion) + + + bin\ + TRACE + true + pdbonly + AnyCPU + prompt + + + + + + + + + + + + True + True + 16528 + / + http://localhost:16528/ + False + False + + + False + + + + + + mkdir "$(SolutionDir)Console\Workspace\Modules\SMM" +C:\Windows\System32\xcopy "$(ProjectDir)Smm.Settings.json" "$(SolutionDir)Console\Workspace\Modules\SMM" /C /Y /I /D + + + \ No newline at end of file diff --git a/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.SMM.UI.Svelte/src/routes/+page.svelte b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.SMM.UI.Svelte/src/routes/+page.svelte index bc202b1c6d..a5d5481483 100644 --- a/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.SMM.UI.Svelte/src/routes/+page.svelte +++ b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.SMM.UI.Svelte/src/routes/+page.svelte @@ -12,10 +12,13 @@ ); - +
- users - groups + Datasets Overview + Species + Header Mappings + Tailor Result + Progress Overview
diff --git a/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.SMM.UI.Svelte/src/routes/groups/+page.svelte b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.SMM.UI.Svelte/src/routes/groups/+page.svelte deleted file mode 100644 index 5b6a2b5321..0000000000 --- a/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.SMM.UI.Svelte/src/routes/groups/+page.svelte +++ /dev/null @@ -1,10 +0,0 @@ - - - - \ No newline at end of file diff --git a/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.SMM.UI.Svelte/src/routes/species/+page.svelte b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.SMM.UI.Svelte/src/routes/species/+page.svelte new file mode 100644 index 0000000000..b9c2fbc9e6 --- /dev/null +++ b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.SMM.UI.Svelte/src/routes/species/+page.svelte @@ -0,0 +1,249 @@ + + + + {#each files as item} +

{item.name}

+ {/each} + + + + + + + + +

Result

+
+ + + + +
+ +

Accepted

+
+
+ +
+ \ No newline at end of file diff --git a/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.SMM.UI.Svelte/src/routes/species/services.ts b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.SMM.UI.Svelte/src/routes/species/services.ts new file mode 100644 index 0000000000..734ec01595 --- /dev/null +++ b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.SMM.UI.Svelte/src/routes/species/services.ts @@ -0,0 +1,3 @@ +// Implementations for all the calls for the pokemon endpoints. +//import Api from "./Api"; +import { Api } from '@bexis2/bexis2-core-ui'; diff --git a/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.SMM.UI.Svelte/src/routes/species/types.ts b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.SMM.UI.Svelte/src/routes/species/types.ts new file mode 100644 index 0000000000..b1c6051842 --- /dev/null +++ b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.SMM.UI.Svelte/src/routes/species/types.ts @@ -0,0 +1,4 @@ +export interface SpeciesModel { + count: number; + name: string; +} \ No newline at end of file diff --git a/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.SMM.UI.Svelte/src/routes/users/+page.svelte b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.SMM.UI.Svelte/src/routes/users/+page.svelte deleted file mode 100644 index a5003d03ff..0000000000 --- a/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.SMM.UI.Svelte/src/routes/users/+page.svelte +++ /dev/null @@ -1,9 +0,0 @@ - - - \ No newline at end of file diff --git a/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.SMM.UI.Svelte/svelte.config.js b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.SMM.UI.Svelte/svelte.config.js index d43ccb685e..d27b7b2bd7 100644 --- a/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.SMM.UI.Svelte/svelte.config.js +++ b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.SMM.UI.Svelte/svelte.config.js @@ -10,8 +10,8 @@ const config = { preprocess: vitePreprocess(), kit: { adapter: adapter({ - pages: '../BExIS.Modules.Sam.UI/Scripts/svelte', // ../BExIS.Modules.Dcm.UI/Scripts/svelte - assets: '../BExIS.Modules.Sam.UI/Scripts/svelte', // ../BExIS.Modules.Dcm.UI/Scripts/svelte + pages: '../BExIS.Modules.Smm.UI/Scripts/svelte', // ../BExIS.Modules.Dcm.UI/Scripts/svelte + assets: '../BExIS.Modules.Smm.UI/Scripts/svelte', // ../BExIS.Modules.Dcm.UI/Scripts/svelte fallback: null, precompress: true, preprocess: true, @@ -19,7 +19,7 @@ const config = { }), paths: { relative: true, - base: process.env.NODE_ENV === 'production' ? '/sam' : '' // add module id here, + base: process.env.NODE_ENV === 'production' ? '/smm' : '' // add module id here, }, alias: { diff --git a/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.SMM.UI/BExIS.Modules.SMM.UI.csproj b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.SMM.UI/BExIS.Modules.SMM.UI.csproj index 8800c81737..cef9e243ef 100644 --- a/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.SMM.UI/BExIS.Modules.SMM.UI.csproj +++ b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.SMM.UI/BExIS.Modules.SMM.UI.csproj @@ -49,6 +49,15 @@ ..\..\..\..\packages\Microsoft.AspNet.Identity.Core.2.2.4\lib\net45\Microsoft.AspNet.Identity.Core.dll + + ..\..\..\..\..\packages\Microsoft.IdentityModel.JsonWebTokens.5.7.0\lib\net461\Microsoft.IdentityModel.JsonWebTokens.dll + + + ..\..\..\..\..\packages\Microsoft.IdentityModel.Logging.5.7.0\lib\net461\Microsoft.IdentityModel.Logging.dll + + + ..\..\..\..\..\packages\Microsoft.IdentityModel.Tokens.5.7.0\lib\net461\Microsoft.IdentityModel.Tokens.dll + ..\..\..\..\..\packages\Microsoft.AspNet.WebHelpers.3.2.9\lib\net45\Microsoft.Web.Helpers.dll @@ -56,11 +65,18 @@ ..\..\..\..\..\packages\Microsoft.Web.Infrastructure.2.0.0\lib\net40\Microsoft.Web.Infrastructure.dll - ..\..\..\..\packages\Newtonsoft.Json.13.0.3\lib\net45\Newtonsoft.Json.dll + ..\..\..\..\..\packages\Newtonsoft.Json.13.0.1\lib\net45\Newtonsoft.Json.dll + + + + ..\..\..\..\..\packages\System.IdentityModel.Tokens.Jwt.5.7.0\lib\net461\System.IdentityModel.Tokens.Jwt.dll + + + ..\..\..\..\packages\Microsoft.AspNet.WebApi.Client.5.2.3\lib\net45\System.Net.Http.Formatting.dll @@ -106,10 +122,28 @@ + + + + + + + + + + + + + + + + + + - + @@ -138,13 +172,14 @@ - - + + - + + Web.config @@ -157,8 +192,6 @@ - - @@ -169,6 +202,10 @@ {681c8bc4-55f0-4f43-a685-90e246a88cb1} BExIS.Security.Services + + {7D7FBF8E-37D7-4A4C-B40E-3F267E9B9760} + BExIS.App.Bootstrap + {b4e7b1bf-01b4-40af-8d19-b8f362167261} BExIS.Dlm.Entities @@ -177,10 +214,22 @@ {c4ca0a99-0af3-4372-a9b7-b9073599bd8b} BExIS.Dlm.Services + + {455EC826-9A92-40FF-BD3B-388C288955CE} + BExIS.IO.Transform.Output + {c8a05313-b960-406e-92ec-c1e5b3f47fcd} BExIS.IO.Transform.Validation + + {DE0AD99C-C559-422F-8132-CC4D7C46FF83} + BExIS.UI + + + {6EAD7D02-02F7-42FF-85E4-90BB892D3846} + BExIS.Utils.Config + {0815d220-3625-4e23-bbbc-8152345637fe} Vaiona.Entities @@ -213,6 +262,14 @@ {252F7872-A69C-43A6-84B4-4D2ABDBDD9AB} BExIS.Xml.Helpers + + {34CAD2A2-6928-458B-B8DC-AF71D55F20CC} + BExIS.Dim.Entities + + + {9BFFFD11-03C6-47DF-9CC9-F458A9A49377} + BExIS.Modules.Dim.UI + diff --git a/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.SMM.UI/Controllers/SpeciesController.cs b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.SMM.UI/Controllers/SpeciesController.cs new file mode 100644 index 0000000000..421f155111 --- /dev/null +++ b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.SMM.UI/Controllers/SpeciesController.cs @@ -0,0 +1,1205 @@ +using BExIS.App.Bootstrap.Attributes; +using BExIS.App.Bootstrap.Helpers; +using BExIS.Dim.Entities.Export.GBIF; +using BExIS.Dlm.Entities.Data; +using BExIS.Dlm.Entities.DataStructure; +using BExIS.Dlm.Entities.SpeciesMatching; +using BExIS.Dlm.Services.Data; +using BExIS.Dlm.Services.SpeciesMatching; +using BExIS.IO.Transform.Output; +using BExIS.Modules.Dim.UI.Models.Api; +using BExIS.Modules.Smm.UI.Helpers; +using BExIS.Modules.Smm.UI.Helpers.MatchingAPIs; +using BExIS.Modules.Smm.UI.Models; +using BExIS.Security.Entities.Authorization; +using BExIS.Security.Entities.Requests; +using BExIS.Security.Entities.Subjects; +using BExIS.Security.Services.Authorization; +using BExIS.Security.Services.Objects; +using BExIS.Security.Services.Subjects; +using BExIS.UI.Helpers; +using BExIS.UI.Models; +using BExIS.Utils.Config; +using Microsoft.IdentityModel.Tokens; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using System; +using System.Collections.Generic; +using System.Data; +using System.Diagnostics; +using System.IdentityModel.Tokens.Jwt; +using System.IO; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Runtime.Remoting.Metadata.W3cXsd2001; +using System.Security.Claims; +using System.Text; +using System.Threading.Tasks; +using System.Web.Mvc; +using Vaiona.Persistence.Api; +using Vaiona.Utils.Cfg; +using Vaiona.Web.Mvc.Modularity; + + +namespace BExIS.Modules.Smm.UI.Controllers +{ + public class SpeciesController : Controller + { + // GET: Species + + // Provides access to file based matching Api functions (file creation, - reading, matching via request, ...) + MatchingApiProvider matchingApiProvider = new Helpers.MatchingAPIs.MatchingApiProvider(); + + public ActionResult Index() + { + string module = "SMM"; + + ViewData["app"] = SvelteHelper.GetApp(module); + ViewData["start"] = SvelteHelper.GetStart(module); + + return View(); + } + + [JsonNetFilter] + [HttpGet] + public async Task StartDownloadResultFile(long datasetId, long versionId, int stepId) + { + var user = ResolveUserAndRights(datasetId, out ActionResult errorResult); + if (user == null) + { + return JsonWithStatus(new { success = false, id = datasetId, message = "Authentification error." }, HttpStatusCode.Unauthorized, JsonRequestBehavior.AllowGet); + } + + var matchingProgressLocal = ProgressHelper.LoadMatchingProgress(datasetId, versionId); + if (matchingProgressLocal == null) + { + return JsonWithStatus(new { success = false, id = datasetId, message = "No matching progress found." }, HttpStatusCode.Conflict, JsonRequestBehavior.AllowGet); + } + + try + { + var stepLocal = matchingProgressLocal.GetStepById(stepId); + var apiIdentifier = stepLocal.ApiIdentifier; + + MatchingApiBase apiBase = matchingApiProvider.GetApi(apiIdentifier); + Debug.WriteLine("Starting DownloadResultFile function with apiIdentifier: ", apiIdentifier); + var downloadedFilepath = await apiBase.DownloadResultFile(datasetId, versionId, stepId, matchingProgressLocal); + + if (string.IsNullOrWhiteSpace(downloadedFilepath)) + { + return JsonWithStatus(new { success = false, id = datasetId, message = "Failed to download or extract matching result file." }, HttpStatusCode.Conflict, JsonRequestBehavior.AllowGet); + } + + // TODO: this should never fail + // update matching progress entry + if (stepLocal != null) + { + stepLocal.ResultFileName = Path.GetFileName(downloadedFilepath); + matchingProgressLocal.UpdateStep(stepLocal); + ProgressHelper.SaveMatchingProgress(matchingProgressLocal, datasetId, versionId); + } + + // return filepath for client to use (or filename) + return Json(new { success = true, id = datasetId }, JsonRequestBehavior.AllowGet); + } + catch (KeyNotFoundException) + { + return JsonWithStatus(new { success = false, id = datasetId, message = "Matching API not found for the given identifier." }, HttpStatusCode.Conflict, JsonRequestBehavior.AllowGet); + } + catch (Exception ex) + { + return JsonWithStatus(new { success = false, id = datasetId, message = "Unexpected error while downloading matching result file: " + ex.Message }, HttpStatusCode.InternalServerError, JsonRequestBehavior.AllowGet); + } + } + + [JsonNetFilter] + [HttpGet] + // Returns status information about a matching result file (exists, downloading, matching progress coherence) + public JsonResult GetMatchingFileStatus(long datasetId, long versionId, int stepId) + { + var user = ResolveUserAndRights(datasetId, out ActionResult errorResult); + if (user == null) + { + return JsonWithStatus(new { success = false, id = datasetId, message = "Authentification error." }, HttpStatusCode.Unauthorized, JsonRequestBehavior.AllowGet); + } + + try + { + // locate versioned matching directory (may be null if folder missing) + var directory = ProgressHelper.GetVersionedMatchingPath(datasetId, versionId); + bool directoryExists = !string.IsNullOrEmpty(directory); + + // expected final filepath for matched csv + var filename = ProgressHelper.GenMatchingFileName(true, datasetId, stepId); + string filepath = directoryExists ? Path.Combine(directory, filename) : Path.Combine(AppConfiguration.DataPath, "Datasets", datasetId.ToString(), ProgressHelper.MatchingFolderName, versionId.ToString(), filename); + + bool fileExists = System.IO.File.Exists(filepath); + + // marker file indicates active download + var markerPath = filepath + ".downloading"; + bool markerExists = System.IO.File.Exists(markerPath); + + // inspect marker for potential staleness + bool markerStale = false; + DateTime? markerStart = null; + if (markerExists) + { + try + { + var content = System.IO.File.ReadAllText(markerPath); + // look for a line starting with download_start= + foreach (var line in content.Split(new[] { '\n', '\r' }, StringSplitOptions.RemoveEmptyEntries)) + { + var trimmed = line.Trim(); + if (trimmed.StartsWith("download_start=", StringComparison.OrdinalIgnoreCase)) + { + var ts = trimmed.Substring("download_start=".Length); + if (DateTime.TryParse(ts, null, System.Globalization.DateTimeStyles.RoundtripKind, out DateTime dt)) + { + markerStart = dt.ToUniversalTime(); + } + break; + } + } + + if (markerStart.HasValue) + { + markerStale = (DateTime.UtcNow - markerStart.Value) > TimeSpan.FromHours(24); + } + } + catch { /* ignore parsing errors */ } + } + + // matching progress and step info + var matchingProgress = ProgressHelper.LoadMatchingProgress(datasetId, versionId); + bool matchingProgressExists = matchingProgress != null; + var step = matchingProgress?.GetStepById(stepId); + bool stepExists = step != null; + + return Json(new + { + success = true, + data = new + { + DirectoryExists = directoryExists, + FileExists = fileExists, + MarkerExists = markerExists, + MarkerStale = markerStale, + MarkerStart = markerStart, + MatchingProgressExists = matchingProgressExists, + StepExists = stepExists, + StepCompleted = step?.IsCompleted() ?? false, + DownloadLinkPresent = !string.IsNullOrWhiteSpace(step?.DownloadLink), + JobKeyPresent = !string.IsNullOrWhiteSpace(step?.JobKey) + } + }, JsonRequestBehavior.AllowGet); + } + catch (Exception ex) + { + Debug.WriteLine("Error while checking matching file status: " + ex.Message); + return JsonWithStatus(new { success = false, id = datasetId, message = "Error while checking matching file status." }, HttpStatusCode.Conflict, JsonRequestBehavior.AllowGet); + } + } + + + + private static readonly HttpClient _httpClient = new HttpClient(); + + [JsonNetFilter] + [HttpGet] + public JsonResult GetMyDatasetsJson() + { + var result = new List(); + const string EntityName = "Dataset"; + const RightType RightTypeCondition = RightType.Write; + + var user = ResolveRouteUser(out ActionResult userError); + if (user == null) + { + return JsonWithStatus(new { success = false, message = "User could not be resolved from route." }, HttpStatusCode.Unauthorized, JsonRequestBehavior.AllowGet); + } + + string username = user.Name; + // TODO: - CHANGE - JUST FOR TESTING + if (string.IsNullOrWhiteSpace(username)) username = "erik"; + + using (var datasetManager = new DatasetManager()) + using (var entityPermissionManager = new EntityPermissionManager()) + using (var entityManager = new EntityManager()) + using (var speciesMatchingResultManager = new SpeciesMatchingResultManager()) + { + // Find entity (defensive) + var entity = entityManager.FindByName(EntityName); + if (entity == null) + { + return JsonWithStatus(new { error = $"Entity '{EntityName}' not found." }, HttpStatusCode.Conflict, JsonRequestBehavior.AllowGet); + } + + // collect dataset ids the current user has the requested right for + List datasetIds = entityPermissionManager.GetKeys(username, EntityName, typeof(Dataset), RightTypeCondition).Result ?? new List(); + + var smrmRepo = speciesMatchingResultManager.GetBulkUnitOfWork().GetReadOnlyRepository(); + + // For each dataset id retrieve all versions (and working copy if present) and include version id and version number + foreach (var dsId in datasetIds) + { + try + { + var versions = datasetManager.GetDatasetVersions(dsId) ?? new List(); + + if (versions == null || versions.Count == 0) continue; + + // choose a representative version for dataset-level info (prefer latest by timestamp) + var representative = versions.OrderByDescending(v => v.Timestamp).First(); + + bool isTabular = representative.Dataset.DataStructure?.Self is StructuredDataStructure; + + // we only consider tabular datasets + if (!isTabular) continue; + + bool metadataComplete = false; + if (representative.StateInfo != null) + { + metadataComplete = string.Equals(representative.StateInfo.State, DatasetStateInfo.Valid.ToString(), StringComparison.OrdinalIgnoreCase); + } + + bool hasSpeciesMatches = smrmRepo.Query().Any(r => r.Dataset.Id == dsId); + + // build versions info list + var orderedVersions = versions.OrderBy(v => v.Timestamp).ToList(); + + List versionsInfo; + + if (!hasSpeciesMatches) + { + // If there are no species matches for this dataset at all, only include the latest version info + var v = orderedVersions.Last(); + versionsInfo = new List + { + new + { + VersionId = v.Id, + VersionNr = datasetManager.GetDatasetVersionNr(v), + Timestamp = v.Timestamp, + Status = v.Status.ToString(), + VersionName = v.VersionName ?? string.Empty, + HasMatchingProgress = false + } + }; + } + else + { + // Iterate from latest to oldest and apply header-mapping rules + var descVersions = orderedVersions.OrderByDescending(v => v.Timestamp).ToList(); + var filtered = new List(); + + for (int i = 0; i < descVersions.Count; i++) + { + var v = descVersions[i]; + bool isLatest = (i == 0); + + bool hasHeader = ProgressHelper.HasHeaderMappings(dsId, v.Id); + + if (!hasHeader) + { + if (isLatest) + { + // keep latest but mark matching progress as false + filtered.Add(new + { + VersionId = v.Id, + VersionNr = datasetManager.GetDatasetVersionNr(v), + Timestamp = v.Timestamp, + Status = v.Status.ToString(), + VersionName = v.VersionName ?? string.Empty, + HasMatchingProgress = false + }); + } + else + { + // drop this older version without header mappings + continue; + } + } + else + { + // keep version and mark that header mappings exist + filtered.Add(new + { + VersionId = v.Id, + VersionNr = datasetManager.GetDatasetVersionNr(v), + Timestamp = v.Timestamp, + Status = v.Status.ToString(), + VersionName = v.VersionName ?? string.Empty, + HasMatchingProgress = true + }); + } + } + + // filtered currently in descending order (latest first). Return ascending to keep previous ordering. + versionsInfo = filtered.OrderBy(v => ((DateTime)((dynamic)v).Timestamp)).ToList(); + } + + result.Add(new + { + Id = representative.Dataset.Id, + Title = representative.Title ?? string.Empty, + Abstract = representative.Description ?? string.Empty, + IsTabular = isTabular, + MetadataComplete = metadataComplete, + HasMatchingProgress = hasSpeciesMatches, + DataStructureId = representative.Dataset.DataStructure?.Id, + Versions = versionsInfo + }); + } + catch (Exception ex) + { + Debug.WriteLine("Error while retrieving versions for dataset " + dsId + ": " + ex.Message); + // ignore dataset on error and continue with others + continue; + } + } + } + + return Json(result, JsonRequestBehavior.AllowGet); + } + + [JsonNetFilter] + [HttpPost] + public JsonResult SubmitHeaderMappings(SubmitHeaderMappingsRequest request) + { + // basic model binding validation: ensure payload present and DatasetId provided (>0) + if (request.Data == null) + { + return JsonWithStatus(new { success = false, message = "Request body missing or invalid." }, HttpStatusCode.BadRequest); + } + + var data = request.Data; + var datasetId = data.DatasetId; + var versionId = request.VersionId; + + if (!ModelState.IsValid) + { + // var errors = ModelState.Values.SelectMany(v => v.Errors).Select(e => e.ErrorMessage).Where(m => !string.IsNullOrWhiteSpace(m)).ToList(); + // if (!errors.Any()) errors.Add("Invalid request payload."); + return JsonWithStatus(new { success = false, message = "Validation failed." }, HttpStatusCode.BadRequest); + } + + var user = ResolveUserAndRights(datasetId, out ActionResult errorResult); + if (user == null) + { + return JsonWithStatus(new { success = false, id = datasetId, message = "Authentification error." }, HttpStatusCode.Unauthorized); + } + + if (ProgressHelper.HasHeaderMappings(datasetId, versionId)) + { + return JsonWithStatus(new { success = false, id = datasetId, message = "Header Mappings already exist and cannot be directly overwritten." }, HttpStatusCode.InternalServerError); + } + + var folderSuccess = ProgressHelper.CreateMatchingFolder(datasetId, versionId); + if (!folderSuccess) + { + return JsonWithStatus(new { success = false, id = datasetId, message = "Failed to create matching folder." }, HttpStatusCode.InternalServerError); + } + + var success = ProgressHelper.CreateHeaderMappingsFile(data, datasetId, versionId, out string errorMessage); + + if (success) + { + return Json(new { success = true, id = datasetId }); + } else + { + return JsonWithStatus(new + { + success = false, + message = errorMessage + }, HttpStatusCode.BadRequest); + } + } + + [JsonNetFilter] + [HttpPost] + // Calls Datastatistic API with the given dataset id and variable id, creates a SpeciesMatchingResult for each unique name and saves to database. + public async Task Tailor(long datasetId, long versionId) + { + /* + This method is supposed to be called after the user has set up the header mappings and wants to start the matching process. + It should then call the DataStatistic API with the given dataset id and variable id and create a SpeciesMatchingResult for each + individual name. The API call needs to be authenticated with a JWT token, which we can generate here with a custom method. + */ + + var user = ResolveUserAndRights(datasetId, out ActionResult errorResult); + if (user == null) + { + return JsonWithStatus(new { success = false, id = datasetId, message = "Authentification error." }, HttpStatusCode.Unauthorized); + } + + // load header mappings for this dataset + var headerMappings = ProgressHelper.LoadHeaderMappings(datasetId, versionId); + if (headerMappings == null) + { + return JsonWithStatus(new { success = false, id = datasetId, message = "Header mappings not found for this dataset." }, HttpStatusCode.Conflict); + } + + // find variable id for scientific name field + long? targetVariableId = headerMappings.GetVariableIdForScientificName(); + if (targetVariableId == null) { + return JsonWithStatus(new { success = false, id = datasetId, message = "No variable mapped for scientific name found in header mappings." }, HttpStatusCode.Conflict); + } + + // check matching progress + if (ProgressHelper.HasMatchingProgress(datasetId, versionId)) + { + return JsonWithStatus(new { success = false, id = datasetId, message = "Matching progress file already exists for this dataset. Please complete or reset existing matching progress before starting a new tailoring process." }, HttpStatusCode.Conflict); + } + + // generate token for local api call + string jwtToken = GenerateCustomJwtToken(); + if (jwtToken == null) + { + Debug.WriteLine("JWT token generation failed."); + return JsonWithStatus(new { success = false, id = datasetId, message = "Could not call internal API. Please try again later." }, HttpStatusCode.Conflict); + } + + var result = await TailorDataset(datasetId, targetVariableId.Value, jwtToken); + var list_result = JsonConvert.DeserializeObject>(result.Content); + ApiDataStatisticModel json_result = list_result.FirstOrDefault(); + + if (!result.IsSuccess) + { + return JsonWithStatus(new { success = false, id = datasetId, message = "Api call failed " + result.Content }, HttpStatusCode.BadRequest); + } + + using (var speciesMatchingResultManager = new SpeciesMatchingResultManager()) + using (var uow = speciesMatchingResultManager.GetBulkUnitOfWork()) + using (var datasetManager = new DatasetManager()) + { + try + { + // check if rows present in API result + if (json_result.uniqueValues == null || json_result.uniqueValues.Rows.Count <= 0) + { + return JsonWithStatus(new { success = false, id = datasetId, message = "DataStatisticApi returned no meaningful result." }, HttpStatusCode.Conflict); + } + + // check if expected column "var" is present in the API result + if (!json_result.uniqueValues.Columns.Contains("var")) + { + return JsonWithStatus(new { success = false, id = datasetId, message = "Column variable 'var' was missing on the Api result." }, HttpStatusCode.Conflict); + } + + var repo = uow.GetRepository(); + var dataset = datasetManager.DatasetRepo.Get(datasetId); + var placeHolderTimeStamp = DateTime.Now; + + // load existing species matching results for this dataset (if any) + var existingMatchesQuery = repo.Query().Where(r => r.Dataset.Id == datasetId); + bool hasMatchesForThisVersion = existingMatchesQuery.Any(r => r.DatasetVersionId == versionId); + + // If there are already matches for the same dataset version, do not allow re-tailoring the same version + if (hasMatchesForThisVersion) + { + return JsonWithStatus(new { success = false, id = datasetId, message = "Species matching results already exist for this dataset version. Please complete or reset existing matching progress before starting a new tailoring process." }, HttpStatusCode.BadRequest); + } + + // Build a set of already processed original names for the dataset (across all versions) + var existingOriginalNames = new HashSet(StringComparer.OrdinalIgnoreCase); + foreach (var em in existingMatchesQuery.Select(r => r.OriginalName)) + { + if (!string.IsNullOrWhiteSpace(em)) existingOriginalNames.Add(em); + } + + int rowCount = 0; + + // create one row in SpeciesMatchingResult per unique row from DataStatistic Api + foreach (DataRow row in json_result.uniqueValues.Rows) + { + if (row["var"] == DBNull.Value) continue; + string varValue = row["var"].ToString(); + + // skip names that were already processed for this dataset (from previous tailoring runs) + if (existingOriginalNames.Contains(varValue)) continue; + + // create row for newly encountered original name + var matchingResult = new SpeciesMatchingResult + { + OriginalName = varValue, + EditedName = "", + MatchedName = "", + Status = "", + MatchType = "", + TimestampMatch = placeHolderTimeStamp, + MatchSource = "", + MatchSourceVersion = "", + ConfirmedByUser = false, + Dataset = dataset, + DatasetVersionId = versionId + }; + + repo.Put(matchingResult); + rowCount++; + existingOriginalNames.Add(varValue); + } + + // TODO: check success (but in general should be built in a way that it never fails) + ProgressHelper.CreateMatchingProgressFile(datasetId, versionId, rowCount); + + // batch commit + uow.Commit(); + } + catch (Exception ex) + { + // ignore on failure to avoid partial commits and inconsistent state; the user can then try again after fixing the underlying issue + uow.Ignore(); + Debug.WriteLine("Custom exception catch in Tailor."); + return JsonWithStatus(new { success = false, id = datasetId, message = "An error occured while processing the Api result: " + ex.Message }, HttpStatusCode.InternalServerError); + } + } + + return Json(new { success = true, id = datasetId, message = json_result }); + } + + [JsonNetFilter] + [HttpGet] + // Get ALL SpeciesMatchingResults for a given dataset. + // Used to display the overall state of the matching results in the frontend, and to allow users to filter and edit. + public JsonResult ViewTailored(long datasetId, long versionId) + { + var user = ResolveUserAndRights(datasetId, out ActionResult errorResult); + if (user == null) + { + return JsonWithStatus(new { success = false, id = datasetId, message = "Authentification error." }, HttpStatusCode.Unauthorized, JsonRequestBehavior.AllowGet); + } + + var result = MatchingResultHelper.GetAll(datasetId, versionId); + + if (result == null) + { + return JsonWithStatus(new { success = false, id = datasetId, message = "No matching results found for this dataset." }, HttpStatusCode.NotFound, JsonRequestBehavior.AllowGet); + } else + { + return Json(new + { + succes = true, + id = datasetId, + message = result + }, JsonRequestBehavior.AllowGet + ); + } + } + + [JsonNetFilter] + [HttpGet] + public JsonResult ViewProgress(long datasetId, long versionId) + { + var user = ResolveUserAndRights(datasetId, out ActionResult errorResult); + if (user == null) + { + return JsonWithStatus(new { success = false, id = datasetId, message = "Authentification error." }, HttpStatusCode.Unauthorized, JsonRequestBehavior.AllowGet); + } + + try + { + // header mappings + bool hasHeaderMappings = ProgressHelper.HasHeaderMappings(datasetId, versionId); + var headerMappings = hasHeaderMappings ? ProgressHelper.LoadHeaderMappings(datasetId, versionId) : null; + + // tailored check: any SpeciesMatchingResult entries for this dataset and version? + bool isTailored = false; + using (var smrm = new SpeciesMatchingResultManager()) + { + var repo = smrm.GetBulkUnitOfWork().GetReadOnlyRepository(); + isTailored = repo.Query().Any(r => r.Dataset.Id == datasetId && r.DatasetVersionId == versionId); + } + + // matching progress + bool hasMatchingProgress = ProgressHelper.HasMatchingProgress(datasetId, versionId); + var matchingProgress = hasMatchingProgress ? ProgressHelper.LoadMatchingProgress(datasetId, versionId) : null; + ExternalApiMetadata externalApiMetadata = ModuleManager.GetModuleSettings("SMM").GetValueByKey("externalApiMetadata"); + + + return Json(new + { + success = true, + hasHeaderMappings, + headerMappings, + isTailored, + hasMatchingProgress, + matchingProgress, + externalApiMetadata + }, JsonRequestBehavior.AllowGet); + } + catch (Exception ex) + { + Debug.WriteLine("Error while building progress view: " + ex); + return JsonWithStatus(new { success = false, id = datasetId, message = "Error while retrieving progress information." }, HttpStatusCode.Conflict, JsonRequestBehavior.AllowGet); + } + + } + + [JsonNetFilter] + [HttpPost] + public JsonResult GenNewMatchInputFile(long datasetId, long versionId, string apiIdentifier) + { + var user = ResolveUserAndRights(datasetId, out ActionResult errorResult); + if (user == null) + { + return JsonWithStatus(new { success = false, id = datasetId, message = "Authentification error." }, HttpStatusCode.Unauthorized); + } + + var matchingProgress = ProgressHelper.LoadMatchingProgress(datasetId, versionId); + if (matchingProgress == null) + { + return JsonWithStatus(new { success = false, id = datasetId, message = "No matching progress found." }, HttpStatusCode.Unauthorized); + } + + if (!matchingProgress.AllStepsCompleted()) + { + return JsonWithStatus(new { success = false, id = datasetId, message = "Not all matching steps are completed yet. Please complete existing steps before generating a new matching input file." }, HttpStatusCode.Conflict); + } + + var newStepId = matchingProgress.GetNewId(); + var datastructureId = GetDatastructureIdFromDatasetId(datasetId); + if (datastructureId == null) + { + return JsonWithStatus(new { success = false, id = datasetId, message = "Dataset or datastructure not found." }, HttpStatusCode.NotFound); + } + + try + { + MatchingApiBase apiBase = matchingApiProvider.GetApi(apiIdentifier); + var (FilePath, RowCount) = apiBase.GenerateInputFile(datasetId, datastructureId.Value, versionId, newStepId); + string filepath = FilePath; + int rows = RowCount; + if (filepath == null) + { + return JsonWithStatus(new { success = false, id = datasetId, message = "Could not generate MatchingInput file." }, HttpStatusCode.Conflict); + } + + // this is double generated, but simpler + var filename = ProgressHelper.GenMatchingFileName(false, datasetId, newStepId); + + matchingProgress.AddStep(newStepId, rows, filename, apiIdentifier); + ProgressHelper.SaveMatchingProgress(matchingProgress, datasetId, versionId); + } catch (KeyNotFoundException ex) + { + return JsonWithStatus(new { success = false, id = datasetId, message = "Matching API not found for the given identifier." }, HttpStatusCode.Conflict); + } catch (Exception ex) + { + return JsonWithStatus(new { success = false, id = datasetId, message = "Unexpected error while generating matching input file: " + ex.Message }, HttpStatusCode.InternalServerError); + } + + return Json(new { success = true, data = new { id = datasetId, stepId = newStepId }, message = "Matching input file generated." }); + } + + [JsonNetFilter] + [HttpPost] + public async Task MatchFileByStepId(long datasetId, long versionId, int stepId) + { + Debug.WriteLine("EXECUTING MatchNextFile"); + var user = ResolveUserAndRights(datasetId, out ActionResult errorResult); + if (user == null) + { + return JsonWithStatus(new { success = false, id = datasetId, message = "Authentification error." }, HttpStatusCode.Unauthorized); + } + + var matchingProgress = ProgressHelper.LoadMatchingProgress(datasetId, versionId); + if (matchingProgress == null) + { + return JsonWithStatus(new { success = false, id = datasetId, message = "No matching progress found." }, HttpStatusCode.Unauthorized); + } + + StepEntry step = matchingProgress.GetStepById(stepId); + if (step == null) + { + return JsonWithStatus(new { success = false, id = datasetId, message = "Step not found." }, HttpStatusCode.Conflict); + } + + if (!step.IsReadyToMatch()) + { + return JsonWithStatus(new { success = false, id = datasetId, message = "Step is not ready to match. Either no input file available or step is already completed." }, HttpStatusCode.Conflict); + } + + string nextFileName = step.InputFileName; + string apiIdentifier = step.ApiIdentifier; + + // TODO: - pfad logik vereinfachen + string directory = Path.Combine(AppConfiguration.DataPath, "Datasets", datasetId.ToString(), ProgressHelper.MatchingFolderName, versionId.ToString()); + + if (!Directory.Exists(directory)) + { + Debug.WriteLine("SaveMatchingProgress: dataset directory does not exist: " + directory); + return JsonWithStatus(new { success = false, id = datasetId, message = "No matching input file found on disk." }, HttpStatusCode.Conflict); + } + + string filepath = Path.Combine(directory, nextFileName); + + try + { + // Read raw request body and try to parse options as JSON object + JObject options = null; + try + { + Request.InputStream.Position = 0; + using (var sr = new StreamReader(Request.InputStream, Encoding.UTF8)) + { + var body = sr.ReadToEnd(); + if (!string.IsNullOrWhiteSpace(body)) + { + try + { + options = JObject.Parse(body); + } + catch (JsonException) + { + // try to extract nested property named "options" + try + { + var parsed = JObject.Parse(body); + if (parsed["options"] != null && parsed["options"].Type == JTokenType.Object) + { + options = (JObject)parsed["options"]; + } + } + catch { } + } + } + } + } + catch { /* ignore read errors and keep options null */ } + + IApiOptions apiOptions = matchingApiProvider.ResolveOptions(apiIdentifier, options); + + Debug.WriteLine(apiOptions); + + MatchingApiBase apiBase = matchingApiProvider.GetApi(apiIdentifier); + MatchingApiResponse response = await apiBase.MatchAsync(datasetId, versionId, filepath, matchingProgress, apiOptions); + response.StepId = step.Id; + + return Json(new { success = true, data = response.Message }); + } + catch (ArgumentException ex) + { + return JsonWithStatus(new { success = false, id = datasetId, message = ex.Message }, HttpStatusCode.Conflict); + } + catch (KeyNotFoundException ex) + { + return JsonWithStatus(new { success = false, id = datasetId, message = "Matching API not found for the given identifier." }, HttpStatusCode.Conflict); + } + catch (Exception ex) + { + return JsonWithStatus(new { success = false, id = datasetId, message = "Unexpected error while generating matching input file: " + ex.Message }, HttpStatusCode.InternalServerError); + } + } + + [JsonNetFilter] + [HttpGet] + public JsonResult ViewMatchingResult(long datasetId, long versionId, int stepId) + { + var user = ResolveUserAndRights(datasetId, out ActionResult errorResult); + if (user == null) + { + return JsonWithStatus(new { success = false, id = datasetId, message = "Authentification error." }, HttpStatusCode.Unauthorized, JsonRequestBehavior.AllowGet); + } + + var matchingProgress = ProgressHelper.LoadMatchingProgress(datasetId, versionId); + if (matchingProgress == null) + { + return JsonWithStatus(new { success = false, id = datasetId, message = "No matching progress found under the given datasetId." }, HttpStatusCode.Conflict, JsonRequestBehavior.AllowGet); + } + + if (!matchingProgress.IsCompletedById(stepId)) + { + return JsonWithStatus(new { success = false, id = datasetId, message = "No valid matching job found in the matching progress data for the given stepId." }, HttpStatusCode.Conflict, JsonRequestBehavior.AllowGet); + } + + var filepath = ProgressHelper.GetMatchedFilepath(datasetId, versionId, stepId); + + if (filepath == null) + { + return JsonWithStatus(new { success = false, id = datasetId, message = "No result file found." }, HttpStatusCode.Conflict, JsonRequestBehavior.AllowGet); + } + + try + { + var apiIdentifier = matchingProgress.GetApiIdentifierById(stepId); + Debug.WriteLine("SEARCHING MATCHING PROGRESS FOR: " + datasetId.ToString() + " " + versionId.ToString()); + Debug.WriteLine("API IDENTIFIER: " + apiIdentifier); + MatchingApiBase apiBase = matchingApiProvider.GetApi(apiIdentifier); + var matchingResults = apiBase.ReadResultFile(filepath); + + var acceptableMatchTypes = apiBase.GetAcceptableMatchTypes(); + + // If matching results were read, build a set of IDs from the file and then + // query the SpeciesMatchingResult table for the subset of rows that belong + // to this dataset/version and whose IDs are present in the matching result file. + List dbResults = null; + + if (matchingResults != null && matchingResults.Count > 0) + { + try + { + // collect ids from matching result rows + var idsFromFile = new HashSet(); + foreach (var m in matchingResults) + { + if (string.IsNullOrWhiteSpace(m.Original_ID)) continue; + if (long.TryParse(m.Original_ID, out long mid)) idsFromFile.Add(mid); + } + + Debug.WriteLine("Creating HashSet from file MatchingResultRows:"); + Debug.WriteLine(idsFromFile.Count); + + using (var smrm = new SpeciesMatchingResultManager()) + { + var repo = smrm.GetBulkUnitOfWork().GetReadOnlyRepository(); + + // query only the subset matching dataset/version and the ids from file + dbResults = repo.Query() + .Where(r => r.Dataset.Id == datasetId && r.DatasetVersionId == versionId && idsFromFile.Contains(r.Id)) + .ToList(); + } + } + catch (Exception ex) + { + Debug.WriteLine("Error while querying SpeciesMatchingResult subset: " + ex.Message); + dbResults = null; // fall back to null on error + } + } + + return Json(new { success = true, matchingResults, acceptableMatchTypes, speciesMatchingResults = dbResults }, JsonRequestBehavior.AllowGet); + } + catch (ArgumentException ex) + { + return JsonWithStatus(new { success = false, id = datasetId, message = ex.Message }, HttpStatusCode.Conflict, JsonRequestBehavior.AllowGet); + } + catch (KeyNotFoundException ex) + { + return JsonWithStatus(new { success = false, id = datasetId, message = "Matching API not found for the given identifier." }, HttpStatusCode.Conflict, JsonRequestBehavior.AllowGet); + } + catch (Exception ex) + { + return JsonWithStatus(new { success = false, id = datasetId, message = "Unexpected error while generating matching input file: " + ex.Message }, HttpStatusCode.InternalServerError, JsonRequestBehavior.AllowGet); + } + } + + [JsonNetFilter] + [HttpPost] + public JsonResult AcceptMatches(AcceptMatchesRequestModel request) + { + if (request == null) return JsonWithStatus(new { success = false, message = "Request body missing or invalid." }, HttpStatusCode.BadRequest); + + if (!ModelState.IsValid) return JsonWithStatus(new { success = false, message = "Validation failed." }, HttpStatusCode.BadRequest); + + var datasetId = request.DatasetId; + var versionId = request.VersionId; + var stepId = request.StepId; + + var user = ResolveUserAndRights(datasetId, out ActionResult errorResult); + if (user == null) + { + return JsonWithStatus(new { success = false, id = datasetId, message = "Authentification error." }, HttpStatusCode.Unauthorized); + } + + var matchingProgress = ProgressHelper.LoadMatchingProgress(datasetId, versionId); + if (matchingProgress == null) + { + return JsonWithStatus(new { success = false, id = datasetId, message = "No matching progress found under the given datasetId." }, HttpStatusCode.Conflict); + } + + if (!matchingProgress.IsCompletedById(stepId)) + { + return JsonWithStatus(new { success = false, id = datasetId, message = "No valid matching job found in the matching progress data for the given stepId." }, HttpStatusCode.Conflict); + } + + try + { + var step = matchingProgress.GetStepById(stepId); + var apiIdentifier = step.ApiIdentifier; + MatchingApiBase apiBase = matchingApiProvider.GetApi(apiIdentifier); + + // Convert incoming List MatchIds into a HashSet for fast lookups + var matchIdsSet = ConversionHelper.ConvertStringListToLongHashSet(request.MatchIds); + + apiBase.AcceptMatches(datasetId, versionId, step, matchIdsSet); + } catch (Exception ex) + { + return JsonWithStatus(new { success = false, id = datasetId, message = "Error: " + ex.Message }, HttpStatusCode.BadRequest); + } + + // matchIdsSet is now available for efficient contains checks further down the method + return Json(new { success = true, id = request.DatasetId }); + } + + [JsonNetFilter] + [HttpPost] + public JsonResult ApplyTailorEdits(long datasetId, long versionId, TailorEdit[] edits) + { + // Debug.WriteLine("Received request to apply tailor edits..."); + + if (edits == null) return JsonWithStatus(new { success = false, message = "Request body missing or invalid." }, HttpStatusCode.BadRequest); + + if (!ModelState.IsValid) return JsonWithStatus(new { success = false, message = "Validation failed." }, HttpStatusCode.BadRequest); + + var user = ResolveUserAndRights(datasetId, out ActionResult errorResult); + if (user == null) + { + return JsonWithStatus(new { success = false, id = datasetId, message = "Authentification error." }, HttpStatusCode.Unauthorized); + } + + // Debug.WriteLine("Received request to apply tailor edits for dataset " + datasetId + " version " + versionId); + + try + { + var result = MatchingResultHelper.ApplyTailorEdits(datasetId, versionId, edits.ToList()); + if (result == false) + { + return JsonWithStatus(new { success = false, id = datasetId, message = "Failed to apply edits." }, HttpStatusCode.InternalServerError); + } + + // Debug.WriteLine("Successfully applied tailor edits for dataset " + datasetId + " version " + versionId); + + return Json(new { success = true, id = datasetId }); + } + catch (Exception ex) + { + // Debug.WriteLine("Error while applying tailor edits for dataset " + datasetId + " version " + versionId + ": " + ex); + return JsonWithStatus(new { success = false, id = datasetId, message = "Error while applying edits: " + ex.Message }, HttpStatusCode.BadRequest); + } + } + + public async Task<(bool IsSuccess, string Content)> TailorDataset(long datasetId, long variableId, string jwtToken) + { + string url = "http://localhost:44345/api/DataStatistic/" + datasetId.ToString() + "/" + variableId.ToString(); + + using (var request = new HttpRequestMessage(HttpMethod.Get, url)) + { + request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", jwtToken); + + using (HttpResponseMessage response = await _httpClient.SendAsync(request)) + { + if (response.IsSuccessStatusCode) + { + string apiResponseContent = await response.Content.ReadAsStringAsync(); + + // since response codes are ambiguous, we need to check with this workaround + if (apiResponseContent.StartsWith("[")) + { + // SUCCESS + return (true, apiResponseContent); + } + else + { + // API SOFT FAILURE + return (false, apiResponseContent); + } + + } + else + { + // API HARD FAILURE + Debug.WriteLine("Error: " + response.StatusCode.ToString()); + return (false, "Response status code was not ok." + response.StatusCode.ToString()); + } + } + } + } + + + // Returns the datastructure id for the given dataset id, or null if not found or on error + private long? GetDatastructureIdFromDatasetId(long datasetId) + { + try + { + using (var datasetManager = new DatasetManager()) + { + var datastructureId = datasetManager.DatasetRepo.Query() + .Where(d => d.Id == datasetId) + .Select(d => d.DataStructure != null ? (long?)d.DataStructure.Id : null) + .FirstOrDefault(); + + return datastructureId; + + + } + } + catch (Exception ex) + { + Debug.WriteLine("Error getting datastructure id: " + ex); + return null; + } + } + + // Returns a custom JWT token for authenticating internal API calls + private string GenerateCustomJwtToken() + { + try + { + var jwtConfiguration = GeneralSettings.JwtConfiguration; + + using (var userManager = new UserManager()) + { + // var user = BExISAuthorizeHelper.GetUserFromAuthorizationAsync(HttpContext).Result; + var user = ResolveRouteUser(out ActionResult userError); + + Debug.WriteLine(user.DisplayName, " ", user.Email, " ", user.Id); + + if (user != null) + { + + var securityKey = new SymmetricSecurityKey(Encoding.UTF8.GetBytes(jwtConfiguration.IssuerSigningKey)); + var credentials = new SigningCredentials(securityKey, SecurityAlgorithms.HmacSha256); + + + //Create a List of Claims, Keep claims name short + var permClaims = new List + { + new Claim(JwtRegisteredClaimNames.Jti, Guid.NewGuid().ToString()), + new Claim(ClaimTypes.NameIdentifier, user.Id.ToString()), + new Claim(ClaimTypes.Name, user.UserName) + }; + + + //Create Security Token object by giving required parameters + var token = new JwtSecurityToken(jwtConfiguration.ValidIssuer, + jwtConfiguration.ValidAudience, + permClaims, + notBefore: DateTime.Now, + expires: jwtConfiguration.ValidLifetime > 0 ? DateTime.Now.AddHours(jwtConfiguration.ValidLifetime) : DateTime.MaxValue, + signingCredentials: credentials); + + var jwtToken = new JwtSecurityTokenHandler().WriteToken(token); + return jwtToken; + } + else + { + return null; + } + } + } + catch (Exception ex) + { + return null; + } + } + + // Resolves user from route and checks write rights for the given datasetId. Returns the user if successful, otherwise null + private User ResolveUserAndRights(long datasetId, out ActionResult errorResult) + { + errorResult = null; + var user = ResolveRouteUser(out ActionResult userError); + if (user == null) + { + errorResult = Json(new { success = false, id = datasetId, message = "User could not be resolved from route." }); + return null; + } + if (!EnsureUserHasWriteRights(user.UserName, datasetId, out ActionResult rightsError)) + { + errorResult = Json(new { success = false, id = datasetId, message = "User has no write rights for this dataset." }); + return null; + } + return user; + } + + // Resolves and returns user from the current HttpContext, or null if not possible + private User ResolveRouteUser(out ActionResult errorResult) + { + errorResult = null; + try + { + using (var userManager = new UserManager()) + { + // try token-based resolution first + var user = BExISAuthorizeHelper.GetUserFromAuthorizationAsync(HttpContext).Result; + if (user != null) + { + Debug.WriteLine("User resolved from token: " + user.Name); + return user; + } + + // fallback: try to find a user named 'erik' + var fallback = userManager.Users.FirstOrDefault(u => u.Name == "erik"); + if (fallback != null) + { + Debug.WriteLine("User 'erik' found and used as fallback."); + return fallback; + } + + // final fallback: any available user (default) + var any = userManager.Users.FirstOrDefault(); + if (any != null) + { + Debug.WriteLine("No specific user found; using any available user: " + any.Name); + return any; + } + + Debug.WriteLine("Resolving Route User failed."); + errorResult = Json(new { success = false, message = "User not found in route data." }); + return null; + } + } + catch (Exception ex) + { + Debug.WriteLine("Error resolving user: " + ex.ToString()); + errorResult = Json(new { success = false, message = "Error while resolving user." }); + return null; + } + } + + // Returns TRUE IF the given username has Write rights on the specified datasetId, ELSE FALSE + private bool EnsureUserHasWriteRights(string username, long datasetId, out ActionResult errorResult) + { + errorResult = null; + + if (string.IsNullOrWhiteSpace(username)) + { + errorResult = Json(new { success = false, id = datasetId, message = "Username is missing." }); + return false; + } + + var entityPermissionManager = new EntityPermissionManager(); + try + { + bool hasRights = entityPermissionManager.HasEffectiveRightsAsync(username, typeof(Dataset), datasetId, RightType.Read).Result; + if (!hasRights) + { + errorResult = Json(new { success = false, id = datasetId, message = "User has no rights to read the given dataset." }); + return false; + } + + return true; + } + catch (Exception ex) + { + Debug.WriteLine("Error while checking permissions: " + ex.ToString()); + errorResult = Json(new { success = false, id = datasetId, message = "Error while checking permissions." }); + return false; + } + finally + { + entityPermissionManager.Dispose(); + } + } + + // Helper overloads to return a JsonResult and set a custom HTTP status code in a consistent way. + // Use the no-behavior overload for POST (will use DenyGet), and the overload with behavior for GET responses. + // GET usage (one-liner): return JsonWithStatus(new { success = false, id = datasetId, message = "..." }, HttpStatusCode.BadRequest, JsonRequestBehavior.AllowGet); + // POST usage (one-liner): return JsonWithStatus(new { success = false, id = datasetId, message = "..." }, HttpStatusCode.BadRequest); + private JsonResult JsonWithStatus(object data, HttpStatusCode statusCode, JsonRequestBehavior behavior) + { + Response.StatusCode = (int)statusCode; + Response.TrySkipIisCustomErrors = true; // ensure IIS does not override the response body + return Json(data, behavior); + } + + private JsonResult JsonWithStatus(object data, HttpStatusCode statusCode) + { + Response.StatusCode = (int)statusCode; + Response.TrySkipIisCustomErrors = true; // ensure IIS does not override the response body + return Json(data, JsonRequestBehavior.DenyGet); + } + + } +} \ No newline at end of file diff --git a/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.SMM.UI/Properties/AssemblyInfo.cs b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.SMM.UI/Properties/AssemblyInfo.cs index 1eecfffc92..861e822098 100644 --- a/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.SMM.UI/Properties/AssemblyInfo.cs +++ b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.SMM.UI/Properties/AssemblyInfo.cs @@ -19,7 +19,7 @@ [assembly: ComVisible(false)] // The following GUID is for the ID of the typelib if this project is exposed to COM -[assembly: Guid("dec22f9c-47be-4146-94be-1c9aa969afc3")] +[assembly: Guid("dec22f9c-47be-4146-94be-1c9aa969afc4")] // Version information for an assembly consists of the following four values: // diff --git a/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.SMM.UI/SMM.Settings.json b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.SMM.UI/SMM.Settings.json index a53aaea853..c1715f4e99 100644 --- a/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.SMM.UI/SMM.Settings.json +++ b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.SMM.UI/SMM.Settings.json @@ -30,8 +30,24 @@ "title": "Party Relationship Type for Owner", "type": "String", "value": "Owner" + }, + { + "key": "externalApiMetadata", + "title": "External API Metadata", + "type": "JSON", + "value": { + "clb": { + "sourceKeyInfo": [ + { + "sourceKey": "", + "title": "", + "alias": "" + } + ] + } + } } ], - "id": "smm", + "id": "Smm", "name": "species mapping" } \ No newline at end of file diff --git a/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.SMM.UI/Views/Species/Index.cshtml b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.SMM.UI/Views/Species/Index.cshtml new file mode 100644 index 0000000000..72ca073204 --- /dev/null +++ b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.SMM.UI/Views/Species/Index.cshtml @@ -0,0 +1,17 @@ +@{ + ViewBag.Title = "Edit"; + Layout = "~/Themes/Default/Layouts/_svelteLayout.cshtml"; + + @*long id = 0; + long version = 0; + + if (ViewData["id"] != null) { id = Convert.ToInt64(ViewData["id"]); } + + if (ViewData["version"] != null) { version = Convert.ToInt64(ViewData["version"]); } + + *@ + } +
+ + @Html.Partial("_sveltePage") +
\ No newline at end of file diff --git a/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.SMM.UI/packages.config b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.SMM.UI/packages.config index b8d2a63a93..a44cd26801 100644 --- a/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.SMM.UI/packages.config +++ b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.SMM.UI/packages.config @@ -7,7 +7,12 @@ + + + + + \ No newline at end of file diff --git a/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI.Svelte/src/lib/helper/custom_diff.ts b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI.Svelte/src/lib/helper/custom_diff.ts new file mode 100644 index 0000000000..4cdd2623fb --- /dev/null +++ b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI.Svelte/src/lib/helper/custom_diff.ts @@ -0,0 +1,64 @@ +// Source: +// https://blog.devgenius.io/implementing-the-myers-diff-algorithm-in-typescript-character-level-precision-5aa0430f6727 + +export interface DiffPart { + value: string; + added?: boolean; + removed?: boolean; +} + +export function getDifference(oldStr: string, newStr: string): DiffPart[] { + const oldLen = oldStr.length; + const newLen = newStr.length; + + // Step 1: Create a 2D grid to compute the Longest Common Subsequence (LCS) + const grid: number[][] = Array.from({ length: oldLen + 1 }, () => + new Array(newLen + 1).fill(0) + ); + + for (let i = 1; i <= oldLen; i++) { + for (let j = 1; j <= newLen; j++) { + if (oldStr[i - 1] === newStr[j - 1]) { + grid[i][j] = grid[i - 1][j - 1] + 1; + } else { + grid[i][j] = Math.max(grid[i - 1][j], grid[i][j - 1]); + } + } + } + + // Step 2: Backtrack through the grid to assemble the differences + const result: DiffPart[] = []; + let i = oldLen; + let j = newLen; + + while (i > 0 || j > 0) { + if (i > 0 && j > 0 && oldStr[i - 1] === newStr[j - 1]) { + // Characters match -> part of the original/clean sequence + result.unshift({ value: oldStr[i - 1] }); + i--; + j--; + } else if (j > 0 && (i === 0 || grid[i][j - 1] >= grid[i - 1][j])) { + // Character was added in the new string + result.unshift({ value: newStr[j - 1], added: true }); + j--; + } else { + // Character was removed from the old string + result.unshift({ value: oldStr[i - 1], removed: true }); + i--; + } + } + + // Step 3: Optional optimization - merge adjacent parts of the same type + // (e.g., merge separate 'a', 'b', 'c' objects into a single 'abc' object) + const mergedResult: DiffPart[] = []; + for (const part of result) { + const last = mergedResult[mergedResult.length - 1]; + if (last && last.added === part.added && last.removed === part.removed) { + last.value += part.value; + } else { + mergedResult.push({ ...part }); + } + } + + return mergedResult; +} \ No newline at end of file diff --git a/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI.Svelte/src/lib/stores/persist.ts b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI.Svelte/src/lib/stores/persist.ts new file mode 100644 index 0000000000..fcad26565a --- /dev/null +++ b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI.Svelte/src/lib/stores/persist.ts @@ -0,0 +1,20 @@ +import { writable } from 'svelte/store'; + +export function persisted(key, initialValue) { + // 1. Check if we have a saved value in localStorage + const saved = typeof window !== 'undefined' ? localStorage.getItem(key) : null; + + // 2. Use saved value if it exists, otherwise use initialValue + const data = saved ? JSON.parse(saved) : initialValue; + + const store = writable(data); + + // 3. Listen for changes and save them to localStorage + if (typeof window !== 'undefined') { + store.subscribe(value => { + localStorage.setItem(key, JSON.stringify(value)); + }); + } + + return store; +} \ No newline at end of file diff --git a/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI.Svelte/src/lib/stores/selectionStore.ts b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI.Svelte/src/lib/stores/selectionStore.ts new file mode 100644 index 0000000000..0a07d1465c --- /dev/null +++ b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI.Svelte/src/lib/stores/selectionStore.ts @@ -0,0 +1,25 @@ +import { persisted } from "./persist"; + +/** + * These values distinctly identify the data the user is currently working on. + * [datasetId, versionId] ... distinct identifier for the actual scientificNames and matching results + * [versionNr] ... only for user readability + * [stepId] ... identifies a matching step (matching file-based against multiple APIs results in multiple such steps) + * [datastructureId] ... only used as helper variable for backend functionalities + * -1 ... not selected + * + * They are used everywhere and guide the flow - and selection of data during the whole matching process. + */ +export const matchingSelection = persisted('matchingSelection', { + // unique const datasetId of the selected dataset + datasetId: -1, + // unique const datastructureId belonging to selected datasetId + versionId pair + datastructureId: -1, + // unique const version identifier + versionId: -1, + // dynamic version number (only for client display) + versionNr: -1, + // unique identifier of the StepEntry that is being selected (Matching step) + stepId: -1 +}); + diff --git a/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI.Svelte/src/lib/types/types.ts b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI.Svelte/src/lib/types/types.ts new file mode 100644 index 0000000000..e2d7fb5ea9 --- /dev/null +++ b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI.Svelte/src/lib/types/types.ts @@ -0,0 +1,186 @@ +// global types + +export interface MappingEntry { + variableId: number, + variableName: string, + headerMapping: string +} + +export interface HeaderMappings { + datastructureId: number, + datasetId: number, + mappings: MappingEntry[], +} + +export interface StepEntry { + id: number, + numRows: number, + inputFileName: string, + resultFileName: string, + jobKey: string, + downloadLink: string, + matchSource: string, + timeStamp: string, + done: boolean +} + +export interface MatchingProgress { + datasetId: number, + numRowsGlobal: number, + steps: StepEntry[], +} + +// this is a helper for typing response content correctly +// success false indicates that either the response failed or the whole request failed +export type ServiceResult = + | { success: true, data: T } + | { success: false, error: string }; + +export interface MatchingFileStatus { + directoryExists: boolean, + fileExists: boolean, + markerExists: boolean, + markerStale: boolean, + markerStart: string, + matchingProgressExists: boolean, + stepExists: boolean, + stepCompleted: boolean, + downloadLinkPresent: boolean, + jobKeyPresent: boolean, +} + +export type SpeciesMatchingRow = { + // database row id in postgres + postgres_id: number, + // indexedDB key + __id: number, + // original unchanged name (used for matching if EditedName is empty, and for display purposes) + originalName: string, + // helper to apply data cleaning and better overview (field itself does not exist in db) + cleanedName: string, + // edited name after data cleaning + manual corrections (used for matching) + editedName: string, + // indicates whether the match has been confirmed by the user + confirmedByUser: boolean, + // unique identifier of the matched name in the external source (e.g. GBIF taxon ID) + matchId: string, + // matched name from the external source (the result) + matchedName: string, + // authorship of the matched name + matchAuthorship: string, + // taxonomic rank of the matched name (e.g. species, genus, etc.) + matchRank: string, + // type of the match (e.g. exact, fuzzy, etc.) + matchType: string, + // taxonomic status of the matched name (e.g. accepted, synonym, etc.) + status: string, + // accepted name if (for example) the matched name is a synonym + acceptedScientificName: string, + // unique identifier of the accepted name in the external source (e.g. GBIF taxon ID) + acceptedId: string, + // authorship of the accepted name + acceptedAuthorship: string, + // higher classification of the matched name (e.g. kingdom, phylum, class, order, family, genus) + taxonKingdom: string, + taxonPhylum: string, + taxonClass: string, + taxonOrder: string, + taxonFamily: string, + taxonGenus: string, + // source of the match (e.g. Catalogue of Life, GBIF, etc.) + matchSource: string, + // version of the source used for matching + matchSourceVersion: string, + // timestamp of the match (can vary by hours due to processing and queue times on different APIs) + timeStampMatch: string +} + +export interface GenericMatchingResult { + original_ID: string, + __id: number, + original_scientificName: string, + scientificName: string, + original_rank?: string, + original_kingdom?: string, + original_authorship?: string, + matchType?: string, + matchIssues?: string, + id: string, + rank?: string, + authorship?: string, + status?: string, + acceptedID?: string, + acceptedScientificName?: string, + acceptedAuthorship?: string, + kingdom?: string, + phylum?: string, + class?: string, + order?: string, + family?: string, + genus?: string, + classification?: string, +} + +export interface CLBMatchingResult { + original_ID: string, + original_scientificName: string, + original_rank: string, + original_kingdom: string, + original_authorship: string, + matchType: string, + matchIssues: string, + id: string, + rank: string, + scientificName: string, + authorship: string, + status: string, + acceptedID: string, + acceptedScientificName: string, + acceptedAuthorship: string, + kingdom: string, + phylum: string, + class: string, + order: string, + family: string, + genus: string, + classification: string, +} + +export type AcceptMatchesRequest = { + datasetId: number, + versionId: number, + stepId: number, + matchIds: (string | undefined)[] +} + + +// Selectable api metadata/options provided by the backend +export interface SourceKeyInfoItem { + sourceKey: string; + title: string; + alias: string; +} + +export interface ExternalApiSource { + sourceKeyInfo: SourceKeyInfoItem[]; +} + +export interface ExternalApiMetadata { + clb: ExternalApiSource; +} + +// apiOptions types that are actually send as a payload together with a file Matching request +export interface ClbOptions { + type: 'clb'; // Discriminator (optional, but highly recommended) + sourceKey: string; + synonyms: boolean; +} + +export interface GbifOptions { + type: 'gbif'; + parameter1: string; + parameter2: string; +} + +// Representing the IApiOptions interface as a Union type +export type IApiOptions = ClbOptions | GbifOptions \ No newline at end of file diff --git a/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI.Svelte/src/routes/datasets_overview/+page.svelte b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI.Svelte/src/routes/datasets_overview/+page.svelte new file mode 100644 index 0000000000..ea0c0698b2 --- /dev/null +++ b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI.Svelte/src/routes/datasets_overview/+page.svelte @@ -0,0 +1,141 @@ + + + +

Datasets Overview

+ +

This page gives an overview of all your datasets and their respective matching progress. This matching process is bound to a specific dataset version (shown by the column VersionNr). Currently only tabular datasets with complete metadata are shown!

+

If there is an Eye icon at the end of the row, the dataset matching has already been started. Click it to get an overview and continue the process as you wish.

+

If there is a Plus icon, no matching has been started. Click it to start a fresh matching process on this dataset version. Keep in mind that a new matching process right now can only be started with the latest version of a dataset.

+ + {#await load()} + + {:then data} +
+
+ + {/await} + +
+ + \ No newline at end of file diff --git a/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI.Svelte/src/routes/datasets_overview/ResultTableOptions.svelte b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI.Svelte/src/routes/datasets_overview/ResultTableOptions.svelte new file mode 100644 index 0000000000..ca7d87fa4b --- /dev/null +++ b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI.Svelte/src/routes/datasets_overview/ResultTableOptions.svelte @@ -0,0 +1,45 @@ + + +
+ {#if row.hasMatchingProgress} + + {:else} + + {/if} +
\ No newline at end of file diff --git a/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI.Svelte/src/routes/datasets_overview/data.ts b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI.Svelte/src/routes/datasets_overview/data.ts new file mode 100644 index 0000000000..591ca81213 --- /dev/null +++ b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI.Svelte/src/routes/datasets_overview/data.ts @@ -0,0 +1,49 @@ +import { writable } from 'svelte/store'; + +export type VersionInfo = { + // title: string, + // description: string, + // changeDescription: string, + timestamp: string, + versionType: string, + versionName: string, + // versionDescription: string, + // publicAccess: boolean, + // publicAccessDate: string, + hasMatchingProgress: boolean, + versionId: number, + versionNr: number, +} + +export type BasicDatasetInfo = { + id: number, + dataStructureId: number, + title: string, + abstract: string, + isTabular: boolean, + metadataComplete: boolean, + hasMatchingProgress: boolean, + versions: VersionInfo[] +} + +export type DisplayDatasetVersion = { + id: number, + dataStructureId: number, + title: string, + abstract: string, + isTabular: boolean, + metadataComplete: boolean, + timestamp: string, + versionType: string, + versionName: string, + hasMatchingProgress: boolean + versionId: number, + versionNr: number, +} + +let datasetRows: DisplayDatasetVersion[] = [ + +] + + +export let datasetsStore = writable(datasetRows); \ No newline at end of file diff --git a/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI.Svelte/src/routes/datasets_overview/services.ts b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI.Svelte/src/routes/datasets_overview/services.ts new file mode 100644 index 0000000000..64d6e80346 --- /dev/null +++ b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI.Svelte/src/routes/datasets_overview/services.ts @@ -0,0 +1,10 @@ +import { Api } from '@bexis2/bexis2-core-ui'; + +export const loadBasicDatasetInfo = async () => { + try { + const response = await Api.get('/smm/species/GetMyDatasetsJson'); + return response.data; + } catch (error) { + console.error(error); + } +}; diff --git a/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI.Svelte/src/routes/datasets_overview/types.ts b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI.Svelte/src/routes/datasets_overview/types.ts new file mode 100644 index 0000000000..e69de29bb2 diff --git a/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI.Svelte/src/routes/headermapping/+page.svelte b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI.Svelte/src/routes/headermapping/+page.svelte new file mode 100644 index 0000000000..0bb6675d00 --- /dev/null +++ b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI.Svelte/src/routes/headermapping/+page.svelte @@ -0,0 +1,186 @@ + + + + +

Select header mapping

+ +

+ You are working on Dataset: {$matchingSelection.datasetId}, Datastructure: {$matchingSelection.datastructureId} and VersionID: {$matchingSelection.versionId}. +

+

+ The original column headers are shown on the left. Selectable mappings are shown on the right. Some mappings might already be pre-assigned based on your datasets metadata. Please try to select as many matching mappings as possible, but at the very least select a scientificName mapping. + For columns that have no clear associated mapping, just select IGNORE. If there are no conflicts, you should be able to submit the mappings with the button below. +

+ +

+ This information is used to cut off unnecessary data and help matching APIs understand your data better. Your original data will NOT be changed at any step and will remain fully functional. +

+ +{#if dataStructure} + {#each dataStructure.variables as variable, i} +
+
{variable.name}
+
+
+
+ +
+
+ {/each} + +
+ +
+{/if} + +
+ + \ No newline at end of file diff --git a/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI.Svelte/src/routes/headermapping/services.ts b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI.Svelte/src/routes/headermapping/services.ts new file mode 100644 index 0000000000..cac20af8f6 --- /dev/null +++ b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI.Svelte/src/routes/headermapping/services.ts @@ -0,0 +1,39 @@ +import { Api } from '@bexis2/bexis2-core-ui'; +import type { HeaderMappings } from '$lib/types/types'; +import type { ServiceResult } from '$lib/types/types'; + +/** + * Submits selected column mappings to backend for storage. (used in later stages) + * @param data HeaderMappings data including a MappingEntry[] with the selected mappings. + * @returns response data + */ +export const submitHeaderMappings = async (data: HeaderMappings, datasetId: number, versionId: number): Promise> => { + try { + const payload = { + data: data, + datasetId: datasetId, + versionId: versionId + } + + const response = await Api.post('/smm/species/SubmitHeaderMappings', payload); + return { success: true, data: response.data }; + } catch (error: any) { + console.error(error); + return { success: false, error: error.data?.message }; + } +} + +/** + * Loads datastructure information for display. + * @param datastructureId .. datastructure id loaded from store + * @returns DataStructureEditModel + */ +export const loadDataStructure = async (datastructureId: number): Promise> => { + try { + const response = await Api.get(`/rpm/DataStructure/get?id=${datastructureId}`); + return { success: true, data: response.data }; + } catch (error: any) { + console.error(error); + return { success: false, error: error.data?.message }; + } +} diff --git a/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI.Svelte/src/routes/headermapping/types.ts b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI.Svelte/src/routes/headermapping/types.ts new file mode 100644 index 0000000000..b156dde82e --- /dev/null +++ b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI.Svelte/src/routes/headermapping/types.ts @@ -0,0 +1,80 @@ +export interface Variable { + id: number, + name: string, + unit: string, + dataType: string, + isKeys: boolean +} + +export interface ListItem { + id: number, + text: string, + group: string, + description: string +} + +export interface UnitItem { + id: number, + text: string, + group: string, + data_types: string[] +} + +export interface VariableTemplateItem { + id: number, + text: string, + group: string, + description: string, + data_type: string, + unit: string, + data_types: string[], + units: string[], + meanings: string[], + constraints: string[] +} + +export interface Link { + label: string, + link: string, + prefix: string, + releation: string +} + +export interface Meaning { + group: string, + id: number, + constraints: string[], + links: Link[], + text: string, +} + +export interface VariableInstanceModel { + is_key: boolean, + is_optional: boolean, + display_pattern: ListItem, + possible_units: UnitItem[], + name: string, + id: number, + meanings: Meaning[], + possible_templates: VariableTemplateItem[], + possible_display_patterns: ListItem[] +} + +export interface MissingValueModel { + display_name: string, + description: string +} + +export interface DataStructureEditModel { + id: number, + title: string, + description: string, + preview: string[], + variables: VariableInstanceModel[], + missing_values: MissingValueModel[] +} + +export interface MultiSelectSourceDetailed { + value: string, + label: string +} diff --git a/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI.Svelte/src/routes/matchingresult/+page.svelte b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI.Svelte/src/routes/matchingresult/+page.svelte new file mode 100644 index 0000000000..6d7494ef27 --- /dev/null +++ b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI.Svelte/src/routes/matchingresult/+page.svelte @@ -0,0 +1,886 @@ + + + +
+ + + + +
+ + {#if statusLoaded && resultFileExists} + {#await load()} + + {:then data} +

Global Actions

+ {#each Object.entries(uniqueMatchTypes) as [id, value] } + + {/each} +
+
+
+ + Done ({doneCount}) +
+
+ + WIP ({wipCount}) +
+
+ + Mismatch ({mismatchCount}) +
+ +
+ Total: {totalCount} +
+
+ +
+ + +
+ +
+ +
+
+
+ +
+ Work in progress +
+

(Acceptable results from this matching step)

+
+
+ + +
+
+ Accepted +
+

(Will be stored on Submit)

+
+
+ + +
+ +
+ +
+

(Can not be accepted)

+
+
+
+ + + +
+ +
+ +
+

(Already accepted and stored previously)

+ +
+
+
+ + + +
+ +
+ {#if submittingEntries} + + {/if} +
+
+ + +
+ {/await} + {:else} + {#if !statusLoaded} + + {:else} + {#if !resultFileExists} + + {:else} + + {/if} + {/if} + {/if} + +
+ \ No newline at end of file diff --git a/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI.Svelte/src/routes/matchingresult/AcceptedTableOptions.svelte b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI.Svelte/src/routes/matchingresult/AcceptedTableOptions.svelte new file mode 100644 index 0000000000..a603bcece6 --- /dev/null +++ b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI.Svelte/src/routes/matchingresult/AcceptedTableOptions.svelte @@ -0,0 +1,30 @@ + + +
+ {#each buttons as button} + + {/each} +
\ No newline at end of file diff --git a/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI.Svelte/src/routes/matchingresult/ResultTableOptions.svelte b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI.Svelte/src/routes/matchingresult/ResultTableOptions.svelte new file mode 100644 index 0000000000..ec048b12b2 --- /dev/null +++ b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI.Svelte/src/routes/matchingresult/ResultTableOptions.svelte @@ -0,0 +1,42 @@ + + +
+ {#each buttons as button} + + {/each} +
\ No newline at end of file diff --git a/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI.Svelte/src/routes/matchingresult/SubmitAcceptedModal.svelte b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI.Svelte/src/routes/matchingresult/SubmitAcceptedModal.svelte new file mode 100644 index 0000000000..7610b94390 --- /dev/null +++ b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI.Svelte/src/routes/matchingresult/SubmitAcceptedModal.svelte @@ -0,0 +1,30 @@ + + +
+ + +
You are about to submit {count} accepted entries. Are you sure?
+
+ + +
+
\ No newline at end of file diff --git a/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI.Svelte/src/routes/matchingresult/data.ts b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI.Svelte/src/routes/matchingresult/data.ts new file mode 100644 index 0000000000..71c3dcc87c --- /dev/null +++ b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI.Svelte/src/routes/matchingresult/data.ts @@ -0,0 +1,32 @@ +import { writable } from 'svelte/store'; +import type { GenericMatchingResult, SpeciesMatchingRow } from '$lib/types/types'; + + + +let acceptedRows: GenericMatchingResult[] = [ + +] + +export let acceptedStore = writable(acceptedRows) + + + +let resultRows: GenericMatchingResult[] = [ + +] + +export let resultStore = writable(resultRows) + + +let mismatchRows: GenericMatchingResult[] = [ + +] + +export let mismatchStore = writable(mismatchRows) + + +let doneRows: SpeciesMatchingRow[] = [ + +] + +export let doneStore = writable(doneRows) \ No newline at end of file diff --git a/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI.Svelte/src/routes/matchingresult/services.ts b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI.Svelte/src/routes/matchingresult/services.ts new file mode 100644 index 0000000000..af3706c649 --- /dev/null +++ b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI.Svelte/src/routes/matchingresult/services.ts @@ -0,0 +1,43 @@ +import { Api } from '@bexis2/bexis2-core-ui'; +import type { AcceptMatchesRequest, ServiceResult } from '$lib/types/types'; + +export const loadMatchingResult = async (datasetId: number, versionId: number, stepId: number): Promise> => { + try { + const response = await Api.get(`/smm/species/ViewMatchingResult?datasetId=${datasetId}&versionId=${versionId}&stepId=${stepId}`); + + return { success: true, data: response.data }; + } catch (error: any) { + return { success: false, error: error.data?.message }; + } +}; + +export const submitAcceptedIds = async (payload: AcceptMatchesRequest): Promise> => { + try { + const response = await Api.post('/smm/species/AcceptMatches', payload); + + return { success: true, data: response.data } + } catch (error: any) { + console.error(error); + return { success: false, error: error.data?.message }; + } +} + +export const loadMatchingFileStatus = async (datasetId: number, versionId: number, stepId: number): Promise> => { + try { + const response = await Api.get(`/smm/species/GetMatchingFileStatus?datasetId=${datasetId}&versionId=${versionId}&stepId=${stepId}`); + + return { success: true, data: response.data }; + } catch (error: any) { + return { success: false, error: error.data?.message }; + } +} + +export const requestResultFileDownload = async (datasetId: number, versionId: number, stepId: number): Promise> => { + try { + const response = await Api.get(`/smm/species/StartDownloadResultFile?datasetId=${datasetId}&versionId=${versionId}&stepId=${stepId}`); + + return { success: true, data: response.data }; + } catch (error: any) { + return { success: false, error: error.data?.message }; + } +} \ No newline at end of file diff --git a/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI.Svelte/src/routes/matchingresult/types.ts b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI.Svelte/src/routes/matchingresult/types.ts new file mode 100644 index 0000000000..e69de29bb2 diff --git a/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI.Svelte/src/routes/progress_overview/+page.svelte b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI.Svelte/src/routes/progress_overview/+page.svelte new file mode 100644 index 0000000000..56ef547b90 --- /dev/null +++ b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI.Svelte/src/routes/progress_overview/+page.svelte @@ -0,0 +1,214 @@ + + + + +
+ + + +
+ +

Progress Overview

+ + {#if tailorError} + + {tailorErrorMessage} + + {/if} + + {#await load()} + + {:then data} + {#if !data.hasHeaderMappings} +

This dataset does not seem to be initialized. Please go back to the Datasets Overview and start from scratch.

+ {:else} +

The dataset has {data.headerMappings.mappings.length} mapped columns.

+ + {#if !data.isTailored} + {#if tailorLoading} + + {/if} +
+ +
+ {:else} + {#if !data.hasMatchingProgress} +

No matching progress data available. Something went wrong.

+ {:else} + {#if data.matchingProgress.steps.length == 0} +

For this dataset, no matching request have been done to external APIs. Feel free to check/edit the current state or begin matching.

+
+ + +
+ {:else} +

Your matching jobs

+ +
+
+ + + {/if} + + + {/if} + {/if} + {/if} + {:catch error} + + {error.message} + + {/await} + + +
+ + \ No newline at end of file diff --git a/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI.Svelte/src/routes/progress_overview/ApiMatchingSelector.svelte b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI.Svelte/src/routes/progress_overview/ApiMatchingSelector.svelte new file mode 100644 index 0000000000..8626b57640 --- /dev/null +++ b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI.Svelte/src/routes/progress_overview/ApiMatchingSelector.svelte @@ -0,0 +1,72 @@ + + + +{#if apiSelectOptions.length > 0} + + + + +{/if} \ No newline at end of file diff --git a/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI.Svelte/src/routes/progress_overview/TableOptions.svelte b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI.Svelte/src/routes/progress_overview/TableOptions.svelte new file mode 100644 index 0000000000..534dc88e52 --- /dev/null +++ b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI.Svelte/src/routes/progress_overview/TableOptions.svelte @@ -0,0 +1,33 @@ + + +
+ {#if row.done} + + {:else} + + {/if} +
\ No newline at end of file diff --git a/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI.Svelte/src/routes/progress_overview/data.ts b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI.Svelte/src/routes/progress_overview/data.ts new file mode 100644 index 0000000000..c319e1ac38 --- /dev/null +++ b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI.Svelte/src/routes/progress_overview/data.ts @@ -0,0 +1,9 @@ +import { writable } from 'svelte/store'; +import { get } from 'svelte/store'; +import type { StepEntry } from '$lib/types/types'; + +let matchingJobRows: StepEntry[] = [ + +] + +export let matchingJobStore = writable(matchingJobRows); \ No newline at end of file diff --git a/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI.Svelte/src/routes/progress_overview/downloadLinkCell.svelte b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI.Svelte/src/routes/progress_overview/downloadLinkCell.svelte new file mode 100644 index 0000000000..2930ac0e5d --- /dev/null +++ b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI.Svelte/src/routes/progress_overview/downloadLinkCell.svelte @@ -0,0 +1,10 @@ + + +
+ Download link +
\ No newline at end of file diff --git a/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI.Svelte/src/routes/progress_overview/jobKeyCell.svelte b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI.Svelte/src/routes/progress_overview/jobKeyCell.svelte new file mode 100644 index 0000000000..7088b27cd0 --- /dev/null +++ b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI.Svelte/src/routes/progress_overview/jobKeyCell.svelte @@ -0,0 +1,32 @@ + + +
+ +
\ No newline at end of file diff --git a/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI.Svelte/src/routes/progress_overview/services.ts b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI.Svelte/src/routes/progress_overview/services.ts new file mode 100644 index 0000000000..795266d9f6 --- /dev/null +++ b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI.Svelte/src/routes/progress_overview/services.ts @@ -0,0 +1,43 @@ +import { Api } from '@bexis2/bexis2-core-ui'; +import type { IApiOptions, ServiceResult } from '$lib/types/types'; + +export const loadDatasetProgress = async (datasetId: number, versionId: number): Promise> => { + try { + const response = await Api.get(`/smm/species/ViewProgress?datasetId=${datasetId}&versionId=${versionId}`); + + return { success: true, data: response.data }; + } catch (error: any) { + return { success: false, error: error.data?.message }; + } +}; + +export const tailorDataset = async (datasetId: number, versionId: number): Promise> => { + try { + const response = await Api.post('/smm/species/Tailor', { datasetId, versionId }); + + return { success: true, data: response.data }; + } catch (error: any) { + console.log(error); + return { success: false, error: error.data?.message }; + } +} + +export const genNewMatchFile = async (datasetId: number, versionId: number, apiIdentifier: string): Promise> => { + try { + const response = await Api.post('/smm/species/GenNewMatchInputFile', { datasetId, versionId, apiIdentifier }); + + return { success: true, data: response.data }; + } catch (error: any) { + return { success: false, error: error.data?.message }; + } +} + +export const matchNextFile = async (datasetId: number, versionId: number, stepId: number, apiOptions: IApiOptions): Promise> => { + try { + const response = await Api.post(`/smm/species/MatchFileByStepId?datasetId=${datasetId}&versionId=${versionId}&stepId=${stepId}`, apiOptions); + + return { success: true, data: response.data }; + } catch (error: any) { + return { success: false, error: error.data?.message }; + } +} \ No newline at end of file diff --git a/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI.Svelte/src/routes/progress_overview/types.ts b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI.Svelte/src/routes/progress_overview/types.ts new file mode 100644 index 0000000000..ac2247f9d4 --- /dev/null +++ b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI.Svelte/src/routes/progress_overview/types.ts @@ -0,0 +1,13 @@ +import type { HeaderMappings, MatchingProgress } from "$lib/types/types" +import type { ExternalApiMetadata } from "$lib/types/types" + +export interface ProgressOverview { + success: boolean, + hasHeaderMappings: boolean, + hasMatchingProgress: boolean, + isTailored: boolean, + headerMappings: HeaderMappings, + matchingProgress: MatchingProgress, + externalApiMetadata: ExternalApiMetadata +} + diff --git a/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI.Svelte/src/routes/species/AcceptedTableOptions.svelte b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI.Svelte/src/routes/species/AcceptedTableOptions.svelte new file mode 100644 index 0000000000..a603bcece6 --- /dev/null +++ b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI.Svelte/src/routes/species/AcceptedTableOptions.svelte @@ -0,0 +1,30 @@ + + +
+ {#each buttons as button} + + {/each} +
\ No newline at end of file diff --git a/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI.Svelte/src/routes/species/EditResult.svelte b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI.Svelte/src/routes/species/EditResult.svelte new file mode 100644 index 0000000000..0e21582b38 --- /dev/null +++ b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI.Svelte/src/routes/species/EditResult.svelte @@ -0,0 +1,12 @@ + + +
+ + +
+ + +
+
\ No newline at end of file diff --git a/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI.Svelte/src/routes/species/ResultTableOptions.svelte b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI.Svelte/src/routes/species/ResultTableOptions.svelte new file mode 100644 index 0000000000..2a807fb7a7 --- /dev/null +++ b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI.Svelte/src/routes/species/ResultTableOptions.svelte @@ -0,0 +1,40 @@ + + +
+ {#each buttons as button} + + {/each} +
\ No newline at end of file diff --git a/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI.Svelte/src/routes/species/data.ts b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI.Svelte/src/routes/species/data.ts new file mode 100644 index 0000000000..c070d2b80b --- /dev/null +++ b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI.Svelte/src/routes/species/data.ts @@ -0,0 +1,29 @@ +import { writable } from 'svelte/store'; + +export type ResultRow = { + inputID: string, + inputRank?: string, + inputName: string, + matchType: string, + id: number, + rank: string, + label?: string, + scientificName: string, + authorship: string, + status: string, + acceptedName?: string, + classification?: string, + issues?: string, +} + +let acceptedRows: ResultRow[] = [ + +] + +export let acceptedStore = writable(acceptedRows) + +let emptyTestRows: ResultRow[] = [ + +] + +export let resultStore = writable(emptyTestRows) diff --git a/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI.Svelte/src/routes/tailor_view/+page.svelte b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI.Svelte/src/routes/tailor_view/+page.svelte new file mode 100644 index 0000000000..5ce2babbdd --- /dev/null +++ b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI.Svelte/src/routes/tailor_view/+page.svelte @@ -0,0 +1,633 @@ + + + +
+ + + +
+ +
+ Select steps for data cleaning (changes applied automatically). Use Global actions to run specific procedures across the whole dataset. +
+
+ Hover the Data cleaning options and Global actions , to get an explanation for what they are doing. +
+
+ Click the pencil icon to edit individual names (if empty, the cleaned name property is used for matching or if empty as well, the original name). +
+
+ When you're done here, be sure to SUBMIT the changes for them to take effect! +
+ + {#await load()} + + {:then data} +

Data cleaning config

+ + + +
+ {#each Object.entries(cleanConfig) as [key, conf]} +
+ {key} +
+ {/each} +
+ +

Global Actions

+ + + + +
+
+ Matched ({confirmedCount}/{totalCount}) + {Math.round(percentage)}% +
+ +
+
+
+
+ +

{showEditsOnly ? 'Edits only (row edits disabled)' : 'Table data'}

+
+ {#if tableInDOM} +
+
+ + + {#if showEditsOnly} +
+ {/if} + {/if} + + {/await} + +
+ +
+ +
+ +
+ +
+ + \ No newline at end of file diff --git a/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI.Svelte/src/routes/tailor_view/EditNameModal.svelte b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI.Svelte/src/routes/tailor_view/EditNameModal.svelte new file mode 100644 index 0000000000..e9f3d55233 --- /dev/null +++ b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI.Svelte/src/routes/tailor_view/EditNameModal.svelte @@ -0,0 +1,44 @@ + + +
+ + +
{row.originalName}
+ +
{row.cleanedName}
+ + +
+ + +
+
\ No newline at end of file diff --git a/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI.Svelte/src/routes/tailor_view/EditSubmitModal.svelte b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI.Svelte/src/routes/tailor_view/EditSubmitModal.svelte new file mode 100644 index 0000000000..5ac5f8e932 --- /dev/null +++ b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI.Svelte/src/routes/tailor_view/EditSubmitModal.svelte @@ -0,0 +1,46 @@ + + +
+ + +
You are about to submit {changedRows.length} changed rows. Are you sure you want to continue?
+
+ + +
+
\ No newline at end of file diff --git a/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI.Svelte/src/routes/tailor_view/ResetEditsModal.svelte b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI.Svelte/src/routes/tailor_view/ResetEditsModal.svelte new file mode 100644 index 0000000000..a8e5225298 --- /dev/null +++ b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI.Svelte/src/routes/tailor_view/ResetEditsModal.svelte @@ -0,0 +1,21 @@ + + +
+
You are about to RESET ALL edited rows. Are you SURE you want to continue?
+
+ + +
+
\ No newline at end of file diff --git a/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI.Svelte/src/routes/tailor_view/ResultTableOptions.svelte b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI.Svelte/src/routes/tailor_view/ResultTableOptions.svelte new file mode 100644 index 0000000000..a56fd12b75 --- /dev/null +++ b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI.Svelte/src/routes/tailor_view/ResultTableOptions.svelte @@ -0,0 +1,31 @@ + + +
+ {#each buttons as button} + + {/each} +
\ No newline at end of file diff --git a/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI.Svelte/src/routes/tailor_view/cleanedName.svelte b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI.Svelte/src/routes/tailor_view/cleanedName.svelte new file mode 100644 index 0000000000..68aa5d2dcf --- /dev/null +++ b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI.Svelte/src/routes/tailor_view/cleanedName.svelte @@ -0,0 +1,45 @@ + + + +
+ {#each diffs as part} + {#if part.removed} + {part.value} + {:else if part.added} + {part.value} + {:else} + {part.value} + {/if} + {/each} +
+ + diff --git a/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI.Svelte/src/routes/tailor_view/data.ts b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI.Svelte/src/routes/tailor_view/data.ts new file mode 100644 index 0000000000..87d8088f76 --- /dev/null +++ b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI.Svelte/src/routes/tailor_view/data.ts @@ -0,0 +1,401 @@ +import { writable } from 'svelte/store'; +import { get } from 'svelte/store'; +import * as CleaningUtils from './dataCleaningUtils'; +import { type SpeciesMatchingRow } from "$lib/types/types"; + + +let rows: SpeciesMatchingRow[] = [] + +export let tailorEditStore = writable(rows); + +let onlyEditsRows: SpeciesMatchingRow[] = [] + +export let tailorOnlyEditsStore = writable(rows); + +export const cleanConfig = { + sanitize_whitespaces: { + apply: true, + description: "Replace non-breaking spaces, zero-width spaces, tabs, and unusual Unicode spaces. Collapse multiple consecutive spaces into a single space." + }, + normalize_chars_and_dashes: { + apply: true, + description: "Normalize en-dashes, em-dashes, and non-breaking hyphens to standard ASCII hyphens. Standardize quotes and apostrophes if present." + }, + standardize_hybrids: { + apply: true, + description: "Infix hybrid: Genus x species or Genus X species or Genus ✕ species -> Genus × species. Prefix hybrid: x Genus species at start of string -> × Genus species." + }, + standardize_infraspecifics: { + apply: true, + description: "Matches exact rank keywords bounded by word boundaries to avoid mangling author initials." + }, + capitalize_genus: { + apply: true, + description: "Capitalize the first letter of the Genus (or the word right after a leading hybrid '×')." + }, + trim: { + apply: true, + description: "Trim double spaces into one." + } +} + +/** + * Carefully cleans a scientific species name string while preserving author data. + * Follows a conservative "do no harm" strategy: normalizes whitespace, dashes, + * rank markers, and hybrid symbols without dropping any text or altering author capitalization. + * + * @param rawName - The raw species string (with or without author info, no commas). + * @returns The safely cleaned canonical string. + */ +export function cleanName(rawName: string): string { + if (!rawName) return ''; + + let name = rawName; + + // -------------------------------------------------------------------------- + // 1. Whitespace & Control Character Sanitization + // -------------------------------------------------------------------------- + if (cleanConfig.sanitize_whitespaces.apply) { + name = name + // Replace non-breaking spaces, zero-width spaces, tabs, and unusual Unicode spaces + .replace(/[\u00A0\u1680\u2000-\u200A\u202F\u205F\u3000\uFEFF\t]/g, ' ') + // Collapse multiple consecutive spaces into a single space + .replace(/\s+/g, ' ') + .trim(); + } + + if (!name) return ''; + + // -------------------------------------------------------------------------- + // 2. Character & Dash Normalization + // -------------------------------------------------------------------------- + if (cleanConfig.normalize_chars_and_dashes.apply) { + name = name + // Normalize en-dashes, em-dashes, and non-breaking hyphens to standard ASCII hyphens + .replace(/[\u2010-\u2015]/g, '-') + // Standardize quotes and apostrophes if present + .replace(/[`'’‘]/g, "'"); + } + + // -------------------------------------------------------------------------- + // 3. Standardization of Hybrid Markers + // -------------------------------------------------------------------------- + if (cleanConfig.standardize_hybrids.apply) { + name = name + // Infix hybrid: "Genus x species" or "Genus X species" or "Genus ✕ species" -> "Genus × species" + .replace(/\s+[xX×✕✖]\s+/g, ' × ') + // Prefix hybrid: "x Genus species" at start of string -> "× Genus species" + .replace(/^[xX×✕✖]\s+/g, '× '); + } + + // -------------------------------------------------------------------------- + // 4. Safe Standardization of Infraspecific Rank Indicators + // -------------------------------------------------------------------------- + if (cleanConfig.standardize_infraspecifics.apply) { + // Matches exact rank keywords bounded by word boundaries to avoid mangling author initials. + name = name + .replace(/\b(subsp|ssp|sub-sp)\.?\b/gi, 'subsp.') + .replace(/\b(var)\.?\b/gi, 'var.') + .replace(/\b(subvar)\.?\b/gi, 'subvar.') + .replace(/\b(forma)\b/gi, 'f.'); + } + + // -------------------------------------------------------------------------- + // 5. Safe Genus Capitalization + // -------------------------------------------------------------------------- + // Capitalize the first letter of the Genus (or the word right after a leading hybrid '×'). + if (cleanConfig.capitalize_genus.apply) { + if (name.startsWith('× ')) { + const rest = name.slice(2); + name = '× ' + rest.charAt(0).toUpperCase() + rest.slice(1); + } else { + name = name.charAt(0).toUpperCase() + name.slice(1); + } + } + + // -------------------------------------------------------------------------- + // 6. Final Polish Trim + // -------------------------------------------------------------------------- + if (cleanConfig.trim.apply) { + return name.replace(/\s+/g, ' ').trim(); + } else { + return name; + } +} + + + + +// export const cleanConfig = { +// stripSymbols: { +// apply: true, +// description: "" +// }, +// removeSymbols: { +// apply: true, +// description: "" +// }, +// replaceDiacritics: { +// apply: true, +// description: "" +// }, +// replaceNonTrailing: { +// apply: true, +// description: "" +// }, +// standardizeHybrids: { +// apply: true, +// description: "" +// }, +// deleteAfterEqual: { +// apply: true, +// description: "" +// }, +// cleanHybridFormulas: { +// apply: true, +// description: "" +// }, +// deleteTripleHybrids: { +// apply: true, +// description: "" +// }, +// removeCultivars: { +// apply: true, +// description: "" +// }, +// cleanMiddleHyphens: { +// apply: true, +// description: "" +// }, +// deleteTaxonomicAbbreviations: { +// apply: true, +// description: "" +// }, +// deleteHabitatDescriptors: { +// apply: true, +// description: "" +// }, +// deleteGeneralNoise: { +// apply: true, +// description: "" +// }, +// deleteLeadingDescriptors: { +// apply: true, +// description: "" +// }, +// truncateFromBeginning: { +// apply: true, +// description: "" +// }, +// truncateFromMarker: { +// apply: true, +// description: "" +// }, +// truncateFromGeographicOrBreeding: { +// apply: true, +// description: "" +// }, +// truncateFromUncertainty: { +// apply: true, +// description: "" +// }, +// changeVernacularNames: { +// apply: true, +// description: "" +// }, +// updateFamilyNames: { +// apply: true, +// description: "" +// }, +// deleteUselessMarkers: { +// apply: true, +// description: "" +// }, +// correctOcrErrors: { +// apply: true, +// description: "" +// }, +// harmonizeAbbreviations: { +// apply: true, +// description: "" +// }, +// deletePointAfterKey: { +// apply: true, +// description: "" +// }, +// deletePointAfterSpecies: { +// apply: true, +// description: "" +// }, +// fixMissingSpaces: { +// apply: true, +// description: "" +// }, +// validateFamilySuffix: { +// apply: true, +// description: "" +// }, +// informationInParentheses: { +// apply: true, +// description: "" +// }, +// correctWritingGenus: { +// apply: true, +// description: "" +// }, +// spacesBeforeAndAfterParentheses: { +// apply: true, +// description: "" +// }, +// correctionHybrid: { +// apply: true, +// description: "" +// }, +// removeAuthors: { +// apply: true, +// description: "" +// }, +// } + +// export const cleanName = (name: string) => { +// if (!name) return ''; +// name = CleaningUtils.removeSpecialEscapes(name); +// name = CleaningUtils.removeSpecialCharacters(name); + +// if (cleanConfig.stripSymbols.apply) { +// name = CleaningUtils.stripInsideSymbols(name, '"'); +// name = CleaningUtils.stripInsideSymbols(name, "'"); +// name = CleaningUtils.stripInsideSymbols(name, "(", ")"); +// } + +// if (cleanConfig.removeSymbols.apply) { +// name = name.replace(/'/g, '').replace(/"/g, '').replace("(", '').replace(")", ''); +// } + +// if (cleanConfig.replaceDiacritics.apply) { +// name = CleaningUtils.replaceDiacritics(name); +// } + +// name = CleaningUtils.removeNumbers(name); + +// if (cleanConfig.replaceNonTrailing.apply) { +// name = CleaningUtils.replaceNonTrailingSymbolsWithSpace(name, "_"); +// name = CleaningUtils.replaceNonTrailingSymbolsWithSpace(name, "."); +// } + +// name = CleaningUtils.deleteNumeral(name); + +// if (cleanConfig.standardizeHybrids.apply) { +// name = CleaningUtils.standardizeHybrids(name); +// } + +// if (cleanConfig.deleteAfterEqual.apply) { +// name = CleaningUtils.deleteAfterEqual(name); +// } + +// if (cleanConfig.cleanHybridFormulas.apply) { +// name = CleaningUtils.cleanHybridFormulas(name); +// } + +// if (cleanConfig.deleteTripleHybrids.apply) { +// name = CleaningUtils.deleteTripleHybrids(name); +// } + +// if (cleanConfig.removeCultivars.apply) { +// name = CleaningUtils.removeCultivars(name); +// } + +// if (cleanConfig.cleanMiddleHyphens.apply) { +// name = CleaningUtils.cleanMiddleHyphens(name); +// } + +// if (cleanConfig.deleteTaxonomicAbbreviations.apply) { +// name = CleaningUtils.deleteTaxonomicAbbreviations(name); +// } + +// if (cleanConfig.stripSymbols.apply) { +// name = CleaningUtils.deleteHabitatDescriptors(name); +// } + +// if (cleanConfig.deleteGeneralNoise.apply) { +// name = CleaningUtils.deleteGeneralNoise(name); +// } + +// if (cleanConfig.deleteLeadingDescriptors.apply) { +// name = CleaningUtils.deleteLeadingDescriptors(name); +// } + +// if (cleanConfig.truncateFromBeginning.apply) { +// name = CleaningUtils.truncateFromBeginning(name); +// } + +// if (cleanConfig.truncateFromMarker.apply) { +// name = CleaningUtils.truncateFromMarker(name); +// } + +// if (cleanConfig.truncateFromGeographicOrBreeding.apply) { +// name = CleaningUtils.truncateFromGeographicOrBreeding(name); +// } + +// if (cleanConfig.truncateFromUncertainty.apply) { +// name = CleaningUtils.truncateFromUncertainty(name); +// } + +// if (cleanConfig.changeVernacularNames.apply) { +// name = CleaningUtils.changeVernacularNames(name); +// } + +// if (cleanConfig.updateFamilyNames.apply) { +// name = CleaningUtils.updateFamilyNames(name); +// } + +// if (cleanConfig.deleteUselessMarkers.apply) { +// name = CleaningUtils.deleteUselessMarkers(name); +// } + +// if (cleanConfig.correctOcrErrors.apply) { +// name = CleaningUtils.correctOcrErrors(name); +// } + +// if (cleanConfig.harmonizeAbbreviations.apply) { +// name = CleaningUtils.harmonizeAbbreviations(name); +// } + +// if (cleanConfig.deletePointAfterKey.apply) { +// name = CleaningUtils.deletePointAfterKey(name); +// } + +// if (cleanConfig.deletePointAfterSpecies.apply) { +// name = CleaningUtils.deletePointAfterSpecies(name); +// } + +// if (cleanConfig.fixMissingSpaces.apply) { +// name = CleaningUtils.fixMissingSpaces(name); +// } + +// if (cleanConfig.validateFamilySuffix.apply) { +// name = CleaningUtils.validateFamilySuffix(name); +// } + +// if (cleanConfig.informationInParentheses.apply) { +// name = CleaningUtils.informationInParentheses(name); +// } + +// if (cleanConfig.correctWritingGenus.apply) { +// name = CleaningUtils.correctWritingGenus(name); +// } + +// if (cleanConfig.spacesBeforeAndAfterParentheses.apply) { +// name = CleaningUtils.spacesBeforeAndAfterParentheses(name); +// } + +// if (cleanConfig.correctionHybrid.apply) { +// name = CleaningUtils.correctionHybrid(name); +// } + +// if (cleanConfig.removeAuthors.apply) { +// name = CleaningUtils.removeAuthors(name); +// } + +// return name; +// } diff --git a/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI.Svelte/src/routes/tailor_view/dataCleaningUtils.ts b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI.Svelte/src/routes/tailor_view/dataCleaningUtils.ts new file mode 100644 index 0000000000..ec20c90161 --- /dev/null +++ b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI.Svelte/src/routes/tailor_view/dataCleaningUtils.ts @@ -0,0 +1,618 @@ +export const stripInsideSymbols = (text: string, startSym: string, endSym = startSym) => { + if (!text) return ''; + + // We need to escape symbols like '(' or '[' so Regex doesn't think they are code + const escape = (s: string) => s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + + const s = escape(startSym); + const e = escape(endSym); + + // Dynamically build: /startSymbol\s*(.*?)\s*endSymbol/g + const regex = new RegExp(`${s}\\s*(.*?)\\s*${e}`, 'g'); + + // Replace the whole match with: startSymbol + capturedText + endSymbol + return text.replace(regex, `${startSym}$1${endSym}`); +}; + +export const removeSpecialEscapes = (text: string) => { + if (!text) return ''; + + // remove newline, tab, carriage return + let result = text.replace(/[\n\r\t]/g, ''); + + // remove potentially created double whitespaces + return result.replace(/\s\s+/g, ' ').trim(); +} + +export const removeSpecialCharacters = (text: string) => { + if (!text) return ''; + + const specialChars = /[!?@#\$%&*\^†,¬Ç¡ˆ◊√ó]/g; + let result = text.replace(specialChars, ''); + + result = result.replace(/\.{2,}/g, '.'); + + return result.replace(/\s\s+/g, ' ').trim(); +} + +export const replaceDiacritics = (text: string) => { + if (!text) return ''; + + const diacriticsMap = { + "á": "a", "é": "e", "√™": "e", "í": "i", "ó": "o", "ú": "u", + "Á": "A", "É": "E", "Í": "I", "Ó": "O", "Ú": "U", + "à": "a", "è": "e", "ì": "i", "ò": "o", "ù": "u", + "À": "A", "È": "E", "Ì": "I", "Ò": "O", "Ù": "U", + "ã": "a", "ẽ": "e", "ĩ": "i", "õ": "o", "ũ": "u", + "Ã": "A", "Ẽ": "E", "Ĩ": "I", "Õ": "O", "Ũ": "U", + "Â": "A", "â": "a", "Ê": "E", "ê": "e", "Î": "I", "î": "i", + "Ô": "O", "ô": "o", "Û": "U", "û": "u", + "Æ": "AE", "æ": "ae", "Ç": "S", "ç": "s", "Œ": "oe", + "Ä": "AE", "ä": "ae", "Ö": "OU", "ö": "ou", "Ü": "U", "ü": "u", + "Ÿ": "I", "ÿ": "i" + }; + + // Create a regex that matches any of the keys in our map + // We use [ ... ] for single chars, but since we have multi-char keys like "√™", + // we join them with the OR operator | + const pattern = new RegExp( + Object.keys(diacriticsMap) + .map(key => key.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')) // escape keys + .join('|'), + 'g' + ); + + // One single pass over the string! + return text.replace(pattern, (matched) => diacriticsMap[matched]); +}; + +export const removeNumbers = (text: string) => { + if (!text) return ''; + + let result = text.replace(/\d/g, ''); + + return result.replace(/\s\s+/g, ' ').trim(); +} + +export const replaceNonTrailingSymbolsWithSpace = (text: string, separator: string) => { + if (!text) return ''; + + // We escape the separator in case it's a dot + const escaped = separator.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + // Lookaround equivalent in JS: replace separator with space if between word characters + const regex = new RegExp(`(?<=\\w)${escaped}(?=\\w)`, 'g'); + return text.replace(regex, ' ').trim(); +}; + +// Deletes the # symbol +export const deleteNumeral = (text: string) => text.replace(/#/g, ''); + +export const standardizeHybrids = (text: string) => { + if (!text) return ''; + + let result = text; + + // 1. Ensure 'ex' followed by Uppercase has spaces: 'exName' -> ' ex Name' + result = result.replace(/\s*ex(?=[A-Z])/g, ' ex '); + + // 2. Remove leading 'x' or 'X' if followed by Uppercase: 'x Gardenia' -> 'Gardenia' + // ^\s* matches start of string plus any whitespace + result = result.replace(/^\s*x\s*(?=[A-Z])/i, ''); // 'i' flag handles x and X + + return result.replace(/\s\s+/g, ' ').trim(); +}; + +export const deleteAfterEqual = (text: string) => { + if (!text) return ''; + + // Replace '=' and everything after it (.*) with a space + return text.replace(/=.*/, '').trim(); +}; + +export const cleanHybridFormulas = (text: string) => { + if (!text) return ''; + + let result = text; + + // 1. Identify the first word (Genus) + const firstSpaceIndex = result.trim().indexOf(' '); + if (firstSpaceIndex !== -1) { + const genus = result.substring(0, firstSpaceIndex); + const escapedGenus = genus.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + // 2. Remove the genus if it repeats after an ' x ' + // Example: "Quercus alba x Quercus robur" -> "Quercus alba x robur" + // We use a dynamic Regex to find: space + x + space + genus + space + const redundantGenusRegex = new RegExp(` x ${escapedGenus} `, 'g'); + result = result.replace(redundantGenusRegex, ' x '); + } + + // 3. Fix double 'x' markers + result = result.replace(/x\s+x/g, 'x'); + + // 4. Handle 'x Q.' or 'x Q ' (where Q is any genus initial) + // This replaces 'x' followed by an initial and a dot/space with just ' x ' + result = result.replace(/x\s+[A-Z]\b\.?/g, ' x '); + + // 5. Final space cleanup + return result.replace(/\s\s+/g, ' ').trim(); +}; + +export const deleteTripleHybrids = (text: string) => { + if (!text) return ''; + + // Split the string by the hybrid marker + const parts = text.split(' x '); + + // If there's more than one ' x ' (meaning 3 or more parts) + if (parts.length > 2) { + // Take only the first two parts and join them back + return `${parts[0]} x ${parts[1]}`.trim(); + } + + return text.trim(); +}; + +export const removeCultivars = (text: string) => { + if (!text) return ''; + + let result = text; + + // This regex matches: + // 1. Optional 'cv.' or 'cv' (case insensitive) + // 2. Followed by text in either 'single' or "double" quotes + // 3. Or just the quoted text alone + const cultivarRegex = /\s*(cv\.?)?\s*(['"])(?:(?!\2).)+\2/gi; + + result = result.replace(cultivarRegex, ' '); + + return result.replace(/\s\s+/g, ' ').trim(); +}; + +export const cleanMiddleHyphens = (text: string) => { + if (!text) return ''; + + let result = text; + + // 1. Collapse spaces around hyphens: 'Word - Word' -> 'Word-Word' + // Matches whitespace before and/or after a hyphen as long as text exists on both sides + result = result.replace(/(?<=\S)\s*-\s*(?=\S)/g, '-'); + + // 2. Truncate at space-dash-space followed by a Capital Letter + // 'Pinus sylvestris - Note' -> 'Pinus sylvestris' + result = result.replace(/\s+-\s+[A-Z].*/, ''); + + // 3. Convert remaining isolated ' - ' to a single space + result = result.replace(/\s+-\s+/g, ' '); + + // 4. Remove trailing hyphen at the end of the string + result = result.replace(/\s*-$/, ''); + + return result.replace(/\s\s+/g, ' ').trim(); +}; + +export const deleteTaxonomicAbbreviations = (text: string) => { + if (!text) return ''; + + const badList = [ + "especie", "taxon", "s(ens)?\\.? ?str\\.", "s(ens)?\\. ?l(at)?\\.", "gen(us)?", "comb", + "agg?r?", "subfo", "subg", "subgen", "subgrp", "aff?", "ef", "cf", "cff", "indet", "indeterminate", + "indeterminad\\w", "inconnue", "ined", "non ?det", "sp\\w?", "sppl?", "spec", "species", "nov(o|a)?", + "sp\\.?nov", "orth", "subspecies" + ]; + + // Join the list into a single (word1|word2|word3) pattern + const joinedPatterns = badList.join('|'); + + // The 'Lasso': Matches start of string, space, hyphen, or dot + // before and after the forbidden words. + const before = '(^|[ \\-\\.\\/])'; + const after = '\\.?([ \\-\\.\\/]|$|\\.)'; + + const regex = new RegExp(`${before}(?:${joinedPatterns})${after}`, 'gi'); + + // Replace with a space to avoid merging words together + let result = text.replace(regex, ' '); + + return result.replace(/\s\s+/g, ' ').trim(); +}; + +export const deleteHabitatDescriptors = (text: string) => { + if (!text) return ''; + + const descriptors = [ + "bunch", "upland", "terrestrial", "rosette", "salt marsh", "spSugden", "including", + "swamp", "bark", "culms?", "terra firme", "chapparral", "catinga", "shortgrass", "steppe", "plateau", + "wetland", "cultivated", "vegetables", "mistletoe", "monocot", "valley", "river", "coastal", "mountain", + "harvest", "residues", "nublados?", "bosques?", "mesophytic", "halophytic", "bamboo", "annual", "perennial", + "secondary", "primary", "rain", "herbaceous", "conifers?", "coniferous", "broadleaf", "broad-leaved", + "canopy", "tall", "low", "mata", "field", "forest", "pseudospecies", "leaf", "leaves", "savanna", + "deciduous", "evergreen", "grassland", "abandoned", "pasture", "meadow", "fine", "broad", "form", + "forbs?", "ferns?", "epiphytes?", "trees?", "lianas?", "palms?", "graminoids?", "grass(es)?", "shrubs?", + "sedges?", "Solling" + ]; + + // Join into a single regex pattern + const joinedDescriptors = descriptors.join('|'); + + // Boundary 'lasso': Handles spaces, start/end of string, and hyphens (-) + const before = '(^|[ \\-])'; + const after = '([ \\-]|$|\\.)'; + + const regex = new RegExp(`${before}(?:${joinedDescriptors})${after}`, 'gi'); + + let result = text.replace(regex, ' '); + + return result.replace(/\s\s+/g, ' ').trim(); +}; + +export const deleteGeneralNoise = (text: string) => { + if (!text) return ''; + + const badList = [ + "herbs?", "red", "white", "blue", "green", "yellow", "black", "name error", + "orthodox( p)?", "hiro", "et al\\.", "none", "null", "small", "dark", "smooth" + ]; + + const joined = badList.join('|'); + // Lasso: Start of string or space | word | space or end of string + const regex = new RegExp(`(^| )(?:${joined})( |$)`, 'gi'); + + // We replace with a space to keep word boundaries clean + let result = text.replace(regex, ' '); + return result.replace(/\s\s+/g, ' ').trim(); +}; + +export const deleteLeadingDescriptors = (text: string) => { + if (!text) return ''; + + const badList = ["wood", "alpine", "non-\\w{2,}", "pubescent"]; + const joined = badList.join('|'); + + // Anchor to the start of the string (^) + const regex = new RegExp(`^(?:${joined})( |$|-)`, 'i'); + + return text.replace(regex, '').trim(); +}; + +export const truncateFromBeginning = (text: string) => { + if (!text) return ''; + + // Matches 'pau' or 'mata/matas/mata' at the very start + const badList = ["pau", "mat\\w*"]; + const joined = badList.join('|'); + + // ^(?:...) matches start, (?: .*)? matches the rest of the string + const regex = new RegExp(`^(?:${joined})(?: .*|$)`, 'i'); + + return text.replace(regex, '').trim(); +}; + +export const truncateFromMarker = (text: string) => { + if (!text) return ''; + + // se. sect. ind. indet. cv. cv + const badList = ["se(ct)?\\.", "ind(et)?", "cv\\.?"]; + const joined = badList.join('|'); + + // (?:^| ) ensures we match the word at start or after space + // (?: .*|$) captures everything until the end + const regex = new RegExp(`(?:^| )(?:${joined})(?: .*|$)`, 'i'); + + return text.replace(regex, '').trim(); +}; + +export const truncateFromGeographicOrBreeding = (text: string) => { + if (!text) return ''; + + const badList = [ + "caatinga", "boreal", "germany", "north(ern)?", "south(ern)?", "west(ern)?", + "east(ern)?", "subpolar", "ural", "southafrica", "tropical", "temperate", "cultivar", + "genotype", "hybride?", "inbred line", "variety" + ]; + + const joined = badList.join('|'); + // Lasso: Start, space, or hyphen | bad words | everything else + const regex = new RegExp(`(^|[ \\-])(?:${joined})(?: .*|$)`, 'gi'); + + return text.replace(regex, '').trim(); +}; + +export const truncateFromUncertainty = (text: string) => { + if (!text) return ''; + + const badList = [ + "group\\w?", "death", "dwarf", "little", "mid", "average", "other", "mixed", "under", + "all", "dry", "wet", "open", "new", "old", "unk\\.?", "not identified", "unknown", "undetermined", + "undefined", "unidentified", "unclassified" + ]; + + const joined = badList.join('|'); + // Lasso: Start or space | bad words | everything else + const regex = new RegExp(`(^| )(?:${joined})(?: .*|$)`, 'gi'); + + return text.replace(regex, '').trim(); +}; + +export const changeVernacularNames = (text: string) => { + if (!text) return ''; + + // 1. Handle the "Starts with" cases (Genus swaps) + const startsWithMap = { + 'Abiu': 'Pouteria', + 'Lily': 'Lilium', + 'Cotton': 'Gossypium', + 'Strawberry': 'Fragaria', + 'Cashew': 'Anacardium' + }; + + let result = text; + + // Check if the string starts with any of our map keys + for (const [common, scientific] of Object.entries(startsWithMap)) { + const regex = new RegExp(`^${common}( .*|$)`, 'i'); + if (regex.test(result)) { + return result.replace(regex, `${scientific}$1`); + } + } + + // 2. Handle the "Contains" cases (Specific replacements) + // Coffee -> Coffea arabica + result = result.replace(/(^| )coffee( .*|$)/i, '$1Coffea arabica$2'); + + // Orchid -> Orchidaceae + result = result.replace(/(^| )orchid( .*|$)/i, '$1Orchidaceae$2'); + + return result.trim(); +}; + +export const updateFamilyNames = (text: string) => { + if (!text) return ''; + + const familyMap = { + 'Compositae': 'Asteraceae', + 'Cruciferae': 'Brassicaceae', + 'Gramineae': 'Poaceae', + 'Guttiferae': 'Clusiaceae', + 'Labiatae': 'Lamiaceae', + 'Leguminosae': 'Fabaceae', + 'Palmae': 'Arecaceae', + 'Umbelliferae': 'Apiaceae' + }; + + let result = text; + + // We loop through the map and use the ^ anchor to match the start of the string + for (const [oldName, newName] of Object.entries(familyMap)) { + const regex = new RegExp(`^${oldName}`, 'i'); + if (regex.test(result)) { + // Replace only the first occurrence at the start + result = result.replace(regex, newName); + break; // Once we find a match at the start, we can stop + } + } + + return result; +}; + +export const deleteUselessMarkers = (text: string) => { + if (!text) return ''; + + let result = text; + + // 1. Remove trailing hybrid marker: 'Quercus x' -> 'Quercus' + result = result.replace(/\s[xX]$/, ''); + + // 2. Standardize 'A-' notation + result = result.replace(/^A-/, ' '); // At start + result = result.replace(/\sA-/g, ' x '); // In middle + + // 3. Remove leading lowercase words (Invalid for Genus) + result = result.replace(/^[a-z]+\s+/, ' '); + + // 4. Remove 3-letter uppercase codes, but PROTECT 'POA' + // (?!POA\s) is a negative lookahead + result = result.replace(/^(?!POA\s)[A-Z]{3}\s/, ''); + + // 5. Remove 'NA' markers + result = result.replace(/\sNA(\s|$)/g, ' '); + + // 6. The d'Urville correction + // Matches 'd', up to 2 chars, 'd', up to 2 chars, 'Urv' and everything after + result = result.replace(/d.{0,2}d.{0,2}Urv.*/, "d'Urv"); + + return result.replace(/\s\s+/g, ' ').trim(); +}; + +export const correctOcrErrors = (text: string) => { + if (!text) return ''; + // Replaces I with l ONLY if surrounded by lowercase letters + return text.replace(/(?<=[a-z])I(?=[a-z])/g, 'l'); +}; + +export const harmonizeAbbreviations = (text: string) => { + if (!text) return ''; + + let result = text; + + // 1. Standardize subspecies variations to ' subsp. ' + // Matches s., ssp, sspp, susbp, etc. + result = result.replace(/(?:\.|\s)?s(ub)?sp(\.)?(?:\s|$|\.)/gi, ' subsp. '); + result = result.replace(/\s(susbp|subs)(\.|\s)/gi, ' subsp. '); + result = result.replace(/(\.)?subspecies(\.)?/gi, ' subsp. '); + result = result.replace(/\s+s\.\s+/g, ' subsp. '); + + // 2. Standardize form variations to ' f. ' + result = result.replace(/fo?(rma)?\.?(\s|$)/gi, ' f. '); + + // 3. Standardize variety to ' var. ' + result = result.replace(/\s+var(\.|\s)/gi, ' var. '); + + // 4. Clean up "stacked" or redundant abbreviations + result = result.replace(/f\.\s+subsp\./g, 'subsp.'); + result = result.replace(/f\.\s+var\./g, 'var.'); + + // 5. Remove abbreviations if they start the string + result = result.replace(/^ ?(subsp|var|f)(\.)?\s+.*/i, ' '); + + // 6. Remove "trailing" abbreviations with no content after them + result = result.replace(/\s(subsp|var|f)\.?\s*$/i, ''); + + return result.replace(/\s\s+/g, ' ').trim(); +}; + +export const deletePointAfterKey = (text: string) => { + if (!text) return ''; + + // Regex breakdown: + // ^([A-Z][a-z\-\s]*[a-z]) -> Group 1: The Genus (Starts with Caps) + // \s+(subsp|var|f) -> Group 2: The Rank + // \. -> The dot we want to remove + const regex = /^([A-Z][a-z\-\s]*[a-z])\s+(subsp|var|f)\./; + + // We replace the whole match with Group 1 + space + Group 2 (no dot) + return text.replace(regex, '$1 $2').trim(); +}; + +export const deletePointAfterSpecies = (text: string) => { + if (!text) return ''; + + // Regex breakdown: + // ^([A-Z][a-z\-\s]*[a-z]) -> Group 1: Genus + // \s+ -> Space + // ([a-z][a-z\-]*[a-z]) -> Group 2: species epithet + // \. -> The dot we want to remove + const regex = /^([A-Z][a-z\-\s]*[a-z])\s+([a-z][a-z\-]*[a-z])\./; + + return text.replace(regex, '$1 $2').trim(); +}; + +export const fixMissingSpaces = (text: string) => { + if (!text) return ''; + + let result = text; + + // 1. Remove space before hyphen or period: 'Canis -' -> 'Canis-' + result = result.replace(/\s-/g, '-'); + result = result.replace(/\s\./g, '.'); + + // 2. Ensure a space follows a period, UNLESS it's a closing bracket + // Example: 'C.lupus' -> 'C. lupus' | '(sp.)' -> '(sp.)' (stays same) + result = result.replace(/\.(?!\))/g, '. '); + + // 3. Special case for period-hyphen: '. -' -> '.-' + result = result.replace(/\.\s-/g, '.-'); + + // 4. Final Spacing Cleanup + // Replace multiple spaces with one, then trim edges + return result.replace(/\s+/g, ' ').trim(); +}; + +export const validateFamilySuffix = (text: string) => { + if (!text) return ''; + + // Condition 2: Does it contain a word ending in 'aceae'? + const familyMatch = text.match(/[A-Za-z]+aceae/); + if (!familyMatch) return text; + + const foundWord = familyMatch[0]; + + // Condition 3: Is it preceded by a Genus/Species pattern? + // This regex looks for: Capitalized Genus + word + (optional rank) + our 'aceae' word + const speciesContextRegex = new RegExp(`[A-Z][a-z\\-]+(?:\\s+[a-z\\-]+)+(?:\\s+[a-z]+\\.)?\\s+${foundWord}`); + + const isSpeciesEpithet = speciesContextRegex.test(text); + + if (!isSpeciesEpithet) { + // It's a TRUE family name. Capitalize the first letter. + const capitalized = foundWord.charAt(0).toUpperCase() + foundWord.slice(1); + return text.replace(foundWord, capitalized).trim(); + } else { + // It's a FALSE family (a species name). Change suffix to 'cea'. + // We handle the three specific cases from the Python code + let corrected = text; + corrected = corrected.replace(/ceae$/g, 'cea'); + corrected = corrected.replace(/ceae\s/g, 'cea '); + corrected = corrected.replace(/ceae\)/g, 'cea)'); + return corrected.trim(); + } +}; + +export const informationInParentheses = (text: string) => { + if (!text) return ""; + + // 1. Remove (lowercase-words-with-hyphens) + // Equivalent to: r'\([a-z]([a-z]|-){1,}[a-z]\)' + const lowercaseInfo = /\([a-z]([a-z]|-)+[a-z]\)/g; + text = text.replace(lowercaseInfo, ''); + + // 2. Remove empty or whitespace-only parentheses: ( ) + // Equivalent to: r'\(\s*\)' + const emptyParens = /\(\s*\)/g; + text = text.replace(emptyParens, ''); + + return text.replace(/\s+/g, ' ').trim(); +} + +export const correctWritingGenus = (text: string) => { + if (!text) return ""; + + const pattern = /^([A-Z]{2,}((\s(x|X))?\s+|\W|$))+/; + const match = text.match(pattern); + + if (match) { + const matchedText = match[0]; + const transformed = matchedText.charAt(0) + matchedText.slice(1).toLowerCase(); + text = transformed + text.slice(matchedText.length); + } + + return text.replace(/\s+/g, ' ').trim(); +}; + +export const spacesBeforeAndAfterParentheses = (text: string) => { + if (!text) return ""; + + text = text.replace(/\( /g, '('); + text = text.replace(/ \)/g, ')'); + + return text.replace(/\s+/g, ' ').trim(); +}; + +export const correctionHybrid = (text: string) => { + if (!text) return ""; + + if (/^x\s/.test(text)) { + // Note: hybrid_1 is unused in the original Python snippet's return, + // but the regex replacement is performed here. + text = text.replace(/^x\s/, ''); + } + + return text.replace(/\s+/g, ' ').trim(); +}; + +export const removeAuthors = (text: string) => { + if (!text) return ""; + + const array = text.split(" "); + const length = array.length; + + // Find the index of the first item starting with '(' + let indexOpen = -1; + for (let i = 0; i < array.length; i++) { + if (array[i].startsWith('(')) { + indexOpen = i; + break; + } + } + + if (indexOpen > 0) { + const a = array.slice(0, indexOpen); + return a.join(' ').replace(/\s+/g, ' ').trim(); + } + + if (length > 2) { + const a = array.slice(0, length - 1); + return a.join(' ').replace(/\s+/g, ' ').trim(); + } else { + return text; + } +}; \ No newline at end of file diff --git a/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI.Svelte/src/routes/tailor_view/services.ts b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI.Svelte/src/routes/tailor_view/services.ts new file mode 100644 index 0000000000..776cb6f2b6 --- /dev/null +++ b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI.Svelte/src/routes/tailor_view/services.ts @@ -0,0 +1,26 @@ +import { Api } from '@bexis2/bexis2-core-ui'; +import type { TailorEdit } from './types'; +import type { ServiceResult } from '$lib/types/types'; + +export const loadResult = async (datasetId: number, versionId: number) => { + try { + const response = await Api.get(`http://localhost:44345/smm/species/ViewTailored?datasetId=${datasetId}&versionId=${versionId}`); + return response.data; + } catch (error) { + console.error(error); + } +} + +export const submitTailorEdits = async (datasetId: number, versionId: number, payload: TailorEdit[]): Promise> => { + try { + console.log("Applying tailor Edits..."); + console.log("Payload: \n", payload); + const response = await Api.post(`/smm/species/ApplyTailorEdits?datasetId=${datasetId}&versionId=${versionId}`, payload); + + return { success: true, data: response.data } + } catch (error: any) { + console.error(error); + return { success: false, error: error.data?.message }; + } + +} \ No newline at end of file diff --git a/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI.Svelte/src/routes/tailor_view/types.ts b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI.Svelte/src/routes/tailor_view/types.ts new file mode 100644 index 0000000000..ca05267c12 --- /dev/null +++ b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI.Svelte/src/routes/tailor_view/types.ts @@ -0,0 +1,8 @@ +// one row-change of a SpeciesMatchingResult in bexis +// used to submit/apply changes to backend database +export interface TailorEdit { + id: number, + originalName: string, + editedName: string, + cleanedName: string +} diff --git a/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI/Helpers/ConversionHelper.cs b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI/Helpers/ConversionHelper.cs new file mode 100644 index 0000000000..403c2e03e4 --- /dev/null +++ b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI/Helpers/ConversionHelper.cs @@ -0,0 +1,33 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Web; + +namespace BExIS.Modules.Smm.UI.Helpers +{ + public class ConversionHelper + { + public static HashSet ConvertStringListToLongHashSet(IEnumerable stringList) + { + var longHashSet = new HashSet(); + + if (stringList == null) return longHashSet; + + foreach (var str in stringList) + { + if (string.IsNullOrWhiteSpace(str)) continue; + if (long.TryParse(str, out long result)) + { + longHashSet.Add(result); + } + else + { + // TODO: + // Handle the case where parsing fails, e.g., log a warning or throw an exception + // For this example, we will simply ignore invalid entries + } + } + return longHashSet; + } + } +} \ No newline at end of file diff --git a/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI/Helpers/MatchingAPIs/CLBApi.cs b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI/Helpers/MatchingAPIs/CLBApi.cs new file mode 100644 index 0000000000..9e0f7e993a --- /dev/null +++ b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI/Helpers/MatchingAPIs/CLBApi.cs @@ -0,0 +1,570 @@ +using BExIS.Dim.Entities.Export.GBIF; +using BExIS.Dlm.Entities.SpeciesMatching; +using BExIS.Dlm.Services.SpeciesMatching; +using BExIS.IO.Transform.Output; +using BExIS.Modules.Smm.UI.Models; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using System; +using System.Collections.Generic; +using System.Data; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Text; +using System.Threading.Tasks; +using System.Web; +using System.Web.Mvc; +using System.IO.Compression; +using Vaiona.Persistence.Api; +using Vaiona.Web.Mvc.Modularity; + + +namespace BExIS.Modules.Smm.UI.Helpers.MatchingAPIs +{ + public class CLBApi : MatchingApiBase + { + public CLBApi(HttpClient http) : base(http) { } + + public override string Identifier => "CLB"; + + public override Type OptionsType => typeof(ClbOptions); + + public override string BaseUrl => "https://api.checklistbank.org/dataset/3LR/match/nameusage/job"; + + public override HashSet AcceptableMatchTypes => new HashSet + { + "ambiguous", + "exact", + "variant", + "canonical" + }; + + public override string GenMatchingUrl() + { + return $"{BaseUrl}?format=csv"; + } + + public override (string FilePath, int RowCount) GenerateInputFile(long datasetId, long dataStructureId, long versionId, int stepId) + { + + DataTable dt = new DataTable("SpeciesUnmatched"); + dt.Columns.Add("ID", typeof(long)); + dt.Columns.Add("scientificName", typeof(string)); + dt.Columns.Add("rank", typeof(string)); + dt.Columns.Add("kingdom", typeof(string)); + dt.Columns.Add("authorship", typeof(string)); + + int writtenCount = 0; + + using (var smrm = new SpeciesMatchingResultManager()) + { + var smrmRepo = smrm.GetBulkUnitOfWork().GetReadOnlyRepository(); + List result = smrmRepo.Query().Where(r => r.Dataset.Id == datasetId && r.DatasetVersionId == versionId && r.ConfirmedByUser == false).ToList(); + + // TODO: - write all columns correctly + foreach (var item in result) + { + if (item.EditedName != null && item.EditedName != "") + { + dt.Rows.Add(item.Id, item.EditedName, "species", "", ""); + writtenCount++; + } + else if (item.OriginalName != null && item.OriginalName != "") + { + dt.Rows.Add(item.Id, item.OriginalName, "species", "", ""); + writtenCount++; + } + else + { + continue; + } + } + } + + var outputManager = new OutputDataManager(); + // put matching files under Datasets//Matching// + string ns = Path.Combine(datasetId.ToString(), "Matching", versionId.ToString()); + string title = ProgressHelper.GenMatchingFileName(false, datasetId, stepId, false); + + string filepath = outputManager.GenerateAsciiFile(ns, dt, title, "text/csv", dataStructureId); + + if (!System.IO.File.Exists(filepath)) return (null, 0); + return (filepath, writtenCount); + } + + public override List ReadResultFile(string filepath) + { + var result = new List(); + + try + { + if (string.IsNullOrWhiteSpace(filepath) || !System.IO.File.Exists(filepath)) return result; + + using (var sr = new StreamReader(filepath, Encoding.UTF8)) + { + string headerLine = sr.ReadLine(); + if (headerLine == null) return result; + + // parse header columns + var headers = TabularFileHelper.ParseCsvLine(headerLine).Select(h => h?.Trim()).ToList(); + // map header name (case-insensitive) to index + var headerIndex = new Dictionary(StringComparer.OrdinalIgnoreCase); + for (int i = 0; i < headers.Count; i++) + { + if (!string.IsNullOrEmpty(headers[i]) && !headerIndex.ContainsKey(headers[i])) + { + headerIndex[headers[i]] = i; + } + } + + string line; + while ((line = sr.ReadLine()) != null) + { + if (string.IsNullOrWhiteSpace(line)) continue; + var fields = TabularFileHelper.ParseCsvLine(line); + + string GetField(string name) + { + if (!headerIndex.TryGetValue(name, out int idx)) return string.Empty; + if (idx < 0 || idx >= fields.Count) return string.Empty; + var v = fields[idx]; + return string.IsNullOrEmpty(v) ? string.Empty : v; + } + + var entry = new MatchingResultRow + { + Original_ID = GetField("Original_ID"), + Original_scientificName = GetField("Original_scientificName"), + Original_rank = GetField("Original_rank"), + Original_kingdom = GetField("Original_kingdom"), + Original_authorship = GetField("Original_authorship"), + MatchType = GetField("MatchType"), + MatchIssues = GetField("MatchIssues"), + ID = GetField("ID"), + Rank = GetField("Rank"), + ScientificName = GetField("ScientificName"), + Authorship = GetField("Authorship"), + Status = GetField("Status"), + AcceptedID = GetField("AcceptedID"), + AcceptedScientificName = GetField("AcceptedScientificName"), + AcceptedAuthorship = GetField("AcceptedAuthorship"), + Kingdom = GetField("Kingdom"), + Phylum = GetField("Phylum"), + Class = GetField("Class"), + Order = GetField("Order"), + Family = GetField("Family"), + Genus = GetField("Genus"), + Classification = GetField("Classification") + }; + + result.Add(entry); + } + } + + return result; + } + catch (Exception ex) + { + return result; + } + } + + public override bool AcceptMatches(long datasetId, long versionId, StepEntry step, HashSet acceptedIds) + { + + var filepath = ProgressHelper.GetMatchedFilepath(datasetId, versionId, step.Id); + if (filepath == null) return false; + + using (var speciesMatchingResultManager = new SpeciesMatchingResultManager()) + // Use a regular (stateful) unit of work here so that modified entities are tracked by NHibernate. + // The bulk unit of work uses a stateless session which does not track changes to entities, + // therefore modifications made to retrieved objects would not be persisted on Commit. + using (var uow = speciesMatchingResultManager.GetUnitOfWork()) + using (var sr = new StreamReader(filepath, Encoding.UTF8)) + { + try + { + string headerLine = sr.ReadLine(); + if (headerLine == null) return false; + + // parse header columns + var headers = TabularFileHelper.ParseCsvLine(headerLine).Select(h => h?.Trim()).ToList(); + // map header name (case-insensitive) to index + var headerIndex = new Dictionary(StringComparer.OrdinalIgnoreCase); + for (int i = 0; i < headers.Count; i++) + { + if (!string.IsNullOrEmpty(headers[i]) && !headerIndex.ContainsKey(headers[i])) + { + headerIndex[headers[i]] = i; + } + } + + // pre-filter the species matching results for this dataset/version so we only operate on this subset + var repo = uow.GetRepository(); + var subsetIds = repo.Query().Where(r => r.Dataset.Id == datasetId && r.DatasetVersionId == versionId); + + Debug.WriteLine("Read file header and mapped columns. Now processing lines..."); + + string line; + while ((line = sr.ReadLine()) != null) + { + if (string.IsNullOrWhiteSpace(line)) continue; + var fields = TabularFileHelper.ParseCsvLine(line); + + string GetField(string name) + { + if (!headerIndex.TryGetValue(name, out int idx)) return string.Empty; + if (idx < 0 || idx >= fields.Count) return string.Empty; + var v = fields[idx]; + return string.IsNullOrEmpty(v) ? string.Empty : v; + } + + var original_id = GetField("Original_ID"); + + var entry = new CLBMatchingResultFile + { + Original_ID = GetField("Original_ID"), + //Original_scientificName = GetField("Original_scientificName"), + //Original_rank = GetField("Original_rank"), + //Original_kingdom = GetField("Original_kingdom"), + //Original_authorship = GetField("Original_authorship"), + MatchType = GetField("MatchType"), + //MatchIssues = GetField("MatchIssues"), + ID = GetField("ID"), + Rank = GetField("Rank"), + ScientificName = GetField("ScientificName"), + Authorship = GetField("Authorship"), + Status = GetField("Status"), + AcceptedID = GetField("AcceptedID"), + AcceptedScientificName = GetField("AcceptedScientificName"), + AcceptedAuthorship = GetField("AcceptedAuthorship"), + Kingdom = GetField("Kingdom"), + Phylum = GetField("Phylum"), + Class = GetField("Class"), + Order = GetField("Order"), + Family = GetField("Family"), + Genus = GetField("Genus"), + //Classification = GetField("Classification") + }; + + if (acceptedIds.Contains(long.Parse(original_id))) + { + // query for the SpeciesMatchingResult with this Original_ID and mark it as confirmed + var result = subsetIds.FirstOrDefault(r => r.Id == long.Parse(original_id)); + if (result != null) + { + result.ConfirmedByUser = true; + result.MatchedName = entry.ScientificName; + result.MatchType = entry.MatchType; + result.Status = entry.Status; + result.StepId = step.Id; + result.MatchId = entry.ID; + result.MatchRank = entry.Rank; + result.MatchAuthorship = entry.Authorship; + result.AcceptedId = entry.AcceptedID; + result.AcceptedScientificName = entry.AcceptedScientificName; + result.AcceptedAuthorship = entry.AcceptedAuthorship; + result.TaxonKingdom = entry.Kingdom; + result.TaxonPhylum = entry.Phylum; + result.TaxonClass = entry.Class; + result.TaxonOrder = entry.Order; + result.TaxonFamily = entry.Family; + result.TaxonGenus = entry.Genus; + result.MatchSource = step.MatchSource; + result.TimestampMatch = step.TimeStamp; + } + } + + Debug.WriteLine("Processed line with Original_ID=" + original_id); + } + + Debug.WriteLine("Commiting results to database..."); + uow.Commit(); + Debug.WriteLine("Finished commiting results."); + return true; + } + catch (Exception ex) + { + uow.Ignore(); + return false; + } + } + } + + public override async Task MatchAsync(long datasetId, long versionId, string filepath, MatchingProgressModel matchingProgress, IApiOptions apiOptions) + { + if (string.IsNullOrWhiteSpace(filepath) || !System.IO.File.Exists(filepath)) + { + return await Task.FromResult(new MatchingApiResponse + { + Success = false, + StatusCode = null, + Message = "Export file not generated.", + Payload = null + }); + } + + var step = matchingProgress.GetLatestStep(); + + Debug.WriteLine("{====================FILEPATH:====================}"); + Debug.WriteLine(filepath); + byte[] fileBytes = System.IO.File.ReadAllBytes(filepath); + + // Debug: inspect apiOptions concrete type and values + try + { + if (apiOptions == null) + { + Debug.WriteLine("apiOptions == null"); + } + else if (apiOptions is ClbOptions clb) + { + Debug.WriteLine($"ClbOptions: SourceKey={clb.SourceKey}, Synonyms={clb.Synonyms}"); + } + else if (apiOptions is GenericOptions gen) + { + Debug.WriteLine("GenericOptions.Raw: " + (gen.Raw != null ? gen.Raw.ToString(Formatting.Indented) : "")); + } + else + { + try + { + Debug.WriteLine("apiOptions: " + JsonConvert.SerializeObject(apiOptions)); + } + catch + { + Debug.WriteLine("apiOptions type: " + apiOptions.GetType().FullName); + } + } + } + catch (Exception ex) + { + Debug.WriteLine("Failed to debug-print apiOptions: " + ex.Message); + } + + + GBFICrendentials credentials = ModuleManager.GetModuleSettings("DIM").GetValueByKey("gbifapicredentials"); + var username = credentials.Username; + var password = credentials.Password; + + var authValue = Convert.ToBase64String(Encoding.UTF8.GetBytes($"{username}:{password}")); + var url = this.GenMatchingUrl(); + // var url = "https://api.checklistbank.org/dataset/3LR/match/nameusage/job?format=csv"; + + using (var content = new ByteArrayContent(fileBytes)) + { + //content.Headers.ContentType = new MediaTypeHeaderValue("text/tab-separated-values"); + content.Headers.ContentType = new MediaTypeHeaderValue("text/csv"); + + using (var request = new HttpRequestMessage(HttpMethod.Post, url)) + { + request.Content = content; + request.Headers.Authorization = new AuthenticationHeaderValue("Basic", authValue); + + try + { + // HttpResponseMessage response = await _http.PostAsync(url, content); + var response = await _http.SendAsync(request); + string responseString = await response.Content.ReadAsStringAsync(); + + // try to parse the response string as JSON. If parsing fails treat the whole call as a failure + // because we cannot interpret the API response reliably. + object responseJson; + try + { + // TODO: source out handling and/or better failure handling + // Try to parse response as JSON object so we can extract result.download and result.key + var responseObject = JObject.Parse(responseString); + responseJson = responseObject; + + // If we have a matching progress step available, update it with download and key + if (step != null) + { + var resultToken = responseObject["result"]; + if (resultToken != null) + { + var download = resultToken["download"]?.ToString(); + var key = resultToken["key"]?.ToString(); + var matchSource = resultToken["dataset"]?["sourceKey"]?.ToString(); + + if (!string.IsNullOrEmpty(download)) step.DownloadLink = download; + if (!string.IsNullOrEmpty(key)) step.JobKey = key; + if (!string.IsNullOrEmpty(matchSource)) step.MatchSource = matchSource; + + step.ApiIdentifier = this.Identifier; + step.TimeStamp = DateTime.UtcNow; + + // persist the updated step back to the matching progress + matchingProgress.UpdateStep(step); + ProgressHelper.SaveMatchingProgress(matchingProgress, datasetId, versionId); + } + } + } + catch (Exception ex) + { + Debug.WriteLine("Failed to deserialize ChecklistBank response as JSON: " + ex.Message); + Debug.WriteLine("RESPONSE STRING: "); + Debug.WriteLine(responseString); + return new MatchingApiResponse + { + Success = false, + StatusCode = (int?)response.StatusCode, + Message = "Failed to parse API response as JSON.", + Payload = responseString + }; + } + + if (response.IsSuccessStatusCode) + { + return new MatchingApiResponse + { + Success = true, + StatusCode = (int?)response.StatusCode, + Message = null, + Payload = responseJson + }; + } + else + { + return new MatchingApiResponse + { + Success = false, + StatusCode = (int?)response.StatusCode, + Message = "API returned non-success status.", + Payload = responseJson + }; + } + } + catch (Exception ex) + { + // failure before request sent or other network error: no HTTP status code available + return new MatchingApiResponse + { + Success = false, + StatusCode = null, + Message = ex.Message, + Payload = null + }; + } + } + } + } + + public override async Task DownloadResultFile(long datasetId, long versionId, int stepId, MatchingProgressModel matchingProgress) + { + var step = matchingProgress.GetStepById(stepId); + if (step == null) return null; + Debug.WriteLine("Succesfully loaded step from matchingProgress..."); + + var downloadLink = step.DownloadLink; + Debug.WriteLine(downloadLink, "DownloadLink: "); + Debug.WriteLine(datasetId.ToString(), versionId.ToString(), stepId.ToString(), "Getting Matched Filepath..."); + var filepath = ProgressHelper.GetMatchedFilepath(datasetId, versionId, stepId, false); + Debug.WriteLine(filepath, "Retrieved filepath..."); + + if (filepath == null) return null; + + if (string.IsNullOrWhiteSpace(downloadLink)) return null; + + Debug.WriteLine("Starting download attempt on filepath: ", filepath); + + try + { + // create a download marker file to indicate active download + var markerPath = filepath + ".downloading"; + if (File.Exists(markerPath)) + { + // if marker is recent assume another process is downloading + var lastWrite = File.GetLastWriteTimeUtc(markerPath); + if (DateTime.UtcNow - lastWrite < TimeSpan.FromHours(24)) + { + Debug.WriteLine("Download already in progress for " + filepath); + return null; + } + else + { + // stale marker, try to remove + try { File.Delete(markerPath); } catch { } + } + } + + try + { + // write basic metadata to marker so other processes can inspect + var markerContent = "download_start=" + DateTime.UtcNow.ToString("o") + + "\njobKey=" + (step?.JobKey ?? string.Empty) + + "\ndownloadLink=" + (step?.DownloadLink ?? string.Empty); + File.WriteAllText(markerPath, markerContent, Encoding.UTF8); + } + catch (Exception ex) + { + Debug.WriteLine("Could not create marker file: " + ex.Message); + } + // ensure target directory exists + var targetDir = Path.GetDirectoryName(filepath); + if (!string.IsNullOrEmpty(targetDir) && !Directory.Exists(targetDir)) + { + Directory.CreateDirectory(targetDir); + } + + // download the zip to a temporary file + var tempZip = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString() + ".zip"); + using (var response = await _http.GetAsync(downloadLink)) + { + if (!response.IsSuccessStatusCode) return null; + + using (var fs = new FileStream(tempZip, FileMode.Create, FileAccess.Write, FileShare.None)) + { + await response.Content.CopyToAsync(fs); + } + } + + // open the zip and find the first .csv entry + using (var archive = ZipFile.OpenRead(tempZip)) + { + ZipArchiveEntry csvEntry = null; + foreach (var entry in archive.Entries) + { + if (entry == null) continue; + if (entry.FullName.EndsWith(".csv", StringComparison.OrdinalIgnoreCase)) + { + csvEntry = entry; + break; + } + } + + if (csvEntry == null) + { + // no csv found + try { File.Delete(markerPath); } catch { } + return null; + } + + // extract the csv entry to the desired filepath (overwrite if exists) + // ZipArchiveEntry.ExtractToFile throws if file exists and overwrite not specified in older frameworks, + // so delete target if exists first. + if (File.Exists(filepath)) File.Delete(filepath); + csvEntry.ExtractToFile(filepath); + } + + // cleanup temp file + try { File.Delete(tempZip); } catch { } + + try { File.Delete(markerPath); } catch { } + + return filepath; + } + catch (Exception ex) + { + Debug.WriteLine("Failed to download or extract CLB result file: " + ex.Message); + // ensure marker cleanup on failure + try { var markerPath = filepath + ".downloading"; if (File.Exists(markerPath)) File.Delete(markerPath); } catch { } + return null; + } + } + + } +} \ No newline at end of file diff --git a/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI/Helpers/MatchingAPIs/HttpClientRegistry.cs b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI/Helpers/MatchingAPIs/HttpClientRegistry.cs new file mode 100644 index 0000000000..fe9c9ff2f1 --- /dev/null +++ b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI/Helpers/MatchingAPIs/HttpClientRegistry.cs @@ -0,0 +1,18 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net.Http; +using System.Web; + +namespace BExIS.Modules.Smm.UI.Helpers.MatchingAPIs +{ + public static class HttpClientRegistry + { + public static readonly HttpClient SharedClient = new HttpClient(); + + static HttpClientRegistry() + { + // Set any default configuration for the shared HttpClient here if needed + } + } +} \ No newline at end of file diff --git a/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI/Helpers/MatchingAPIs/IApiOptions.cs b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI/Helpers/MatchingAPIs/IApiOptions.cs new file mode 100644 index 0000000000..abfd8aea00 --- /dev/null +++ b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI/Helpers/MatchingAPIs/IApiOptions.cs @@ -0,0 +1,33 @@ +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using System.ComponentModel.DataAnnotations; + +namespace BExIS.Modules.Smm.UI.Helpers.MatchingAPIs +{ + public interface IApiOptions { } + + public class ClbOptions : IApiOptions + { + [JsonProperty("sourceKey")] + [Required] + public string SourceKey { get; set; } + + [JsonProperty("synonyms")] + public bool Synonyms { get; set; } + } + + // TODO: - remove (just an example) + public class GbifOptions : IApiOptions + { + [JsonProperty("parameter1")] + public string Parameter1 { get; set; } + + [JsonProperty("parameter2")] + public string Parameter2 { get; set; } + } + + public class GenericOptions : IApiOptions + { + public JObject Raw { get; set; } + } +} diff --git a/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI/Helpers/MatchingAPIs/MatchingApiBase.cs b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI/Helpers/MatchingAPIs/MatchingApiBase.cs new file mode 100644 index 0000000000..cf60ed28d4 --- /dev/null +++ b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI/Helpers/MatchingAPIs/MatchingApiBase.cs @@ -0,0 +1,62 @@ +using BExIS.Modules.Smm.UI.Models; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net.Http; +using System.Threading.Tasks; +using System.Web; +using System.Web.Mvc; + +namespace BExIS.Modules.Smm.UI.Helpers.MatchingAPIs +{ + // Abstract base class for matching APIs (file based matching), defining the common interface and properties + // NOTE: each new implemented (file based) API should inherit from this + // NOTE: for now, each configuration/implementation change here needs a rebuild + public abstract class MatchingApiBase + { + protected readonly HttpClient _http; + + protected MatchingApiBase(HttpClient http) + { + _http = http ?? throw new ArgumentNullException(nameof(http)); + } + + // used to provide access and store information on which apiBase has been used e.g. in matchingProgress steps + // this is not optimal but works for now. + // NOTE IMPORTANT: once set and used, an Identifier should not be changed because this would invalidate the identifiers stored in json + public abstract string Identifier { get; } + + public abstract string BaseUrl { get; } + + public abstract HashSet AcceptableMatchTypes { get; } + + // Method to perform the matching based on the provided file path + // (this actually makes the post request to the API and returns the result as a JsonResult) + public abstract Task MatchAsync(long datasetId, long versionId, string filepath, MatchingProgressModel matchingProgress, IApiOptions apiOptions); + + // Method to generate the unmatched input file (source file for matching) + // NOTE: different APIs need different file structure and input format + public abstract (string FilePath, int RowCount) GenerateInputFile(long datasetId, long dataStructureId, long versionId, int stepId); + + public abstract Task DownloadResultFile(long datasetId, long versionId, int stepId, MatchingProgressModel matchingProgress); + + // Method to read the result file and return a list of matching results + // NOTE: different APIs have different result file structure and output format + // NOTE: Try to always parse the file into a List of MatchingResultRow + public abstract List ReadResultFile(string filepath); + + // Method to iterate result file and accept a subset of results + // NOTE: to 'accept' means updating the respective SpeciesMatchingResult object in the database + public abstract bool AcceptMatches(long datasetId, long versionId, StepEntry step, HashSet acceptedIds); + + public abstract string GenMatchingUrl(); + + public HashSet GetAcceptableMatchTypes() + { + return AcceptableMatchTypes; + } + + // options type for the API, used for deserialization of options from JSON + public virtual Type OptionsType => null; + } +} \ No newline at end of file diff --git a/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI/Helpers/MatchingAPIs/MatchingApiProvider.cs b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI/Helpers/MatchingAPIs/MatchingApiProvider.cs new file mode 100644 index 0000000000..7be58b62c5 --- /dev/null +++ b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI/Helpers/MatchingAPIs/MatchingApiProvider.cs @@ -0,0 +1,91 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Web; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using System.ComponentModel.DataAnnotations; + +namespace BExIS.Modules.Smm.UI.Helpers.MatchingAPIs +{ + public class MatchingApiProvider + { + private readonly Dictionary _apiRegistry; + + public MatchingApiProvider() + { + var sharedClient = HttpClientRegistry.SharedClient; + + var apiList = new List + { + new CLBApi(sharedClient), + }; + + // Register available APIs here + _apiRegistry = apiList.ToDictionary( + api => api.Identifier, + api => api, + StringComparer.OrdinalIgnoreCase + ); + } + + public MatchingApiBase GetApi(string identifier) + { + if (string.IsNullOrWhiteSpace(identifier)) + throw new ArgumentException("API identifier cannot be null or empty.", nameof(identifier)); + if (_apiRegistry.TryGetValue(identifier, out var api)) + { + return api; + } + throw new KeyNotFoundException($"No matching API found for identifier: {identifier}"); + } + + // Resolves the options for a given API identifier and options payload + public IApiOptions ResolveOptions(string apiIdentifier, JObject options) + { + if (options == null) return null; + if (string.IsNullOrWhiteSpace(apiIdentifier)) + throw new ArgumentException("API identifier cannot be null or empty.", nameof(apiIdentifier)); + + MatchingApiBase api; + try + { + api = GetApi(apiIdentifier); + } + catch (KeyNotFoundException) + { + // unknown api -> keep raw + return new GenericOptions { Raw = options }; + } + + var targetType = api?.OptionsType; + if (targetType != null) + { + try + { + var typed = (IApiOptions)options.ToObject(targetType); + Validate(typed); // keep your existing Validate method + return typed; + } + catch (JsonException ex) + { + throw new ArgumentException("Invalid JSON for options payload.", ex); + } + } + + return new GenericOptions { Raw = options }; + } + + // Validates an options object using data annotations + private void Validate(object obj) + { + if (obj == null) return; + var ctx = new ValidationContext(obj); + var results = new List(); + if (!Validator.TryValidateObject(obj, ctx, results, true)) + { + throw new ValidationException(results.First().ErrorMessage); + } + } + } +} \ No newline at end of file diff --git a/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI/Helpers/MatchingResultHelper.cs b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI/Helpers/MatchingResultHelper.cs new file mode 100644 index 0000000000..d065f27210 --- /dev/null +++ b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI/Helpers/MatchingResultHelper.cs @@ -0,0 +1,77 @@ +using BExIS.Dlm.Entities.SpeciesMatching; +using BExIS.Dlm.Services.SpeciesMatching; +using BExIS.IO.Transform.Output; +using BExIS.Modules.Smm.UI.Models; +using BExIS.Utils.Models; +using System; +using System.Collections.Generic; +using System.Data; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Text; +using System.Web; +using System.Web.Http.Results; +using System.Web.Mvc; +using Vaiona.Persistence.Api; + +namespace BExIS.Modules.Smm.UI.Helpers +{ + // This helper class is used for reading and parsing the matching result files. Currently only ChecklistBank (CLB) but later also other APIs. + public class MatchingResultHelper + { + // returns all SpeciesMatchingResult entries for a given datasetId, or null if an error occurs + public static List GetAll(long datasetId, long versionId) + { + try + { + using (var smrm = new SpeciesMatchingResultManager()) + { + var smrmRepo = smrm.GetBulkUnitOfWork().GetReadOnlyRepository(); + List result = smrmRepo.Query().Where(r => r.Dataset.Id == datasetId && r.DatasetVersionId == versionId).ToList(); + + return result; + } + } + catch (Exception ex) + { + return null; + } + } + + public static bool ApplyTailorEdits(long datasetId, long versionId, List edits) + { + try + { + using (var smrm = new SpeciesMatchingResultManager()) + using (var uow = smrm.GetUnitOfWork()) + { + var repo = uow.GetRepository(); + foreach (var edit in edits) + { + var entity = repo.Query().FirstOrDefault(e => e.Id == edit.Id && e.Dataset.Id == datasetId && e.DatasetVersionId == versionId && e.ConfirmedByUser == false); + if (entity != null) + { + // TODO: - adapt this when further data cleaning constraints are clearer + // optionally also update other fields + if (edit.EditedName != "") + { + entity.EditedName = edit.EditedName; + } else + { + entity.EditedName = edit.CleanedName; + } + } + } + uow.Commit(); + return true; + } + } + catch (Exception ex) + { + return false; + } + } + + } +} \ No newline at end of file diff --git a/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI/Helpers/ProgressHelper.cs b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI/Helpers/ProgressHelper.cs new file mode 100644 index 0000000000..3a6c2c6fee --- /dev/null +++ b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI/Helpers/ProgressHelper.cs @@ -0,0 +1,295 @@ +using BExIS.Modules.Smm.UI.Models; +using Newtonsoft.Json; +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Net; +using System.Text; +using System.Web; +using Vaiona.Utils.Cfg; + +namespace BExIS.Modules.Smm.UI.Helpers +{ + public class ProgressHelper + { + public const string MappingFilename = "header_mappings.json"; + public const string MatchingFilename = "matching_progress.json"; + public const string MatchedPrefix = "species_matched"; + public const string UnmatchedPrefix = "species_unmatched"; + public const string MatchingFolderName = "Matching"; + + public static MatchingProgressModel LoadMatchingProgress(long datasetId, long versionId) + { + try + { + string directory = GetVersionedMatchingPath(datasetId, versionId); + if (directory == null) + { + Debug.WriteLine("LoadMatchingProgress: dataset directory does not exist."); + return null; + } + + string filepath = Path.Combine(directory, MatchingFilename); + + if (!System.IO.File.Exists(filepath)) + { + Debug.WriteLine($"Matching Progress file not found: {filepath}"); + return null; + } + + string content = System.IO.File.ReadAllText(filepath); + if (string.IsNullOrWhiteSpace(content)) + { + Debug.WriteLine($"Matching progress file empty: {filepath}"); + return null; + } + + var model = JsonConvert.DeserializeObject(content); + return model; + } + catch (Exception ex) + { + Debug.WriteLine("Failed to load matching progress: " + ex); + return null; + } + } + + + // Loads the header mappings JSON file for the given dataset id. + // Returns the deserialized HeaderMappingsModel or null when the file + // does not exist, is empty or cannot be parsed. + public static HeaderMappingsModel LoadHeaderMappings(long datasetId, long versionId) + { + try + { + string directory = GetVersionedMatchingPath(datasetId, versionId); + if (directory == null) + { + Debug.WriteLine("LoadHeaderMappings: dataset directory does not exist."); + return null; + } + + string filepath = Path.Combine(directory, MappingFilename); + + if (!System.IO.File.Exists(filepath)) + { + Debug.WriteLine($"Header mappings file not found: {filepath}"); + return null; + } + + string content = System.IO.File.ReadAllText(filepath); + if (string.IsNullOrWhiteSpace(content)) + { + Debug.WriteLine($"Header mappings file empty: {filepath}"); + return null; + } + + var model = JsonConvert.DeserializeObject(content); + return model; + } + catch (Exception ex) + { + Debug.WriteLine("Failed to load header mappings: " + ex); + return null; + } + } + + public static bool CreateMatchingFolder(long datasetId, long versionId) + { + try + { + string subdirectory = Path.Combine(AppConfiguration.DataPath, "Datasets", datasetId.ToString()); + if (!Directory.Exists(subdirectory)) + { + Debug.WriteLine("CreateMatchingFolder: dataset directory does not exist: " + subdirectory); + return false; + } + + string directory = Path.Combine(AppConfiguration.DataPath, "Datasets", datasetId.ToString(), MatchingFolderName, versionId.ToString()); + if (!Directory.Exists(directory)) + { + Directory.CreateDirectory(directory); + } + else + { + Debug.WriteLine("Matching folder already exists: " + directory); + } + return true; + } + catch (Exception ex) + { + Debug.WriteLine("Failed to create matching folder: " + ex); + return false; + } + } + + // Creates a matching_progress.json file for the given dataset with an empty Steps list. + // Returns true when the file was created successfully, false on error. + public static bool CreateMatchingProgressFile(long datasetId, long versionId, int numRowsGlobal) + { + try + { + string directory = GetVersionedMatchingPath(datasetId, versionId); + + if (directory == null) + { + Debug.WriteLine("CreateMatchingProgressFile: dataset directory does not exist."); + return false; + } + + string filepath = Path.Combine(directory, MatchingFilename); + + var model = new MatchingProgressModel + { + DatasetId = datasetId, + NumRowsGlobal = numRowsGlobal, + Steps = new List() + }; + + string json = JsonConvert.SerializeObject(model, Formatting.Indented); + System.IO.File.WriteAllText(filepath, json); + + Debug.WriteLine("Created matching progress file: " + filepath); + return true; + } + catch (Exception ex) + { + Debug.WriteLine("Failed to create matching progress file: " + ex); + return false; + } + } + + public static bool CreateHeaderMappingsFile(HeaderMappingsModel data, long datasetId, long versionId, out string errorMessage) + { + foreach (var entry in data.Mappings) + { + if (!MappingValidator.IsValid(entry.HeaderMapping)) + { + errorMessage = "The selected HeaderMapping " + entry.HeaderMapping + " does not exist."; + return false; + } + } + + string directory = GetVersionedMatchingPath(datasetId, versionId); + string filepath = Path.Combine(directory, MappingFilename); + + if (directory == null) + { + errorMessage = "The dataset folder with id " + datasetId + " does not exist."; + return false; + } + else + { + System.IO.File.WriteAllText(filepath, JsonConvert.SerializeObject(data)); + errorMessage = null; + return true; + } + } + + // Persist the provided MatchingProgressModel to the dataset's matching_progress.json file. + // This method will overwrite the file regardless of whether it already exists. + // Returns true on success, false on failure. + public static bool SaveMatchingProgress(MatchingProgressModel model, long datasetId, long versionId) + { + if (model == null) + { + Debug.WriteLine("SaveMatchingProgress: model is null."); + return false; + } + + try + { + string directory = GetVersionedMatchingPath(datasetId, versionId); + + if (directory == null) + { + Debug.WriteLine("SaveMatchingProgress: dataset directory does not exist."); + return false; + } + + string filepath = Path.Combine(directory, MatchingFilename); + + string json = JsonConvert.SerializeObject(model, Formatting.Indented); + + // Overwrite the file (or create it if missing) + System.IO.File.WriteAllText(filepath, json, Encoding.UTF8); + + Debug.WriteLine("Saved matching progress file: " + filepath); + return true; + } + catch (Exception ex) + { + Debug.WriteLine("Failed to save matching progress: " + ex); + return false; + } + } + + public static string GenMatchingFileName(bool matched, long datasetId, int suffixId, bool withFileEnding = true) + { + string prefix = matched ? MatchedPrefix : UnmatchedPrefix; + if (withFileEnding) + { + return $"{prefix}_{datasetId}_{suffixId}.csv"; + } + else + { + return $"{prefix}_{datasetId}_{suffixId}"; + } + } + + public static string GetMatchedFilepath(long datasetId, long versionId, int stepId, bool exists = true) + { + string directory = GetVersionedMatchingPath(datasetId, versionId); + if (directory == null) { + return null; + } + + string filename = GenMatchingFileName(true, datasetId, stepId); + string filepath = Path.Combine(directory, filename); + if (System.IO.File.Exists(filepath)) + { + return filepath; + } + else + { + if (exists) + { + return null; + } else + { + return filepath; + } + } + + } + + public static string GetVersionedMatchingPath(long datasetId, long versionId) + { + string directory = Path.Combine(AppConfiguration.DataPath, "Datasets", datasetId.ToString(), MatchingFolderName, versionId.ToString()); + if (Directory.Exists(directory)) + { + return directory; + } + else + { + return null; + } + } + + public static bool HasMatchingProgress(long datasetId, long versionId) + { + string directory = Path.Combine(AppConfiguration.DataPath, "Datasets", datasetId.ToString(), MatchingFolderName, versionId.ToString()); + string filepath = Path.Combine(directory, MatchingFilename); + return System.IO.File.Exists(filepath); + } + + public static bool HasHeaderMappings(long datasetId, long versionId) + { + string directory = Path.Combine(AppConfiguration.DataPath, "Datasets", datasetId.ToString(), MatchingFolderName, versionId.ToString()); + string filepath = Path.Combine(directory, MappingFilename); + return System.IO.File.Exists(filepath); + } + } +} \ No newline at end of file diff --git a/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI/Helpers/TabularFileHelper.cs b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI/Helpers/TabularFileHelper.cs new file mode 100644 index 0000000000..cb18e339f4 --- /dev/null +++ b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI/Helpers/TabularFileHelper.cs @@ -0,0 +1,50 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Web; + +namespace BExIS.Modules.Smm.UI.Helpers +{ + public class TabularFileHelper + { + // Simple CSV line parser that handles quoted fields and commas inside quotes. + public static List ParseCsvLine(string line) + { + var fields = new List(); + if (line == null) return fields; + + var sb = new StringBuilder(); + bool inQuotes = false; + for (int i = 0; i < line.Length; i++) + { + char c = line[i]; + if (c == '"') + { + if (inQuotes && i + 1 < line.Length && line[i + 1] == '"') + { + // escaped quote + sb.Append('"'); + i++; // skip next + } + else + { + inQuotes = !inQuotes; + } + } + else if (c == ',' && !inQuotes) + { + fields.Add(sb.ToString()); + sb.Clear(); + } + else + { + sb.Append(c); + } + } + + fields.Add(sb.ToString()); + return fields; + } + } +} \ No newline at end of file diff --git a/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI/Models/AcceptMatchesRequestModel.cs b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI/Models/AcceptMatchesRequestModel.cs new file mode 100644 index 0000000000..390ec59d0e --- /dev/null +++ b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI/Models/AcceptMatchesRequestModel.cs @@ -0,0 +1,25 @@ +using System; +using System.Collections.Generic; +using System.ComponentModel.DataAnnotations; +using System.Linq; +using System.Web; + +namespace BExIS.Modules.Smm.UI.Models +{ + public class AcceptMatchesRequestModel + { + [Range(1, long.MaxValue, ErrorMessage = "DatasetId must be provided and greater than 0.")] + public long DatasetId { get; set; } + + [Range(1, long.MaxValue, ErrorMessage = "VersionId must be provided and greater than 0.")] + public long VersionId { get; set; } + + [Range(0, int.MaxValue, ErrorMessage = "StepId must be provided and at least 0.")] + public int StepId { get; set; } + + [Required(ErrorMessage = "At least one MatchId must be provided.")] + [MinLength(1, ErrorMessage = "At least one MatchId must be provided.")] + // Use an array here so MinLengthAttribute can validate the collection correctly during model binding + public string[] MatchIds { get; set; } + } +} \ No newline at end of file diff --git a/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI/Models/CLBMatchingResultFile.cs b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI/Models/CLBMatchingResultFile.cs new file mode 100644 index 0000000000..3652127139 --- /dev/null +++ b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI/Models/CLBMatchingResultFile.cs @@ -0,0 +1,34 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Web; + +namespace BExIS.Modules.Smm.UI.Models +{ + public class CLBMatchingResultFile + { + public string Original_ID { get; set; } + public string Original_scientificName { get; set; } + public string Original_rank { get; set; } + public string Original_kingdom { get; set; } + public string Original_authorship { get; set; } + public string MatchType { get; set; } + public string MatchIssues { get; set; } + public string ID { get; set; } + public string Rank { get; set; } + public string ScientificName { get; set; } + public string Authorship { get; set; } + public string Status { get; set; } + public string AcceptedID { get; set; } + public string AcceptedScientificName { get; set; } + public string AcceptedAuthorship { get; set; } + public string Kingdom { get; set; } + public string Phylum { get; set; } + public string Class { get; set; } + public string Order { get; set; } + public string Family { get; set; } + public string Genus { get; set; } + public string Classification { get; set; } + + } +} \ No newline at end of file diff --git a/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI/Models/ExternalApiMetadata.cs b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI/Models/ExternalApiMetadata.cs new file mode 100644 index 0000000000..464164a09f --- /dev/null +++ b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI/Models/ExternalApiMetadata.cs @@ -0,0 +1,31 @@ +using System; +using System.Collections.Generic; +using Newtonsoft.Json; +using System.Linq; +using System.Web; + +namespace BExIS.Modules.Smm.UI.Models +{ + public class ExternalApiMetadata + { + [JsonProperty("clb")] + public ExternalApiSource Clb { get; set; } + } + + public class ExternalApiSource + { + [JsonProperty("sourceKeyInfo")] + public List SourceKeyInfo { get; set; } + } + + public class SourceKeyInfoItem + { + [JsonProperty("sourceKey")] + public string SourceKey { get; set; } + [JsonProperty("title")] + public string Title { get; set; } + + [JsonProperty("alias")] + public string Alias { get; set; } + } +} \ No newline at end of file diff --git a/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI/Models/HeaderMappingsModel.cs b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI/Models/HeaderMappingsModel.cs new file mode 100644 index 0000000000..f66b93f9d7 --- /dev/null +++ b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI/Models/HeaderMappingsModel.cs @@ -0,0 +1,66 @@ +using Newtonsoft.Json; +using System.ComponentModel.DataAnnotations; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Web; + +namespace BExIS.Modules.Smm.UI.Models +{ + public class HeaderMappingsModel + { + public List Mappings { get; set; } = new List(); + + public long DatastructureId { get; set; } + + [Range(1, long.MaxValue, ErrorMessage = "DatasetId must be provided and greater than 0.")] + public long DatasetId { get; set; } + + // Returns the VariableId of the mapping entry whose HeaderMapping equals + // "scientificName". If no such entry exists the method returns null. + public long? GetVariableIdForScientificName() + { + var entry = Mappings?.FirstOrDefault(m => string.Equals(m.HeaderMapping, "scientificName", StringComparison.OrdinalIgnoreCase)); + return entry?.VariableId; + } + } + + public class SubmitHeaderMappingsRequest + { + [JsonProperty("data")] + public HeaderMappingsModel Data { get; set; } + + [JsonProperty("versionId")] + public long VersionId { get; set; } + + [JsonProperty("datasetId")] + [Range(1, long.MaxValue, ErrorMessage = "DatasetId must be provided and greater than 0.")] + public long DatasetId { get; set; } + } + + public class MappingEntry + { + public long VariableId { get; set; } + + public string VariableName { get; set; } + + public string HeaderMapping { get; set; } + } + + public static class MappingValidator + { + private static readonly HashSet ValidOptions = new HashSet + { + "scientificName", + "authorship", + "rank", + "kingdom", + "IGNORE" + }; + + public static bool IsValid(string value) + { + return ValidOptions.Contains(value); + } + } +} \ No newline at end of file diff --git a/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI/Models/MatchingApiResponse.cs b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI/Models/MatchingApiResponse.cs new file mode 100644 index 0000000000..c795ed63ad --- /dev/null +++ b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI/Models/MatchingApiResponse.cs @@ -0,0 +1,16 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Web; + +namespace BExIS.Modules.Smm.UI.Models +{ + public class MatchingApiResponse + { + public bool Success { get; set; } + public int? StatusCode { get; set; } + public string Message { get; set; } + public object Payload { get; set; } + public int StepId { get; set; } + } +} \ No newline at end of file diff --git a/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI/Models/MatchingProgressModel.cs b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI/Models/MatchingProgressModel.cs new file mode 100644 index 0000000000..bbdb768a92 --- /dev/null +++ b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI/Models/MatchingProgressModel.cs @@ -0,0 +1,155 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Web; + +namespace BExIS.Modules.Smm.UI.Models +{ + public class MatchingProgressModel + { + + public List Steps { get; set; } = new List(); + + // total number of rows in the original data, should be set at the beginning of the matching process + public int NumRowsGlobal { get; set; } + + // identifier for the dataset being matched, should be set at the beginning of the matching process + public long DatasetId { get; set; } + + // identifier for the specific version of the dataset being matched, should be set at the beginning of the matching process + public long VersionId { get; set; } + + public int GetNewId() + { + return Steps.Count; + } + + public StepEntry GetLatestStep() + { + // Return the last step in the list or null when there are no steps + if (Steps == null || Steps.Count == 0) return null; + + return Steps.Last(); + } + + public void AddStep(int id, int numRows, string inputFileName, string apiIdentifier) + { + var entry = new StepEntry + { + Id = id, + NumRows = numRows, + InputFileName = inputFileName, + ResultFileName = string.Empty, + ApiIdentifier = apiIdentifier, + DownloadLink = string.Empty, + JobKey = string.Empty, + MatchSource = string.Empty, + TimeStamp = DateTime.MinValue, + }; + + Steps.Add(entry); + } + + public bool AllStepsCompleted() + { + // Return true when there are no unfinished steps (i.e. no step with Done == false) + return Steps == null || Steps.All(s => s.IsCompleted()); + } + + public bool IsCompletedById(int stepId) + { + // Return false when there are no steps + if (Steps == null || Steps.Count == 0) return false; + + var entry = Steps.FirstOrDefault(s => s.Id == stepId); + + if (entry == null) return false; + + return entry.IsCompleted(); + } + + public string GetApiIdentifierById(int stepId) + { + // Return null when there are no steps + if (Steps == null || Steps.Count == 0) return null; + + var entry = Steps.FirstOrDefault(s => s.Id == stepId); + + return entry?.ApiIdentifier; + } + + public StepEntry GetStepById(int stepId) + { + if (Steps == null || Steps.Count == 0) return null; + + return Steps.FirstOrDefault(s => s.Id == stepId); + } + + public bool UpdateStep(StepEntry updatedStep) + { + // Validate input and existing steps + if (updatedStep == null) return false; + if (Steps == null || Steps.Count == 0) return false; + + var existing = Steps.FirstOrDefault(s => s.Id == updatedStep.Id); + if (existing == null) return false; + + // Update fields of the existing entry + existing.NumRows = updatedStep.NumRows; + existing.InputFileName = updatedStep.InputFileName; + existing.ResultFileName = updatedStep.ResultFileName; + existing.ApiIdentifier = updatedStep.ApiIdentifier; + existing.DownloadLink = updatedStep.DownloadLink; + existing.MatchSource = updatedStep.MatchSource; + existing.JobKey = updatedStep.JobKey; + + return true; + } + } + + // Represents a single step in the matching process + // Each step corresponds to a matching operation, which involves an input file, result file and an API call + // to a file based matching service (e.g. CheckListBank). The step is considered completed when the result file is available and the API call is done. + public class StepEntry + { + // identifier for this step, should be unique within the context of a MatchingProgressModel + public int Id { get; set; } + + // number of rows in the input file + public int NumRows { get; set; } + + // name of the input file for this step + public string InputFileName { get; set; } + + // name of the result file for this step, should be non-empty when the step is completed + public string ResultFileName { get; set; } + + // identifier for the API call associated with this step, should be non-empty when the step is completed + public string ApiIdentifier { get; set; } + + // download link for the result file + public string DownloadLink { get; set; } + + // source of the matching results (e.g. string of dataset sourceKey in CheckListBank) + public string MatchSource { get; set; } + + // timestamp when the match request is sent + public DateTime TimeStamp { get; set; } + + // job key for tracking the matching job (if asynchronous) + public string JobKey { get; set; } + + public bool IsReadyToMatch() + { + // input file exists + // matching process has not started for this step + return !string.IsNullOrEmpty(InputFileName) && string.IsNullOrEmpty(JobKey) && string.IsNullOrEmpty(ResultFileName) && string.IsNullOrEmpty(DownloadLink); + } + + public bool IsCompleted() + { + // step is completed when the result file is available and the API call is done + return !string.IsNullOrEmpty(ResultFileName); + } + } +} \ No newline at end of file diff --git a/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI/Models/MatchingResultRow.cs b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI/Models/MatchingResultRow.cs new file mode 100644 index 0000000000..beabffb4d3 --- /dev/null +++ b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI/Models/MatchingResultRow.cs @@ -0,0 +1,33 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Web; + +namespace BExIS.Modules.Smm.UI.Models +{ + public class MatchingResultRow + { + public string Original_ID { get; set; } + public string Original_scientificName { get; set; } + public string Original_rank { get; set; } + public string Original_kingdom { get; set; } + public string Original_authorship { get; set; } + public string MatchType { get; set; } + public string MatchIssues { get; set; } + public string ID { get; set; } + public string Rank { get; set; } + public string ScientificName { get; set; } + public string Authorship { get; set; } + public string Status { get; set; } + public string AcceptedID { get; set; } + public string AcceptedScientificName { get; set; } + public string AcceptedAuthorship { get; set; } + public string Kingdom { get; set; } + public string Phylum { get; set; } + public string Class { get; set; } + public string Order { get; set; } + public string Family { get; set; } + public string Genus { get; set; } + public string Classification { get; set; } + } +} \ No newline at end of file diff --git a/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI/Models/TailorEditsModel.cs b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI/Models/TailorEditsModel.cs new file mode 100644 index 0000000000..b489968d65 --- /dev/null +++ b/Console/BExIS.Web.Shell/Areas/SMM/BExIS.Modules.Smm.UI/Models/TailorEditsModel.cs @@ -0,0 +1,22 @@ +using System; +using System.Collections.Generic; +using System.ComponentModel.DataAnnotations; +using System.Linq; +using System.Web; + +namespace BExIS.Modules.Smm.UI.Models +{ + public class TailorEdit + { + [Range(1, long.MaxValue, ErrorMessage = "Id must be provided and greater than 0.")] + public long Id { get; set; } + + public string OriginalName { get; set; } + + [DisplayFormat(ConvertEmptyStringToNull = false)] + public string EditedName { get; set; } + + [DisplayFormat(ConvertEmptyStringToNull = false)] + public string CleanedName { get; set; } + } +} \ No newline at end of file diff --git a/Console/BExIS.Web.Shell/BExIS.Web.Shell.csproj b/Console/BExIS.Web.Shell/BExIS.Web.Shell.csproj index 1b5495a252..84ac9802d2 100644 --- a/Console/BExIS.Web.Shell/BExIS.Web.Shell.csproj +++ b/Console/BExIS.Web.Shell/BExIS.Web.Shell.csproj @@ -1114,7 +1114,7 @@ - + diff --git a/Console/Workspace b/Console/Workspace index 81a0357473..03749988f1 160000 --- a/Console/Workspace +++ b/Console/Workspace @@ -1 +1 @@ -Subproject commit 81a035747389563f7760c34d93f6b213e8e52e80 +Subproject commit 03749988f1bae7d2d2885081bcf42b2ec0a5bbf8