diff --git a/fineract-core/src/main/java/org/apache/fineract/infrastructure/dataqueries/data/DatatableData.java b/fineract-core/src/main/java/org/apache/fineract/infrastructure/dataqueries/data/DatatableData.java index 8e22cbbca4e..f0d0f4f996e 100644 --- a/fineract-core/src/main/java/org/apache/fineract/infrastructure/dataqueries/data/DatatableData.java +++ b/fineract-core/src/main/java/org/apache/fineract/infrastructure/dataqueries/data/DatatableData.java @@ -34,18 +34,21 @@ public final class DatatableData implements Serializable { private final String entitySubType; @SuppressWarnings("unused") private final List columnHeaderData; + @SuppressWarnings("unused") + private final boolean multiRow; public static DatatableData create(final String applicationTableName, final String registeredTableName, final String entitySubType, - final List columnHeaderData) { - return new DatatableData(applicationTableName, registeredTableName, entitySubType, columnHeaderData); + final List columnHeaderData, final boolean multiRow) { + return new DatatableData(applicationTableName, registeredTableName, entitySubType, columnHeaderData, multiRow); } private DatatableData(final String applicationTableName, final String registeredTableName, final String entitySubType, - final List columnHeaderData) { + final List columnHeaderData, final boolean multiRow) { this.applicationTableName = applicationTableName; this.registeredTableName = registeredTableName; this.entitySubType = entitySubType; this.columnHeaderData = columnHeaderData; + this.multiRow = multiRow; } @@ -65,4 +68,12 @@ public String getRegisteredTableName() { return registeredTableName; } + public List getColumnHeaderData() { + return columnHeaderData; + } + + public boolean isMultiRow() { + return multiRow; + } + } diff --git a/fineract-core/src/main/java/org/apache/fineract/infrastructure/dataqueries/data/ResultsetColumnHeaderData.java b/fineract-core/src/main/java/org/apache/fineract/infrastructure/dataqueries/data/ResultsetColumnHeaderData.java index c196954c6a9..592a39fe316 100644 --- a/fineract-core/src/main/java/org/apache/fineract/infrastructure/dataqueries/data/ResultsetColumnHeaderData.java +++ b/fineract-core/src/main/java/org/apache/fineract/infrastructure/dataqueries/data/ResultsetColumnHeaderData.java @@ -149,6 +149,10 @@ public boolean hasColumnValues() { return columnValues != null && !columnValues.isEmpty(); } + public List getColumnValues() { + return columnValues; + } + public boolean isColumnValueAllowed(final String match) { for (final ResultsetColumnValueData allowedValue : this.columnValues) { if (allowedValue.matches(match)) { diff --git a/fineract-core/src/main/java/org/apache/fineract/infrastructure/dataqueries/data/ResultsetColumnValueData.java b/fineract-core/src/main/java/org/apache/fineract/infrastructure/dataqueries/data/ResultsetColumnValueData.java index 9a9e8a10074..b0436c68a72 100644 --- a/fineract-core/src/main/java/org/apache/fineract/infrastructure/dataqueries/data/ResultsetColumnValueData.java +++ b/fineract-core/src/main/java/org/apache/fineract/infrastructure/dataqueries/data/ResultsetColumnValueData.java @@ -49,4 +49,12 @@ public boolean matches(final String match) { public boolean codeMatches(final Integer match) { return match.intValue() == this.id; } + + public int getId() { + return id; + } + + public String getValue() { + return value; + } } diff --git a/fineract-core/src/main/java/org/apache/fineract/portfolio/search/service/SearchUtil.java b/fineract-core/src/main/java/org/apache/fineract/portfolio/search/service/SearchUtil.java index 0107308eb6a..0c5b1ef6b11 100644 --- a/fineract-core/src/main/java/org/apache/fineract/portfolio/search/service/SearchUtil.java +++ b/fineract-core/src/main/java/org/apache/fineract/portfolio/search/service/SearchUtil.java @@ -271,7 +271,15 @@ public Object parseColumnValue(@NonNull ResultsetColumnHeaderData columnHeader, } return columnValue; } else if (columnHeader.isCodeLookupDisplayType()) { - final Integer codeLookup = Integer.valueOf(columnValue); + // Extract ID from display value format: "Label (ID)" if present + String extractedId = extractIdFromDisplayValue(columnValue); + Integer codeLookup; + try { + codeLookup = Integer.valueOf(extractedId); + } catch (NumberFormatException e) { + // If extraction didn't change the value or parsing failed, try original value + codeLookup = Integer.valueOf(columnValue); + } if (!columnHeader.isColumnCodeAllowed(codeLookup)) { ApiParameterError error = ApiParameterError.parameterError("error.msg.invalid.columnValue", "Value not found in Allowed Value list", columnHeader.getColumnName(), columnValue); @@ -293,6 +301,17 @@ public Object parseColumnValue(@NonNull ResultsetColumnHeaderData columnHeader, return JsonParserHelper.convertDateTimeFrom(columnValue, columnHeader.getColumnName(), format, locale); } if (colType.isAnyIntegerType()) { + // Extract ID from display value format: "Label (ID)" if present + // This handles FK/INTEGER columns that show human-readable values in Excel + String extractedId = extractIdFromDisplayValue(columnValue); + if (!extractedId.equals(columnValue)) { + // ID was extracted, try to parse it + try { + return Integer.parseInt(extractedId); + } catch (NumberFormatException e) { + // If parsing fails, fall back to original conversion + } + } return helper.convertToInteger(columnValue, columnHeader.getColumnName(), locale); } if (colType.isDecimalType()) { @@ -323,4 +342,28 @@ public String camelToSnake(final String camelStr) { return camelStr == null ? null : camelStr.replaceAll("([A-Z]+)([A-Z][a-z])", "$1_$2").replaceAll("([a-z])([A-Z])", "$1_$2").toLowerCase(); } + + /** + * Extracts numeric ID from strings ending with "(id)" format. + * Example: "Permanent (32)" -> "32" + * If the string doesn't match the pattern, returns the original value. + * + * @param value The string value that may contain an ID in parentheses + * @return The extracted ID as a string, or the original value if no ID pattern is found + */ + private static String extractIdFromDisplayValue(String value) { + if (value == null || value.trim().isEmpty()) { + return value; + } + // Match pattern: ".*(\d+)$" where the number is in parentheses at the end + String trimmed = value.trim(); + int lastParenIndex = trimmed.lastIndexOf('('); + if (lastParenIndex > 0 && trimmed.endsWith(")")) { + String idPart = trimmed.substring(lastParenIndex + 1, trimmed.length() - 1); + if (idPart.matches("\\d+")) { + return idPart; + } + } + return value; + } } diff --git a/fineract-core/src/test/java/org/apache/fineract/infrastructure/dataqueries/data/DatatableDataTest.java b/fineract-core/src/test/java/org/apache/fineract/infrastructure/dataqueries/data/DatatableDataTest.java new file mode 100644 index 00000000000..0aef602bac9 --- /dev/null +++ b/fineract-core/src/test/java/org/apache/fineract/infrastructure/dataqueries/data/DatatableDataTest.java @@ -0,0 +1,90 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.fineract.infrastructure.dataqueries.data; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.ArrayList; +import java.util.List; +import org.apache.fineract.infrastructure.core.service.database.DatabaseType; +import org.junit.jupiter.api.Test; + +class DatatableDataTest { + + @Test + void testSingleRowDatatable() { + // Given: A single-row datatable (no "id" column) + String appTableName = "m_client"; + String registeredTableName = "extra_client_details"; + String entitySubType = null; + List columnHeaders = new ArrayList<>(); + // Single-row datatable has client_id as primary key, no "id" column + columnHeaders.add(ResultsetColumnHeaderData.basic("client_id", "bigint", DatabaseType.POSTGRESQL)); + columnHeaders.add(ResultsetColumnHeaderData.basic("field1", "text", DatabaseType.POSTGRESQL)); + boolean multiRow = false; + + // When: Creating DatatableData + DatatableData datatableData = DatatableData.create(appTableName, registeredTableName, entitySubType, + columnHeaders, multiRow); + + // Then: It should be identified as single-row + assertFalse(datatableData.isMultiRow(), "Single-row datatable should return false for isMultiRow()"); + assertEquals(registeredTableName, datatableData.getRegisteredTableName()); + assertEquals(columnHeaders, datatableData.getColumnHeaderData()); + } + + @Test + void testMultiRowDatatable() { + // Given: A multi-row datatable (has "id" column) + String appTableName = "m_client"; + String registeredTableName = "extra_family_details"; + String entitySubType = null; + List columnHeaders = new ArrayList<>(); + // Multi-row datatable has "id" as primary key + columnHeaders.add(ResultsetColumnHeaderData.basic("id", "bigint", DatabaseType.POSTGRESQL)); + columnHeaders.add(ResultsetColumnHeaderData.basic("client_id", "bigint", DatabaseType.POSTGRESQL)); + columnHeaders.add(ResultsetColumnHeaderData.basic("field1", "text", DatabaseType.POSTGRESQL)); + boolean multiRow = true; + + // When: Creating DatatableData + DatatableData datatableData = DatatableData.create(appTableName, registeredTableName, entitySubType, + columnHeaders, multiRow); + + // Then: It should be identified as multi-row + assertTrue(datatableData.isMultiRow(), "Multi-row datatable should return true for isMultiRow()"); + assertEquals(registeredTableName, datatableData.getRegisteredTableName()); + assertEquals(columnHeaders, datatableData.getColumnHeaderData()); + } + + @Test + void testHasColumn() { + // Given: A datatable with columns + List columnHeaders = new ArrayList<>(); + columnHeaders.add(ResultsetColumnHeaderData.basic("client_id", "bigint", DatabaseType.POSTGRESQL)); + columnHeaders.add(ResultsetColumnHeaderData.basic("field1", "text", DatabaseType.POSTGRESQL)); + DatatableData datatableData = DatatableData.create("m_client", "test_table", null, columnHeaders, false); + + // When/Then: Checking for column existence + assertTrue(datatableData.hasColumn("client_id"), "Should find existing column"); + assertTrue(datatableData.hasColumn("field1"), "Should find existing column"); + assertFalse(datatableData.hasColumn("nonexistent"), "Should not find non-existent column"); + } +} diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/constants/ClientEntityConstants.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/constants/ClientEntityConstants.java index a06eac907f5..73c1e772c29 100644 --- a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/constants/ClientEntityConstants.java +++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/constants/ClientEntityConstants.java @@ -51,15 +51,55 @@ private ClientEntityConstants() { public static final int COUNTRY_COL = 24;// Y public static final int POSTAL_CODE_COL = 25;// Z public static final int IS_ACTIVE_ADDRESS_COL = 26;// AA - public static final int WARNING_COL = 26;// AA public static final int STATUS_COL = 27;// AB + /** + * @deprecated Lookup columns have been moved to a hidden sheet. Use dynamic column resolution instead. + */ + @Deprecated + public static final int WARNING_COL = 26;// AA + /** + * @deprecated Lookup columns have been moved to a hidden sheet. Use dynamic column resolution instead. + */ + @Deprecated public static final int RELATIONAL_OFFICE_NAME_COL = 35;// AJ + /** + * @deprecated Lookup columns have been moved to a hidden sheet. Use dynamic column resolution instead. + */ + @Deprecated public static final int RELATIONAL_OFFICE_OPENING_DATE_COL = 36;// AK + /** + * @deprecated Lookup columns have been moved to a hidden sheet. Use dynamic column resolution instead. + */ + @Deprecated public static final int LOOKUP_CONSTITUTION_COL = 37;// AL + /** + * @deprecated Lookup columns have been moved to a hidden sheet. Use dynamic column resolution instead. + */ + @Deprecated public static final int LOOKUP_CLIENT_CLASSIFICATION = 38;// AM + /** + * @deprecated Lookup columns have been moved to a hidden sheet. Use dynamic column resolution instead. + */ + @Deprecated public static final int LOOKUP_CLIENT_TYPES = 39;// AN + /** + * @deprecated Lookup columns have been moved to a hidden sheet. Use dynamic column resolution instead. + */ + @Deprecated public static final int LOOKUP_ADDRESS_TYPE = 40;// AO + /** + * @deprecated Lookup columns have been moved to a hidden sheet. Use dynamic column resolution instead. + */ + @Deprecated public static final int LOOKUP_STATE_PROVINCE = 41;// AP + /** + * @deprecated Lookup columns have been moved to a hidden sheet. Use dynamic column resolution instead. + */ + @Deprecated public static final int LOOKUP_COUNTRY = 42;// AQ + /** + * @deprecated Lookup columns have been moved to a hidden sheet. Use dynamic column resolution instead. + */ + @Deprecated public static final int LOOKUP_MAIN_BUSINESS_LINE = 43;// AR } diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/constants/ClientPersonConstants.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/constants/ClientPersonConstants.java index ed4a6fb7bd4..12110b06976 100644 --- a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/constants/ClientPersonConstants.java +++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/constants/ClientPersonConstants.java @@ -50,14 +50,50 @@ private ClientPersonConstants() { public static final int COUNTRY_COL = 23;// X public static final int POSTAL_CODE_COL = 24;// Y public static final int IS_ACTIVE_ADDRESS_COL = 25;// Z - public static final int WARNING_COL = 26;// AA public static final int STATUS_COL = 27;// AB + /** + * @deprecated Lookup columns have been moved to a hidden sheet. Use dynamic column resolution instead. + */ + @Deprecated + public static final int WARNING_COL = 26;// AA + /** + * @deprecated Lookup columns have been moved to a hidden sheet. Use dynamic column resolution instead. + */ + @Deprecated public static final int RELATIONAL_OFFICE_NAME_COL = 35;// AJ + /** + * @deprecated Lookup columns have been moved to a hidden sheet. Use dynamic column resolution instead. + */ + @Deprecated public static final int RELATIONAL_OFFICE_OPENING_DATE_COL = 36;// AK + /** + * @deprecated Lookup columns have been moved to a hidden sheet. Use dynamic column resolution instead. + */ + @Deprecated public static final int LOOKUP_GENDER_COL = 37;// AL + /** + * @deprecated Lookup columns have been moved to a hidden sheet. Use dynamic column resolution instead. + */ + @Deprecated public static final int LOOKUP_CLIENT_CLASSIFICATION_COL = 38;// AM + /** + * @deprecated Lookup columns have been moved to a hidden sheet. Use dynamic column resolution instead. + */ + @Deprecated public static final int LOOKUP_CLIENT_TYPES_COL = 39;// AN + /** + * @deprecated Lookup columns have been moved to a hidden sheet. Use dynamic column resolution instead. + */ + @Deprecated public static final int LOOKUP_ADDRESS_TYPE_COL = 40;// AO + /** + * @deprecated Lookup columns have been moved to a hidden sheet. Use dynamic column resolution instead. + */ + @Deprecated public static final int LOOKUP_STATE_PROVINCE_COL = 41;// AP + /** + * @deprecated Lookup columns have been moved to a hidden sheet. Use dynamic column resolution instead. + */ + @Deprecated public static final int LOOKUP_COUNTRY_COL = 42;// AQ } diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/constants/TemplatePopulateImportConstants.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/constants/TemplatePopulateImportConstants.java index c7226e1c771..a3b90d59b15 100644 --- a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/constants/TemplatePopulateImportConstants.java +++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/constants/TemplatePopulateImportConstants.java @@ -57,6 +57,7 @@ private TemplatePopulateImportConstants() { public static final String EMPLOYEE_SHEET_NAME = "Employee"; public static final String ROLES_SHEET_NAME = "Roles"; public static final String USER_SHEET_NAME = "Users"; + public static final String CLIENT_LOOKUPS_SHEET_NAME = "_CLIENT_LOOKUPS"; public static final int ROWHEADER_INDEX = 0; public static final short ROW_HEADER_HEIGHT = 500; diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/importhandler/ImportHandlerUtils.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/importhandler/ImportHandlerUtils.java index 1090dcbe543..ff8841f2273 100644 --- a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/importhandler/ImportHandlerUtils.java +++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/importhandler/ImportHandlerUtils.java @@ -29,6 +29,18 @@ import org.apache.fineract.infrastructure.core.exception.AbstractPlatformException; import org.apache.fineract.infrastructure.core.exception.PlatformApiDataValidationException; import org.apache.fineract.infrastructure.core.exception.UnsupportedParameterException; +import org.apache.fineract.infrastructure.dataqueries.data.DatatableData; +import org.apache.fineract.infrastructure.dataqueries.data.ResultsetColumnHeaderData; +import org.apache.fineract.infrastructure.dataqueries.domain.EntityDatatableChecks; +import org.apache.fineract.infrastructure.dataqueries.domain.EntityDatatableChecksRepository; +import org.apache.fineract.infrastructure.dataqueries.data.EntityTables; +import org.apache.fineract.infrastructure.dataqueries.data.StatusEnum; +import org.apache.fineract.infrastructure.dataqueries.service.DatatableReadService; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeFormatterBuilder; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Map; import org.apache.fineract.infrastructure.core.service.DateUtils; import org.apache.poi.ss.usermodel.Cell; import org.apache.poi.ss.usermodel.CellStyle; @@ -145,6 +157,31 @@ public static String trimEmptyDecimalPortion(String result) { } } + /** + * Extracts numeric ID from strings ending with "(id)" format. + * Example: "Permanent (32)" -> "32" + * If the string doesn't match the pattern, returns the original value. + * + * @param value The string value that may contain an ID in parentheses + * @return The extracted ID as a string, or the original value if no ID pattern is found + */ + public static String extractIdFromDisplayValue(String value) { + if (value == null || value.trim().isEmpty()) { + return value; + } + // Match pattern: ".*(\d+)$" where the number is in parentheses at the end + // More specifically: ".*\\(\\d+\\)$" + String trimmed = value.trim(); + int lastParenIndex = trimmed.lastIndexOf('('); + if (lastParenIndex > 0 && trimmed.endsWith(")")) { + String idPart = trimmed.substring(lastParenIndex + 1, trimmed.length() - 1); + if (idPart.matches("\\d+")) { + return idPart; + } + } + return value; + } + public static LocalDate readAsDate(int colIndex, Row row) { Cell c = row.getCell(colIndex); if (c == null || c.getCellType() == CellType.BLANK) { @@ -447,4 +484,702 @@ public static String getRepeatsOnDayId(String repeatsOnDay) { return null; } } + + /** + * Validates that all required datatables are present in the Excel workbook for the given row. + * + * Validation logic: + * 1. Multi-row datatables (1:N with entity) are always skipped - they are optional/repeatable child data. + * 2. If NO column header exists for a required datatable, validation is skipped entirely. + * This allows required datatables to be optional in the Excel sheet if no columns are present. + * 3. If column headers EXIST for a required datatable: + * a. Retrieve the datatable schema to identify mandatory (non-nullable) columns. + * b. If the datatable has ZERO mandatory columns (all columns nullable), validation passes even if all values are blank. + * c. If there are mandatory columns, validate that each mandatory column either: + * - Exists in the Excel header AND has a non-blank value, OR + * - Is missing from the Excel header (treated as missing field). + * d. System columns (id, client_id, created_at, updated_at) and locale/dateFormat are ignored. + * + * This ensures that: + * - Optional datatables (not present in Excel) → no validation error + * - Required datatables with all nullable columns (present but empty) → no validation error + * - Required datatables with mandatory columns (present but missing mandatory values) → validation error + * - Multi-row datatables → always skipped + * + * @param workbook The Excel workbook + * @param sheet The sheet containing the client data + * @param row The data row to validate + * @param entitySubtype The entity subtype (PERSON or ENTITY) + * @param entityDatatableChecksRepository Repository to fetch required datatables + * @param datatableReadService Service to retrieve datatable metadata (can be null, will skip multi-row and mandatory field checks if null) + * @return Error message if validation fails (format: "Missing required datatable fields in : , "), null if validation passes + */ + public static String validateRequiredDatatables(Workbook workbook, Sheet sheet, Row row, String entitySubtype, + EntityDatatableChecksRepository entityDatatableChecksRepository, DatatableReadService datatableReadService) { + if (entityDatatableChecksRepository == null) { + return null; + } + + // Get required datatables for CLIENT entity with CREATE status + List requiredChecks; + if (entitySubtype != null) { + requiredChecks = entityDatatableChecksRepository.findByEntityAndStatusAndSubtype( + EntityTables.CLIENT.getName(), StatusEnum.CREATE.getValue(), entitySubtype); + } else { + requiredChecks = entityDatatableChecksRepository.findByEntityAndStatus( + EntityTables.CLIENT.getName(), StatusEnum.CREATE.getValue()); + } + + if (requiredChecks == null || requiredChecks.isEmpty()) { + return null; // No required datatables + } + + // Get header row (row 0) + Row headerRow = sheet.getRow(0); + if (headerRow == null) { + return "Missing required datatable(s): " + getDatatableNames(requiredChecks); + } + + // Build a map of column headers to column indices + Map headerColumnMap = new HashMap<>(); + for (int colIndex = 0; colIndex <= headerRow.getLastCellNum(); colIndex++) { + Cell headerCell = headerRow.getCell(colIndex); + if (headerCell != null && headerCell.getCellType() == CellType.STRING) { + String headerValue = headerCell.getStringCellValue(); + if (headerValue != null) { + headerColumnMap.put(headerValue.trim(), colIndex); + } + } + } + + // Check each required datatable + List missingDatatables = new ArrayList<>(); + for (EntityDatatableChecks check : requiredChecks) { + String datatableName = check.getDatatableName(); + + // Skip validation for multi-row datatables - they are optional/repeatable child data + if (datatableReadService != null) { + try { + DatatableData datatableData = datatableReadService.retrieveDatatable(datatableName); + if (datatableData != null && datatableData.isMultiRow()) { + // Multi-row datatables are optional, skip validation + log.debug("Skipping validation for multi-row datatable '{}' - treated as optional/repeatable child data", + datatableName); + continue; + } + } catch (Exception e) { + // If we can't retrieve datatable metadata, continue with validation + // This preserves backward compatibility + log.debug("Could not retrieve datatable metadata for '{}' to check multi-row status: {}", + datatableName, e.getMessage()); + } + } + + // First, check if ANY column header exists for this datatable (presence check) + boolean headerExists = false; + List matchingColumnIndices = new ArrayList<>(); + + // Check if any column header matches the datatable name pattern + // Prefer dot notation (new format), fallback to underscore notation (legacy) + for (Map.Entry entry : headerColumnMap.entrySet()) { + String header = entry.getKey(); + int colIndex = entry.getValue(); + + // Remove trailing * if present + String headerWithoutStar = header.endsWith("*") ? header.substring(0, header.length() - 1) : header; + + boolean matches = false; + + // Check dot notation first (preferred format): registeredTableName.columnName + if (headerWithoutStar.contains(".")) { + int dotIndex = headerWithoutStar.indexOf("."); + if (dotIndex > 0) { + String headerDatatableName = headerWithoutStar.substring(0, dotIndex); + if (headerDatatableName.equals(datatableName)) { + matches = true; + } + } + } + // Fallback to underscore notation (legacy format) for backward compatibility + else if (headerWithoutStar.contains("_")) { + int lastUnderscoreIndex = headerWithoutStar.lastIndexOf("_"); + if (lastUnderscoreIndex > 0) { + String headerDatatableName = headerWithoutStar.substring(0, lastUnderscoreIndex); + if (headerDatatableName.equals(datatableName)) { + matches = true; + } + } + } + + if (matches) { + headerExists = true; + matchingColumnIndices.add(colIndex); + } + } + + // If no header exists for this datatable, skip validation entirely + // This handles the case where a required datatable is configured but not present in the Excel sheet + if (!headerExists) { + log.debug("Skipping validation for required datatable '{}' - no column headers found in Excel sheet. " + + "Validation only applies when at least one column header for the datatable is present.", + datatableName); + continue; + } + + // Header exists, so now validate mandatory columns + // Build a map of Excel column names (without datatable prefix) to column indices + Map excelColumnMap = new HashMap<>(); + for (Map.Entry entry : headerColumnMap.entrySet()) { + String header = entry.getKey(); + int colIndex = entry.getValue(); + + // Remove trailing * if present + String headerWithoutStar = header.endsWith("*") ? header.substring(0, header.length() - 1) : header; + + String columnName = null; + // Check dot notation first (preferred format): registeredTableName.columnName + if (headerWithoutStar.contains(".")) { + int dotIndex = headerWithoutStar.indexOf("."); + if (dotIndex > 0 && dotIndex < headerWithoutStar.length() - 1) { + String headerDatatableName = headerWithoutStar.substring(0, dotIndex); + if (headerDatatableName.equals(datatableName)) { + columnName = headerWithoutStar.substring(dotIndex + 1); + } + } + } + // Fallback to underscore notation (legacy format) for backward compatibility + else if (headerWithoutStar.contains("_")) { + int lastUnderscoreIndex = headerWithoutStar.lastIndexOf("_"); + if (lastUnderscoreIndex > 0 && lastUnderscoreIndex < headerWithoutStar.length() - 1) { + String headerDatatableName = headerWithoutStar.substring(0, lastUnderscoreIndex); + if (headerDatatableName.equals(datatableName)) { + columnName = headerWithoutStar.substring(lastUnderscoreIndex + 1); + } + } + } + + if (columnName != null) { + excelColumnMap.put(columnName, colIndex); + } + } + + // Retrieve datatable schema to check for mandatory columns + List missingMandatoryFields = new ArrayList<>(); + if (datatableReadService != null) { + try { + DatatableData datatableData = datatableReadService.retrieveDatatable(datatableName); + if (datatableData != null && datatableData.getColumnHeaderData() != null) { + // Find all mandatory columns (non-nullable, excluding system columns) + List mandatoryColumns = new ArrayList<>(); + for (ResultsetColumnHeaderData column : datatableData.getColumnHeaderData()) { + String columnName = column.getColumnName(); + + // Ignore system columns + if (columnName.equalsIgnoreCase("id") + || columnName.equalsIgnoreCase("client_id") + || columnName.equalsIgnoreCase("created_at") + || columnName.equalsIgnoreCase("updated_at") + || columnName.equalsIgnoreCase("locale") + || columnName.equalsIgnoreCase("dateFormat")) { + continue; + } + + // Check if column is mandatory (not nullable) + if (!column.getIsColumnNullable()) { + mandatoryColumns.add(columnName); + } + } + + // If there are no mandatory columns, skip validation (all columns are nullable) + if (mandatoryColumns.isEmpty()) { + log.debug("Skipping validation for required datatable '{}' - no mandatory fields found. " + + "All columns are nullable, so blank values are allowed.", + datatableName); + continue; + } + + // Check each mandatory column + for (String mandatoryColumn : mandatoryColumns) { + Integer colIndex = excelColumnMap.get(mandatoryColumn); + + if (colIndex == null) { + // Mandatory column is missing from Excel header + missingMandatoryFields.add(mandatoryColumn); + } else { + // Check if the cell has a non-blank value + Cell dataCell = row.getCell(colIndex); + boolean hasValue = false; + if (dataCell != null && dataCell.getCellType() != CellType.BLANK) { + String cellValue = readAsString(colIndex, row); + if (cellValue != null && !cellValue.trim().isEmpty()) { + hasValue = true; + } + } + + if (!hasValue) { + missingMandatoryFields.add(mandatoryColumn); + } + } + } + + // If mandatory fields are missing, add to error list + if (!missingMandatoryFields.isEmpty()) { + String friendlyName = getFriendlyDatatableName(datatableName); + String fieldsList = String.join(", ", missingMandatoryFields); + missingDatatables.add(friendlyName + ": " + fieldsList); + log.debug("Failing validation for required datatable '{}' - missing mandatory fields: {}", + datatableName, fieldsList); + } + } else { + // If we can't retrieve datatable schema, fall back to old behavior + // Check if at least one column has a non-empty value + boolean hasValue = false; + for (Integer colIndex : matchingColumnIndices) { + Cell dataCell = row.getCell(colIndex); + if (dataCell != null && dataCell.getCellType() != CellType.BLANK) { + String cellValue = readAsString(colIndex, row); + if (cellValue != null && !cellValue.trim().isEmpty()) { + hasValue = true; + break; + } + } + } + + if (!hasValue) { + String friendlyName = getFriendlyDatatableName(datatableName); + missingDatatables.add(friendlyName); + log.debug("Required datatable '{}' has column headers in Excel but no values provided " + + "(datatable schema unavailable for mandatory field check)", + datatableName); + } + } + } catch (Exception e) { + // If we can't retrieve datatable schema, fall back to old behavior + // Check if at least one column has a non-empty value + boolean hasValue = false; + for (Integer colIndex : matchingColumnIndices) { + Cell dataCell = row.getCell(colIndex); + if (dataCell != null && dataCell.getCellType() != CellType.BLANK) { + String cellValue = readAsString(colIndex, row); + if (cellValue != null && !cellValue.trim().isEmpty()) { + hasValue = true; + break; + } + } + } + + if (!hasValue) { + String friendlyName = getFriendlyDatatableName(datatableName); + missingDatatables.add(friendlyName); + log.debug("Required datatable '{}' has column headers in Excel but no values provided " + + "(error retrieving datatable schema: {})", + datatableName, e.getMessage()); + } + } + } else { + // If datatableReadService is null, fall back to old behavior + // Check if at least one column has a non-empty value + boolean hasValue = false; + for (Integer colIndex : matchingColumnIndices) { + Cell dataCell = row.getCell(colIndex); + if (dataCell != null && dataCell.getCellType() != CellType.BLANK) { + String cellValue = readAsString(colIndex, row); + if (cellValue != null && !cellValue.trim().isEmpty()) { + hasValue = true; + break; + } + } + } + + if (!hasValue) { + String friendlyName = getFriendlyDatatableName(datatableName); + missingDatatables.add(friendlyName); + log.debug("Required datatable '{}' has column headers in Excel but no values provided " + + "(datatableReadService unavailable for mandatory field check)", + datatableName); + } + } + } + + if (!missingDatatables.isEmpty()) { + // Format: "Missing required datatable fields in : , " + // or "Missing required datatable(s): " if no field details available + List errorMessages = new ArrayList<>(); + for (String missing : missingDatatables) { + if (missing.contains(": ")) { + // Has field details: "Missing required datatable fields in : " + errorMessages.add("Missing required datatable fields in " + missing); + } else { + // No field details: fallback to old format + errorMessages.add("Missing required datatable(s): " + missing); + } + } + return String.join("; ", errorMessages); + } + + return null; + } + + private static String getDatatableNames(List checks) { + List names = new ArrayList<>(); + for (EntityDatatableChecks check : checks) { + names.add(getFriendlyDatatableName(check.getDatatableName())); + } + return String.join(", ", names); + } + + private static String getFriendlyDatatableName(String datatableName) { + // Convert datatable name to a more friendly format + // e.g., "extra_client_details" -> "Extra Client Details" + if (datatableName == null || datatableName.isEmpty()) { + return datatableName; + } + // Replace underscores with spaces and capitalize words + List parts = Splitter.on('_').splitToList(datatableName); + StringBuilder friendly = new StringBuilder(); + for (int i = 0; i < parts.size(); i++) { + if (i > 0) { + friendly.append(" "); + } + String part = parts.get(i); + if (!part.isEmpty()) { + friendly.append(Character.toUpperCase(part.charAt(0))); + if (part.length() > 1) { + friendly.append(part.substring(1)); + } + } + } + return friendly.toString(); + } + + /** + * Reads datatable columns from the Excel row and groups them by registered table name. + * + * Preferred format (new): . or .* + * Legacy format (backward compatibility): _ or _* + * + * For dot notation: splits at the first dot (only one is expected). + * For legacy underscore notation: splits at the last underscore. + * + * @param sheet The sheet containing the client data + * @param row The data row to read from + * @param locale The locale for datatable data + * @param dateFormat The date format for datatable data + * @return List of maps representing datatables, each with "registeredTableName" and "data" keys. + * Only includes datatables with at least one non-empty value. + */ + public static List> readDatatablesFromRow(Sheet sheet, Row row, String locale, String dateFormat) { + List> datatablesList = new ArrayList<>(); + + if (sheet == null || row == null) { + return datatablesList; + } + + // Get header row (row 0) + Row headerRow = sheet.getRow(0); + if (headerRow == null) { + return datatablesList; + } + + // Build a map of column headers to column indices + Map headerColumnMap = new HashMap<>(); + for (int colIndex = 0; colIndex <= headerRow.getLastCellNum(); colIndex++) { + Cell headerCell = headerRow.getCell(colIndex); + if (headerCell != null && headerCell.getCellType() == CellType.STRING) { + String headerValue = headerCell.getStringCellValue(); + if (headerValue != null) { + headerColumnMap.put(headerValue.trim(), colIndex); + } + } + } + + // Group datatable columns by datatable name + Map> datatablesMap = new HashMap<>(); + boolean legacyHeaderDetected = false; + + for (Map.Entry entry : headerColumnMap.entrySet()) { + String header = entry.getKey(); + int colIndex = entry.getValue(); + + // Remove trailing * if present + String headerWithoutStar = header.endsWith("*") ? header.substring(0, header.length() - 1) : header; + + String registeredTableName; + String columnName = null; + + // Prefer dot notation (new format): registeredTableName.columnName + if (headerWithoutStar.contains(".")) { + int dotIndex = headerWithoutStar.indexOf("."); + if (dotIndex > 0 && dotIndex < headerWithoutStar.length() - 1) { + registeredTableName = headerWithoutStar.substring(0, dotIndex); + columnName = headerWithoutStar.substring(dotIndex + 1); + } else { + registeredTableName = null; + } + } + // Fallback to underscore notation (legacy format) for backward compatibility + else if (headerWithoutStar.contains("_")) { + legacyHeaderDetected = true; + // Use lastIndexOf to split at the last underscore + // Everything before the last _ is the registeredTableName + // Everything after the last _ is the columnName + int lastUnderscoreIndex = headerWithoutStar.lastIndexOf("_"); + if (lastUnderscoreIndex > 0 && lastUnderscoreIndex < headerWithoutStar.length() - 1) { + registeredTableName = headerWithoutStar.substring(0, lastUnderscoreIndex); + columnName = headerWithoutStar.substring(lastUnderscoreIndex + 1); + } else { + registeredTableName = null; + } + } else { + registeredTableName = null; + } + + // Process if we have a valid datatable column + if (registeredTableName != null && columnName != null) { + // Skip system columns + if (columnName.equalsIgnoreCase("id") || columnName.equalsIgnoreCase("client_id") + || columnName.equalsIgnoreCase("created_at") || columnName.equalsIgnoreCase("updated_at")) { + continue; + } + + // Read cell value + Cell dataCell = row.getCell(colIndex); + Object cellValue = null; + if (dataCell != null && dataCell.getCellType() != CellType.BLANK) { + // Try to read as different types + if (dataCell.getCellType() == CellType.NUMERIC) { + if (org.apache.poi.ss.usermodel.DateUtil.isCellDateFormatted(dataCell)) { + // Date value - format as string using dateFormat + LocalDate dateValue = readAsDate(colIndex, row); + if (dateValue != null && dateFormat != null) { + try { + DateTimeFormatter formatter = new DateTimeFormatterBuilder() + .appendPattern(dateFormat) + .toFormatter(); + cellValue = dateValue.format(formatter); + } catch (Exception e) { + // Fallback to ISO format if dateFormat is invalid + cellValue = dateValue.toString(); + } + } else if (dateValue != null) { + cellValue = dateValue.toString(); + } + } else { + // Numeric value - check if it's a whole number + double numValue = dataCell.getNumericCellValue(); + if (numValue == Math.floor(numValue)) { + cellValue = (long) numValue; + } else { + cellValue = numValue; + } + } + } else if (dataCell.getCellType() == CellType.BOOLEAN) { + cellValue = dataCell.getBooleanCellValue(); + } else { + // String value + String stringValue = readAsString(colIndex, row); + if (stringValue != null && !stringValue.trim().isEmpty()) { + // Try to extract ID from display value format: "Label (ID)" + // This handles CODELOOKUP and FK/INTEGER columns that show human-readable values + String extractedId = extractIdFromDisplayValue(stringValue.trim()); + // If extraction succeeded (returned a different value), try to parse as integer + if (!extractedId.equals(stringValue.trim())) { + try { + cellValue = Integer.parseInt(extractedId); + } catch (NumberFormatException e) { + // If parsing fails, use the original string value + cellValue = stringValue.trim(); + } + } else { + // No ID pattern found, use original value + cellValue = stringValue.trim(); + } + } + } + } + + // Only add if value is not null/empty + if (cellValue != null) { + // Get or create datatable entry using registeredTableName + Map datatableData = datatablesMap.computeIfAbsent(registeredTableName, k -> { + Map newDatatable = new HashMap<>(); + newDatatable.put("registeredTableName", registeredTableName); + Map data = new HashMap<>(); + data.put("locale", locale); + data.put("dateFormat", dateFormat); + newDatatable.put("data", data); + return newDatatable; + }); + + // Add column value to data + @SuppressWarnings("unchecked") + Map data = (Map) datatableData.get("data"); + data.put(columnName, cellValue); + } + } + } + + // Log warning if legacy underscore-based headers were detected + if (legacyHeaderDetected) { + log.warn("Legacy underscore-based datatable column headers detected. Please update template to use dot notation (registeredTableName.columnName) for unambiguous parsing."); + } + + // Convert map values to list (only include datatables with at least one data field beyond locale/dateFormat) + for (Map datatable : datatablesMap.values()) { + @SuppressWarnings("unchecked") + Map data = (Map) datatable.get("data"); + // Check if there are any fields beyond locale and dateFormat + if (data.size() > 2) { + datatablesList.add(datatable); + } + } + + return datatablesList; + } + + /** + * Ensures all configured client datatables are included in the payload, even if they have no values. + * This is required for bulk import to match Fineract's contract where all entity-linked datatables must be present. + * Only includes datatables that are explicitly linked to Client via EntityDatatableChecks. + * + * Construction rules: + * - If datatable has at least one required column (non-nullable): existing behavior applies (must be filled by user) + * - If datatable has NO required columns: always construct the datatable in payload, even if multi-row + * - Multi-row datatables with required columns are skipped (user must provide data) + * - Multi-row datatables with no required columns are included with empty entry + * + * @param sheet The sheet containing the client data + * @param row The data row to read from + * @param locale The locale for datatable data + * @param dateFormat The date format for datatable data + * @param entityDatatableChecksRepository Repository to fetch datatables linked to Client entity (can be null, will skip if null) + * @param datatableReadService Service to retrieve datatable metadata (can be null, will skip multi-row check if null) + * @param entitySubtype The entity subtype (PERSON or ENTITY), can be null + * @return List of maps representing datatables, each with "registeredTableName" and "data" keys. + * Includes only datatables linked via EntityDatatableChecks, with empty entries for those without values. + */ + public static List> readDatatablesFromRowWithAllConfigured( + Sheet sheet, Row row, String locale, String dateFormat, + EntityDatatableChecksRepository entityDatatableChecksRepository, + DatatableReadService datatableReadService, String entitySubtype) { + // First, read datatables that have values from Excel + List> datatablesWithValues = readDatatablesFromRow(sheet, row, locale, dateFormat); + + // Create a map of registeredTableName -> datatable for quick lookup + Map> datatablesMap = new HashMap<>(); + for (Map datatable : datatablesWithValues) { + String registeredTableName = (String) datatable.get("registeredTableName"); + if (registeredTableName != null) { + datatablesMap.put(registeredTableName, datatable); + } + } + + // If entityDatatableChecksRepository is available, ensure all linked client datatables are included + if (entityDatatableChecksRepository != null) { + try { + // Retrieve only datatables linked to Client via EntityDatatableChecks with CREATE status + List linkedChecks; + if (entitySubtype != null) { + linkedChecks = entityDatatableChecksRepository.findByEntityAndStatusAndSubtype( + EntityTables.CLIENT.getName(), StatusEnum.CREATE.getValue(), entitySubtype); + } else { + linkedChecks = entityDatatableChecksRepository.findByEntityAndStatus( + EntityTables.CLIENT.getName(), StatusEnum.CREATE.getValue()); + } + + // For each linked datatable, ensure it's in the result + for (EntityDatatableChecks check : linkedChecks) { + String registeredTableName = check.getDatatableName(); + + // Skip if already included (has values) + if (datatablesMap.containsKey(registeredTableName)) { + continue; + } + + // Retrieve datatable schema to check multi-row status and required columns + boolean isMultiRow = false; + boolean hasRequiredColumns = false; + org.apache.fineract.infrastructure.dataqueries.data.DatatableData datatableData = null; + + if (datatableReadService != null) { + try { + datatableData = datatableReadService.retrieveDatatable(registeredTableName); + if (datatableData != null) { + isMultiRow = datatableData.isMultiRow(); + + // Check if datatable has at least one required column + if (datatableData.getColumnHeaderData() != null) { + for (ResultsetColumnHeaderData column : datatableData.getColumnHeaderData()) { + String columnName = column.getColumnName(); + + // Skip system columns + if (columnName.equalsIgnoreCase("id") + || columnName.equalsIgnoreCase("client_id") + || columnName.equalsIgnoreCase("created_at") + || columnName.equalsIgnoreCase("updated_at") + || columnName.equalsIgnoreCase("locale") + || columnName.equalsIgnoreCase("dateFormat")) { + continue; + } + + // Check if column is required (not nullable) + if (!column.getIsColumnNullable()) { + hasRequiredColumns = true; + break; + } + } + } + } + } catch (Exception e) { + // If we can't retrieve datatable metadata, continue (assume not multi-row, no required columns) + log.debug("Could not retrieve datatable metadata for '{}': {}", + registeredTableName, e.getMessage()); + } + } + + // Skip multi-row datatables ONLY if they have required columns + // Multi-row datatables with NO required columns should be included (empty entry) + if (isMultiRow && hasRequiredColumns) { + // Multi-row with required columns: skip (user must provide data) + continue; + } + + // Create empty datatable entry with locale and dateFormat + Map emptyDatatable = new HashMap<>(); + emptyDatatable.put("registeredTableName", registeredTableName); + Map data = new HashMap<>(); + data.put("locale", locale); + data.put("dateFormat", dateFormat); + + // Add empty strings for known columns if schema is available + if (datatableData != null && datatableData.getColumnHeaderData() != null) { + for (ResultsetColumnHeaderData column : datatableData.getColumnHeaderData()) { + String columnName = column.getColumnName(); + // Skip system columns + if (columnName.equalsIgnoreCase("id") + || columnName.equalsIgnoreCase("client_id") + || columnName.equalsIgnoreCase("created_at") + || columnName.equalsIgnoreCase("updated_at") + || columnName.equalsIgnoreCase("locale") + || columnName.equalsIgnoreCase("dateFormat")) { + continue; + } + // Add empty string for known columns + data.put(columnName, ""); + } + } + + emptyDatatable.put("data", data); + datatablesMap.put(registeredTableName, emptyDatatable); + } + } catch (Exception e) { + // If we can't retrieve linked datatables, log and continue with what we have + log.debug("Could not retrieve linked client datatables for bulk import payload: {}", e.getMessage()); + } + } + + // Return all datatables (with values and empty ones) + return new ArrayList<>(datatablesMap.values()); + } + } diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/importhandler/client/ClientEntityImportHandler.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/importhandler/client/ClientEntityImportHandler.java index 9b08efaa04a..af8be5057d7 100644 --- a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/importhandler/client/ClientEntityImportHandler.java +++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/importhandler/client/ClientEntityImportHandler.java @@ -23,8 +23,11 @@ import java.time.LocalDate; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.List; import java.util.Objects; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import lombok.AllArgsConstructor; import org.apache.fineract.commands.domain.CommandWrapper; import org.apache.fineract.commands.service.CommandWrapperBuilder; @@ -35,12 +38,19 @@ import org.apache.fineract.infrastructure.bulkimport.importhandler.ImportHandler; import org.apache.fineract.infrastructure.bulkimport.importhandler.ImportHandlerUtils; import org.apache.fineract.infrastructure.bulkimport.importhandler.helper.DateSerializer; +import org.apache.fineract.infrastructure.core.data.ApiParameterError; import org.apache.fineract.infrastructure.core.domain.ExternalId; +import org.apache.fineract.infrastructure.core.exception.PlatformApiDataValidationException; import org.apache.fineract.infrastructure.core.serialization.GoogleGsonSerializerHelper; import org.apache.fineract.infrastructure.core.service.ExternalIdFactory; +import org.apache.fineract.infrastructure.dataqueries.domain.EntityDatatableChecksRepository; +import org.apache.fineract.infrastructure.dataqueries.service.DatatableReadService; import org.apache.fineract.portfolio.address.data.AddressData; import org.apache.fineract.portfolio.client.data.ClientData; import org.apache.fineract.portfolio.client.data.ClientNonPersonData; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import java.util.Map; import org.apache.poi.ss.usermodel.Cell; import org.apache.poi.ss.usermodel.IndexedColors; import org.apache.poi.ss.usermodel.Row; @@ -58,6 +68,8 @@ public class ClientEntityImportHandler implements ImportHandler { private static final Logger LOG = LoggerFactory.getLogger(ClientEntityImportHandler.class); private final PortfolioCommandSourceWritePlatformService commandsSourceWritePlatformService; private final ExternalIdFactory externalIdFactory; + private final EntityDatatableChecksRepository entityDatatableChecksRepository; + private final DatatableReadService datatableReadService; @Override public Count process(final Workbook workbook, final String locale, final String dateFormat) { @@ -73,6 +85,9 @@ private List readExcelFile(final Workbook workbook, final String loc for (int rowIndex = 1; rowIndex <= noOfEntries; rowIndex++) { Row row; row = clientSheet.getRow(rowIndex); + if (row != null) { + LOG.info("Processing bulk upload row {} with {} cells", row.getRowNum(), row.getLastCellNum()); + } if (ImportHandlerUtils.isNotImported(row, ClientEntityConstants.STATUS_COL)) { clients.add(readClient(workbook, row, locale, dateFormat)); } @@ -104,16 +119,24 @@ private ClientData readClient(final Workbook workbook, final Row row, final Stri Long clientTypeId = null; if (clientType != null) { List clientTypeAr = Splitter.on(SEPARATOR).splitToList(clientType); - if (clientTypeAr.get(1) != null) { - clientTypeId = Long.parseLong(clientTypeAr.get(1)); + if (clientTypeAr.size() > 1) { + if (clientTypeAr.get(1) != null) { + clientTypeId = Long.parseLong(clientTypeAr.get(1)); + } + } else { + LOG.error("Bulk upload error: Missing expected column at index 1 in clientType. Row data: {}", clientTypeAr); } } String clientClassification = ImportHandlerUtils.readAsString(ClientEntityConstants.CLIENT_CLASSIFICATION_COL, row); Long clientClassicationId = null; if (clientClassification != null) { List clientClassificationAr = Splitter.on(SEPARATOR).splitToList(clientClassification); - if (clientClassificationAr.get(1) != null) { - clientClassicationId = Long.parseLong(clientClassificationAr.get(1)); + if (clientClassificationAr.size() > 1) { + if (clientClassificationAr.get(1) != null) { + clientClassicationId = Long.parseLong(clientClassificationAr.get(1)); + } + } else { + LOG.error("Bulk upload error: Missing expected column at index 1 in clientClassification. Row data: {}", clientClassificationAr); } } String incorporationNo = ImportHandlerUtils.readAsString(ClientEntityConstants.INCOPORATION_NUMBER_COL, row); @@ -123,18 +146,34 @@ private ClientData readClient(final Workbook workbook, final Row row, final Stri if (mainBusinessLine != null) { List mainBusinessLineAr = Splitter.on(SEPARATOR) .splitToList(Objects.requireNonNull(ImportHandlerUtils.readAsString(ClientEntityConstants.MAIN_BUSINESS_LINE, row))); - if (mainBusinessLineAr.get(1) != null) { - mainBusinessId = Long.parseLong(mainBusinessLineAr.get(1)); + if (mainBusinessLineAr.size() > 1) { + if (mainBusinessLineAr.get(1) != null) { + mainBusinessId = Long.parseLong(mainBusinessLineAr.get(1)); + } + } else { + LOG.error("Bulk upload error: Missing expected column at index 1 in mainBusinessLine. Row data: {}", mainBusinessLineAr); } } - String constitution = ImportHandlerUtils.readAsString(ClientEntityConstants.CONSTITUTION_COL, row); + String constitutionCell = ImportHandlerUtils.readAsString(ClientEntityConstants.CONSTITUTION_COL, row); + Long constitutionId = null; - if (constitution != null) { - List constitutionAr = Splitter.on(SEPARATOR).splitToList(constitution); - if (constitutionAr.get(1) != null) { - constitutionId = Long.parseLong(constitutionAr.get(1)); + + if (constitutionCell != null && constitutionCell.contains("(") && constitutionCell.contains(")")) { + Pattern pattern = Pattern.compile("\\((\\d+)\\)"); + Matcher matcher = pattern.matcher(constitutionCell); + + if (matcher.find()) { + constitutionId = Long.valueOf(matcher.group(1)); } } + + if (constitutionId == null && constitutionCell != null && !constitutionCell.trim().isEmpty()) { + LOG.error("Invalid constitution format in row {}: {}", row.getRowNum(), constitutionCell); + ApiParameterError error = ApiParameterError.parameterError("error.msg.bulk.import.constitution.invalid", + "Invalid Constitution format. Expected format: Name (ID)", "constitution", constitutionCell); + throw new PlatformApiDataValidationException("error.msg.bulk.import.constitution.invalid", + "Invalid Constitution format. Expected format: Name (ID)", Collections.singletonList(error)); + } String remarks = ImportHandlerUtils.readAsString(ClientEntityConstants.REMARKS_COL, row); ClientNonPersonData clientNonPersonData = ClientNonPersonData.importInstance(incorporationNo, incorporationTill, remarks, @@ -157,8 +196,12 @@ private ClientData readClient(final Workbook workbook, final Row row, final Stri Long addressTypeId = null; if (addressType != null) { List addressTypeAr = Splitter.on(SEPARATOR).splitToList(addressType); - if (addressTypeAr.get(1) != null) { - addressTypeId = Long.parseLong(addressTypeAr.get(1)); + if (addressTypeAr.size() > 1) { + if (addressTypeAr.get(1) != null) { + addressTypeId = Long.parseLong(addressTypeAr.get(1)); + } + } else { + LOG.error("Bulk upload error: Missing expected column at index 1 in addressType. Row data: {}", addressTypeAr); } } String street = ImportHandlerUtils.readAsString(ClientEntityConstants.STREET_COL, row); @@ -174,16 +217,24 @@ private ClientData readClient(final Workbook workbook, final Row row, final Stri Long stateProvinceId = null; if (stateProvince != null) { List stateProvinceAr = Splitter.on(SEPARATOR).splitToList(stateProvince); - if (stateProvinceAr.get(1) != null) { - stateProvinceId = Long.parseLong(stateProvinceAr.get(1)); + if (stateProvinceAr.size() > 1) { + if (stateProvinceAr.get(1) != null) { + stateProvinceId = Long.parseLong(stateProvinceAr.get(1)); + } + } else { + LOG.error("Bulk upload error: Missing expected column at index 1 in stateProvince. Row data: {}", stateProvinceAr); } } String country = ImportHandlerUtils.readAsString(ClientEntityConstants.COUNTRY_COL, row); Long countryId = null; if (country != null) { List countryAr = Splitter.on(SEPARATOR).splitToList(country); - if (countryAr.get(1) != null) { - countryId = Long.parseLong(countryAr.get(1)); + if (countryAr.size() > 1) { + if (countryAr.get(1) != null) { + countryId = Long.parseLong(countryAr.get(1)); + } + } else { + LOG.error("Bulk upload error: Missing expected column at index 1 in country. Row data: {}", countryAr); } } addressDataObj = new AddressData(addressTypeId, street, addressLine1, addressLine2, addressLine3, city, postalCode, @@ -207,7 +258,33 @@ private Count importEntity(final Workbook workbook, final List clien for (ClientData client : clients) { try { + // Get the client row once for validation and datatable reading + Row clientRow = clientSheet.getRow(client.getRowIndex()); + if (clientRow != null) { + // Validate required datatables before attempting client creation + String validationError = ImportHandlerUtils.validateRequiredDatatables( + workbook, clientSheet, clientRow, "Entity", entityDatatableChecksRepository, datatableReadService); + if (validationError != null) { + throw new RuntimeException(validationError); + } + } + String payload = gsonBuilder.create().toJson(client); + + // Read datatables from the Excel row and add to payload + // Ensure all configured client datatables are included, even if empty + if (clientRow != null) { + List> datatables = ImportHandlerUtils.readDatatablesFromRowWithAllConfigured( + clientSheet, clientRow, locale, dateFormat, entityDatatableChecksRepository, + datatableReadService, "Entity"); + if (datatables != null && !datatables.isEmpty()) { + // Parse JSON and add datatables array + JsonObject jsonObject = JsonParser.parseString(payload).getAsJsonObject(); + jsonObject.add("datatables", gsonBuilder.create().toJsonTree(datatables)); + payload = gsonBuilder.create().toJson(jsonObject); + } + } + final CommandWrapper commandRequest = new CommandWrapperBuilder() // .createClient() // .withJson(payload) // diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/importhandler/client/ClientPersonImportHandler.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/importhandler/client/ClientPersonImportHandler.java index e35027a192c..f7349de475a 100644 --- a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/importhandler/client/ClientPersonImportHandler.java +++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/importhandler/client/ClientPersonImportHandler.java @@ -38,8 +38,14 @@ import org.apache.fineract.infrastructure.core.domain.ExternalId; import org.apache.fineract.infrastructure.core.serialization.GoogleGsonSerializerHelper; import org.apache.fineract.infrastructure.core.service.ExternalIdFactory; +import org.apache.fineract.infrastructure.dataqueries.domain.EntityDatatableChecksRepository; +import org.apache.fineract.infrastructure.dataqueries.service.DatatableReadService; import org.apache.fineract.portfolio.address.data.AddressData; import org.apache.fineract.portfolio.client.data.ClientData; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import java.util.List; +import java.util.Map; import org.apache.poi.ss.usermodel.Cell; import org.apache.poi.ss.usermodel.IndexedColors; import org.apache.poi.ss.usermodel.Row; @@ -57,6 +63,8 @@ public class ClientPersonImportHandler implements ImportHandler { private static final Logger LOG = LoggerFactory.getLogger(ClientPersonImportHandler.class); private final PortfolioCommandSourceWritePlatformService commandsSourceWritePlatformService; private final ExternalIdFactory externalIdFactory; + private final EntityDatatableChecksRepository entityDatatableChecksRepository; + private final DatatableReadService datatableReadService; @Override public Count process(final Workbook workbook, final String locale, final String dateFormat) { @@ -192,7 +200,33 @@ public Count importEntity(final Workbook workbook, final List client gsonBuilder.registerTypeAdapter(LocalDate.class, new DateSerializer(dateFormat, locale)); for (ClientData client : clients) { try { + // Get the client row once for validation and datatable reading + Row clientRow = clientSheet.getRow(client.getRowIndex()); + if (clientRow != null) { + // Validate required datatables before attempting client creation + String validationError = ImportHandlerUtils.validateRequiredDatatables( + workbook, clientSheet, clientRow, "Person", entityDatatableChecksRepository, datatableReadService); + if (validationError != null) { + throw new RuntimeException(validationError); + } + } + String payload = gsonBuilder.create().toJson(client); + + // Read datatables from the Excel row and add to payload + // Ensure all configured client datatables are included, even if empty + if (clientRow != null) { + List> datatables = ImportHandlerUtils.readDatatablesFromRowWithAllConfigured( + clientSheet, clientRow, locale, dateFormat, entityDatatableChecksRepository, + datatableReadService, "Person"); + if (datatables != null && !datatables.isEmpty()) { + // Parse JSON and add datatables array + JsonObject jsonObject = JsonParser.parseString(payload).getAsJsonObject(); + jsonObject.add("datatables", gsonBuilder.create().toJsonTree(datatables)); + payload = gsonBuilder.create().toJson(jsonObject); + } + } + final CommandWrapper commandRequest = new CommandWrapperBuilder() // .createClient() // .withJson(payload) // diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/populator/client/ClientEntityWorkbookPopulator.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/populator/client/ClientEntityWorkbookPopulator.java index 985d209f89b..82a8b3a84be 100644 --- a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/populator/client/ClientEntityWorkbookPopulator.java +++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/populator/client/ClientEntityWorkbookPopulator.java @@ -25,7 +25,10 @@ import org.apache.fineract.infrastructure.bulkimport.populator.OfficeSheetPopulator; import org.apache.fineract.infrastructure.bulkimport.populator.PersonnelSheetPopulator; import org.apache.fineract.infrastructure.codes.data.CodeValueData; +import org.apache.fineract.infrastructure.dataqueries.data.DatatableData; +import org.apache.fineract.infrastructure.dataqueries.data.ResultsetColumnHeaderData; import org.apache.fineract.organisation.office.data.OfficeData; +import java.util.ArrayList; import org.apache.poi.hssf.usermodel.HSSFDataValidationHelper; import org.apache.poi.hssf.usermodel.HSSFSheet; import org.apache.poi.ss.SpreadsheetVersion; @@ -38,11 +41,20 @@ import org.apache.poi.ss.usermodel.Name; import org.apache.poi.ss.usermodel.Row; import org.apache.poi.ss.usermodel.Sheet; +import org.apache.poi.ss.usermodel.SheetVisibility; import org.apache.poi.ss.usermodel.Workbook; import org.apache.poi.ss.util.CellRangeAddressList; +import org.apache.poi.ss.util.CellReference; +import org.apache.fineract.infrastructure.dataqueries.data.ResultsetColumnValueData; +import java.util.Map; +import java.util.HashMap; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class ClientEntityWorkbookPopulator extends AbstractWorkbookPopulator { + private static final Logger log = LoggerFactory.getLogger(ClientEntityWorkbookPopulator.class); + private final OfficeSheetPopulator officeSheetPopulator; private final PersonnelSheetPopulator personnelSheetPopulator; private final List clientTypeCodeValues; @@ -52,11 +64,53 @@ public class ClientEntityWorkbookPopulator extends AbstractWorkbookPopulator { private final List stateProvinceCodeValues; private final List countryCodeValues; private final List mainBusinesslineCodeValues; + private final List requiredDatatables; + // Track datatable dropdown columns: namedRangeName -> (clientSheetColumnIndex, lookupSheetColumnIndex, values) + private final Map datatableDropdowns = new HashMap<>(); + + // Track datatable date columns: columnIndex -> columnInfo + private final Map datatableDateColumns = new HashMap<>(); + + // Track datatable boolean columns: columnIndex -> columnInfo + private final Map datatableBooleanColumns = new HashMap<>(); + + // Inner class to track datatable dropdown information + private static class DatatableDropdownInfo { + final int clientSheetColumnIndex; + final int lookupSheetColumnIndex; + final List values; + + DatatableDropdownInfo(int clientSheetColumnIndex, int lookupSheetColumnIndex, List values) { + this.clientSheetColumnIndex = clientSheetColumnIndex; + this.lookupSheetColumnIndex = lookupSheetColumnIndex; + this.values = values; + } + } + + // Inner class to track datatable date column information + private static class DatatableDateInfo { + final int clientSheetColumnIndex; + final boolean isDateTime; + + DatatableDateInfo(int clientSheetColumnIndex, boolean isDateTime) { + this.clientSheetColumnIndex = clientSheetColumnIndex; + this.isDateTime = isDateTime; + } + } + + // Inner class to track datatable boolean column information + private static class DatatableBooleanInfo { + final int clientSheetColumnIndex; + + DatatableBooleanInfo(int clientSheetColumnIndex) { + this.clientSheetColumnIndex = clientSheetColumnIndex; + } + } public ClientEntityWorkbookPopulator(OfficeSheetPopulator officeSheetPopulator, PersonnelSheetPopulator personnelSheetPopulator, List clientTypeCodeValues, List constitutionCodeValues, List mainBusinessline, List clientClassification, List addressTypesCodeValues, - List stateProvinceCodeValues, List countryCodeValues) { + List stateProvinceCodeValues, List countryCodeValues, List requiredDatatables) { this.officeSheetPopulator = officeSheetPopulator; this.personnelSheetPopulator = personnelSheetPopulator; this.clientTypeCodeValues = clientTypeCodeValues; @@ -66,6 +120,7 @@ public ClientEntityWorkbookPopulator(OfficeSheetPopulator officeSheetPopulator, this.stateProvinceCodeValues = stateProvinceCodeValues; this.countryCodeValues = countryCodeValues; this.mainBusinesslineCodeValues = mainBusinessline; + this.requiredDatatables = requiredDatatables != null ? requiredDatatables : new ArrayList<>(); } @Override @@ -73,12 +128,17 @@ public void populate(Workbook workbook, String dateFormat) { Sheet clientSheet = workbook.createSheet(TemplatePopulateImportConstants.CLIENT_ENTITY_SHEET_NAME); personnelSheetPopulator.populate(workbook, dateFormat); officeSheetPopulator.populate(workbook, dateFormat); - setLayout(clientSheet); - setOfficeDateLookupTable(clientSheet, officeSheetPopulator.getOffices(), ClientEntityConstants.RELATIONAL_OFFICE_NAME_COL, - ClientEntityConstants.RELATIONAL_OFFICE_OPENING_DATE_COL, dateFormat); - setClientDataLookupTable(clientSheet); + + // Create hidden lookup sheet for lookup data + Sheet lookupSheet = workbook.createSheet(TemplatePopulateImportConstants.CLIENT_LOOKUPS_SHEET_NAME); + // TODO: For debugging, temporarily set to VISIBLE. Revert to VERY_HIDDEN once validated. + workbook.setSheetVisibility(workbook.getSheetIndex(lookupSheet), SheetVisibility.VERY_HIDDEN); + + setLayout(clientSheet); // fills datatableDropdowns + setClientDataLookupTable(lookupSheet); // now values exist setFormatStyle(workbook, clientSheet); setRules(clientSheet, dateFormat); + handleDatatableColumnsComplete(clientSheet, lookupSheet); } private void setFormatStyle(Workbook workbook, Sheet worksheet) { @@ -96,6 +156,10 @@ private void setFormatStyle(Workbook workbook, Sheet worksheet) { setFormatActivationAndSubmittedDate(row, ClientEntityConstants.SUBMITTED_ON_COL, dateCellStyle); setFormatActivationAndSubmittedDate(row, ClientEntityConstants.INCOPORATION_VALID_TILL_COL, dateCellStyle); setFormatActivationAndSubmittedDate(row, ClientEntityConstants.INCOPORATION_DATE_COL, dateCellStyle); + // Apply date formatting to datatable date columns + for (ClientEntityWorkbookPopulator.DatatableDateInfo dateInfo : datatableDateColumns.values()) { + setFormatActivationAndSubmittedDate(row, dateInfo.clientSheetColumnIndex, dateCellStyle); + } } } @@ -107,68 +171,91 @@ private void setFormatActivationAndSubmittedDate(Row row, int columnIndex, CellS cell.setCellStyle(cellStyle); } - private void setClientDataLookupTable(Sheet clientSheet) { - int rowIndex = 0; + private void setClientDataLookupTable(Sheet lookupSheet) { + // Data starts at row 2 (row index 1) to match named range references + // Column 0 (A): Client Types + int rowIndex = 1; // Start at row 2 (Excel row 2, POI index 1) for (CodeValueData clientTypeCodeValue : clientTypeCodeValues) { - Row row = clientSheet.getRow(++rowIndex); + Row row = lookupSheet.getRow(rowIndex); if (row == null) { - row = clientSheet.createRow(rowIndex); + row = lookupSheet.createRow(rowIndex); } - writeString(ClientEntityConstants.LOOKUP_CLIENT_TYPES, row, clientTypeCodeValue.getName() + "-" + clientTypeCodeValue.getId()); + writeString(0, row, clientTypeCodeValue.getName() + " (" + clientTypeCodeValue.getId() + ")"); + rowIndex++; } - rowIndex = 0; + + // Column 1 (B): Client Classification + rowIndex = 1; for (CodeValueData clientClassificationCodeValue : clientClassificationCodeValues) { - Row row = clientSheet.getRow(++rowIndex); + Row row = lookupSheet.getRow(rowIndex); if (row == null) { - row = clientSheet.createRow(rowIndex); + row = lookupSheet.createRow(rowIndex); } - writeString(ClientEntityConstants.LOOKUP_CLIENT_CLASSIFICATION, row, - clientClassificationCodeValue.getName() + "-" + clientClassificationCodeValue.getId()); + writeString(1, row, clientClassificationCodeValue.getName() + " (" + clientClassificationCodeValue.getId() + ")"); + rowIndex++; } - rowIndex = 0; + + // Column 2 (C): Constitution + rowIndex = 1; for (CodeValueData constitutionCodeValue : constitutionCodeValues) { - Row row = clientSheet.getRow(++rowIndex); + Row row = lookupSheet.getRow(rowIndex); if (row == null) { - row = clientSheet.createRow(rowIndex); + row = lookupSheet.createRow(rowIndex); } - writeString(ClientEntityConstants.LOOKUP_CONSTITUTION_COL, row, - constitutionCodeValue.getName() + "-" + constitutionCodeValue.getId()); + writeString(2, row, constitutionCodeValue.getName() + " (" + constitutionCodeValue.getId() + ")"); + rowIndex++; } - rowIndex = 0; + + // Column 3 (D): Main Business Line + rowIndex = 1; for (CodeValueData mainBusinessCodeValue : mainBusinesslineCodeValues) { - Row row = clientSheet.getRow(++rowIndex); + Row row = lookupSheet.getRow(rowIndex); if (row == null) { - row = clientSheet.createRow(rowIndex); + row = lookupSheet.createRow(rowIndex); } - writeString(ClientEntityConstants.LOOKUP_MAIN_BUSINESS_LINE, row, - mainBusinessCodeValue.getName() + "-" + mainBusinessCodeValue.getId()); + writeString(3, row, mainBusinessCodeValue.getName() + " (" + mainBusinessCodeValue.getId() + ")"); + rowIndex++; } - rowIndex = 0; + + // Column 4 (E): Address Type + rowIndex = 1; for (CodeValueData addressTypeCodeValue : addressTypesCodeValues) { - Row row = clientSheet.getRow(++rowIndex); + Row row = lookupSheet.getRow(rowIndex); if (row == null) { - row = clientSheet.createRow(rowIndex); + row = lookupSheet.createRow(rowIndex); } - writeString(ClientEntityConstants.LOOKUP_ADDRESS_TYPE, row, - addressTypeCodeValue.getName() + "-" + addressTypeCodeValue.getId()); + writeString(4, row, addressTypeCodeValue.getName() + " (" + addressTypeCodeValue.getId() + ")"); + rowIndex++; } - rowIndex = 0; + + // Column 5 (F): State/Province + rowIndex = 1; for (CodeValueData stateCodeValue : stateProvinceCodeValues) { - Row row = clientSheet.getRow(++rowIndex); + Row row = lookupSheet.getRow(rowIndex); if (row == null) { - row = clientSheet.createRow(rowIndex); + row = lookupSheet.createRow(rowIndex); } - writeString(ClientEntityConstants.LOOKUP_STATE_PROVINCE, row, stateCodeValue.getName() + "-" + stateCodeValue.getId()); + writeString(5, row, stateCodeValue.getName() + " (" + stateCodeValue.getId() + ")"); + rowIndex++; } - rowIndex = 0; + + // Column 6 (G): Country + rowIndex = 1; for (CodeValueData countryCodeValue : countryCodeValues) { - Row row = clientSheet.getRow(++rowIndex); + Row row = lookupSheet.getRow(rowIndex); if (row == null) { - row = clientSheet.createRow(rowIndex); + row = lookupSheet.createRow(rowIndex); } - writeString(ClientEntityConstants.LOOKUP_COUNTRY, row, countryCodeValue.getName() + "-" + countryCodeValue.getId()); + writeString(6, row, countryCodeValue.getName() + " (" + countryCodeValue.getId() + ")"); + rowIndex++; } - + + } + + private String sanitizeNamedRangeName(String name) { + // Excel named ranges cannot contain certain characters + // Replace invalid characters with underscore + return name.replaceAll("[ @#&()<>,;.:$£€§°\\\\/=!\\?\\-\\+\\*\"\\[\\]]", "_"); } private void setLayout(Sheet worksheet) { @@ -202,17 +289,7 @@ private void setLayout(Sheet worksheet) { worksheet.setColumnWidth(ClientEntityConstants.COUNTRY_COL, TemplatePopulateImportConstants.SMALL_COL_SIZE); worksheet.setColumnWidth(ClientEntityConstants.POSTAL_CODE_COL, TemplatePopulateImportConstants.SMALL_COL_SIZE); worksheet.setColumnWidth(ClientEntityConstants.IS_ACTIVE_ADDRESS_COL, TemplatePopulateImportConstants.SMALL_COL_SIZE); - worksheet.setColumnWidth(ClientEntityConstants.WARNING_COL, TemplatePopulateImportConstants.MEDIUM_COL_SIZE); - - worksheet.setColumnWidth(ClientEntityConstants.RELATIONAL_OFFICE_NAME_COL, TemplatePopulateImportConstants.MEDIUM_COL_SIZE); - worksheet.setColumnWidth(ClientEntityConstants.RELATIONAL_OFFICE_OPENING_DATE_COL, TemplatePopulateImportConstants.SMALL_COL_SIZE); - worksheet.setColumnWidth(ClientEntityConstants.LOOKUP_CONSTITUTION_COL, TemplatePopulateImportConstants.SMALL_COL_SIZE); - worksheet.setColumnWidth(ClientEntityConstants.LOOKUP_CLIENT_TYPES, TemplatePopulateImportConstants.SMALL_COL_SIZE); - worksheet.setColumnWidth(ClientEntityConstants.LOOKUP_CLIENT_CLASSIFICATION, TemplatePopulateImportConstants.SMALL_COL_SIZE); - worksheet.setColumnWidth(ClientEntityConstants.LOOKUP_ADDRESS_TYPE, TemplatePopulateImportConstants.SMALL_COL_SIZE); - worksheet.setColumnWidth(ClientEntityConstants.LOOKUP_STATE_PROVINCE, TemplatePopulateImportConstants.SMALL_COL_SIZE); - worksheet.setColumnWidth(ClientEntityConstants.LOOKUP_COUNTRY, TemplatePopulateImportConstants.SMALL_COL_SIZE); - worksheet.setColumnWidth(ClientEntityConstants.LOOKUP_MAIN_BUSINESS_LINE, TemplatePopulateImportConstants.MEDIUM_COL_SIZE); + worksheet.setColumnWidth(ClientEntityConstants.STATUS_COL, TemplatePopulateImportConstants.SMALL_COL_SIZE); writeString(ClientEntityConstants.NAME_COL, rowHeader, "Name"); writeString(ClientEntityConstants.OFFICE_NAME_COL, rowHeader, "Office Name*"); writeString(ClientEntityConstants.STAFF_NAME_COL, rowHeader, "Staff Name"); @@ -240,18 +317,299 @@ private void setLayout(Sheet worksheet) { writeString(ClientEntityConstants.COUNTRY_COL, rowHeader, "Country"); writeString(ClientEntityConstants.POSTAL_CODE_COL, rowHeader, "Postal Code"); writeString(ClientEntityConstants.IS_ACTIVE_ADDRESS_COL, rowHeader, "Is active Address ? "); - writeString(ClientEntityConstants.WARNING_COL, rowHeader, "All * marked fields are compulsory."); - - writeString(ClientEntityConstants.RELATIONAL_OFFICE_NAME_COL, rowHeader, "Lookup office Name "); - writeString(ClientEntityConstants.RELATIONAL_OFFICE_OPENING_DATE_COL, rowHeader, "Lookup Office Opened Date "); - writeString(ClientEntityConstants.LOOKUP_CONSTITUTION_COL, rowHeader, "Lookup Constitution "); - writeString(ClientEntityConstants.LOOKUP_CLIENT_TYPES, rowHeader, "Lookup Client Types "); - writeString(ClientEntityConstants.LOOKUP_CLIENT_CLASSIFICATION, rowHeader, "Lookup Client Classification "); - writeString(ClientEntityConstants.LOOKUP_ADDRESS_TYPE, rowHeader, "Lookup AddressType "); - writeString(ClientEntityConstants.LOOKUP_STATE_PROVINCE, rowHeader, "Lookup State/Province "); - writeString(ClientEntityConstants.LOOKUP_COUNTRY, rowHeader, "Lookup Country "); - writeString(ClientEntityConstants.LOOKUP_MAIN_BUSINESS_LINE, rowHeader, "Lookup Business Line"); + writeString(ClientEntityConstants.STATUS_COL, rowHeader, TemplatePopulateImportConstants.STATUS_COLUMN_HEADER); + + // Handle datatable columns (headers will be inserted here) + // This is called from within setLayout to ensure proper column positioning + int currentCol = handleDatatableColumnHeaders(worksheet, rowHeader); + + // Add warning message after all dynamic columns + writeString(currentCol, rowHeader, "All * marked fields are compulsory."); + worksheet.setColumnWidth(currentCol, TemplatePopulateImportConstants.MEDIUM_COL_SIZE); + currentCol++; + + // Insert 8 empty columns as a gap before lookup headers + currentCol += 8; + + // Add lookup columns dynamically after datatable columns + // These columns are for user reference and data validation + // Data values remain on the hidden lookup sheet + worksheet.setColumnWidth(currentCol, TemplatePopulateImportConstants.MEDIUM_COL_SIZE); + writeString(currentCol, rowHeader, "Lookup office Name "); + int lookupOfficeNameCol = currentCol++; + + worksheet.setColumnWidth(currentCol, TemplatePopulateImportConstants.SMALL_COL_SIZE); + writeString(currentCol, rowHeader, "Lookup Office Opened Date "); + int lookupOfficeDateCol = currentCol++; + + worksheet.setColumnWidth(currentCol, TemplatePopulateImportConstants.SMALL_COL_SIZE); + writeString(currentCol, rowHeader, "Lookup Constitution "); + int lookupConstitutionCol = currentCol++; + + worksheet.setColumnWidth(currentCol, TemplatePopulateImportConstants.SMALL_COL_SIZE); + writeString(currentCol, rowHeader, "Lookup Client Types "); + int lookupClientTypesCol = currentCol++; + worksheet.setColumnWidth(currentCol, TemplatePopulateImportConstants.SMALL_COL_SIZE); + writeString(currentCol, rowHeader, "Lookup Client Classification "); + int lookupClientClassificationCol = currentCol++; + + worksheet.setColumnWidth(currentCol, TemplatePopulateImportConstants.SMALL_COL_SIZE); + writeString(currentCol, rowHeader, "Lookup AddressType "); + int lookupAddressTypeCol = currentCol++; + + worksheet.setColumnWidth(currentCol, TemplatePopulateImportConstants.SMALL_COL_SIZE); + writeString(currentCol, rowHeader, "Lookup State/Province "); + int lookupStateProvinceCol = currentCol++; + + worksheet.setColumnWidth(currentCol, TemplatePopulateImportConstants.SMALL_COL_SIZE); + writeString(currentCol, rowHeader, "Lookup Country "); + int lookupCountryCol = currentCol++; + + worksheet.setColumnWidth(currentCol, TemplatePopulateImportConstants.MEDIUM_COL_SIZE); + writeString(currentCol, rowHeader, "Lookup Business Line"); + int lookupBusinessLineCol = currentCol++; + + // Populate visible lookup values under lookup headers (for user reference) + // These are read-only reference values; validations still use the hidden lookup sheet + populateLookupValues(worksheet, lookupOfficeNameCol, lookupOfficeDateCol, lookupConstitutionCol, + lookupClientTypesCol, lookupClientClassificationCol, lookupAddressTypeCol, + lookupStateProvinceCol, lookupCountryCol, lookupBusinessLineCol); + } + + private void populateLookupValues(Sheet worksheet, int lookupOfficeNameCol, int lookupOfficeDateCol, + int lookupConstitutionCol, int lookupClientTypesCol, int lookupClientClassificationCol, + int lookupAddressTypeCol, int lookupStateProvinceCol, int lookupCountryCol, int lookupBusinessLineCol) { + + // Populate Office Name lookup values + int rowIndex = 1; // Start at Excel row 2 (POI index 1) + List offices = officeSheetPopulator.getOffices(); + for (OfficeData office : offices) { + Row row = worksheet.getRow(rowIndex); + if (row == null) { + row = worksheet.createRow(rowIndex); + } + writeString(lookupOfficeNameCol, row, office.getName() + " (" + office.getId() + ")"); + if (office.getOpeningDate() != null) { + writeString(lookupOfficeDateCol, row, office.getOpeningDate().toString()); + } + rowIndex++; + } + + // Populate Constitution lookup values + rowIndex = 1; + for (CodeValueData constitutionCodeValue : constitutionCodeValues) { + Row row = worksheet.getRow(rowIndex); + if (row == null) { + row = worksheet.createRow(rowIndex); + } + writeString(lookupConstitutionCol, row, constitutionCodeValue.getName() + " (" + constitutionCodeValue.getId() + ")"); + rowIndex++; + } + + // Populate Client Types lookup values + rowIndex = 1; + for (CodeValueData clientTypeCodeValue : clientTypeCodeValues) { + Row row = worksheet.getRow(rowIndex); + if (row == null) { + row = worksheet.createRow(rowIndex); + } + writeString(lookupClientTypesCol, row, clientTypeCodeValue.getName() + " (" + clientTypeCodeValue.getId() + ")"); + rowIndex++; + } + + // Populate Client Classification lookup values + rowIndex = 1; + for (CodeValueData clientClassificationCodeValue : clientClassificationCodeValues) { + Row row = worksheet.getRow(rowIndex); + if (row == null) { + row = worksheet.createRow(rowIndex); + } + writeString(lookupClientClassificationCol, row, clientClassificationCodeValue.getName() + " (" + clientClassificationCodeValue.getId() + ")"); + rowIndex++; + } + + // Populate Address Type lookup values + rowIndex = 1; + for (CodeValueData addressTypeCodeValue : addressTypesCodeValues) { + Row row = worksheet.getRow(rowIndex); + if (row == null) { + row = worksheet.createRow(rowIndex); + } + writeString(lookupAddressTypeCol, row, addressTypeCodeValue.getName() + " (" + addressTypeCodeValue.getId() + ")"); + rowIndex++; + } + + // Populate State/Province lookup values + rowIndex = 1; + for (CodeValueData stateCodeValue : stateProvinceCodeValues) { + Row row = worksheet.getRow(rowIndex); + if (row == null) { + row = worksheet.createRow(rowIndex); + } + writeString(lookupStateProvinceCol, row, stateCodeValue.getName() + " (" + stateCodeValue.getId() + ")"); + rowIndex++; + } + + // Populate Country lookup values + rowIndex = 1; + for (CodeValueData countryCodeValue : countryCodeValues) { + Row row = worksheet.getRow(rowIndex); + if (row == null) { + row = worksheet.createRow(rowIndex); + } + writeString(lookupCountryCol, row, countryCodeValue.getName() + " (" + countryCodeValue.getId() + ")"); + rowIndex++; + } + + // Populate Business Line lookup values + rowIndex = 1; + for (CodeValueData mainBusinessCodeValue : mainBusinesslineCodeValues) { + Row row = worksheet.getRow(rowIndex); + if (row == null) { + row = worksheet.createRow(rowIndex); + } + writeString(lookupBusinessLineCol, row, mainBusinessCodeValue.getName() + " (" + mainBusinessCodeValue.getId() + ")"); + rowIndex++; + } + } + + /** + * Handles datatable column header creation and dropdown tracking. + * This is called from within setLayout() to insert headers at the correct position. + * + * @param worksheet The client sheet + * @param rowHeader The header row + * @return The next column index after datatable columns (or STATUS_COL + 1 if no datatables) + */ + private int handleDatatableColumnHeaders(Sheet worksheet, Row rowHeader) { + int startCol = ClientEntityConstants.STATUS_COL + 1; + try { + int currentCol = startCol; + int lookupSheetCol = 7; // Start after Country (column G = index 6), so next available is H = index 7 + + for (DatatableData datatable : requiredDatatables) { + if (datatable == null) { + continue; + } + String datatableName = datatable.getRegisteredTableName(); + List columns = datatable.getColumnHeaderData(); + if (columns != null) { + for (ResultsetColumnHeaderData column : columns) { + try { + // Skip system columns (id, client_id, etc.) + String columnName = column.getColumnName(); + if (columnName == null || columnName.equalsIgnoreCase("id") + || columnName.equalsIgnoreCase("client_id") + || columnName.equalsIgnoreCase("created_at") + || columnName.equalsIgnoreCase("updated_at")) { + continue; + } + + worksheet.setColumnWidth(currentCol, TemplatePopulateImportConstants.MEDIUM_COL_SIZE); + // Use dot notation: registeredTableName.columnName for unambiguous parsing + // For single-row datatables: treat as mandatory inline data (append "*" if column is not nullable) + // For multi-row datatables: treat as optional/repeatable child data (no "*" even if not nullable) + String headerLabel = datatableName + "." + columnName; + if (!datatable.isMultiRow() && !column.getIsColumnNullable()) { + // Single-row datatables: mark mandatory columns with "*" + headerLabel += "*"; + } + // Multi-row datatables are always optional (no "*" marker) + writeString(currentCol, rowHeader, headerLabel); + + // Check if this is a dropdown column (CODELOOKUP) with values + if (column.isCodeLookupDisplayType() && column.hasColumnValues()) { + String namedRangeName = sanitizeNamedRangeName(datatableName + "_" + columnName); + List columnValues = column.getColumnValues(); + if (columnValues != null && !columnValues.isEmpty()) { + datatableDropdowns.put(namedRangeName, new DatatableDropdownInfo(currentCol, lookupSheetCol, columnValues)); + lookupSheetCol++; + } + } + // Check if this is a date or datetime column + else if (column.isDateDisplayType() || column.isDateTimeDisplayType()) { + datatableDateColumns.put(currentCol, new ClientEntityWorkbookPopulator.DatatableDateInfo(currentCol, column.isDateTimeDisplayType())); + } + // Check if this is a boolean column + else if (column.isBooleanDisplayType()) { + datatableBooleanColumns.put(currentCol, new ClientEntityWorkbookPopulator.DatatableBooleanInfo(currentCol)); + } + + currentCol++; + } catch (Exception e) { + String datatableNameForLog = datatableName != null ? datatableName : "unknown"; + String columnNameForLog = column != null && column.getColumnName() != null ? column.getColumnName() : "unknown"; + log.warn("Failed to process datatable column '{}' in datatable '{}': {}", + columnNameForLog, datatableNameForLog, e.getMessage()); + // Continue with next column + } + } + } + } + return currentCol; + } catch (Exception e) { + log.warn("Failed to handle datatable column headers in client bulk import template: {}. Template generation will continue without datatable columns.", + e.getMessage()); + return startCol; // Return start position if datatables failed + } + } + + /** + * Populates datatable dropdown values into the hidden lookup sheet and adds data validation on the + * client sheet's dropdown columns. Named ranges for these columns are already created in + * {@link #setNames(Sheet, List)}; this only fills in the values they refer to and wires up validation. + * All operations are wrapped in try/catch to ensure datatable failures never break template generation. + * + * @param worksheet The client sheet + * @param lookupSheet The hidden lookup sheet + */ + private void handleDatatableColumnsComplete(Sheet worksheet, Sheet lookupSheet) { + try { + // Populate datatable dropdown values into lookup sheet + for (Map.Entry entry : datatableDropdowns.entrySet()) { + try { + DatatableDropdownInfo dropdownInfo = entry.getValue(); + int rowIndex = 1; // Start at row 2 (Excel row 2, POI index 1) + for (ResultsetColumnValueData valueData : dropdownInfo.values) { + Row row = lookupSheet.getRow(rowIndex); + if (row == null) { + row = lookupSheet.createRow(rowIndex); + } + writeString(dropdownInfo.lookupSheetColumnIndex, row, valueData.getValue() + " (" + valueData.getId() + ")"); + rowIndex++; + } + } catch (Exception e) { + String namedRangeName = entry.getKey(); + log.warn("Failed to populate dropdown values for datatable named range '{}': {}", + namedRangeName, e.getMessage()); + // Continue with next dropdown + } + } + + // Add data validation for datatable dropdown columns + DataValidationHelper validationHelper = new HSSFDataValidationHelper((HSSFSheet) worksheet); + for (Map.Entry entry : datatableDropdowns.entrySet()) { + try { + String namedRangeName = entry.getKey(); + DatatableDropdownInfo dropdownInfo = entry.getValue(); + CellRangeAddressList datatableDropdownRange = new CellRangeAddressList(1, SpreadsheetVersion.EXCEL97.getLastRowIndex(), + dropdownInfo.clientSheetColumnIndex, dropdownInfo.clientSheetColumnIndex); + DataValidationConstraint datatableDropdownConstraint = validationHelper.createFormulaListConstraint(namedRangeName); + DataValidation datatableDropdownValidation = validationHelper.createValidation(datatableDropdownConstraint, datatableDropdownRange); + worksheet.addValidationData(datatableDropdownValidation); + } catch (Exception e) { + String namedRangeName = entry.getKey(); + log.warn("Failed to add data validation for datatable dropdown '{}': {}", + namedRangeName, e.getMessage()); + // Continue with next validation + } + } + } catch (Exception e) { + log.warn("Failed to complete datatable setup in client bulk import template: {}. Template generation will continue without datatable enhancements.", + e.getMessage()); + // Do not throw - allow template generation to continue + } } private void setRules(Sheet worksheet, String dateFormat) { @@ -350,48 +708,116 @@ private void setRules(Sheet worksheet, String dateFormat) { worksheet.addValidationData(activeAddressValidation); worksheet.addValidationData(incorporateDateValidation); worksheet.addValidationData(incorporateDateTillValidation); + + + // Add date validation for datatable date columns + for (ClientEntityWorkbookPopulator.DatatableDateInfo dateInfo : datatableDateColumns.values()) { + try { + CellRangeAddressList datatableDateRange = new CellRangeAddressList(1, SpreadsheetVersion.EXCEL97.getLastRowIndex(), + dateInfo.clientSheetColumnIndex, dateInfo.clientSheetColumnIndex); + // Use same date constraint as submittedOnDate (less than or equal to today) + DataValidationConstraint datatableDateConstraint = validationHelper + .createDateConstraint(DataValidationConstraint.OperatorType.LESS_OR_EQUAL, "=TODAY()", null, dateFormat); + DataValidation datatableDateValidation = validationHelper.createValidation(datatableDateConstraint, datatableDateRange); + worksheet.addValidationData(datatableDateValidation); + } catch (Exception e) { + log.warn("Failed to add date validation for datatable date column at index {}: {}", + dateInfo.clientSheetColumnIndex, e.getMessage()); + } + } + + // Add boolean validation for datatable boolean columns + for (ClientEntityWorkbookPopulator.DatatableBooleanInfo booleanInfo : datatableBooleanColumns.values()) { + try { + CellRangeAddressList datatableBooleanRange = new CellRangeAddressList(1, SpreadsheetVersion.EXCEL97.getLastRowIndex(), + booleanInfo.clientSheetColumnIndex, booleanInfo.clientSheetColumnIndex); + // Use same boolean constraint as active field (True/False) + DataValidationConstraint datatableBooleanConstraint = validationHelper.createExplicitListConstraint(new String[] { "True", "False" }); + DataValidation datatableBooleanValidation = validationHelper.createValidation(datatableBooleanConstraint, datatableBooleanRange); + worksheet.addValidationData(datatableBooleanValidation); + } catch (Exception e) { + log.warn("Failed to add boolean validation for datatable boolean column at index {}: {}", + booleanInfo.clientSheetColumnIndex, e.getMessage()); + } + } } private void setNames(Sheet worksheet, List offices) { Workbook clientWorkbook = worksheet.getWorkbook(); + String lookupSheetName = TemplatePopulateImportConstants.CLIENT_LOOKUPS_SHEET_NAME; + Name officeGroup = clientWorkbook.createName(); officeGroup.setNameName("Office"); officeGroup.setRefersToFormula(TemplatePopulateImportConstants.OFFICE_SHEET_NAME + "!$B$2:$B$" + (offices.size() + 1)); + // All lookup named ranges explicitly reference the hidden lookup sheet + // Column 0 (A): Client Types Name clientTypeGroup = clientWorkbook.createName(); clientTypeGroup.setNameName("ClientTypes"); + int clientTypesLastRow = clientTypeCodeValues.size() + 1; // +1 because data starts at row 2 + String clientTypesCol = CellReference.convertNumToColString(0); clientTypeGroup.setRefersToFormula( - TemplatePopulateImportConstants.CLIENT_ENTITY_SHEET_NAME + "!$AN$2:$AN$" + (clientTypeCodeValues.size() + 1)); + "'" + lookupSheetName + "'!$" + clientTypesCol + "$2:$" + clientTypesCol + "$" + clientTypesLastRow); + // Column 2 (C): Constitution Name constitutionGroup = clientWorkbook.createName(); constitutionGroup.setNameName("Constitution"); + int constitutionLastRow = constitutionCodeValues.size() + 1; + String constitutionCol = CellReference.convertNumToColString(2); constitutionGroup.setRefersToFormula( - TemplatePopulateImportConstants.CLIENT_ENTITY_SHEET_NAME + "!$AL$2:$AL$" + (constitutionCodeValues.size() + 1)); + "'" + lookupSheetName + "'!$" + constitutionCol + "$2:$" + constitutionCol + "$" + constitutionLastRow); + // Column 3 (D): Main Business Line Name mainBusinessLineGroup = clientWorkbook.createName(); mainBusinessLineGroup.setNameName("MainBusinessLine"); + int mainBusinessLineLastRow = mainBusinesslineCodeValues.size() + 1; + String mainBusinessLineCol = CellReference.convertNumToColString(3); mainBusinessLineGroup.setRefersToFormula( - TemplatePopulateImportConstants.CLIENT_ENTITY_SHEET_NAME + "!$AR$2:$AR$" + (mainBusinesslineCodeValues.size() + 1)); + "'" + lookupSheetName + "'!$" + mainBusinessLineCol + "$2:$" + mainBusinessLineCol + "$" + mainBusinessLineLastRow); + // Column 1 (B): Client Classification Name clientClassficationGroup = clientWorkbook.createName(); clientClassficationGroup.setNameName("ClientClassification"); + int clientClassificationLastRow = clientClassificationCodeValues.size() + 1; + String clientClassificationCol = CellReference.convertNumToColString(1); clientClassficationGroup.setRefersToFormula( - TemplatePopulateImportConstants.CLIENT_ENTITY_SHEET_NAME + "!$AM$2:$AM$" + (clientClassificationCodeValues.size() + 1)); + "'" + lookupSheetName + "'!$" + clientClassificationCol + "$2:$" + clientClassificationCol + "$" + clientClassificationLastRow); + // Column 4 (E): Address Type Name addressTypeGroup = clientWorkbook.createName(); addressTypeGroup.setNameName("AddressType"); + int addressTypeLastRow = addressTypesCodeValues.size() + 1; + String addressTypeCol = CellReference.convertNumToColString(4); addressTypeGroup.setRefersToFormula( - TemplatePopulateImportConstants.CLIENT_ENTITY_SHEET_NAME + "!$AO$2:$AO$" + (addressTypesCodeValues.size() + 1)); + "'" + lookupSheetName + "'!$" + addressTypeCol + "$2:$" + addressTypeCol + "$" + addressTypeLastRow); + // Column 5 (F): State/Province Name stateProvinceGroup = clientWorkbook.createName(); stateProvinceGroup.setNameName("StateProvince"); + int stateProvinceLastRow = stateProvinceCodeValues.size() + 1; + String stateProvinceCol = CellReference.convertNumToColString(5); stateProvinceGroup.setRefersToFormula( - TemplatePopulateImportConstants.CLIENT_ENTITY_SHEET_NAME + "!$AP$2:$AP$" + (stateProvinceCodeValues.size() + 1)); + "'" + lookupSheetName + "'!$" + stateProvinceCol + "$2:$" + stateProvinceCol + "$" + stateProvinceLastRow); + // Column 6 (G): Country Name countryGroup = clientWorkbook.createName(); countryGroup.setNameName("Country"); + int countryLastRow = countryCodeValues.size() + 1; + String countryCol = CellReference.convertNumToColString(6); countryGroup.setRefersToFormula( - TemplatePopulateImportConstants.CLIENT_ENTITY_SHEET_NAME + "!$AQ$2:$AQ$" + (countryCodeValues.size() + 1)); + "'" + lookupSheetName + "'!$" + countryCol + "$2:$" + countryCol + "$" + countryLastRow); + + // Create named ranges for datatable dropdown columns + for (Map.Entry entry : datatableDropdowns.entrySet()) { + String namedRangeName = entry.getKey(); + DatatableDropdownInfo dropdownInfo = entry.getValue(); + Name datatableDropdownGroup = clientWorkbook.createName(); + setSanitized(datatableDropdownGroup, namedRangeName); + int dropdownLastRow = dropdownInfo.values.size() + 1; + String dropdownCol = CellReference.convertNumToColString(dropdownInfo.lookupSheetColumnIndex); + datatableDropdownGroup.setRefersToFormula( + "'" + lookupSheetName + "'!$" + dropdownCol + "$2:$" + dropdownCol + "$" + dropdownLastRow); + } for (Integer i = 0; i < offices.size(); i++) { Integer[] officeNameToBeginEndIndexesOfStaff = personnelSheetPopulator.getOfficeNameToBeginEndIndexesOfStaff().get(i); diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/populator/client/ClientPersonWorkbookPopulator.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/populator/client/ClientPersonWorkbookPopulator.java index d3d43f45642..02bc0004a3d 100644 --- a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/populator/client/ClientPersonWorkbookPopulator.java +++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/populator/client/ClientPersonWorkbookPopulator.java @@ -25,7 +25,10 @@ import org.apache.fineract.infrastructure.bulkimport.populator.OfficeSheetPopulator; import org.apache.fineract.infrastructure.bulkimport.populator.PersonnelSheetPopulator; import org.apache.fineract.infrastructure.codes.data.CodeValueData; +import org.apache.fineract.infrastructure.dataqueries.data.DatatableData; +import org.apache.fineract.infrastructure.dataqueries.data.ResultsetColumnHeaderData; import org.apache.fineract.organisation.office.data.OfficeData; +import java.util.ArrayList; import org.apache.poi.hssf.usermodel.HSSFDataValidationHelper; import org.apache.poi.hssf.usermodel.HSSFSheet; import org.apache.poi.ss.SpreadsheetVersion; @@ -38,11 +41,20 @@ import org.apache.poi.ss.usermodel.Name; import org.apache.poi.ss.usermodel.Row; import org.apache.poi.ss.usermodel.Sheet; +import org.apache.poi.ss.usermodel.SheetVisibility; import org.apache.poi.ss.usermodel.Workbook; import org.apache.poi.ss.util.CellRangeAddressList; +import org.apache.poi.ss.util.CellReference; +import org.apache.fineract.infrastructure.dataqueries.data.ResultsetColumnValueData; +import java.util.Map; +import java.util.HashMap; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class ClientPersonWorkbookPopulator extends AbstractWorkbookPopulator { + private static final Logger log = LoggerFactory.getLogger(ClientPersonWorkbookPopulator.class); + private final OfficeSheetPopulator officeSheetPopulator; private final PersonnelSheetPopulator personnelSheetPopulator; private final List clientTypeCodeValues; @@ -51,11 +63,54 @@ public class ClientPersonWorkbookPopulator extends AbstractWorkbookPopulator { private final List addressTypesCodeValues; private final List stateProvinceCodeValues; private final List countryCodeValues; + private final List requiredDatatables; + + // Track datatable dropdown columns: namedRangeName -> (clientSheetColumnIndex, lookupSheetColumnIndex, values) + private final Map datatableDropdowns = new HashMap<>(); + + // Track datatable date columns: columnIndex -> columnInfo + private final Map datatableDateColumns = new HashMap<>(); + + // Track datatable boolean columns: columnIndex -> columnInfo + private final Map datatableBooleanColumns = new HashMap<>(); + + // Inner class to track datatable dropdown information + private static class DatatableDropdownInfo { + final int clientSheetColumnIndex; + final int lookupSheetColumnIndex; + final List values; + + DatatableDropdownInfo(int clientSheetColumnIndex, int lookupSheetColumnIndex, List values) { + this.clientSheetColumnIndex = clientSheetColumnIndex; + this.lookupSheetColumnIndex = lookupSheetColumnIndex; + this.values = values; + } + } + + // Inner class to track datatable date column information + private static class DatatableDateInfo { + final int clientSheetColumnIndex; + final boolean isDateTime; + + DatatableDateInfo(int clientSheetColumnIndex, boolean isDateTime) { + this.clientSheetColumnIndex = clientSheetColumnIndex; + this.isDateTime = isDateTime; + } + } + + // Inner class to track datatable boolean column information + private static class DatatableBooleanInfo { + final int clientSheetColumnIndex; + + DatatableBooleanInfo(int clientSheetColumnIndex) { + this.clientSheetColumnIndex = clientSheetColumnIndex; + } + } public ClientPersonWorkbookPopulator(OfficeSheetPopulator officeSheetPopulator, PersonnelSheetPopulator personnelSheetPopulator, List clientTypeCodeValues, List genderCodeValues, List clientClassification, List addressTypesCodeValues, List stateProvinceCodeValues, - List countryCodeValues) { + List countryCodeValues, List requiredDatatables) { this.officeSheetPopulator = officeSheetPopulator; this.personnelSheetPopulator = personnelSheetPopulator; this.clientTypeCodeValues = clientTypeCodeValues; @@ -64,6 +119,7 @@ public ClientPersonWorkbookPopulator(OfficeSheetPopulator officeSheetPopulator, this.addressTypesCodeValues = addressTypesCodeValues; this.stateProvinceCodeValues = stateProvinceCodeValues; this.countryCodeValues = countryCodeValues; + this.requiredDatatables = requiredDatatables != null ? requiredDatatables : new ArrayList<>(); } @Override @@ -71,12 +127,22 @@ public void populate(Workbook workbook, String dateFormat) { Sheet clientSheet = workbook.createSheet(TemplatePopulateImportConstants.CLIENT_PERSON_SHEET_NAME); personnelSheetPopulator.populate(workbook, dateFormat); officeSheetPopulator.populate(workbook, dateFormat); + // Create hidden lookup sheet for lookup data + Sheet lookupSheet = workbook.createSheet(TemplatePopulateImportConstants.CLIENT_LOOKUPS_SHEET_NAME); + // TODO: For debugging, temporarily set to VISIBLE. Revert to VERY_HIDDEN once validated. + workbook.setSheetVisibility(workbook.getSheetIndex(lookupSheet), SheetVisibility.VERY_HIDDEN); + setLayout(clientSheet); - setOfficeDateLookupTable(clientSheet, officeSheetPopulator.getOffices(), ClientPersonConstants.RELATIONAL_OFFICE_NAME_COL, - ClientPersonConstants.RELATIONAL_OFFICE_OPENING_DATE_COL, dateFormat); - setClientDataLookupTable(clientSheet); + setClientDataLookupTable(lookupSheet); setFormatStyle(workbook, clientSheet); setRules(clientSheet, dateFormat); + + // Complete datatable handling (lookup population, named ranges, validation) + // This is called after setLayout to complete datatable setup + Row rowHeader = clientSheet.getRow(TemplatePopulateImportConstants.ROWHEADER_INDEX); + if (rowHeader != null) { + handleDatatableColumnsComplete(clientSheet, rowHeader, lookupSheet, workbook, dateFormat); + } } private void setFormatStyle(Workbook workbook, Sheet worksheet) { @@ -92,6 +158,11 @@ private void setFormatStyle(Workbook workbook, Sheet worksheet) { setFormatActivationAndSubmittedDate(row, ClientPersonConstants.ACTIVATION_DATE_COL, dateCellStyle); setFormatActivationAndSubmittedDate(row, ClientPersonConstants.SUBMITTED_ON_COL, dateCellStyle); + + // Apply date formatting to datatable date columns + for (DatatableDateInfo dateInfo : datatableDateColumns.values()) { + setFormatActivationAndSubmittedDate(row, dateInfo.clientSheetColumnIndex, dateCellStyle); + } } } @@ -103,59 +174,80 @@ private void setFormatActivationAndSubmittedDate(Row row, int columnIndex, CellS cell.setCellStyle(cellStyle); } - private void setClientDataLookupTable(Sheet clientSheet) { - int rowIndex = 0; + private void setClientDataLookupTable(Sheet lookupSheet) { + // Data starts at row 2 (row index 1) to match named range references + // Column 0 (A): Client Types + int rowIndex = 1; // Start at row 2 (Excel row 2, POI index 1) for (CodeValueData clientTypeCodeValue : clientTypeCodeValues) { - Row row = clientSheet.getRow(++rowIndex); + Row row = lookupSheet.getRow(rowIndex); if (row == null) { - row = clientSheet.createRow(rowIndex); + row = lookupSheet.createRow(rowIndex); } - writeString(ClientPersonConstants.LOOKUP_CLIENT_TYPES_COL, row, - clientTypeCodeValue.getName() + "-" + clientTypeCodeValue.getId()); + writeString(0, row, clientTypeCodeValue.getName() + " (" + clientTypeCodeValue.getId() + ")"); + rowIndex++; } - rowIndex = 0; + + // Column 1 (B): Client Classification + rowIndex = 1; for (CodeValueData clientClassificationCodeValue : clientClassificationCodeValues) { - Row row = clientSheet.getRow(++rowIndex); + Row row = lookupSheet.getRow(rowIndex); if (row == null) { - row = clientSheet.createRow(rowIndex); + row = lookupSheet.createRow(rowIndex); } - writeString(ClientPersonConstants.LOOKUP_CLIENT_CLASSIFICATION_COL, row, - clientClassificationCodeValue.getName() + "-" + clientClassificationCodeValue.getId()); + writeString(1, row, clientClassificationCodeValue.getName() + " (" + clientClassificationCodeValue.getId() + ")"); + rowIndex++; } - rowIndex = 0; + + // Column 2 (C): Gender + rowIndex = 1; for (CodeValueData genderCodeValue : genderCodeValues) { - Row row = clientSheet.getRow(++rowIndex); + Row row = lookupSheet.getRow(rowIndex); if (row == null) { - row = clientSheet.createRow(rowIndex); + row = lookupSheet.createRow(rowIndex); } - writeString(ClientPersonConstants.LOOKUP_GENDER_COL, row, genderCodeValue.getName() + "-" + genderCodeValue.getId()); + writeString(2, row, genderCodeValue.getName() + " (" + genderCodeValue.getId() + ")"); + rowIndex++; } - rowIndex = 0; + + // Column 3 (D): Address Type + rowIndex = 1; for (CodeValueData addressTypeCodeValue : addressTypesCodeValues) { - Row row = clientSheet.getRow(++rowIndex); + Row row = lookupSheet.getRow(rowIndex); if (row == null) { - row = clientSheet.createRow(rowIndex); + row = lookupSheet.createRow(rowIndex); } - writeString(ClientPersonConstants.LOOKUP_ADDRESS_TYPE_COL, row, - addressTypeCodeValue.getName() + "-" + addressTypeCodeValue.getId()); + writeString(3, row, addressTypeCodeValue.getName() + " (" + addressTypeCodeValue.getId() + ")"); + rowIndex++; } - rowIndex = 0; + + // Column 4 (E): State/Province + rowIndex = 1; for (CodeValueData stateCodeValue : stateProvinceCodeValues) { - Row row = clientSheet.getRow(++rowIndex); + Row row = lookupSheet.getRow(rowIndex); if (row == null) { - row = clientSheet.createRow(rowIndex); + row = lookupSheet.createRow(rowIndex); } - writeString(ClientPersonConstants.LOOKUP_STATE_PROVINCE_COL, row, stateCodeValue.getName() + "-" + stateCodeValue.getId()); + writeString(4, row, stateCodeValue.getName() + " (" + stateCodeValue.getId() + ")"); + rowIndex++; } - rowIndex = 0; + + // Column 5 (F): Country + rowIndex = 1; for (CodeValueData countryCodeValue : countryCodeValues) { - Row row = clientSheet.getRow(++rowIndex); + Row row = lookupSheet.getRow(rowIndex); if (row == null) { - row = clientSheet.createRow(rowIndex); + row = lookupSheet.createRow(rowIndex); } - writeString(ClientPersonConstants.LOOKUP_COUNTRY_COL, row, countryCodeValue.getName() + "-" + countryCodeValue.getId()); + writeString(5, row, countryCodeValue.getName() + " (" + countryCodeValue.getId() + ")"); + rowIndex++; } - + + } + + private String sanitizeNamedRangeName(String name) { + // Excel named ranges cannot contain certain characters + // Replace invalid characters with underscore + return name.replaceAll("[ @#&()<>,;.:$£€§°\\\\/=!\\?\\-\\+\\*\"\\[\\]]", "_"); } private void setLayout(Sheet worksheet) { @@ -190,16 +282,7 @@ private void setLayout(Sheet worksheet) { worksheet.setColumnWidth(ClientPersonConstants.COUNTRY_COL, TemplatePopulateImportConstants.SMALL_COL_SIZE); worksheet.setColumnWidth(ClientPersonConstants.POSTAL_CODE_COL, TemplatePopulateImportConstants.SMALL_COL_SIZE); worksheet.setColumnWidth(ClientPersonConstants.IS_ACTIVE_ADDRESS_COL, TemplatePopulateImportConstants.SMALL_COL_SIZE); - worksheet.setColumnWidth(ClientPersonConstants.WARNING_COL, TemplatePopulateImportConstants.MEDIUM_COL_SIZE); - - worksheet.setColumnWidth(ClientPersonConstants.RELATIONAL_OFFICE_NAME_COL, TemplatePopulateImportConstants.MEDIUM_COL_SIZE); - worksheet.setColumnWidth(ClientPersonConstants.RELATIONAL_OFFICE_OPENING_DATE_COL, TemplatePopulateImportConstants.SMALL_COL_SIZE); - worksheet.setColumnWidth(ClientPersonConstants.LOOKUP_GENDER_COL, TemplatePopulateImportConstants.SMALL_COL_SIZE); - worksheet.setColumnWidth(ClientPersonConstants.LOOKUP_CLIENT_TYPES_COL, TemplatePopulateImportConstants.SMALL_COL_SIZE); - worksheet.setColumnWidth(ClientPersonConstants.LOOKUP_CLIENT_CLASSIFICATION_COL, TemplatePopulateImportConstants.SMALL_COL_SIZE); - worksheet.setColumnWidth(ClientPersonConstants.LOOKUP_ADDRESS_TYPE_COL, TemplatePopulateImportConstants.SMALL_COL_SIZE); - worksheet.setColumnWidth(ClientPersonConstants.LOOKUP_STATE_PROVINCE_COL, TemplatePopulateImportConstants.SMALL_COL_SIZE); - worksheet.setColumnWidth(ClientPersonConstants.LOOKUP_COUNTRY_COL, TemplatePopulateImportConstants.SMALL_COL_SIZE); + worksheet.setColumnWidth(ClientPersonConstants.STATUS_COL, TemplatePopulateImportConstants.SMALL_COL_SIZE); writeString(ClientPersonConstants.OFFICE_NAME_COL, rowHeader, "Office Name*"); writeString(ClientPersonConstants.STAFF_NAME_COL, rowHeader, "Staff Name"); writeString(ClientPersonConstants.EXTERNAL_ID_COL, rowHeader, "External ID "); @@ -223,17 +306,307 @@ private void setLayout(Sheet worksheet) { writeString(ClientPersonConstants.COUNTRY_COL, rowHeader, "Country "); writeString(ClientPersonConstants.POSTAL_CODE_COL, rowHeader, "Postal Code "); writeString(ClientPersonConstants.IS_ACTIVE_ADDRESS_COL, rowHeader, "Is active Address ? "); - writeString(ClientPersonConstants.WARNING_COL, rowHeader, "All * marked fields are compulsory."); + writeString(ClientPersonConstants.STATUS_COL, rowHeader, TemplatePopulateImportConstants.STATUS_COLUMN_HEADER); + + // Handle datatable columns (headers will be inserted here) + // This is called from within setLayout to ensure proper column positioning + int currentCol = handleDatatableColumnHeaders(worksheet, rowHeader); + + // Add warning message after all dynamic columns + writeString(currentCol, rowHeader, "All * marked fields are compulsory."); + worksheet.setColumnWidth(currentCol, TemplatePopulateImportConstants.MEDIUM_COL_SIZE); + currentCol++; + + // Insert 8 empty columns as a gap before lookup headers + currentCol += 8; + + // Add lookup columns dynamically after datatable columns + // These columns are for user reference and data validation + // Data values remain on the hidden lookup sheet + worksheet.setColumnWidth(currentCol, TemplatePopulateImportConstants.MEDIUM_COL_SIZE); + writeString(currentCol, rowHeader, "Lookup office Name "); + int lookupOfficeNameCol = currentCol++; + + worksheet.setColumnWidth(currentCol, TemplatePopulateImportConstants.SMALL_COL_SIZE); + writeString(currentCol, rowHeader, "Lookup Office Opened Date "); + int lookupOfficeDateCol = currentCol++; + + worksheet.setColumnWidth(currentCol, TemplatePopulateImportConstants.SMALL_COL_SIZE); + writeString(currentCol, rowHeader, "Lookup Gender "); + int lookupGenderCol = currentCol++; + + worksheet.setColumnWidth(currentCol, TemplatePopulateImportConstants.SMALL_COL_SIZE); + writeString(currentCol, rowHeader, "Lookup Client Types "); + int lookupClientTypesCol = currentCol++; + + worksheet.setColumnWidth(currentCol, TemplatePopulateImportConstants.SMALL_COL_SIZE); + writeString(currentCol, rowHeader, "Lookup Client Classification "); + int lookupClientClassificationCol = currentCol++; + + worksheet.setColumnWidth(currentCol, TemplatePopulateImportConstants.SMALL_COL_SIZE); + writeString(currentCol, rowHeader, "Lookup AddressType "); + int lookupAddressTypeCol = currentCol++; + + worksheet.setColumnWidth(currentCol, TemplatePopulateImportConstants.SMALL_COL_SIZE); + writeString(currentCol, rowHeader, "Lookup State/Province "); + int lookupStateProvinceCol = currentCol++; + + worksheet.setColumnWidth(currentCol, TemplatePopulateImportConstants.SMALL_COL_SIZE); + writeString(currentCol, rowHeader, "Lookup Country "); + int lookupCountryCol = currentCol++; + + // Populate visible lookup values under lookup headers (for user reference) + // These are read-only reference values; validations still use the hidden lookup sheet + populateLookupValues(worksheet, lookupOfficeNameCol, lookupOfficeDateCol, lookupGenderCol, + lookupClientTypesCol, lookupClientClassificationCol, lookupAddressTypeCol, + lookupStateProvinceCol, lookupCountryCol); + } + + private void populateLookupValues(Sheet worksheet, int lookupOfficeNameCol, int lookupOfficeDateCol, + int lookupGenderCol, int lookupClientTypesCol, int lookupClientClassificationCol, + int lookupAddressTypeCol, int lookupStateProvinceCol, int lookupCountryCol) { + + // Populate Office Name lookup values + int rowIndex = 1; // Start at Excel row 2 (POI index 1) + List offices = officeSheetPopulator.getOffices(); + for (OfficeData office : offices) { + Row row = worksheet.getRow(rowIndex); + if (row == null) { + row = worksheet.createRow(rowIndex); + } + writeString(lookupOfficeNameCol, row, office.getName() + " (" + office.getId() + ")"); + if (office.getOpeningDate() != null) { + writeString(lookupOfficeDateCol, row, office.getOpeningDate().toString()); + } + rowIndex++; + } + + // Populate Gender lookup values + rowIndex = 1; + for (CodeValueData genderCodeValue : genderCodeValues) { + Row row = worksheet.getRow(rowIndex); + if (row == null) { + row = worksheet.createRow(rowIndex); + } + writeString(lookupGenderCol, row, genderCodeValue.getName() + " (" + genderCodeValue.getId() + ")"); + rowIndex++; + } + + // Populate Client Types lookup values + rowIndex = 1; + for (CodeValueData clientTypeCodeValue : clientTypeCodeValues) { + Row row = worksheet.getRow(rowIndex); + if (row == null) { + row = worksheet.createRow(rowIndex); + } + writeString(lookupClientTypesCol, row, clientTypeCodeValue.getName() + " (" + clientTypeCodeValue.getId() + ")"); + rowIndex++; + } + + // Populate Client Classification lookup values + rowIndex = 1; + for (CodeValueData clientClassificationCodeValue : clientClassificationCodeValues) { + Row row = worksheet.getRow(rowIndex); + if (row == null) { + row = worksheet.createRow(rowIndex); + } + writeString(lookupClientClassificationCol, row, clientClassificationCodeValue.getName() + " (" + clientClassificationCodeValue.getId() + ")"); + rowIndex++; + } + + // Populate Address Type lookup values + rowIndex = 1; + for (CodeValueData addressTypeCodeValue : addressTypesCodeValues) { + Row row = worksheet.getRow(rowIndex); + if (row == null) { + row = worksheet.createRow(rowIndex); + } + writeString(lookupAddressTypeCol, row, addressTypeCodeValue.getName() + " (" + addressTypeCodeValue.getId() + ")"); + rowIndex++; + } + + // Populate State/Province lookup values + rowIndex = 1; + for (CodeValueData stateCodeValue : stateProvinceCodeValues) { + Row row = worksheet.getRow(rowIndex); + if (row == null) { + row = worksheet.createRow(rowIndex); + } + writeString(lookupStateProvinceCol, row, stateCodeValue.getName() + " (" + stateCodeValue.getId() + ")"); + rowIndex++; + } + + // Populate Country lookup values + rowIndex = 1; + for (CodeValueData countryCodeValue : countryCodeValues) { + Row row = worksheet.getRow(rowIndex); + if (row == null) { + row = worksheet.createRow(rowIndex); + } + writeString(lookupCountryCol, row, countryCodeValue.getName() + " (" + countryCodeValue.getId() + ")"); + rowIndex++; + } + } - writeString(ClientPersonConstants.RELATIONAL_OFFICE_NAME_COL, rowHeader, "Lookup office Name "); - writeString(ClientPersonConstants.RELATIONAL_OFFICE_OPENING_DATE_COL, rowHeader, "Lookup Office Opened Date "); - writeString(ClientPersonConstants.LOOKUP_GENDER_COL, rowHeader, "Lookup Gender "); - writeString(ClientPersonConstants.LOOKUP_CLIENT_TYPES_COL, rowHeader, "Lookup Client Types "); - writeString(ClientPersonConstants.LOOKUP_CLIENT_CLASSIFICATION_COL, rowHeader, "Lookup Client Classification "); - writeString(ClientPersonConstants.LOOKUP_ADDRESS_TYPE_COL, rowHeader, "Lookup AddressType "); - writeString(ClientPersonConstants.LOOKUP_STATE_PROVINCE_COL, rowHeader, "Lookup State/Province "); - writeString(ClientPersonConstants.LOOKUP_COUNTRY_COL, rowHeader, "Lookup Country "); + /** + * Handles datatable column header creation and dropdown tracking. + * This is called from within setLayout() to insert headers at the correct position. + * + * @param worksheet The client sheet + * @param rowHeader The header row + * @return The next column index after datatable columns (or STATUS_COL + 1 if no datatables) + */ + private int handleDatatableColumnHeaders(Sheet worksheet, Row rowHeader) { + int startCol = ClientPersonConstants.STATUS_COL + 1; + try { + int currentCol = startCol; + int lookupSheetCol = 6; // Start after Country (column F = index 5), so next available is G = index 6 + + for (DatatableData datatable : requiredDatatables) { + if (datatable == null) { + continue; + } + String datatableName = datatable.getRegisteredTableName(); + List columns = datatable.getColumnHeaderData(); + if (columns != null) { + for (ResultsetColumnHeaderData column : columns) { + try { + // Skip system columns (id, client_id, etc.) + String columnName = column.getColumnName(); + if (columnName == null || columnName.equalsIgnoreCase("id") + || columnName.equalsIgnoreCase("client_id") + || columnName.equalsIgnoreCase("created_at") + || columnName.equalsIgnoreCase("updated_at")) { + continue; + } + + worksheet.setColumnWidth(currentCol, TemplatePopulateImportConstants.MEDIUM_COL_SIZE); + // Use dot notation: registeredTableName.columnName for unambiguous parsing + // For single-row datatables: treat as mandatory inline data (append "*" if column is not nullable) + // For multi-row datatables: treat as optional/repeatable child data (no "*" even if not nullable) + String headerLabel = datatableName + "." + columnName; + if (!datatable.isMultiRow() && !column.getIsColumnNullable()) { + // Single-row datatables: mark mandatory columns with "*" + headerLabel += "*"; + } + // Multi-row datatables are always optional (no "*" marker) + writeString(currentCol, rowHeader, headerLabel); + + // Check if this is a dropdown column (CODELOOKUP) with values + if (column.isCodeLookupDisplayType() && column.hasColumnValues()) { + String namedRangeName = sanitizeNamedRangeName(datatableName + "_" + columnName); + List columnValues = column.getColumnValues(); + if (columnValues != null && !columnValues.isEmpty()) { + datatableDropdowns.put(namedRangeName, new DatatableDropdownInfo(currentCol, lookupSheetCol, columnValues)); + lookupSheetCol++; + } + } + // Check if this is a date or datetime column + else if (column.isDateDisplayType() || column.isDateTimeDisplayType()) { + datatableDateColumns.put(currentCol, new DatatableDateInfo(currentCol, column.isDateTimeDisplayType())); + } + // Check if this is a boolean column + else if (column.isBooleanDisplayType()) { + datatableBooleanColumns.put(currentCol, new DatatableBooleanInfo(currentCol)); + } + + currentCol++; + } catch (Exception e) { + String datatableNameForLog = datatableName != null ? datatableName : "unknown"; + String columnNameForLog = column != null && column.getColumnName() != null ? column.getColumnName() : "unknown"; + log.warn("Failed to process datatable column '{}' in datatable '{}': {}", + columnNameForLog, datatableNameForLog, e.getMessage()); + // Continue with next column + } + } + } + } + return currentCol; + } catch (Exception e) { + log.warn("Failed to handle datatable column headers in client bulk import template: {}. Template generation will continue without datatable columns.", + e.getMessage()); + return startCol; // Return start position if datatables failed + } + } + /** + * Completes datatable setup: lookup sheet population, named ranges, and validation. + * This is called after setLayout() to complete datatable enhancements. + * All operations are wrapped in try/catch to ensure datatable failures never break template generation. + * + * @param worksheet The client sheet + * @param rowHeader The header row + * @param lookupSheet The hidden lookup sheet + * @param workbook The workbook + * @param dateFormat The date format + */ + private void handleDatatableColumnsComplete(Sheet worksheet, Row rowHeader, Sheet lookupSheet, Workbook workbook, String dateFormat) { + try { + // Step 1: Populate datatable dropdown values into lookup sheet + for (Map.Entry entry : datatableDropdowns.entrySet()) { + try { + DatatableDropdownInfo dropdownInfo = entry.getValue(); + int rowIndex = 1; // Start at row 2 (Excel row 2, POI index 1) + for (ResultsetColumnValueData valueData : dropdownInfo.values) { + Row row = lookupSheet.getRow(rowIndex); + if (row == null) { + row = lookupSheet.createRow(rowIndex); + } + writeString(dropdownInfo.lookupSheetColumnIndex, row, valueData.getValue() + " (" + valueData.getId() + ")"); + rowIndex++; + } + } catch (Exception e) { + String namedRangeName = entry.getKey(); + log.warn("Failed to populate dropdown values for datatable named range '{}': {}", + namedRangeName, e.getMessage()); + // Continue with next dropdown + } + } + + // Step 3: Create named ranges for datatable dropdown columns + String lookupSheetName = TemplatePopulateImportConstants.CLIENT_LOOKUPS_SHEET_NAME; + Workbook clientWorkbook = worksheet.getWorkbook(); + for (Map.Entry entry : datatableDropdowns.entrySet()) { + try { + String namedRangeName = entry.getKey(); + DatatableDropdownInfo dropdownInfo = entry.getValue(); + Name datatableDropdownGroup = clientWorkbook.createName(); + setSanitized(datatableDropdownGroup, namedRangeName); + int dropdownLastRow = dropdownInfo.values.size() + 1; // +1 because data starts at row 2 + String dropdownCol = CellReference.convertNumToColString(dropdownInfo.lookupSheetColumnIndex); + datatableDropdownGroup.setRefersToFormula( + "'" + lookupSheetName + "'!$" + dropdownCol + "$2:$" + dropdownCol + "$" + dropdownLastRow); + } catch (Exception e) { + String namedRangeName = entry.getKey(); + log.warn("Failed to create named range '{}' for datatable dropdown: {}", + namedRangeName, e.getMessage()); + // Continue with next named range + } + } + + // Step 4: Add data validation for datatable dropdown columns + DataValidationHelper validationHelper = new HSSFDataValidationHelper((HSSFSheet) worksheet); + for (Map.Entry entry : datatableDropdowns.entrySet()) { + try { + String namedRangeName = entry.getKey(); + DatatableDropdownInfo dropdownInfo = entry.getValue(); + CellRangeAddressList datatableDropdownRange = new CellRangeAddressList(1, SpreadsheetVersion.EXCEL97.getLastRowIndex(), + dropdownInfo.clientSheetColumnIndex, dropdownInfo.clientSheetColumnIndex); + DataValidationConstraint datatableDropdownConstraint = validationHelper.createFormulaListConstraint(namedRangeName); + DataValidation datatableDropdownValidation = validationHelper.createValidation(datatableDropdownConstraint, datatableDropdownRange); + worksheet.addValidationData(datatableDropdownValidation); + } catch (Exception e) { + String namedRangeName = entry.getKey(); + log.warn("Failed to add data validation for datatable dropdown '{}': {}", + namedRangeName, e.getMessage()); + // Continue with next validation + } + } + } catch (Exception e) { + log.warn("Failed to complete datatable setup in client bulk import template: {}. Template generation will continue without datatable enhancements.", + e.getMessage()); + // Do not throw - allow template generation to continue + } } private void setRules(Sheet worksheet, String dateformat) { @@ -325,43 +698,95 @@ private void setRules(Sheet worksheet, String dateformat) { worksheet.addValidationData(stateProvinceValidation); worksheet.addValidationData(countryValidation); worksheet.addValidationData(activeAddressValidation); + + // Add date validation for datatable date columns + for (DatatableDateInfo dateInfo : datatableDateColumns.values()) { + try { + CellRangeAddressList datatableDateRange = new CellRangeAddressList(1, SpreadsheetVersion.EXCEL97.getLastRowIndex(), + dateInfo.clientSheetColumnIndex, dateInfo.clientSheetColumnIndex); + // Use same date constraint as submittedOnDate (less than or equal to today) + DataValidationConstraint datatableDateConstraint = validationHelper + .createDateConstraint(DataValidationConstraint.OperatorType.LESS_OR_EQUAL, "=TODAY()", null, dateformat); + DataValidation datatableDateValidation = validationHelper.createValidation(datatableDateConstraint, datatableDateRange); + worksheet.addValidationData(datatableDateValidation); + } catch (Exception e) { + log.warn("Failed to add date validation for datatable date column at index {}: {}", + dateInfo.clientSheetColumnIndex, e.getMessage()); + } + } + + // Add boolean validation for datatable boolean columns + for (DatatableBooleanInfo booleanInfo : datatableBooleanColumns.values()) { + try { + CellRangeAddressList datatableBooleanRange = new CellRangeAddressList(1, SpreadsheetVersion.EXCEL97.getLastRowIndex(), + booleanInfo.clientSheetColumnIndex, booleanInfo.clientSheetColumnIndex); + // Use same boolean constraint as active field (True/False) + DataValidationConstraint datatableBooleanConstraint = validationHelper.createExplicitListConstraint(new String[] { "True", "False" }); + DataValidation datatableBooleanValidation = validationHelper.createValidation(datatableBooleanConstraint, datatableBooleanRange); + worksheet.addValidationData(datatableBooleanValidation); + } catch (Exception e) { + log.warn("Failed to add boolean validation for datatable boolean column at index {}: {}", + booleanInfo.clientSheetColumnIndex, e.getMessage()); + } + } } private void setNames(Sheet worksheet, List offices) { Workbook clientWorkbook = worksheet.getWorkbook(); + String lookupSheetName = TemplatePopulateImportConstants.CLIENT_LOOKUPS_SHEET_NAME; + Name officeGroup = clientWorkbook.createName(); officeGroup.setNameName("Office"); officeGroup.setRefersToFormula(TemplatePopulateImportConstants.OFFICE_SHEET_NAME + "!$B$2:$B$" + (offices.size() + 1)); + // All lookup named ranges explicitly reference the hidden lookup sheet + // Column 0 (A): Client Types Name clientTypeGroup = clientWorkbook.createName(); clientTypeGroup.setNameName("ClientTypes"); + int clientTypesLastRow = clientTypeCodeValues.size() + 1; // +1 because data starts at row 2 + String clientTypesCol = CellReference.convertNumToColString(0); clientTypeGroup.setRefersToFormula( - TemplatePopulateImportConstants.CLIENT_PERSON_SHEET_NAME + "!$AN$2:$AN$" + (clientTypeCodeValues.size() + 1)); + "'" + lookupSheetName + "'!$" + clientTypesCol + "$2:$" + clientTypesCol + "$" + clientTypesLastRow); + // Column 2 (C): Gender Name genderGroup = clientWorkbook.createName(); genderGroup.setNameName("Gender"); + int genderLastRow = genderCodeValues.size() + 1; + String genderCol = CellReference.convertNumToColString(2); genderGroup.setRefersToFormula( - TemplatePopulateImportConstants.CLIENT_PERSON_SHEET_NAME + "!$AL$2:$AL$" + (genderCodeValues.size() + 1)); + "'" + lookupSheetName + "'!$" + genderCol + "$2:$" + genderCol + "$" + genderLastRow); + // Column 1 (B): Client Classification Name clientClassficationGroup = clientWorkbook.createName(); clientClassficationGroup.setNameName("ClientClassification"); + int clientClassificationLastRow = clientClassificationCodeValues.size() + 1; + String clientClassificationCol = CellReference.convertNumToColString(1); clientClassficationGroup.setRefersToFormula( - TemplatePopulateImportConstants.CLIENT_PERSON_SHEET_NAME + "!$AM$2:$AM$" + (clientClassificationCodeValues.size() + 1)); + "'" + lookupSheetName + "'!$" + clientClassificationCol + "$2:$" + clientClassificationCol + "$" + clientClassificationLastRow); + // Column 3 (D): Address Type Name addressTypeGroup = clientWorkbook.createName(); addressTypeGroup.setNameName("AddressType"); + int addressTypeLastRow = addressTypesCodeValues.size() + 1; + String addressTypeCol = CellReference.convertNumToColString(3); addressTypeGroup.setRefersToFormula( - TemplatePopulateImportConstants.CLIENT_PERSON_SHEET_NAME + "!$AO$2:$AO$" + (addressTypesCodeValues.size() + 1)); + "'" + lookupSheetName + "'!$" + addressTypeCol + "$2:$" + addressTypeCol + "$" + addressTypeLastRow); + // Column 4 (E): State/Province Name stateProvinceGroup = clientWorkbook.createName(); stateProvinceGroup.setNameName("StateProvince"); + int stateProvinceLastRow = stateProvinceCodeValues.size() + 1; + String stateProvinceCol = CellReference.convertNumToColString(4); stateProvinceGroup.setRefersToFormula( - TemplatePopulateImportConstants.CLIENT_PERSON_SHEET_NAME + "!$AP$2:$AP$" + (stateProvinceCodeValues.size() + 1)); + "'" + lookupSheetName + "'!$" + stateProvinceCol + "$2:$" + stateProvinceCol + "$" + stateProvinceLastRow); + // Column 5 (F): Country Name countryGroup = clientWorkbook.createName(); countryGroup.setNameName("Country"); + int countryLastRow = countryCodeValues.size() + 1; + String countryCol = CellReference.convertNumToColString(5); countryGroup.setRefersToFormula( - TemplatePopulateImportConstants.CLIENT_PERSON_SHEET_NAME + "!$AQ$2:$AQ$" + (countryCodeValues.size() + 1)); + "'" + lookupSheetName + "'!$" + countryCol + "$2:$" + countryCol + "$" + countryLastRow); for (Integer i = 0; i < offices.size(); i++) { Integer[] officeNameToBeginEndIndexesOfStaff = personnelSheetPopulator.getOfficeNameToBeginEndIndexesOfStaff().get(i); diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/service/BulkImportWorkbookPopulatorServiceImpl.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/service/BulkImportWorkbookPopulatorServiceImpl.java index 14351c97b5b..d3ddaef73c5 100644 --- a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/service/BulkImportWorkbookPopulatorServiceImpl.java +++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/service/BulkImportWorkbookPopulatorServiceImpl.java @@ -104,6 +104,13 @@ import org.apache.fineract.portfolio.savings.service.DepositProductReadPlatformService; import org.apache.fineract.portfolio.savings.service.SavingsAccountReadPlatformService; import org.apache.fineract.portfolio.savings.service.SavingsProductReadPlatformService; +import org.apache.fineract.infrastructure.dataqueries.data.DatatableData; +import org.apache.fineract.infrastructure.dataqueries.data.EntityTables; +import org.apache.fineract.infrastructure.dataqueries.data.StatusEnum; +import org.apache.fineract.infrastructure.dataqueries.domain.EntityDatatableChecks; +import org.apache.fineract.infrastructure.dataqueries.domain.EntityDatatableChecksRepository; +import org.apache.fineract.infrastructure.dataqueries.service.DatatableReadService; +import org.apache.fineract.infrastructure.dataqueries.service.EntityDatatableChecksReadService; import org.apache.fineract.portfolio.shareproducts.data.ShareProductData; import org.apache.fineract.useradministration.data.RoleData; import org.apache.fineract.useradministration.service.RoleReadPlatformService; @@ -137,6 +144,9 @@ public class BulkImportWorkbookPopulatorServiceImpl implements BulkImportWorkboo private final ChargeReadPlatformService chargeReadPlatformService; private final DepositProductReadPlatformService depositProductReadPlatformService; private final RoleReadPlatformService roleReadPlatformService; + private final EntityDatatableChecksReadService entityDatatableChecksReadService; + private final DatatableReadService datatableReadService; + private final EntityDatatableChecksRepository entityDatatableChecksRepository; @Autowired public BulkImportWorkbookPopulatorServiceImpl(final PlatformSecurityContext context, @@ -153,7 +163,10 @@ public BulkImportWorkbookPopulatorServiceImpl(final PlatformSecurityContext cont final ShareProductReadPlatformService shareProductReadPlatformService, final ChargeReadPlatformService chargeReadPlatformService, final DepositProductReadPlatformService depositProductReadPlatformService, - final RoleReadPlatformService roleReadPlatformService) { + final RoleReadPlatformService roleReadPlatformService, + final EntityDatatableChecksReadService entityDatatableChecksReadService, + final DatatableReadService datatableReadService, + final EntityDatatableChecksRepository entityDatatableChecksRepository) { this.officeReadPlatformService = officeReadPlatformService; this.staffReadPlatformService = staffReadPlatformService; this.context = context; @@ -173,6 +186,9 @@ public BulkImportWorkbookPopulatorServiceImpl(final PlatformSecurityContext cont this.chargeReadPlatformService = chargeReadPlatformService; this.depositProductReadPlatformService = depositProductReadPlatformService; this.roleReadPlatformService = roleReadPlatformService; + this.entityDatatableChecksReadService = entityDatatableChecksReadService; + this.datatableReadService = datatableReadService; + this.entityDatatableChecksRepository = entityDatatableChecksRepository; } @Override @@ -237,20 +253,59 @@ private WorkbookPopulator populateClientWorkbook(final String entityType, final List addressTypesCodeValues = fetchCodeValuesByCodeName("ADDRESS_TYPE"); List stateProvinceCodeValues = fetchCodeValuesByCodeName("STATE"); List countryCodeValues = fetchCodeValuesByCodeName("COUNTRY"); + + // Fetch required datatables for CLIENT entity with CREATE status + String entitySubtype = null; + if (entityType.trim().equalsIgnoreCase(GlobalEntityType.CLIENTS_PERSON.toString())) { + entitySubtype = "Person"; + } else if (entityType.trim().equalsIgnoreCase(GlobalEntityType.CLIENTS_ENTITY.toString())) { + entitySubtype = "Entity"; + } + List requiredDatatables = fetchRequiredDatatables(entitySubtype); + if (entityType.trim().equalsIgnoreCase(GlobalEntityType.CLIENTS_PERSON.toString())) { List genderCodeValues = fetchCodeValuesByCodeName("Gender"); return new ClientPersonWorkbookPopulator(new OfficeSheetPopulator(offices), new PersonnelSheetPopulator(staff, offices), clientTypeCodeValues, genderCodeValues, clientClassification, addressTypesCodeValues, stateProvinceCodeValues, - countryCodeValues); + countryCodeValues, requiredDatatables); } else if (entityType.trim().equalsIgnoreCase(GlobalEntityType.CLIENTS_ENTITY.toString())) { List constitutionCodeValues = fetchCodeValuesByCodeName("Constitution"); List mainBusinessline = fetchCodeValuesByCodeName("Main Business Line"); return new ClientEntityWorkbookPopulator(new OfficeSheetPopulator(offices), new PersonnelSheetPopulator(staff, offices), clientTypeCodeValues, constitutionCodeValues, mainBusinessline, clientClassification, addressTypesCodeValues, - stateProvinceCodeValues, countryCodeValues); + stateProvinceCodeValues, countryCodeValues, requiredDatatables); } return null; } + + private List fetchRequiredDatatables(final String entitySubtype) { + List requiredDatatables = new ArrayList<>(); + try { + // Get required datatables for CLIENT entity with CREATE status + List entityDatatableChecks; + if (entitySubtype != null) { + // Filter by entity subtype + entityDatatableChecks = entityDatatableChecksRepository.findByEntityAndStatusAndSubtype( + EntityTables.CLIENT.getName(), StatusEnum.CREATE.getValue(), entitySubtype); + } else { + // Get all required datatables regardless of subtype + entityDatatableChecks = entityDatatableChecksRepository.findByEntityAndStatus( + EntityTables.CLIENT.getName(), StatusEnum.CREATE.getValue()); + } + + // Convert EntityDatatableChecks to DatatableData + for (EntityDatatableChecks check : entityDatatableChecks) { + DatatableData datatable = datatableReadService.retrieveDatatable(check.getDatatableName()); + if (datatable != null) { + requiredDatatables.add(datatable); + } + } + } catch (Exception e) { + LOG.warn("Error fetching required datatables for bulk import template", e); + // Continue without datatables if there's an error + } + return requiredDatatables; + } private Response buildResponse(final Workbook workbook, final String entity) { String filename = entity + DateUtils.getBusinessLocalDate().toString() + ".xls"; diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/service/DatatableReadServiceImpl.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/service/DatatableReadServiceImpl.java index 5d38c4a9a2e..26cd4cc7fc7 100644 --- a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/service/DatatableReadServiceImpl.java +++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/service/DatatableReadServiceImpl.java @@ -92,8 +92,9 @@ public List retrieveDatatableNames(final String appTable) { final String registeredDatatableName = rowSet.getString("registered_table_name"); final String entitySubType = rowSet.getString("entity_subtype"); final List columnHeaderData = genericDataService.fillResultsetColumnHeaders(registeredDatatableName); + final boolean multiRow = datatableUtil.isMultirowDatatable(columnHeaderData); - datatables.add(DatatableData.create(appTableName, registeredDatatableName, entitySubType, columnHeaderData)); + datatables.add(DatatableData.create(appTableName, registeredDatatableName, entitySubType, columnHeaderData, multiRow)); } return datatables; @@ -119,8 +120,9 @@ public DatatableData retrieveDatatable(final String datatable) { final String entitySubType = rowSet.getString("entity_subtype"); final List columnHeaderData = this.genericDataService .fillResultsetColumnHeaders(registeredDatatableName); + final boolean multiRow = datatableUtil.isMultirowDatatable(columnHeaderData); - datatableData = DatatableData.create(appTableName, registeredDatatableName, entitySubType, columnHeaderData); + datatableData = DatatableData.create(appTableName, registeredDatatableName, entitySubType, columnHeaderData, multiRow); } return datatableData; diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/survey/service/ReadSurveyServiceImpl.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/survey/service/ReadSurveyServiceImpl.java index 3d3607b575b..18d2fc16ff7 100644 --- a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/survey/service/ReadSurveyServiceImpl.java +++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/survey/service/ReadSurveyServiceImpl.java @@ -27,6 +27,7 @@ import org.apache.fineract.infrastructure.dataqueries.data.GenericResultsetData; import org.apache.fineract.infrastructure.dataqueries.data.ResultsetColumnHeaderData; import org.apache.fineract.infrastructure.dataqueries.service.DatatableReadService; +import org.apache.fineract.infrastructure.dataqueries.service.DatatableUtil; import org.apache.fineract.infrastructure.dataqueries.service.GenericDataService; import org.apache.fineract.infrastructure.security.service.PlatformSecurityContext; import org.apache.fineract.infrastructure.security.service.SqlValidator; @@ -47,6 +48,7 @@ public class ReadSurveyServiceImpl implements ReadSurveyService { private final SqlValidator sqlValidator; private final GenericDataService genericDataService; private final DatatableReadService datatableReadService; + private final DatatableUtil datatableUtil; @Override public List retrieveAllSurveys() { @@ -63,9 +65,10 @@ public List retrieveAllSurveys() { final boolean enabled = rs.getBoolean("enabled"); final List columnHeaderData = this.genericDataService .fillResultsetColumnHeaders(registeredDatatableName); + final boolean multiRow = datatableUtil.isMultirowDatatable(columnHeaderData); surveyDataTables.add(SurveyDataTableData - .create(DatatableData.create(appTableName, registeredDatatableName, entitySubType, columnHeaderData), enabled)); + .create(DatatableData.create(appTableName, registeredDatatableName, entitySubType, columnHeaderData, multiRow), enabled)); } return surveyDataTables; @@ -104,8 +107,9 @@ public SurveyDataTableData retrieveSurvey(String surveyName) { final boolean enabled = rs.getBoolean("enabled"); final List columnHeaderData = this.genericDataService .fillResultsetColumnHeaders(registeredDatatableName); + final boolean multiRow = datatableUtil.isMultirowDatatable(columnHeaderData); datatableData = SurveyDataTableData - .create(DatatableData.create(appTableName, registeredDatatableName, entitySubType, columnHeaderData), enabled); + .create(DatatableData.create(appTableName, registeredDatatableName, entitySubType, columnHeaderData, multiRow), enabled); } diff --git a/fineract-provider/src/test/java/org/apache/fineract/infrastructure/survey/service/ReadSurveyServiceImplTest.java b/fineract-provider/src/test/java/org/apache/fineract/infrastructure/survey/service/ReadSurveyServiceImplTest.java index 54118681247..a3f6b968d30 100644 --- a/fineract-provider/src/test/java/org/apache/fineract/infrastructure/survey/service/ReadSurveyServiceImplTest.java +++ b/fineract-provider/src/test/java/org/apache/fineract/infrastructure/survey/service/ReadSurveyServiceImplTest.java @@ -27,6 +27,7 @@ import java.util.List; import org.apache.fineract.infrastructure.dataqueries.service.DatatableReadService; +import org.apache.fineract.infrastructure.dataqueries.service.DatatableUtil; import org.apache.fineract.infrastructure.dataqueries.service.GenericDataService; import org.apache.fineract.infrastructure.security.service.PlatformSecurityContext; import org.apache.fineract.infrastructure.security.service.SqlValidator; @@ -64,6 +65,8 @@ class ReadSurveyServiceImplTest { @Mock private DatatableReadService datatableReadService; @Mock + private DatatableUtil datatableUtil; + @Mock private AppUser appUser; private ReadSurveyServiceImpl underTest; @@ -72,7 +75,8 @@ class ReadSurveyServiceImplTest { void setUp() { when(context.authenticatedUser()).thenReturn(appUser); when(appUser.getId()).thenReturn(USER_ID); - underTest = new ReadSurveyServiceImpl(context, jdbcTemplate, sqlValidator, genericDataService, datatableReadService); + underTest = new ReadSurveyServiceImpl(context, jdbcTemplate, sqlValidator, genericDataService, datatableReadService, + datatableUtil); } @Test