From d43dc2f12910e864424a9d875a2d57f7c61be4b2 Mon Sep 17 00:00:00 2001 From: Chukwudumebi Onwuli Date: Tue, 30 Jun 2026 03:34:20 -0400 Subject: [PATCH 1/6] feat: annotate import errors with Excel row location --- i18n/en.pot | 21 ++- i18n/es.po | 47 ++++- i18n/fr.po | 48 ++++-- i18n/pt.po | 26 ++- i18n/ru.po | 28 ++- src/data/Dhis2Events.ts | 8 +- src/data/Dhis2Import.ts | 41 +++-- src/data/Dhis2TrackedEntityInstances.ts | 1 + src/data/InstanceDhisRepository.ts | 27 ++- src/domain/entities/ImportRowLookup.ts | 89 ++++++++++ src/domain/entities/Template.ts | 1 + src/domain/entities/TrackedEntityInstance.ts | 2 + .../__tests__/ImportRowLookup.spec.ts | 160 ++++++++++++++++++ src/domain/helpers/ExcelReader.ts | 5 + src/domain/repositories/InstanceRepository.ts | 9 +- src/domain/usecases/ImportTemplateUseCase.ts | 11 +- 16 files changed, 463 insertions(+), 61 deletions(-) create mode 100644 src/domain/entities/ImportRowLookup.ts create mode 100644 src/domain/entities/__tests__/ImportRowLookup.spec.ts diff --git a/i18n/en.pot b/i18n/en.pot index f63984f1..9ea0a67e 100644 --- a/i18n/en.pot +++ b/i18n/en.pot @@ -5,8 +5,8 @@ msgstr "" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" "Plural-Forms: nplurals=2; plural=(n != 1)\n" -"POT-Creation-Date: 2025-11-19T17:15:26.356Z\n" -"PO-Revision-Date: 2025-11-19T17:15:26.356Z\n" +"POT-Creation-Date: 2026-06-29T11:38:42.923Z\n" +"PO-Revision-Date: 2026-06-29T11:38:42.924Z\n" msgid "Events - Create/update" msgstr "" @@ -97,6 +97,23 @@ msgstr "" msgid "Tracker Program" msgstr "" +msgid "row {{lines}}" +msgstr "" + +msgid "rows {{lines}}" +msgstr "" + +msgid "and {{count}} more" +msgid_plural "and {{count}} more" +msgstr[0] "" +msgstr[1] "" + +msgid "sheet {{sheet}}, {{lines}}" +msgstr "" + +msgid "Found in {{locations}} of the Excel file" +msgstr "" + msgid "Cannot read data form id" msgstr "" diff --git a/i18n/es.po b/i18n/es.po index 2f03a077..6dfbe7bc 100644 --- a/i18n/es.po +++ b/i18n/es.po @@ -1,7 +1,7 @@ msgid "" msgstr "" "Project-Id-Version: Bulk Load\n" -"POT-Creation-Date: 2025-11-19T17:15:26.356Z\n" +"POT-Creation-Date: 2026-06-29T11:38:42.923Z\n" "Language: es\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=UTF-8\n" @@ -109,6 +109,24 @@ msgstr "Programa de eventos" msgid "Tracker Program" msgstr "Programa de tracker" +msgid "row {{lines}}" +msgstr "" + +msgid "rows {{lines}}" +msgstr "" + +#, fuzzy +msgid "and {{count}} more" +msgid_plural "and {{count}} more" +msgstr[0] "Borrar {{count}} temas" +msgstr[1] "" + +msgid "sheet {{sheet}}, {{lines}}" +msgstr "" + +msgid "Found in {{locations}} of the Excel file" +msgstr "" + msgid "Cannot read data form id" msgstr "No se pueden leer los datos del formulario id" @@ -1031,7 +1049,9 @@ msgid "Synchronization Results" msgstr "Resultados de la sincronización" msgid "Selected template has unsupported period type: {{periodType}}" -msgstr "La plantilla seleccionada tiene el tipo de período no soportado: {{periodType}}" +msgstr "" +"La plantilla seleccionada tiene el tipo de período no soportado: " +"{{periodType}}" msgid "Select organisation unit to populate data" msgstr "Seleccione la unidad de organización para rellenar los datos" @@ -1330,9 +1350,9 @@ msgid "" "Samaritan’s Purse, Medecins Sans Frontières (MSF), the the Norwegian Refugee " "Council (NRC) and the Clinton Health Access Initiative (CHAI) to support " "countries in strengthening the collection and use of health data by using " -"DHIS2. The application has been developed by [EyeSeeTea SL](http://eyeseetea." -"com). Source code, documentation and release notes can be found at the " -"[EyeSeetea GitHub Project Page](https://eyeseetea.github.io/Bulk-Load-" +"DHIS2. The application has been developed by [EyeSeeTea SL](http://" +"eyeseetea.com). Source code, documentation and release notes can be found at " +"the [EyeSeetea GitHub Project Page](https://eyeseetea.github.io/Bulk-Load-" "blessed/)." msgstr "" @@ -1427,19 +1447,30 @@ msgid "" "All data values in the spreadsheet will be imported to the system, but any " "data that was existing for such organisation unit and periods in the system " "will be deleted first, so none will be kept before doing the import." -msgstr "Todos los valores de la hoja de cálculo se importarán al sistema, pero cualquier dato existente para dicha unidad organizativa y periodos se eliminará primero, por lo que no se conservará nada antes de realizar la importación." +msgstr "" +"Todos los valores de la hoja de cálculo se importarán al sistema, pero " +"cualquier dato existente para dicha unidad organizativa y periodos se " +"eliminará primero, por lo que no se conservará nada antes de realizar la " +"importación." msgid "" "Import only new data values, without updating nor deleting any existing one. " "Only values in the spreadsheet that do not currently exist in the system " "will be imported" -msgstr "Importar solo valores de datos nuevos, sin actualizar ni eliminar los existentes. Solo se importarán los valores de la hoja de cálculo que no existan actualmente en el sistema." +msgstr "" +"Importar solo valores de datos nuevos, sin actualizar ni eliminar los " +"existentes. Solo se importarán los valores de la hoja de cálculo que no " +"existan actualmente en el sistema." msgid "" "Import new data values and also update existing ones. All data values in the " "spreadsheet will be imported to the system, but other data values present in " "the system that are not provided in the spreadsheet will be kept." -msgstr "Importar nuevos valores de datos y actualizar los existentes. Todos los valores de la hoja de cálculo se importarán al sistema, pero se mantendrán los demás valores presentes en el sistema que no figuren en la hoja de cálculo." +msgstr "" +"Importar nuevos valores de datos y actualizar los existentes. Todos los " +"valores de la hoja de cálculo se importarán al sistema, pero se mantendrán " +"los demás valores presentes en el sistema que no figuren en la hoja de " +"cálculo." msgid "Warning: Your upload may result in the generation of duplicates" msgstr "Alerta: Su importación puede generar duplicados" diff --git a/i18n/fr.po b/i18n/fr.po index 1853d3bc..90c71b47 100644 --- a/i18n/fr.po +++ b/i18n/fr.po @@ -1,7 +1,7 @@ msgid "" msgstr "" "Project-Id-Version: Bulk Load App\n" -"POT-Creation-Date: 2025-11-19T17:15:26.356Z\n" +"POT-Creation-Date: 2026-06-29T11:38:42.923Z\n" "Language: fr\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=UTF-8\n" @@ -103,6 +103,24 @@ msgstr "Programme" msgid "Tracker Program" msgstr "Programme" +msgid "row {{lines}}" +msgstr "" + +msgid "rows {{lines}}" +msgstr "" + +#, fuzzy +msgid "and {{count}} more" +msgid_plural "and {{count}} more" +msgstr[0] "Supprimer {{count}} thèmes" +msgstr[1] "" + +msgid "sheet {{sheet}}, {{lines}}" +msgstr "" + +msgid "Found in {{locations}} of the Excel file" +msgstr "" + msgid "Cannot read data form id" msgstr "" @@ -1050,7 +1068,8 @@ msgid "Synchronization Results" msgstr "" msgid "Selected template has unsupported period type: {{periodType}}" -msgstr "Modèle sélectionné avec un type de période non supporté: {{periodType}}" +msgstr "" +"Modèle sélectionné avec un type de période non supporté: {{periodType}}" msgid "Select organisation unit to populate data" msgstr "" @@ -1349,9 +1368,9 @@ msgid "" "Samaritan’s Purse, Medecins Sans Frontières (MSF), the the Norwegian Refugee " "Council (NRC) and the Clinton Health Access Initiative (CHAI) to support " "countries in strengthening the collection and use of health data by using " -"DHIS2. The application has been developed by [EyeSeeTea SL](http://eyeseetea." -"com). Source code, documentation and release notes can be found at the " -"[EyeSeetea GitHub Project Page](https://eyeseetea.github.io/Bulk-Load-" +"DHIS2. The application has been developed by [EyeSeeTea SL](http://" +"eyeseetea.com). Source code, documentation and release notes can be found at " +"the [EyeSeetea GitHub Project Page](https://eyeseetea.github.io/Bulk-Load-" "blessed/)." msgstr "" @@ -1450,26 +1469,29 @@ msgid "" "data that was existing for such organisation unit and periods in the system " "will be deleted first, so none will be kept before doing the import." msgstr "" -"Toutes les données de la feuille de calcul seront importées dans le système," -"mais les données existantes pour ces unités d'organisation et ces périodes seront d'abord supprimées" -"; aucune donnée existante ne sera donc conservée avant l'importation." +"Toutes les données de la feuille de calcul seront importées dans le " +"système,mais les données existantes pour ces unités d'organisation et ces " +"périodes seront d'abord supprimées; aucune donnée existante ne sera donc " +"conservée avant l'importation." msgid "" "Import only new data values, without updating nor deleting any existing one. " "Only values in the spreadsheet that do not currently exist in the system " "will be imported" msgstr "" -"Importer uniquement les nouvelles valeurs, sans mettre à jour ni supprimer les données existantes." -"Seules les valeurs de la feuille de calcul qui n'existent pas encore dans le système seront importées." +"Importer uniquement les nouvelles valeurs, sans mettre à jour ni supprimer " +"les données existantes.Seules les valeurs de la feuille de calcul qui " +"n'existent pas encore dans le système seront importées." msgid "" "Import new data values and also update existing ones. All data values in the " "spreadsheet will be imported to the system, but other data values present in " "the system that are not provided in the spreadsheet will be kept." msgstr "" -"Importer les nouvelles valeurs et mettre à jour les valeurs existantes." -"Toutes les données de la feuille de calcul seront importées," -"mais les autres valeurs présentes dans le système et absentes de la feuille de calcul seront conservées." +"Importer les nouvelles valeurs et mettre à jour les valeurs " +"existantes.Toutes les données de la feuille de calcul seront importées,mais " +"les autres valeurs présentes dans le système et absentes de la feuille de " +"calcul seront conservées." msgid "Warning: Your upload may result in the generation of duplicates" msgstr "" diff --git a/i18n/pt.po b/i18n/pt.po index 47412f6f..ce245b5a 100644 --- a/i18n/pt.po +++ b/i18n/pt.po @@ -1,7 +1,7 @@ msgid "" msgstr "" "Project-Id-Version: Bulk Load\n" -"POT-Creation-Date: 2025-11-19T17:15:26.356Z\n" +"POT-Creation-Date: 2026-06-29T11:38:42.923Z\n" "Language: pt\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=UTF-8\n" @@ -111,6 +111,24 @@ msgstr "Programa" msgid "Tracker Program" msgstr "Programa" +msgid "row {{lines}}" +msgstr "" + +msgid "rows {{lines}}" +msgstr "" + +#, fuzzy +msgid "and {{count}} more" +msgid_plural "and {{count}} more" +msgstr[0] "Excluir {{count}} temas" +msgstr[1] "" + +msgid "sheet {{sheet}}, {{lines}}" +msgstr "" + +msgid "Found in {{locations}} of the Excel file" +msgstr "" + msgid "Cannot read data form id" msgstr "Não é possível ler os dados do formulário de identificação" @@ -1385,9 +1403,9 @@ msgid "" "Samaritan’s Purse, Medecins Sans Frontières (MSF), the the Norwegian Refugee " "Council (NRC) and the Clinton Health Access Initiative (CHAI) to support " "countries in strengthening the collection and use of health data by using " -"DHIS2. The application has been developed by [EyeSeeTea SL](http://eyeseetea." -"com). Source code, documentation and release notes can be found at the " -"[EyeSeetea GitHub Project Page](https://eyeseetea.github.io/Bulk-Load-" +"DHIS2. The application has been developed by [EyeSeeTea SL](http://" +"eyeseetea.com). Source code, documentation and release notes can be found at " +"the [EyeSeetea GitHub Project Page](https://eyeseetea.github.io/Bulk-Load-" "blessed/)." msgstr "" diff --git a/i18n/ru.po b/i18n/ru.po index d8b02fcc..45d49bfc 100644 --- a/i18n/ru.po +++ b/i18n/ru.po @@ -1,7 +1,7 @@ msgid "" msgstr "" "Project-Id-Version: Bulk Load\n" -"POT-Creation-Date: 2025-11-19T17:15:26.356Z\n" +"POT-Creation-Date: 2026-06-29T11:38:42.923Z\n" "Language: ru\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=UTF-8\n" @@ -113,6 +113,26 @@ msgstr "Программа" msgid "Tracker Program" msgstr "Программа" +msgid "row {{lines}}" +msgstr "" + +msgid "rows {{lines}}" +msgstr "" + +#, fuzzy +msgid "and {{count}} more" +msgid_plural "and {{count}} more" +msgstr[0] "Удалить {{count}} темы" +msgstr[1] "" +msgstr[2] "" +msgstr[3] "" + +msgid "sheet {{sheet}}, {{lines}}" +msgstr "" + +msgid "Found in {{locations}} of the Excel file" +msgstr "" + msgid "Cannot read data form id" msgstr "Невозможно прочитать данные формы id" @@ -1390,9 +1410,9 @@ msgid "" "Samaritan’s Purse, Medecins Sans Frontières (MSF), the the Norwegian Refugee " "Council (NRC) and the Clinton Health Access Initiative (CHAI) to support " "countries in strengthening the collection and use of health data by using " -"DHIS2. The application has been developed by [EyeSeeTea SL](http://eyeseetea." -"com). Source code, documentation and release notes can be found at the " -"[EyeSeetea GitHub Project Page](https://eyeseetea.github.io/Bulk-Load-" +"DHIS2. The application has been developed by [EyeSeeTea SL](http://" +"eyeseetea.com). Source code, documentation and release notes can be found at " +"the [EyeSeetea GitHub Project Page](https://eyeseetea.github.io/Bulk-Load-" "blessed/)." msgstr "" diff --git a/src/data/Dhis2Events.ts b/src/data/Dhis2Events.ts index 1e0d1c5d..6c5b3329 100644 --- a/src/data/Dhis2Events.ts +++ b/src/data/Dhis2Events.ts @@ -5,8 +5,13 @@ import { SynchronizationResult } from "../domain/entities/SynchronizationResult" import i18n from "../utils/i18n"; import { promiseMap } from "../utils/promises"; import { ImportPostResponse, postImport } from "./Dhis2Import"; +import { ImportRowLookup } from "../domain/entities/ImportRowLookup"; -export async function postEvents(api: D2Api, events: Event[]): Promise { +export async function postEvents( + api: D2Api, + events: Event[], + rowLookup?: ImportRowLookup +): Promise { return promiseMap(_.chunk(events, 200), eventsToSave => { return postImport( api, @@ -15,6 +20,7 @@ export async function postEvents(api: D2Api, events: Event[]): Promise { - const { api, title, model, importResult, splitStatsList } = options; + const { api, title, model, importResult, splitStatsList, rowLookup } = options; const { bundleReport, status, validationReport } = importResult; const message = status === "OK" ? i18n.t("Import was successful") : i18n.t("Import failed"); @@ -73,7 +75,7 @@ export async function processImportResponse(options: { details: "", })); - const detailedErrors = await getMetadataDetailsFromErrors(api, errors); + const detailedErrors = await getMetadataDetailsFromErrors(api, errors, rowLookup); if (!bundleReport) { return { @@ -115,7 +117,7 @@ export async function processImportResponse(options: { title: title, status: status === "OK" ? "SUCCESS" : "ERROR", message: message, - errors: errors, + errors: detailedErrors, stats: stats, rawResponse: importResult, }; @@ -124,9 +126,9 @@ export async function processImportResponse(options: { export async function postImport( api: D2Api, postFn: () => Promise, - options: { title: string; model: string; splitStatsList: boolean } + options: { title: string; model: string; splitStatsList: boolean; rowLookup?: ImportRowLookup } ): Promise { - const { title, model, splitStatsList } = options; + const { title, model, splitStatsList, rowLookup } = options; try { const response = await postFn(); @@ -149,6 +151,7 @@ export async function postImport( model: model, importResult: trackerJobReport, splitStatsList, + rowLookup, }); } catch (error: any) { if (error?.response?.data) { @@ -159,6 +162,7 @@ export async function postImport( model: model, importResult: error.response.data, splitStatsList, + rowLookup, }); } else { return { @@ -178,13 +182,14 @@ async function getTrackerJobs(api: D2Api, trackerImportResponseId: Id): Promise< return await api.get<{ completed: boolean }[]>(`/tracker/jobs/${trackerImportResponseId}`).getData(); } -export async function getMetadataDetailsFromErrors(api: D2Api, errors: ErrorMessage[]): Promise { - const metadataIds = errors.flatMap(error => { - const uids = UidErrorMessage.extractUids(error.message); - return { id: error.id, uids }; - }); +export async function getMetadataDetailsFromErrors( + api: D2Api, + errors: ErrorMessage[], + rowLookup?: ImportRowLookup +): Promise { + const uidsByError = errors.map(error => UidErrorMessage.extractUids(error.message)); - const allIds = _.uniq(metadataIds.flatMap(item => item.uids)); + const allIds = _.uniq(uidsByError.flat()); const allItemsDetails = await promiseMap(_.chunk(allIds, 100), async (ids): Promise => { const data = await api @@ -197,15 +202,19 @@ export async function getMetadataDetailsFromErrors(api: D2Api, errors: ErrorMess }); const allItems = _.flatten(allItemsDetails); + const allItemsById = new Map(allItems.map(item => [item.id, item.name])); - if (allItems.length === 0) return errors; + return errors.map((error, index) => { + const message = + allItems.length === 0 ? error.message : UidErrorMessage.replaceUidsInMessage(error.message, allItemsById); - const allItemsById = new Map(allItems.map(item => [item.id, item.name])); + const rowDetails = rowLookup + ? rowLookup.formatLocations(rowLookup.getLocations([error.id, ...(uidsByError[index] ?? [])])) + : ""; + const details = rowDetails || error.details; - const errorMessages = errors.map(error => { - return { ...error, message: UidErrorMessage.replaceUidsInMessage(error.message, allItemsById) }; + return { ...error, message, details }; }); - return errorMessages; } type MetadataSimple = { id: string; displayFormName?: string; displayName?: string; code?: string }; diff --git a/src/data/Dhis2TrackedEntityInstances.ts b/src/data/Dhis2TrackedEntityInstances.ts index 34c50793..33aaf8f5 100644 --- a/src/data/Dhis2TrackedEntityInstances.ts +++ b/src/data/Dhis2TrackedEntityInstances.ts @@ -274,6 +274,7 @@ async function uploadTeis(options: { title: `${model} - ${title}`, model: model, splitStatsList: false, + rowLookup: importOptions.rowLookup, } ); }); diff --git a/src/data/InstanceDhisRepository.ts b/src/data/InstanceDhisRepository.ts index 20367bf0..249c1068 100644 --- a/src/data/InstanceDhisRepository.ts +++ b/src/data/InstanceDhisRepository.ts @@ -34,6 +34,7 @@ import { SynchronizationStatus, } from "../domain/entities/SynchronizationResult"; import { Program, TrackedEntityInstance } from "../domain/entities/TrackedEntityInstance"; +import { ImportRowLookup } from "../domain/entities/ImportRowLookup"; import { BuilderMetadata, GetDataFormsParams, @@ -257,25 +258,29 @@ export class InstanceDhisRepository implements InstanceRepository { .value(); } - public async deleteAggregatedData(dataPackage: DataPackage): Promise { - return this.importAggregatedData("DELETE", dataPackage); + public async deleteAggregatedData( + dataPackage: DataPackage, + rowLookup?: ImportRowLookup + ): Promise { + return this.importAggregatedData("DELETE", dataPackage, rowLookup); } public async importDataPackage( dataPackage: DataPackage, options: ImportDataPackageOptions ): Promise { - const { createAndUpdate } = options; + const { createAndUpdate, rowLookup } = options; switch (dataPackage.type) { case dataFormTypeMap.dataSets: { const result = await this.importAggregatedData( createAndUpdate ? "CREATE_AND_UPDATE" : "CREATE", - dataPackage + dataPackage, + rowLookup ); return [result]; } case dataFormTypeMap.programs: { - return this.importEventsData(dataPackage); + return this.importEventsData(dataPackage, rowLookup); } case dataFormTypeMap.trackerPrograms: { return this.importTrackerProgramData(dataPackage, options); @@ -446,7 +451,8 @@ export class InstanceDhisRepository implements InstanceRepository { private async importAggregatedData( importStrategy: "CREATE" | "UPDATE" | "CREATE_AND_UPDATE" | "DELETE", - dataPackage: DataPackage + dataPackage: DataPackage, + rowLookup?: ImportRowLookup ): Promise { if (dataPackage.type !== dataFormTypeMap.dataSets) throw new Error("Invalid data package type"); @@ -499,7 +505,7 @@ export class InstanceDhisRepository implements InstanceRepository { const allConflicts = _.flatMap(summaries, s => s.conflicts ?? []); const errors = allConflicts.map(({ object, value }) => ({ id: object, message: value, details: "" })); - const errorDetails = await getMetadataDetailsFromErrors(this.api, errors); + const errorDetails = await getMetadataDetailsFromErrors(this.api, errors, rowLookup); return { title, @@ -585,7 +591,10 @@ export class InstanceDhisRepository implements InstanceRepository { ); } - private async importEventsData(dataPackage: DataPackage): Promise { + private async importEventsData( + dataPackage: DataPackage, + rowLookup?: ImportRowLookup + ): Promise { const events = this.buildEventsPayload(dataPackage); const programs = _(events) @@ -612,7 +621,7 @@ export class InstanceDhisRepository implements InstanceRepository { }; }); - return postEvents(this.api, eventsToSave); + return postEvents(this.api, eventsToSave, rowLookup); } private async getEventProgramStage(programId: Id): Promise { diff --git a/src/domain/entities/ImportRowLookup.ts b/src/domain/entities/ImportRowLookup.ts new file mode 100644 index 00000000..fc4cb1ad --- /dev/null +++ b/src/domain/entities/ImportRowLookup.ts @@ -0,0 +1,89 @@ +import _ from "lodash"; +import i18n from "../../utils/i18n"; +import { Id } from "./ReferenceObject"; +import { TemplateDataPackage } from "./Template"; + +export interface RowLocation { + sheet?: string; + row: number; +} + +const MAX_LINES_PER_SHEET = 5; + +/** + * Maps every metadata/object id present in an imported template to the Excel + * row(s) it came from, so import errors (which only reference raw ids) can be + * annotated with "Found in sheet 'X', line Y". + */ +export class ImportRowLookup { + constructor(private readonly locationsById: Map) {} + + static fromTemplateDataPackage(dataPackage: TemplateDataPackage): ImportRowLookup { + const locationsById = new Map(); + + const add = (id: string | undefined | null, location: RowLocation): void => { + if (!id) return; + locationsById.set(id, [...(locationsById.get(id) ?? []), location]); + }; + + dataPackage.dataEntries.forEach(entry => { + const rawRow = typeof entry.group === "number" ? entry.group : parseInt(String(entry.group), 10); + const row = Number.isFinite(rawRow) ? rawRow : undefined; + if (row === undefined) return; + + const location: RowLocation = { sheet: entry.sheet, row }; + add(entry.id, location); + add(entry.orgUnit, location); + add(entry.attribute, location); + add(entry.programStage, location); + add(entry.trackedEntityInstance, location); + entry.dataValues.forEach(dataValue => { + add(dataValue.dataElement, location); + add(dataValue.category, location); + add(dataValue.optionId, location); + }); + }); + + if (dataPackage.type === "trackerPrograms") { + dataPackage.trackedEntityInstances.forEach(tei => { + if (tei.row === undefined) return; + const location: RowLocation = { sheet: tei.sheet, row: tei.row }; + add(tei.id, location); + add(tei.orgUnit.id, location); + tei.attributeValues.forEach(attributeValue => { + add(attributeValue.attribute.id, location); + add(attributeValue.optionId, location); + }); + }); + } + + return new ImportRowLookup(locationsById); + } + + getLocations(ids: Id[]): RowLocation[] { + const locations = _.compact(ids).flatMap(id => this.locationsById.get(id) ?? []); + return _.uniqWith(locations, _.isEqual); + } + + formatLocations(locations: RowLocation[]): string { + if (locations.length === 0) return ""; + + const bySheet = _(locations) + .groupBy(location => location.sheet ?? "") + .map((sheetLocations, sheet) => { + const rows = _.sortBy(_.uniq(sheetLocations.map(location => location.row))); + const shown = rows.slice(0, MAX_LINES_PER_SHEET); + const remaining = rows.length - shown.length; + const linesLabel = + shown.length === 1 + ? i18n.t("row {{lines}}", { lines: String(shown[0]) }) + : i18n.t("rows {{lines}}", { lines: shown.join(", ") }); + const moreLabel = remaining > 0 ? ` ${i18n.t("and {{count}} more", { count: remaining })}` : ""; + const lines = `${linesLabel}${moreLabel}`; + return sheet ? i18n.t("sheet {{sheet}}, {{lines}}", { sheet, lines }) : lines; + }) + .value(); + + return i18n.t("Found in {{locations}} of the Excel file", { locations: bySheet.join("; ") }); + } +} diff --git a/src/domain/entities/Template.ts b/src/domain/entities/Template.ts index 8c2621e1..240f3f7d 100644 --- a/src/domain/entities/Template.ts +++ b/src/domain/entities/Template.ts @@ -443,6 +443,7 @@ export type TemplateDataValue = { }; export type TemplateDataPackageData = { group: number | Maybe; + sheet?: string; dataForm: string; id: Maybe; orgUnit: string; diff --git a/src/domain/entities/TrackedEntityInstance.ts b/src/domain/entities/TrackedEntityInstance.ts index b04e38d1..6c9ecb35 100644 --- a/src/domain/entities/TrackedEntityInstance.ts +++ b/src/domain/entities/TrackedEntityInstance.ts @@ -15,6 +15,8 @@ export interface TrackedEntityInstance { enrollment: Enrollment | undefined; relationships: Relationship[]; geometry: Geometry; + row?: number; + sheet?: string; } export interface Enrollment { diff --git a/src/domain/entities/__tests__/ImportRowLookup.spec.ts b/src/domain/entities/__tests__/ImportRowLookup.spec.ts new file mode 100644 index 00000000..bf382892 --- /dev/null +++ b/src/domain/entities/__tests__/ImportRowLookup.spec.ts @@ -0,0 +1,160 @@ +import { ImportRowLookup } from "../ImportRowLookup"; +import { TemplateDataPackage, TemplateDataPackageData } from "../Template"; +import { TrackedEntityInstance } from "../TrackedEntityInstance"; + +function dataEntry(partial: Partial): TemplateDataPackageData { + return { + group: 5, + sheet: "Data Entry", + dataForm: "form1", + id: undefined, + orgUnit: "orgUnitA", + period: "202401", + attribute: undefined, + coordinate: undefined, + trackedEntityInstance: undefined, + programStage: undefined, + geometry: undefined, + dataValues: [], + ...partial, + }; +} + +describe("ImportRowLookup", () => { + describe("fromTemplateDataPackage + getLocations", () => { + it("maps org unit and data element ids to their row and sheet", () => { + const pkg: TemplateDataPackage = { + type: "dataSets", + dataEntries: [ + dataEntry({ + group: 7, + sheet: "Data Entry", + orgUnit: "orgUnitA", + dataValues: [{ dataElement: "deX", category: undefined, value: 1, optionId: undefined, contentType: undefined }], + }), + ], + }; + + const lookup = ImportRowLookup.fromTemplateDataPackage(pkg); + + expect(lookup.getLocations(["orgUnitA"])).toEqual([{ sheet: "Data Entry", row: 7 }]); + expect(lookup.getLocations(["deX"])).toEqual([{ sheet: "Data Entry", row: 7 }]); + expect(lookup.getLocations(["unknown"])).toEqual([]); + }); + + it("indexes the event id, attribute, programStage and option id", () => { + const pkg: TemplateDataPackage = { + type: "programs", + dataEntries: [ + dataEntry({ + group: 4, + id: "eventA", + attribute: "attrA", + programStage: "stageA", + dataValues: [{ dataElement: "deY", category: "cocA", value: "v", optionId: "optA", contentType: undefined }], + }), + ], + }; + + const lookup = ImportRowLookup.fromTemplateDataPackage(pkg); + + expect(lookup.getLocations(["eventA"])).toEqual([{ sheet: "Data Entry", row: 4 }]); + expect(lookup.getLocations(["attrA"])).toEqual([{ sheet: "Data Entry", row: 4 }]); + expect(lookup.getLocations(["stageA"])).toEqual([{ sheet: "Data Entry", row: 4 }]); + expect(lookup.getLocations(["cocA"])).toEqual([{ sheet: "Data Entry", row: 4 }]); + expect(lookup.getLocations(["optA"])).toEqual([{ sheet: "Data Entry", row: 4 }]); + }); + + it("collects every row a shared id appears in (deduped)", () => { + const pkg: TemplateDataPackage = { + type: "dataSets", + dataEntries: [ + dataEntry({ group: 5, orgUnit: "ou" }), + dataEntry({ group: 9, orgUnit: "ou" }), + dataEntry({ group: 5, orgUnit: "ou" }), + ], + }; + + const lookup = ImportRowLookup.fromTemplateDataPackage(pkg); + + expect(lookup.getLocations(["ou"])).toEqual([ + { sheet: "Data Entry", row: 5 }, + { sheet: "Data Entry", row: 9 }, + ]); + }); + + it("skips entries without a numeric row (group)", () => { + const pkg: TemplateDataPackage = { + type: "programs", + dataEntries: [dataEntry({ group: undefined, orgUnit: "ou" }), dataEntry({ group: "custom", orgUnit: "ou" })], + }; + + const lookup = ImportRowLookup.fromTemplateDataPackage(pkg); + + expect(lookup.getLocations(["ou"])).toEqual([]); + }); + + it("indexes tracked entity instances using their own row/sheet", () => { + const tei: TrackedEntityInstance = { + program: { id: "p" }, + id: "teiA", + orgUnit: { id: "ouTei" }, + disabled: false, + row: 12, + sheet: "TEI Instances", + attributeValues: [ + { attribute: { id: "attrTei", valueType: "TEXT" }, value: "x", optionId: "optTei" }, + ], + enrollment: undefined, + relationships: [], + geometry: { type: "none" }, + }; + + const pkg: TemplateDataPackage = { + type: "trackerPrograms", + dataEntries: [], + trackedEntityInstances: [tei], + }; + + const lookup = ImportRowLookup.fromTemplateDataPackage(pkg); + + expect(lookup.getLocations(["teiA"])).toEqual([{ sheet: "TEI Instances", row: 12 }]); + expect(lookup.getLocations(["ouTei"])).toEqual([{ sheet: "TEI Instances", row: 12 }]); + expect(lookup.getLocations(["attrTei"])).toEqual([{ sheet: "TEI Instances", row: 12 }]); + expect(lookup.getLocations(["optTei"])).toEqual([{ sheet: "TEI Instances", row: 12 }]); + }); + }); + + describe("formatLocations", () => { + const lookup = new ImportRowLookup(new Map()); + + it("returns an empty string when there are no locations", () => { + expect(lookup.formatLocations([])).toBe(""); + }); + + it("formats a single line with its sheet", () => { + expect(lookup.formatLocations([{ sheet: "Data Entry", row: 12 }])).toBe( + "Found in sheet Data Entry, row 12 of the Excel file" + ); + }); + + it("formats multiple lines on the same sheet", () => { + const result = lookup.formatLocations([ + { sheet: "Data Entry", row: 9 }, + { sheet: "Data Entry", row: 5 }, + ]); + expect(result).toBe("Found in sheet Data Entry, rows 5, 9 of the Excel file"); + }); + + it("caps the list and reports the remainder", () => { + const locations = [1, 2, 3, 4, 5, 6, 7].map(row => ({ sheet: "Data Entry", row })); + expect(lookup.formatLocations(locations)).toBe( + "Found in sheet Data Entry, rows 1, 2, 3, 4, 5 and 2 more of the Excel file" + ); + }); + + it("omits the sheet label when no sheet is known", () => { + expect(lookup.formatLocations([{ row: 3 }])).toBe("Found in row 3 of the Excel file"); + }); + }); +}); diff --git a/src/domain/helpers/ExcelReader.ts b/src/domain/helpers/ExcelReader.ts index 9b848698..7c9b59a0 100644 --- a/src/domain/helpers/ExcelReader.ts +++ b/src/domain/helpers/ExcelReader.ts @@ -114,6 +114,7 @@ export class ExcelReader { dataValues: _.flatMap(items, ({ dataValues }) => dataValues), coordinate: items[0]?.coordinate, geometry: items[0]?.geometry, + sheet: items[0]?.sheet, }; }) .compact() @@ -166,6 +167,7 @@ export class ExcelReader { return { group: this.excelRepository.buildRowNumber(cell.ref), + sheet: String(dataSource.range.sheet), dataForm: this.formatValue(dataFormId), id: eventId ? this.formatValue(eventId) : undefined, orgUnit: this.formatValue(orgUnit), @@ -345,6 +347,7 @@ export class ExcelReader { const data: TemplateDataPackageData = { group: rowIdx, + sheet: String(dataSource.dataValues.sheet), dataForm: String(programId), id: eventId ? String(eventId) : undefined, orgUnit: tei.orgUnit.id, @@ -475,6 +478,8 @@ export class ExcelReader { id: String(teiId), orgUnit: { id: orgUnitId }, disabled: false, + row: rowIdx, + sheet: String(dataSource.attributes.sheet), attributeValues, enrollment: { enrolledAt: this.formatValue(enrollmentDate), diff --git a/src/domain/repositories/InstanceRepository.ts b/src/domain/repositories/InstanceRepository.ts index de7cdd49..8962c77a 100644 --- a/src/domain/repositories/InstanceRepository.ts +++ b/src/domain/repositories/InstanceRepository.ts @@ -10,6 +10,7 @@ import { OrgUnit } from "../entities/OrgUnit"; import { Id, NamedRef } from "../entities/ReferenceObject"; import { SynchronizationResult } from "../entities/SynchronizationResult"; import { Program, TrackedEntityInstance } from "../entities/TrackedEntityInstance"; +import { ImportRowLookup } from "../entities/ImportRowLookup"; export interface GetDataPackageParams { type: DataFormType; @@ -37,7 +38,7 @@ export interface InstanceRepository { getDataPackage(params: GetDataPackageParams): Promise; getLocales(): Promise; getDefaultIds(filter?: string): Promise; - deleteAggregatedData(dataPackage: DataPackage): Promise; + deleteAggregatedData(dataPackage: DataPackage, rowLookup?: ImportRowLookup): Promise; importDataPackage(dataPackage: DataPackage, options: ImportDataPackageOptions): Promise; getProgram(programId: Id): Promise; convertDataPackage(dataPackage: DataPackage): EventsPackage | AggregatedPackage; @@ -45,7 +46,11 @@ export interface InstanceRepository { getDataFormPermissions(dataFormIds: Id[]): Promise; } -export type ImportDataPackageOptions = { createAndUpdate: boolean; multiTextTeiDelimiter: Maybe }; +export type ImportDataPackageOptions = { + createAndUpdate: boolean; + multiTextTeiDelimiter: Maybe; + rowLookup?: ImportRowLookup; +}; export interface BuilderMetadata { orgUnits: Record; diff --git a/src/domain/usecases/ImportTemplateUseCase.ts b/src/domain/usecases/ImportTemplateUseCase.ts index 6e433870..08e2cc06 100644 --- a/src/domain/usecases/ImportTemplateUseCase.ts +++ b/src/domain/usecases/ImportTemplateUseCase.ts @@ -20,6 +20,7 @@ import { templateToDataPackage, TemplateTrackerProgramPackage, } from "../entities/Template"; +import { ImportRowLookup } from "../entities/ImportRowLookup"; import { ExcelReader } from "../helpers/ExcelReader"; import { MULTI_TEXT_OPTION_DELIMITER } from "../helpers/ExcelBuilder"; import { ExcelRepository } from "../repositories/ExcelRepository"; @@ -246,16 +247,19 @@ export class ImportTemplateUseCase implements UseCase { }); } + const rowLookup = ImportRowLookup.fromTemplateDataPackage(dataValues); + const shouldDeleteExistingData = dataForm.type === dataFormTypeMap.dataSets ? this.shouldDeleteAggregatedData(duplicateStrategy) : false; const deleteResult = shouldDeleteExistingData - ? await this.instanceRepository.deleteAggregatedData(templateToDataPackage(instanceDataValues)) + ? await this.instanceRepository.deleteAggregatedData(templateToDataPackage(instanceDataValues), rowLookup) : undefined; const importResult = await this.instanceRepository.importDataPackage(templateToDataPackage(dataValues), { createAndUpdate: duplicateStrategy === "IMPORT_WITHOUT_DELETE" || duplicateStrategy === "ERROR", multiTextTeiDelimiter: this.getMultiTextTeiDelimiter(template), + rowLookup, }); const importResultHasErrors = importResult.flatMap(result => result.errors); @@ -632,9 +636,12 @@ export class ImportTemplateUseCase implements UseCase { return errors.map(error => { const orgUnit = orgUnits.find(ou => ou.id === error.id); const matches = allowedMessages.find(regex => error.message.match(regex.regex)); + const orgUnitMessage = matches && orgUnit ? matches.getErrorMessage(orgUnit.name) : ""; + const details = _.compact([error.details, orgUnitMessage]).join(". "); + return { ...error, - details: matches && orgUnit ? matches.getErrorMessage(orgUnit.name) : "", + details, }; }); } From ca75156e4bde3eb3c88f81978e0daba3acb90008 Mon Sep 17 00:00:00 2001 From: Chukwudumebi Onwuli Date: Tue, 30 Jun 2026 13:37:33 -0400 Subject: [PATCH 2/6] feat: show cell reference instead of row number in import error details --- i18n/en.pot | 14 +++-- i18n/es.po | 12 +++-- i18n/fr.po | 12 +++-- i18n/pt.po | 12 +++-- i18n/ru.po | 12 +++-- src/data/Dhis2Import.ts | 12 ++++- src/domain/entities/ImportRowLookup.ts | 51 +++++++++++++------ src/domain/entities/Template.ts | 1 + src/domain/entities/TrackedEntityInstance.ts | 1 + .../__tests__/ImportRowLookup.spec.ts | 41 +++++++++++++++ src/domain/helpers/ExcelReader.ts | 11 +++- 11 files changed, 145 insertions(+), 34 deletions(-) diff --git a/i18n/en.pot b/i18n/en.pot index 9ea0a67e..26086b4b 100644 --- a/i18n/en.pot +++ b/i18n/en.pot @@ -5,8 +5,8 @@ msgstr "" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" "Plural-Forms: nplurals=2; plural=(n != 1)\n" -"POT-Creation-Date: 2026-06-29T11:38:42.923Z\n" -"PO-Revision-Date: 2026-06-29T11:38:42.924Z\n" +"POT-Creation-Date: 2026-06-30T17:31:02.698Z\n" +"PO-Revision-Date: 2026-06-30T17:31:02.699Z\n" msgid "Events - Create/update" msgstr "" @@ -97,10 +97,16 @@ msgstr "" msgid "Tracker Program" msgstr "" -msgid "row {{lines}}" +msgid "cell {{ref}}" msgstr "" -msgid "rows {{lines}}" +msgid "cells {{refs}}" +msgstr "" + +msgid "row {{ref}}" +msgstr "" + +msgid "rows {{refs}}" msgstr "" msgid "and {{count}} more" diff --git a/i18n/es.po b/i18n/es.po index 6dfbe7bc..f2de0ee6 100644 --- a/i18n/es.po +++ b/i18n/es.po @@ -1,7 +1,7 @@ msgid "" msgstr "" "Project-Id-Version: Bulk Load\n" -"POT-Creation-Date: 2026-06-29T11:38:42.923Z\n" +"POT-Creation-Date: 2026-06-30T17:31:02.698Z\n" "Language: es\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=UTF-8\n" @@ -109,10 +109,16 @@ msgstr "Programa de eventos" msgid "Tracker Program" msgstr "Programa de tracker" -msgid "row {{lines}}" +msgid "cell {{ref}}" msgstr "" -msgid "rows {{lines}}" +msgid "cells {{refs}}" +msgstr "" + +msgid "row {{ref}}" +msgstr "" + +msgid "rows {{refs}}" msgstr "" #, fuzzy diff --git a/i18n/fr.po b/i18n/fr.po index 90c71b47..184225b1 100644 --- a/i18n/fr.po +++ b/i18n/fr.po @@ -1,7 +1,7 @@ msgid "" msgstr "" "Project-Id-Version: Bulk Load App\n" -"POT-Creation-Date: 2026-06-29T11:38:42.923Z\n" +"POT-Creation-Date: 2026-06-30T17:31:02.698Z\n" "Language: fr\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=UTF-8\n" @@ -103,10 +103,16 @@ msgstr "Programme" msgid "Tracker Program" msgstr "Programme" -msgid "row {{lines}}" +msgid "cell {{ref}}" msgstr "" -msgid "rows {{lines}}" +msgid "cells {{refs}}" +msgstr "" + +msgid "row {{ref}}" +msgstr "" + +msgid "rows {{refs}}" msgstr "" #, fuzzy diff --git a/i18n/pt.po b/i18n/pt.po index ce245b5a..58496131 100644 --- a/i18n/pt.po +++ b/i18n/pt.po @@ -1,7 +1,7 @@ msgid "" msgstr "" "Project-Id-Version: Bulk Load\n" -"POT-Creation-Date: 2026-06-29T11:38:42.923Z\n" +"POT-Creation-Date: 2026-06-30T17:31:02.698Z\n" "Language: pt\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=UTF-8\n" @@ -111,10 +111,16 @@ msgstr "Programa" msgid "Tracker Program" msgstr "Programa" -msgid "row {{lines}}" +msgid "cell {{ref}}" msgstr "" -msgid "rows {{lines}}" +msgid "cells {{refs}}" +msgstr "" + +msgid "row {{ref}}" +msgstr "" + +msgid "rows {{refs}}" msgstr "" #, fuzzy diff --git a/i18n/ru.po b/i18n/ru.po index 45d49bfc..343e0d54 100644 --- a/i18n/ru.po +++ b/i18n/ru.po @@ -1,7 +1,7 @@ msgid "" msgstr "" "Project-Id-Version: Bulk Load\n" -"POT-Creation-Date: 2026-06-29T11:38:42.923Z\n" +"POT-Creation-Date: 2026-06-30T17:31:02.698Z\n" "Language: ru\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=UTF-8\n" @@ -113,10 +113,16 @@ msgstr "Программа" msgid "Tracker Program" msgstr "Программа" -msgid "row {{lines}}" +msgid "cell {{ref}}" msgstr "" -msgid "rows {{lines}}" +msgid "cells {{refs}}" +msgstr "" + +msgid "row {{ref}}" +msgstr "" + +msgid "rows {{refs}}" msgstr "" #, fuzzy diff --git a/src/data/Dhis2Import.ts b/src/data/Dhis2Import.ts index bd6cbdc0..9c48040b 100644 --- a/src/data/Dhis2Import.ts +++ b/src/data/Dhis2Import.ts @@ -75,7 +75,7 @@ export async function processImportResponse(options: { details: "", })); - const detailedErrors = await getMetadataDetailsFromErrors(api, errors, rowLookup); + const detailedErrors = deduplicateErrors(await getMetadataDetailsFromErrors(api, errors, rowLookup)); if (!bundleReport) { return { @@ -222,6 +222,16 @@ type MetadataApiResponse = { system: { id: string } } & Record e.message) + .flatMap(group => { + const withDetails = group.filter(e => e.details); + return withDetails.length > 0 ? withDetails : group; + }) + .value(); +} + const getItemsFromMetadataResponse = (response: MetadataApiResponse): Item[] => { const entries = Object.entries(response); diff --git a/src/domain/entities/ImportRowLookup.ts b/src/domain/entities/ImportRowLookup.ts index fc4cb1ad..b9a1d052 100644 --- a/src/domain/entities/ImportRowLookup.ts +++ b/src/domain/entities/ImportRowLookup.ts @@ -6,6 +6,7 @@ import { TemplateDataPackage } from "./Template"; export interface RowLocation { sheet?: string; row: number; + column?: string; } const MAX_LINES_PER_SHEET = 5; @@ -38,9 +39,10 @@ export class ImportRowLookup { add(entry.programStage, location); add(entry.trackedEntityInstance, location); entry.dataValues.forEach(dataValue => { - add(dataValue.dataElement, location); - add(dataValue.category, location); - add(dataValue.optionId, location); + const dvLocation: RowLocation = { ...location, column: dataValue.column }; + add(dataValue.dataElement, dvLocation); + add(dataValue.category, dvLocation); + add(dataValue.optionId, dvLocation); }); }); @@ -51,8 +53,9 @@ export class ImportRowLookup { add(tei.id, location); add(tei.orgUnit.id, location); tei.attributeValues.forEach(attributeValue => { - add(attributeValue.attribute.id, location); - add(attributeValue.optionId, location); + const avLocation: RowLocation = { ...location, column: attributeValue.column }; + add(attributeValue.attribute.id, avLocation); + add(attributeValue.optionId, avLocation); }); }); } @@ -71,16 +74,34 @@ export class ImportRowLookup { const bySheet = _(locations) .groupBy(location => location.sheet ?? "") .map((sheetLocations, sheet) => { - const rows = _.sortBy(_.uniq(sheetLocations.map(location => location.row))); - const shown = rows.slice(0, MAX_LINES_PER_SHEET); - const remaining = rows.length - shown.length; - const linesLabel = - shown.length === 1 - ? i18n.t("row {{lines}}", { lines: String(shown[0]) }) - : i18n.t("rows {{lines}}", { lines: shown.join(", ") }); - const moreLabel = remaining > 0 ? ` ${i18n.t("and {{count}} more", { count: remaining })}` : ""; - const lines = `${linesLabel}${moreLabel}`; - return sheet ? i18n.t("sheet {{sheet}}, {{lines}}", { sheet, lines }) : lines; + // Per row: prefer cell-level locations over row-only ones + const byRow = _.groupBy(sheetLocations, loc => loc.row); + const refined = _.flatMap(byRow, rowLocs => { + const withColumn = rowLocs.filter(l => l.column); + return withColumn.length > 0 ? withColumn : rowLocs; + }); + + const sortedLocs = _(refined) + .uniqBy(loc => `${loc.sheet ?? ""}|${loc.row}|${loc.column ?? ""}`) + .sortBy([loc => loc.row, loc => loc.column]) + .value(); + const shownLocs = sortedLocs.slice(0, MAX_LINES_PER_SHEET); + const remaining = sortedLocs.length - shownLocs.length; + const allHaveColumn = shownLocs.every(l => l.column); + const shown = shownLocs.map(loc => (loc.column ? `${loc.column}${loc.row}` : String(loc.row))); + const refs = shown.length === 1 ? shown[0] : shown.join(", "); + const refsLabel = allHaveColumn + ? shown.length === 1 + ? i18n.t("cell {{ref}}", { ref: refs }) + : i18n.t("cells {{refs}}", { refs }) + : shown.length === 1 + ? i18n.t("row {{ref}}", { ref: refs }) + : i18n.t("rows {{refs}}", { refs }); + const label = + remaining > 0 + ? `${refsLabel} ${i18n.t("and {{count}} more", { count: remaining })}` + : refsLabel; + return sheet ? i18n.t("sheet {{sheet}}, {{lines}}", { sheet, lines: label }) : label; }) .value(); diff --git a/src/domain/entities/Template.ts b/src/domain/entities/Template.ts index 240f3f7d..fd5edf55 100644 --- a/src/domain/entities/Template.ts +++ b/src/domain/entities/Template.ts @@ -440,6 +440,7 @@ export type TemplateDataValue = { optionId: Maybe; contentType: Maybe; comment?: string; + column?: string; }; export type TemplateDataPackageData = { group: number | Maybe; diff --git a/src/domain/entities/TrackedEntityInstance.ts b/src/domain/entities/TrackedEntityInstance.ts index 6c9ecb35..8107b57f 100644 --- a/src/domain/entities/TrackedEntityInstance.ts +++ b/src/domain/entities/TrackedEntityInstance.ts @@ -34,6 +34,7 @@ export interface AttributeValue { value: string; optionId?: Id; contentType?: ContentType; + column?: string; } export interface Program { diff --git a/src/domain/entities/__tests__/ImportRowLookup.spec.ts b/src/domain/entities/__tests__/ImportRowLookup.spec.ts index bf382892..514f4db6 100644 --- a/src/domain/entities/__tests__/ImportRowLookup.spec.ts +++ b/src/domain/entities/__tests__/ImportRowLookup.spec.ts @@ -42,6 +42,25 @@ describe("ImportRowLookup", () => { expect(lookup.getLocations(["unknown"])).toEqual([]); }); + it("carries column letter from data element values but not from row-level ids", () => { + const pkg: TemplateDataPackage = { + type: "dataSets", + dataEntries: [ + dataEntry({ + group: 6, + sheet: "Data Entry", + orgUnit: "ouB", + dataValues: [{ dataElement: "deZ", category: undefined, value: 42, optionId: undefined, contentType: undefined, column: "C" }], + }), + ], + }; + + const lookup = ImportRowLookup.fromTemplateDataPackage(pkg); + + expect(lookup.getLocations(["ouB"])).toEqual([{ sheet: "Data Entry", row: 6 }]); + expect(lookup.getLocations(["deZ"])).toEqual([{ sheet: "Data Entry", row: 6, column: "C" }]); + }); + it("indexes the event id, attribute, programStage and option id", () => { const pkg: TemplateDataPackage = { type: "programs", @@ -156,5 +175,27 @@ describe("ImportRowLookup", () => { it("omits the sheet label when no sheet is known", () => { expect(lookup.formatLocations([{ row: 3 }])).toBe("Found in row 3 of the Excel file"); }); + + it("formats a single cell ref when column is known", () => { + expect(lookup.formatLocations([{ sheet: "Data Entry", row: 6, column: "C" }])).toBe( + "Found in sheet Data Entry, cell C6 of the Excel file" + ); + }); + + it("formats multiple cell refs when all have columns", () => { + const result = lookup.formatLocations([ + { sheet: "Data Entry", row: 9, column: "C" }, + { sheet: "Data Entry", row: 5, column: "B" }, + ]); + expect(result).toBe("Found in sheet Data Entry, cells B5, C9 of the Excel file"); + }); + + it("prefers cell ref over row-only location for the same row", () => { + const result = lookup.formatLocations([ + { sheet: "Data Entry", row: 6, column: "B" }, + { sheet: "Data Entry", row: 6 }, + ]); + expect(result).toBe("Found in sheet Data Entry, cell B6 of the Excel file"); + }); }); }); diff --git a/src/domain/helpers/ExcelReader.ts b/src/domain/helpers/ExcelReader.ts index 7c9b59a0..ba7f0a49 100644 --- a/src/domain/helpers/ExcelReader.ts +++ b/src/domain/helpers/ExcelReader.ts @@ -185,8 +185,8 @@ export class ExcelReader { category: category ? this.formatValue(category) : undefined, value: this.formatValue(value), optionId: optionId ? removeCharacters(optionId) : undefined, - contentType: contentType, + column: this.parseColumn(cell.ref), }, ], }; @@ -312,6 +312,7 @@ export class ExcelReader { ]); return { row: this.excelRepository.buildRowNumber(cell.ref), + column: this.parseColumn(cell.ref), value: value, optionId: optionId, contentType: contentType, @@ -343,7 +344,7 @@ export class ExcelReader { // If column id does not exist on program, exclude values => Attributes if (!dataForm.dataElements.find(({ id }) => id === dataElementId)) return null; - const { value, optionId, contentType } = item; + const { value, optionId, contentType, column } = item; const data: TemplateDataPackageData = { group: rowIdx, @@ -364,6 +365,7 @@ export class ExcelReader { value: this.formatValue(value), optionId: optionId ? removeCharacters(optionId) : undefined, contentType: contentType, + column, }, ], }; @@ -439,6 +441,7 @@ export class ExcelReader { return { row: this.excelRepository.buildRowNumber(cell.ref), + column: this.parseColumn(cell.ref), attribute: { id: attributeId, valueType: dataForm.teiAttributes?.find(attribute => attribute.id === attributeId)?.valueType, @@ -525,6 +528,10 @@ export class ExcelReader { } } + private parseColumn(ref: string): string | undefined { + return /^[A-Z]+/.exec(ref)?.[0]; + } + private formatValue(value: ExcelValue | undefined): string { if (value instanceof Date) { return moment(value).format("YYYY-MM-DD[T]HH:mm"); From 3fc1f2dad54c498d5625b33815006e694d18e68d Mon Sep 17 00:00:00 2001 From: Chukwudumebi Onwuli <37223065+deeonwuli@users.noreply.github.com> Date: Tue, 7 Jul 2026 12:43:37 +0300 Subject: [PATCH 3/6] feat: update translations for "and {{count}} more" in Spanish, French, Portuguese, and Russian --- i18n/es.po | 5 ++--- i18n/fr.po | 5 ++--- i18n/pt.po | 5 ++--- i18n/ru.po | 9 ++++----- 4 files changed, 10 insertions(+), 14 deletions(-) diff --git a/i18n/es.po b/i18n/es.po index f2de0ee6..6ffe825d 100644 --- a/i18n/es.po +++ b/i18n/es.po @@ -121,11 +121,10 @@ msgstr "" msgid "rows {{refs}}" msgstr "" -#, fuzzy msgid "and {{count}} more" msgid_plural "and {{count}} more" -msgstr[0] "Borrar {{count}} temas" -msgstr[1] "" +msgstr[0] "y {{count}} más" +msgstr[1] "y {{count}} más" msgid "sheet {{sheet}}, {{lines}}" msgstr "" diff --git a/i18n/fr.po b/i18n/fr.po index 184225b1..04efd4fc 100644 --- a/i18n/fr.po +++ b/i18n/fr.po @@ -115,11 +115,10 @@ msgstr "" msgid "rows {{refs}}" msgstr "" -#, fuzzy msgid "and {{count}} more" msgid_plural "and {{count}} more" -msgstr[0] "Supprimer {{count}} thèmes" -msgstr[1] "" +msgstr[0] "et {{count}} de plus" +msgstr[1] "et {{count}} de plus" msgid "sheet {{sheet}}, {{lines}}" msgstr "" diff --git a/i18n/pt.po b/i18n/pt.po index 58496131..4eefd2be 100644 --- a/i18n/pt.po +++ b/i18n/pt.po @@ -123,11 +123,10 @@ msgstr "" msgid "rows {{refs}}" msgstr "" -#, fuzzy msgid "and {{count}} more" msgid_plural "and {{count}} more" -msgstr[0] "Excluir {{count}} temas" -msgstr[1] "" +msgstr[0] "e mais {{count}}" +msgstr[1] "e mais {{count}}" msgid "sheet {{sheet}}, {{lines}}" msgstr "" diff --git a/i18n/ru.po b/i18n/ru.po index 343e0d54..059cf422 100644 --- a/i18n/ru.po +++ b/i18n/ru.po @@ -125,13 +125,12 @@ msgstr "" msgid "rows {{refs}}" msgstr "" -#, fuzzy msgid "and {{count}} more" msgid_plural "and {{count}} more" -msgstr[0] "Удалить {{count}} темы" -msgstr[1] "" -msgstr[2] "" -msgstr[3] "" +msgstr[0] "и ещё {{count}}" +msgstr[1] "и ещё {{count}}" +msgstr[2] "и ещё {{count}}" +msgstr[3] "и ещё {{count}}" msgid "sheet {{sheet}}, {{lines}}" msgstr "" From 77a74e8563df512a97e7e5b10a79a37d187bdfe0 Mon Sep 17 00:00:00 2001 From: Chukwudumebi Onwuli <37223065+deeonwuli@users.noreply.github.com> Date: Tue, 7 Jul 2026 15:35:22 +0300 Subject: [PATCH 4/6] feat: enhance import error handling by indexing cell data sources with their own location --- src/domain/entities/ImportRowLookup.ts | 34 +++++--- src/domain/entities/Template.ts | 1 + .../__tests__/ImportRowLookup.spec.ts | 80 +++++++++++++++++-- src/domain/helpers/ExcelReader.ts | 7 ++ 4 files changed, 102 insertions(+), 20 deletions(-) diff --git a/src/domain/entities/ImportRowLookup.ts b/src/domain/entities/ImportRowLookup.ts index b9a1d052..669acc1f 100644 --- a/src/domain/entities/ImportRowLookup.ts +++ b/src/domain/entities/ImportRowLookup.ts @@ -29,17 +29,26 @@ export class ImportRowLookup { dataPackage.dataEntries.forEach(entry => { const rawRow = typeof entry.group === "number" ? entry.group : parseInt(String(entry.group), 10); - const row = Number.isFinite(rawRow) ? rawRow : undefined; - if (row === undefined) return; - - const location: RowLocation = { sheet: entry.sheet, row }; - add(entry.id, location); - add(entry.orgUnit, location); - add(entry.attribute, location); - add(entry.programStage, location); - add(entry.trackedEntityInstance, location); + const entryRow = Number.isFinite(rawRow) ? rawRow : undefined; + + if (entryRow !== undefined) { + const location: RowLocation = { sheet: entry.sheet, row: entryRow }; + add(entry.id, location); + add(entry.orgUnit, location); + add(entry.attribute, location); + add(entry.programStage, location); + add(entry.trackedEntityInstance, location); + } + entry.dataValues.forEach(dataValue => { - const dvLocation: RowLocation = { ...location, column: dataValue.column }; + const row = dataValue.row ?? entryRow; + if (row === undefined) return; + + const dvLocation: RowLocation = { + sheet: entry.sheet, + row: row, + column: dataValue.column, + }; add(dataValue.dataElement, dvLocation); add(dataValue.category, dvLocation); add(dataValue.optionId, dvLocation); @@ -98,9 +107,8 @@ export class ImportRowLookup { ? i18n.t("row {{ref}}", { ref: refs }) : i18n.t("rows {{refs}}", { refs }); const label = - remaining > 0 - ? `${refsLabel} ${i18n.t("and {{count}} more", { count: remaining })}` - : refsLabel; + remaining > 0 ? `${refsLabel} ${i18n.t("and {{count}} more", { count: remaining })}` : refsLabel; + return sheet ? i18n.t("sheet {{sheet}}, {{lines}}", { sheet, lines: label }) : label; }) .value(); diff --git a/src/domain/entities/Template.ts b/src/domain/entities/Template.ts index fd5edf55..6c898ed7 100644 --- a/src/domain/entities/Template.ts +++ b/src/domain/entities/Template.ts @@ -441,6 +441,7 @@ export type TemplateDataValue = { contentType: Maybe; comment?: string; column?: string; + row?: number; }; export type TemplateDataPackageData = { group: number | Maybe; diff --git a/src/domain/entities/__tests__/ImportRowLookup.spec.ts b/src/domain/entities/__tests__/ImportRowLookup.spec.ts index 514f4db6..78f14bd4 100644 --- a/src/domain/entities/__tests__/ImportRowLookup.spec.ts +++ b/src/domain/entities/__tests__/ImportRowLookup.spec.ts @@ -30,7 +30,15 @@ describe("ImportRowLookup", () => { group: 7, sheet: "Data Entry", orgUnit: "orgUnitA", - dataValues: [{ dataElement: "deX", category: undefined, value: 1, optionId: undefined, contentType: undefined }], + dataValues: [ + { + dataElement: "deX", + category: undefined, + value: 1, + optionId: undefined, + contentType: undefined, + }, + ], }), ], }; @@ -50,7 +58,16 @@ describe("ImportRowLookup", () => { group: 6, sheet: "Data Entry", orgUnit: "ouB", - dataValues: [{ dataElement: "deZ", category: undefined, value: 42, optionId: undefined, contentType: undefined, column: "C" }], + dataValues: [ + { + dataElement: "deZ", + category: undefined, + value: 42, + optionId: undefined, + contentType: undefined, + column: "C", + }, + ], }), ], }; @@ -70,7 +87,15 @@ describe("ImportRowLookup", () => { id: "eventA", attribute: "attrA", programStage: "stageA", - dataValues: [{ dataElement: "deY", category: "cocA", value: "v", optionId: "optA", contentType: undefined }], + dataValues: [ + { + dataElement: "deY", + category: "cocA", + value: "v", + optionId: "optA", + contentType: undefined, + }, + ], }), ], }; @@ -105,7 +130,10 @@ describe("ImportRowLookup", () => { it("skips entries without a numeric row (group)", () => { const pkg: TemplateDataPackage = { type: "programs", - dataEntries: [dataEntry({ group: undefined, orgUnit: "ou" }), dataEntry({ group: "custom", orgUnit: "ou" })], + dataEntries: [ + dataEntry({ group: undefined, orgUnit: "ou" }), + dataEntry({ group: "custom", orgUnit: "ou" }), + ], }; const lookup = ImportRowLookup.fromTemplateDataPackage(pkg); @@ -113,6 +141,46 @@ describe("ImportRowLookup", () => { expect(lookup.getLocations(["ou"])).toEqual([]); }); + it("indexes cell data sources by the data value's own location (empty group)", () => { + const pkg: TemplateDataPackage = { + type: "dataSets", + dataEntries: [ + dataEntry({ + group: undefined, + sheet: "Data Entry", + orgUnit: "ouAgg", + dataValues: [ + { + dataElement: "deA", + category: "cocA", + value: 1, + optionId: undefined, + contentType: undefined, + column: "C", + row: 8, + }, + { + dataElement: "deB", + category: undefined, + value: 2, + optionId: undefined, + contentType: undefined, + column: "D", + row: 9, + }, + ], + }), + ], + }; + + const lookup = ImportRowLookup.fromTemplateDataPackage(pkg); + + expect(lookup.getLocations(["deA"])).toEqual([{ sheet: "Data Entry", row: 8, column: "C" }]); + expect(lookup.getLocations(["cocA"])).toEqual([{ sheet: "Data Entry", row: 8, column: "C" }]); + expect(lookup.getLocations(["deB"])).toEqual([{ sheet: "Data Entry", row: 9, column: "D" }]); + expect(lookup.getLocations(["ouAgg"])).toEqual([]); + }); + it("indexes tracked entity instances using their own row/sheet", () => { const tei: TrackedEntityInstance = { program: { id: "p" }, @@ -121,9 +189,7 @@ describe("ImportRowLookup", () => { disabled: false, row: 12, sheet: "TEI Instances", - attributeValues: [ - { attribute: { id: "attrTei", valueType: "TEXT" }, value: "x", optionId: "optTei" }, - ], + attributeValues: [{ attribute: { id: "attrTei", valueType: "TEXT" }, value: "x", optionId: "optTei" }], enrollment: undefined, relationships: [], geometry: { type: "none" }, diff --git a/src/domain/helpers/ExcelReader.ts b/src/domain/helpers/ExcelReader.ts index ba7f0a49..d9f45df1 100644 --- a/src/domain/helpers/ExcelReader.ts +++ b/src/domain/helpers/ExcelReader.ts @@ -235,9 +235,14 @@ export class ExcelReader { this.excelRepository.getContentType(template.id, cell), ]); + const sheet = String(dataSource.ref.sheet); + const row = cell ? this.excelRepository.buildRowNumber(cell.ref) : undefined; + const column = cell ? this.parseColumn(cell.ref) : undefined; + return [ { group: undefined, // TODO: Add a way for custom templates to group by event + sheet: sheet, dataForm: String(dataFormId), id: eventId ? String(eventId) : undefined, orgUnit: String(orgUnit), @@ -254,6 +259,8 @@ export class ExcelReader { value: this.formatValue(value), optionId: optionId ? removeCharacters(optionId) : undefined, contentType: contentType, + column: column, + row: row, }, ], }, From 58e52053cdc1058bb739f1f883941f9977b8d996 Mon Sep 17 00:00:00 2001 From: Chukwudumebi Onwuli <37223065+deeonwuli@users.noreply.github.com> Date: Tue, 7 Jul 2026 15:45:37 +0300 Subject: [PATCH 5/6] refactor: build ImportRowLookup via flatMap + groupBy --- src/domain/entities/ImportRowLookup.ts | 117 ++++++++++-------- .../__tests__/ImportRowLookup.spec.ts | 2 +- 2 files changed, 66 insertions(+), 53 deletions(-) diff --git a/src/domain/entities/ImportRowLookup.ts b/src/domain/entities/ImportRowLookup.ts index 669acc1f..6fbc30df 100644 --- a/src/domain/entities/ImportRowLookup.ts +++ b/src/domain/entities/ImportRowLookup.ts @@ -1,7 +1,9 @@ import _ from "lodash"; import i18n from "../../utils/i18n"; +import { Maybe } from "../../types/utils"; import { Id } from "./ReferenceObject"; -import { TemplateDataPackage } from "./Template"; +import { TemplateDataPackage, TemplateDataPackageData } from "./Template"; +import { TrackedEntityInstance } from "./TrackedEntityInstance"; export interface RowLocation { sheet?: string; @@ -9,6 +11,8 @@ export interface RowLocation { column?: string; } +type IdLocation = { id: Maybe; location: RowLocation }; + const MAX_LINES_PER_SHEET = 5; /** @@ -17,63 +21,27 @@ const MAX_LINES_PER_SHEET = 5; * annotated with "Found in sheet 'X', line Y". */ export class ImportRowLookup { - constructor(private readonly locationsById: Map) {} + constructor(private readonly locationsById: Record) {} static fromTemplateDataPackage(dataPackage: TemplateDataPackage): ImportRowLookup { - const locationsById = new Map(); - - const add = (id: string | undefined | null, location: RowLocation): void => { - if (!id) return; - locationsById.set(id, [...(locationsById.get(id) ?? []), location]); - }; - - dataPackage.dataEntries.forEach(entry => { - const rawRow = typeof entry.group === "number" ? entry.group : parseInt(String(entry.group), 10); - const entryRow = Number.isFinite(rawRow) ? rawRow : undefined; - - if (entryRow !== undefined) { - const location: RowLocation = { sheet: entry.sheet, row: entryRow }; - add(entry.id, location); - add(entry.orgUnit, location); - add(entry.attribute, location); - add(entry.programStage, location); - add(entry.trackedEntityInstance, location); - } - - entry.dataValues.forEach(dataValue => { - const row = dataValue.row ?? entryRow; - if (row === undefined) return; - - const dvLocation: RowLocation = { - sheet: entry.sheet, - row: row, - column: dataValue.column, - }; - add(dataValue.dataElement, dvLocation); - add(dataValue.category, dvLocation); - add(dataValue.optionId, dvLocation); - }); - }); - - if (dataPackage.type === "trackerPrograms") { - dataPackage.trackedEntityInstances.forEach(tei => { - if (tei.row === undefined) return; - const location: RowLocation = { sheet: tei.sheet, row: tei.row }; - add(tei.id, location); - add(tei.orgUnit.id, location); - tei.attributeValues.forEach(attributeValue => { - const avLocation: RowLocation = { ...location, column: attributeValue.column }; - add(attributeValue.attribute.id, avLocation); - add(attributeValue.optionId, avLocation); - }); - }); - } + const teis = dataPackage.type === "trackerPrograms" ? dataPackage.trackedEntityInstances : []; + + const pairs: IdLocation[] = [ + ...dataPackage.dataEntries.flatMap(dataEntryLocations), + ...teis.flatMap(trackedEntityLocations), + ]; + + const locationsById = _(pairs) + .filter(({ id }) => Boolean(id)) + .groupBy(({ id }) => id) + .mapValues(idLocations => idLocations.map(({ location }) => location)) + .value(); return new ImportRowLookup(locationsById); } getLocations(ids: Id[]): RowLocation[] { - const locations = _.compact(ids).flatMap(id => this.locationsById.get(id) ?? []); + const locations = _.compact(ids).flatMap(id => this.locationsById[id] ?? []); return _.uniqWith(locations, _.isEqual); } @@ -108,7 +76,7 @@ export class ImportRowLookup { : i18n.t("rows {{refs}}", { refs }); const label = remaining > 0 ? `${refsLabel} ${i18n.t("and {{count}} more", { count: remaining })}` : refsLabel; - + return sheet ? i18n.t("sheet {{sheet}}, {{lines}}", { sheet, lines: label }) : label; }) .value(); @@ -116,3 +84,48 @@ export class ImportRowLookup { return i18n.t("Found in {{locations}} of the Excel file", { locations: bySheet.join("; ") }); } } + +function dataEntryLocations(entry: TemplateDataPackageData): IdLocation[] { + const rawRow = typeof entry.group === "number" ? entry.group : parseInt(String(entry.group), 10); + const entryRow = Number.isFinite(rawRow) ? rawRow : undefined; + + const entryPairs: IdLocation[] = + entryRow !== undefined + ? locationPairs({ sheet: entry.sheet, row: entryRow }, [ + entry.id, + entry.orgUnit, + entry.attribute, + entry.programStage, + entry.trackedEntityInstance, + ]) + : []; + + const dataValuePairs: IdLocation[] = entry.dataValues.flatMap(dataValue => { + const row = dataValue.row ?? entryRow; + if (row === undefined) return []; + const location: RowLocation = { sheet: entry.sheet, row, column: dataValue.column }; + return locationPairs(location, [dataValue.dataElement, dataValue.category, dataValue.optionId]); + }); + + return [...entryPairs, ...dataValuePairs]; +} + +function trackedEntityLocations(tei: TrackedEntityInstance): IdLocation[] { + if (tei.row === undefined) return []; + + const location: RowLocation = { sheet: tei.sheet, row: tei.row }; + const teiPairs = locationPairs(location, [tei.id, tei.orgUnit.id]); + + const attributePairs = tei.attributeValues.flatMap(attributeValue => + locationPairs({ ...location, column: attributeValue.column }, [ + attributeValue.attribute.id, + attributeValue.optionId, + ]) + ); + + return [...teiPairs, ...attributePairs]; +} + +function locationPairs(location: RowLocation, ids: Maybe[]): IdLocation[] { + return ids.map(id => ({ id, location })); +} diff --git a/src/domain/entities/__tests__/ImportRowLookup.spec.ts b/src/domain/entities/__tests__/ImportRowLookup.spec.ts index 78f14bd4..7c2f053c 100644 --- a/src/domain/entities/__tests__/ImportRowLookup.spec.ts +++ b/src/domain/entities/__tests__/ImportRowLookup.spec.ts @@ -211,7 +211,7 @@ describe("ImportRowLookup", () => { }); describe("formatLocations", () => { - const lookup = new ImportRowLookup(new Map()); + const lookup = new ImportRowLookup({}); it("returns an empty string when there are no locations", () => { expect(lookup.formatLocations([])).toBe(""); From 67dfe1494e09fcdae5675d20b487cd4a7a93e71f Mon Sep 17 00:00:00 2001 From: Chukwudumebi Onwuli <37223065+deeonwuli@users.noreply.github.com> Date: Tue, 7 Jul 2026 16:25:39 +0300 Subject: [PATCH 6/6] docs: explain deduplicateErrors groups by message only --- src/data/Dhis2Import.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/data/Dhis2Import.ts b/src/data/Dhis2Import.ts index 9c48040b..faba7284 100644 --- a/src/data/Dhis2Import.ts +++ b/src/data/Dhis2Import.ts @@ -222,6 +222,8 @@ type MetadataApiResponse = { system: { id: string } } & Record e.message)