[Feature] Add csv events input for update events

README.md

@@ -44,15 +44,19 @@ Available levels: 'debug' | 'info' | 'warn' | 'error'
     -   [Create an SQL file to remove any orgunit below the country level](#create-an-sql-file-to-remove-any-orgunit-below-the-country-level)
     -   [Create an SQL file to remove all org subunits of Canada](#create-an-sql-file-to-remove-all-org-subunits-of-canada)
     -   [Copy the organisation units from a data set to one or more datasets](#copy-the-organisation-units-from-a-data-set-to-one-or-more-datasets)
+    -   [Set all DataSets `skipOffline` where last data input period year was 'year' indicated or earlier](#set-all-datasets-skipoffline-where-last-data-input-period-year-was-year-indicated-or-earlier)
 -   [Translations](#translations)
 -   [Events](#events)
+    -   [Detect events assigned to organisation units outside their enrollment](#detect-events-assigned-to-organisation-units-outside-their-enrollment)
     -   [Move events from one orgunit to another](#move-events-from-one-orgunit-to-another)
     -   [Update events which met the condition](#update-events-which-met-the-condition)
+    -   [Recode boolean to ternary optionSet values](#recode-boolean-to-ternary-optionset-values)
 -   [Data values](#data-values)
     -   [Dangling data values](#dangling-data-values)
     -   [Revert data values](#revert-data-values)
     -   [Delete duplicated event data values](#delete-duplicated-event-data-values)
     -   [Email notification for data values](#email-notification-for-data-values)
+    -   [Delete all data values given a data elements file](#delete-all-data-values-given-a-data-elements-file)
 -   [Notifications](#notifications)
     -   [Send user info email](#send-user-info-email)
 -   [Load testing](#load-testing)
@@ -73,10 +77,14 @@ Available levels: 'debug' | 'info' | 'warn' | 'error'
     -   [Transfer](#transfer)
 -   [Options](#options)
     -   [Rename](#rename)
+    -   [Analyze](#analyze)
 -   [Data](#data)
     -   [Get report](#get-report)
 -   [Enrollments](#enrollments)
     -   [Close Enrollments with Events older than a date](#close-enrollments-with-events-older-than-a-date)
+-   [Category option combos](#category-option-combos)
+    -   [Translations](#translations-1)
+    -   [Regenerate](#regenerate)
 
 ## Execute Program Rules
 
@@ -270,15 +278,34 @@ $ yarn start events move-to-org-unit \
 
 ### Update events which met the condition
 
+Updates the value for a collection of events that belongs to the provided data element and satisfies the condition.
+The events must be provided as either a comma separated list (`--event-ids`) or a CSV with _id_ header (`--event-ids`).
+Use `--report-path` to generate a CSV with a report of the changes. Unless `--post` is used the changes are not saved.
+To update events that already have the desired value use `--update-same-value`.
+
+Using events list:
 ```shell
 $ yarn start events update-events \
 --url='http://USER:PASSWORD@HOST:PORT' \
---root-org-unit='org-unit-id'
+--root-org-unit='org-unit-id' \
 --event-ids='event_id_1,event_id_2,event_id_3' \
 --data-element-id='data_element_id' \
 --condition='true' \
 --new-value='' \
---csv-path='./events.csv' \
+--report-path='./report.csv' \
+--post
+```
+
+Using events CSV:
+```shell
+$ yarn start events update-events \
+--url='http://USER:PASSWORD@HOST:PORT' \
+--root-org-unit='org-unit-id' \
+--events-csv='./events.csv' \
+--data-element-id='data_element_id' \
+--condition='true' \
+--new-value='' \
+--report-path='./report.csv' \
 --post
 ```
 

src/data/EventExportSpreadsheetRepository.ts

@@ -8,7 +8,7 @@ import { ProgramEvent } from "domain/entities/ProgramEvent";
 export class EventExportSpreadsheetRepository implements EventExportRepository {
     async saveReport(events: ProgramEvent[], options: MigrateOptions): Async<void> {
         const csvWriter = createObjectCsvWriter({
-            path: options.csvPath,
+            path: options.reportPath,
             header: [
                 {
                     id: "event",

src/domain/usecases/UpdateEventDataValueUseCase.ts

@@ -2,7 +2,7 @@ import _ from "lodash";
 import { Async } from "domain/entities/Async";
 import { Id } from "domain/entities/Base";
 import { Result } from "domain/entities/Result";
-import logger from "utils/log";
+import { Logger } from "domain/logger/Logger";
 import { EventExportSpreadsheetRepository } from "data/EventExportSpreadsheetRepository";
 import { ProgramEventsRepository } from "domain/repositories/ProgramEventsRepository";
 import { ProgramEvent } from "domain/entities/ProgramEvent";
@@ -13,34 +13,61 @@ export type MigrateOptions = {
     dataElementId: Id;
     condition: string;
     newValue: string;
-    csvPath: string;
+    reportPath: string;
     post: boolean;
+    updateSameValue: boolean;
 };
 
 export class UpdateEventDataValueUseCase {
+    private readonly eventChunkSize = 200;
+
     constructor(
+        private logger: Logger,
         private programEventsRepository: ProgramEventsRepository,
         private eventExportSpreadsheetRepository: EventExportSpreadsheetRepository
     ) {}
 
     async execute(options: MigrateOptions): Async<Result> {
-        const eventMetadata = await this.programEventsRepository.get({
-            eventsIds: options.eventIds,
-            orgUnitsIds: [options.rootOrgUnit],
-            orgUnitMode: "DESCENDANTS",
-        });
+        const eventIdsLength = options.eventIds.length;
+
+        let eventMetadata: ProgramEvent[] = [];
+        for (let i = 0; i < eventIdsLength; i += this.eventChunkSize) {
+            this.logger.debug(
+                `Fetching events metadata for events ${i + 1} to ${Math.min(
+                    i + this.eventChunkSize,
+                    eventIdsLength
+                )} of ${eventIdsLength}`
+            );
+
+            const eventIdsChunk = options.eventIds.slice(i, i + this.eventChunkSize);
+            const eventMetadataChunk = await this.programEventsRepository.get({
+                eventsIds: eventIdsChunk,
+                orgUnitsIds: [options.rootOrgUnit],
+                orgUnitMode: "DESCENDANTS",
+            });
+
+            eventMetadata = eventMetadata.concat(eventMetadataChunk);
+        }
 
         const eventsWithDvInCondition = this.getEventsInCondition(eventMetadata, options);
 
-        logger.info(`Matching events: ${eventsWithDvInCondition.length}`);
+        if (eventsWithDvInCondition.length === 0) {
+            this.logger.info("No events found with the specified condition");
+            return {
+                type: "success",
+                message: "No events found with the specified condition",
+            };
+        }
+
+        this.logger.info(`Matching events: ${eventsWithDvInCondition.length}`);
 
-        if (options.csvPath) {
-            logger.debug(`Generate report: ${options.csvPath}`);
+        if (options.reportPath) {
+            this.logger.debug(`Generate report: ${options.reportPath}`);
             await this.eventExportSpreadsheetRepository.saveReport(eventsWithDvInCondition, options);
         }
 
         if (options.post) {
-            logger.debug(`Events to change: ${eventsWithDvInCondition.length}`);
+            this.logger.debug(`Events to change: ${eventsWithDvInCondition.length}`);
             const result = await this.programEventsRepository.save(eventsWithDvInCondition);
             return result;
         } else {
@@ -71,7 +98,10 @@ export class UpdateEventDataValueUseCase {
             .filter(
                 event =>
                     event.dataValues.filter(
-                        dv => dv.dataElement.id === options.dataElementId && dv.value === options.newValue
+                        dv =>
+                            dv.dataElement.id === options.dataElementId &&
+                            dv.value === options.newValue &&
+                            (options.updateSameValue || dv.oldValue !== dv.value)
                     ).length > 0
             );
 

src/scripts/commands/events.ts

@@ -1,4 +1,6 @@
+import fs from "fs";
 import _ from "lodash";
+import CsvReadableStream from "csv-reader";
 import { command, string, subcommands, option, optional, flag } from "cmd-ts";
 
 import {
@@ -11,13 +13,15 @@ import {
 import { ProgramEventsD2Repository } from "data/ProgramEventsD2Repository";
 import { MoveEventsToOrgUnitUseCase } from "domain/usecases/MoveEventsToOrgUnitUseCase";
 import logger from "utils/log";
+import { TerminalLogger } from "utils/TerminalLogger";
 import { UpdateEventDataValueUseCase } from "domain/usecases/UpdateEventDataValueUseCase";
 import { EventExportSpreadsheetRepository } from "data/EventExportSpreadsheetRepository";
 import { DetectExternalOrgUnitUseCase } from "domain/usecases/ProcessEventsOutsideEnrollmentOrgUnitUseCase";
 import { ProgramsD2Repository } from "data/ProgramsD2Repository";
 import { RecodeBooleanDataValuesInEventsUseCase } from "domain/usecases/RecodeBooleanDataValuesInEventsUseCase";
 import { NotificationsEmailRepository } from "data/NotificationsEmailRepository";
 import { TrackedEntityD2Repository } from "data/TrackedEntityD2Repository";
+import { Id, Ref } from "domain/entities/Base";
 
 export function getCommand() {
     return subcommands({
@@ -110,11 +114,16 @@ const updateEventsDataValues = command({
     name: "Update events",
     description: "Update events that meet a condition",
     args: {
-        url: getApiUrlOption(),
-        eventIds: option({
-            type: StringsSeparatedByCommas,
+        ...getApiUrlOptions(),
+        eventIdsArray: option({
+            type: optional(StringsSeparatedByCommas),
             long: "event-ids",
-            description: "event id's separated by commas",
+            description: "event id's separated by commas (mutually exclusive with --events-csv)",
+        }),
+        eventsCsvPath: option({
+            type: optional(string),
+            long: "events-csv",
+            description: "Path to CSV file containing event IDs (mutually exclusive with --event-ids)",
         }),
         rootOrgUnit: option({
             type: string,
@@ -136,9 +145,9 @@ const updateEventsDataValues = command({
             long: "new-value",
             description: "New value for the data element",
         }),
-        csvPath: option({
+        reportPath: option({
             type: string,
-            long: "csv-path",
+            long: "report-path",
             description: "Path for the CSV report",
             defaultValue: () => "",
         }),
@@ -147,24 +156,48 @@ const updateEventsDataValues = command({
             description: "Save changes",
             defaultValue: () => false,
         }),
+        updateSameValue: flag({
+            long: "update-same-value",
+            description: "Update data values even if they are the same as the new value",
+            defaultValue: () => false,
+        }),
     },
     handler: async args => {
-        const api = getD2Api(args.url);
-        const programEventsRepository = new ProgramEventsD2Repository(api);
-        const eventExportSpreadsheetRepository = new EventExportSpreadsheetRepository();
-        const result = await new UpdateEventDataValueUseCase(
-            programEventsRepository,
-            eventExportSpreadsheetRepository
-        ).execute(args);
+        try {
+            if (args.eventIdsArray && args.eventsCsvPath) {
+                throw new Error("Cannot use both --event-ids and --events-csv at the same time");
+            }
 
-        logger.info(`Result: ${JSON.stringify(result, null, 2)}`);
+            let eventIds: Id[] = [];
+            if (args.eventIdsArray) {
+                eventIds = args.eventIdsArray;
+            } else if (args.eventsCsvPath) {
+                eventIds = await readEventsFile(args.eventsCsvPath);
+            } else {
+                throw new Error("Either --event-ids or --events-csv must be provided");
+            }
 
-        if (!args.post) {
-            logger.info(`Add --post to save changes`);
-        }
+            const api = getD2ApiFromArgs(args);
+            const programEventsRepository = new ProgramEventsD2Repository(api);
+            const eventExportSpreadsheetRepository = new EventExportSpreadsheetRepository();
+            const result = await new UpdateEventDataValueUseCase(
+                new TerminalLogger(),
+                programEventsRepository,
+                eventExportSpreadsheetRepository
+            ).execute({ ...args, eventIds });
+
+            logger.info(`Result: ${JSON.stringify(result, null, 2)}`);
 
-        if (!args.csvPath) {
-            logger.info(`Add --csv-path to generate a csv report`);
+            if (!args.post) {
+                logger.info(`Add --post to save changes`);
+            }
+
+            if (!args.reportPath) {
+                logger.info(`Add --report-path to generate a csv report`);
+            }
+        } catch (error) {
+            console.error((error as Error).message);
+            process.exit(1);
         }
     },
 });
@@ -196,3 +229,31 @@ const recodeBooleanDataValues = command({
         return new RecodeBooleanDataValuesInEventsUseCase(api, programsRepository).execute(args);
     },
 });
+
+async function readEventsFile(csvPath: string): Promise<Id[]> {
+    if (!fs.existsSync(csvPath) || !fs.statSync(csvPath).isFile()) {
+        throw new Error(`Can't find file: ${csvPath}`);
+    }
+
+    return new Promise((resolve, reject) => {
+        const eventIds: Id[] = [];
+
+        fs.createReadStream(csvPath, "utf8")
+            .pipe(new CsvReadableStream({ asObject: true, trim: true }))
+            .on("data", rawRow => {
+                const row = rawRow as unknown as Ref;
+                if (row.id) {
+                    eventIds.push(row.id);
+                }
+            })
+            .on("error", msg => {
+                return reject(msg);
+            })
+            .on("end", () => {
+                if (eventIds.length === 0) {
+                    return reject(new Error("No event IDs found to process"));
+                }
+                return resolve(eventIds);
+            });
+    });
+}