diff --git a/README.md b/README.md index 78cde7a..3238fa7 100644 --- a/README.md +++ b/README.md @@ -44,15 +44,19 @@ Available levels: 'debug' | 'info' | 'warn' | 'error' - [Create an SQL file to remove any orgunit below the country level](#create-an-sql-file-to-remove-any-orgunit-below-the-country-level) - [Create an SQL file to remove all org subunits of Canada](#create-an-sql-file-to-remove-all-org-subunits-of-canada) - [Copy the organisation units from a data set to one or more datasets](#copy-the-organisation-units-from-a-data-set-to-one-or-more-datasets) + - [Set all DataSets `skipOffline` where last data input period year was 'year' indicated or earlier](#set-all-datasets-skipoffline-where-last-data-input-period-year-was-year-indicated-or-earlier) - [Translations](#translations) - [Events](#events) + - [Detect events assigned to organisation units outside their enrollment](#detect-events-assigned-to-organisation-units-outside-their-enrollment) - [Move events from one orgunit to another](#move-events-from-one-orgunit-to-another) - [Update events which met the condition](#update-events-which-met-the-condition) + - [Recode boolean to ternary optionSet values](#recode-boolean-to-ternary-optionset-values) - [Data values](#data-values) - [Dangling data values](#dangling-data-values) - [Revert data values](#revert-data-values) - [Delete duplicated event data values](#delete-duplicated-event-data-values) - [Email notification for data values](#email-notification-for-data-values) + - [Delete all data values given a data elements file](#delete-all-data-values-given-a-data-elements-file) - [Notifications](#notifications) - [Send user info email](#send-user-info-email) - [Load testing](#load-testing) @@ -73,10 +77,14 @@ Available levels: 'debug' | 'info' | 'warn' | 'error' - [Transfer](#transfer) - [Options](#options) - [Rename](#rename) + - [Analyze](#analyze) - [Data](#data) - [Get report](#get-report) - [Enrollments](#enrollments) - [Close Enrollments with Events older than a date](#close-enrollments-with-events-older-than-a-date) +- [Category option combos](#category-option-combos) + - [Translations](#translations-1) + - [Regenerate](#regenerate) ## Execute Program Rules @@ -270,15 +278,34 @@ $ yarn start events move-to-org-unit \ ### Update events which met the condition +Updates the value for a collection of events that belongs to the provided data element and satisfies the condition. +The events must be provided as either a comma separated list (`--event-ids`) or a CSV with _id_ header (`--event-ids`). +Use `--report-path` to generate a CSV with a report of the changes. Unless `--post` is used the changes are not saved. +To update events that already have the desired value use `--update-same-value`. + +Using events list: ```shell $ yarn start events update-events \ --url='http://USER:PASSWORD@HOST:PORT' \ ---root-org-unit='org-unit-id' +--root-org-unit='org-unit-id' \ --event-ids='event_id_1,event_id_2,event_id_3' \ --data-element-id='data_element_id' \ --condition='true' \ --new-value='' \ ---csv-path='./events.csv' \ +--report-path='./report.csv' \ +--post +``` + +Using events CSV: +```shell +$ yarn start events update-events \ +--url='http://USER:PASSWORD@HOST:PORT' \ +--root-org-unit='org-unit-id' \ +--events-csv='./events.csv' \ +--data-element-id='data_element_id' \ +--condition='true' \ +--new-value='' \ +--report-path='./report.csv' \ --post ``` diff --git a/src/data/EventExportSpreadsheetRepository.ts b/src/data/EventExportSpreadsheetRepository.ts index ece10de..82d4695 100644 --- a/src/data/EventExportSpreadsheetRepository.ts +++ b/src/data/EventExportSpreadsheetRepository.ts @@ -8,7 +8,7 @@ import { ProgramEvent } from "domain/entities/ProgramEvent"; export class EventExportSpreadsheetRepository implements EventExportRepository { async saveReport(events: ProgramEvent[], options: MigrateOptions): Async { const csvWriter = createObjectCsvWriter({ - path: options.csvPath, + path: options.reportPath, header: [ { id: "event", diff --git a/src/domain/usecases/UpdateEventDataValueUseCase.ts b/src/domain/usecases/UpdateEventDataValueUseCase.ts index c2ebbde..2fec39b 100644 --- a/src/domain/usecases/UpdateEventDataValueUseCase.ts +++ b/src/domain/usecases/UpdateEventDataValueUseCase.ts @@ -2,7 +2,7 @@ import _ from "lodash"; import { Async } from "domain/entities/Async"; import { Id } from "domain/entities/Base"; import { Result } from "domain/entities/Result"; -import logger from "utils/log"; +import { Logger } from "domain/logger/Logger"; import { EventExportSpreadsheetRepository } from "data/EventExportSpreadsheetRepository"; import { ProgramEventsRepository } from "domain/repositories/ProgramEventsRepository"; import { ProgramEvent } from "domain/entities/ProgramEvent"; @@ -13,34 +13,61 @@ export type MigrateOptions = { dataElementId: Id; condition: string; newValue: string; - csvPath: string; + reportPath: string; post: boolean; + updateSameValue: boolean; }; export class UpdateEventDataValueUseCase { + private readonly eventChunkSize = 200; + constructor( + private logger: Logger, private programEventsRepository: ProgramEventsRepository, private eventExportSpreadsheetRepository: EventExportSpreadsheetRepository ) {} async execute(options: MigrateOptions): Async { - const eventMetadata = await this.programEventsRepository.get({ - eventsIds: options.eventIds, - orgUnitsIds: [options.rootOrgUnit], - orgUnitMode: "DESCENDANTS", - }); + const eventIdsLength = options.eventIds.length; + + let eventMetadata: ProgramEvent[] = []; + for (let i = 0; i < eventIdsLength; i += this.eventChunkSize) { + this.logger.debug( + `Fetching events metadata for events ${i + 1} to ${Math.min( + i + this.eventChunkSize, + eventIdsLength + )} of ${eventIdsLength}` + ); + + const eventIdsChunk = options.eventIds.slice(i, i + this.eventChunkSize); + const eventMetadataChunk = await this.programEventsRepository.get({ + eventsIds: eventIdsChunk, + orgUnitsIds: [options.rootOrgUnit], + orgUnitMode: "DESCENDANTS", + }); + + eventMetadata = eventMetadata.concat(eventMetadataChunk); + } const eventsWithDvInCondition = this.getEventsInCondition(eventMetadata, options); - logger.info(`Matching events: ${eventsWithDvInCondition.length}`); + if (eventsWithDvInCondition.length === 0) { + this.logger.info("No events found with the specified condition"); + return { + type: "success", + message: "No events found with the specified condition", + }; + } + + this.logger.info(`Matching events: ${eventsWithDvInCondition.length}`); - if (options.csvPath) { - logger.debug(`Generate report: ${options.csvPath}`); + if (options.reportPath) { + this.logger.debug(`Generate report: ${options.reportPath}`); await this.eventExportSpreadsheetRepository.saveReport(eventsWithDvInCondition, options); } if (options.post) { - logger.debug(`Events to change: ${eventsWithDvInCondition.length}`); + this.logger.debug(`Events to change: ${eventsWithDvInCondition.length}`); const result = await this.programEventsRepository.save(eventsWithDvInCondition); return result; } else { @@ -71,7 +98,10 @@ export class UpdateEventDataValueUseCase { .filter( event => event.dataValues.filter( - dv => dv.dataElement.id === options.dataElementId && dv.value === options.newValue + dv => + dv.dataElement.id === options.dataElementId && + dv.value === options.newValue && + (options.updateSameValue || dv.oldValue !== dv.value) ).length > 0 ); diff --git a/src/scripts/commands/events.ts b/src/scripts/commands/events.ts index 2d3ab22..5e11f0a 100644 --- a/src/scripts/commands/events.ts +++ b/src/scripts/commands/events.ts @@ -1,4 +1,6 @@ +import fs from "fs"; import _ from "lodash"; +import CsvReadableStream from "csv-reader"; import { command, string, subcommands, option, optional, flag } from "cmd-ts"; import { @@ -11,6 +13,7 @@ import { import { ProgramEventsD2Repository } from "data/ProgramEventsD2Repository"; import { MoveEventsToOrgUnitUseCase } from "domain/usecases/MoveEventsToOrgUnitUseCase"; import logger from "utils/log"; +import { TerminalLogger } from "utils/TerminalLogger"; import { UpdateEventDataValueUseCase } from "domain/usecases/UpdateEventDataValueUseCase"; import { EventExportSpreadsheetRepository } from "data/EventExportSpreadsheetRepository"; import { DetectExternalOrgUnitUseCase } from "domain/usecases/ProcessEventsOutsideEnrollmentOrgUnitUseCase"; @@ -18,6 +21,7 @@ import { ProgramsD2Repository } from "data/ProgramsD2Repository"; import { RecodeBooleanDataValuesInEventsUseCase } from "domain/usecases/RecodeBooleanDataValuesInEventsUseCase"; import { NotificationsEmailRepository } from "data/NotificationsEmailRepository"; import { TrackedEntityD2Repository } from "data/TrackedEntityD2Repository"; +import { Id, Ref } from "domain/entities/Base"; export function getCommand() { return subcommands({ @@ -110,11 +114,16 @@ const updateEventsDataValues = command({ name: "Update events", description: "Update events that meet a condition", args: { - url: getApiUrlOption(), - eventIds: option({ - type: StringsSeparatedByCommas, + ...getApiUrlOptions(), + eventIdsArray: option({ + type: optional(StringsSeparatedByCommas), long: "event-ids", - description: "event id's separated by commas", + description: "event id's separated by commas (mutually exclusive with --events-csv)", + }), + eventsCsvPath: option({ + type: optional(string), + long: "events-csv", + description: "Path to CSV file containing event IDs (mutually exclusive with --event-ids)", }), rootOrgUnit: option({ type: string, @@ -136,9 +145,9 @@ const updateEventsDataValues = command({ long: "new-value", description: "New value for the data element", }), - csvPath: option({ + reportPath: option({ type: string, - long: "csv-path", + long: "report-path", description: "Path for the CSV report", defaultValue: () => "", }), @@ -147,24 +156,48 @@ const updateEventsDataValues = command({ description: "Save changes", defaultValue: () => false, }), + updateSameValue: flag({ + long: "update-same-value", + description: "Update data values even if they are the same as the new value", + defaultValue: () => false, + }), }, handler: async args => { - const api = getD2Api(args.url); - const programEventsRepository = new ProgramEventsD2Repository(api); - const eventExportSpreadsheetRepository = new EventExportSpreadsheetRepository(); - const result = await new UpdateEventDataValueUseCase( - programEventsRepository, - eventExportSpreadsheetRepository - ).execute(args); + try { + if (args.eventIdsArray && args.eventsCsvPath) { + throw new Error("Cannot use both --event-ids and --events-csv at the same time"); + } - logger.info(`Result: ${JSON.stringify(result, null, 2)}`); + let eventIds: Id[] = []; + if (args.eventIdsArray) { + eventIds = args.eventIdsArray; + } else if (args.eventsCsvPath) { + eventIds = await readEventsFile(args.eventsCsvPath); + } else { + throw new Error("Either --event-ids or --events-csv must be provided"); + } - if (!args.post) { - logger.info(`Add --post to save changes`); - } + const api = getD2ApiFromArgs(args); + const programEventsRepository = new ProgramEventsD2Repository(api); + const eventExportSpreadsheetRepository = new EventExportSpreadsheetRepository(); + const result = await new UpdateEventDataValueUseCase( + new TerminalLogger(), + programEventsRepository, + eventExportSpreadsheetRepository + ).execute({ ...args, eventIds }); + + logger.info(`Result: ${JSON.stringify(result, null, 2)}`); - if (!args.csvPath) { - logger.info(`Add --csv-path to generate a csv report`); + if (!args.post) { + logger.info(`Add --post to save changes`); + } + + if (!args.reportPath) { + logger.info(`Add --report-path to generate a csv report`); + } + } catch (error) { + console.error((error as Error).message); + process.exit(1); } }, }); @@ -196,3 +229,31 @@ const recodeBooleanDataValues = command({ return new RecodeBooleanDataValuesInEventsUseCase(api, programsRepository).execute(args); }, }); + +async function readEventsFile(csvPath: string): Promise { + if (!fs.existsSync(csvPath) || !fs.statSync(csvPath).isFile()) { + throw new Error(`Can't find file: ${csvPath}`); + } + + return new Promise((resolve, reject) => { + const eventIds: Id[] = []; + + fs.createReadStream(csvPath, "utf8") + .pipe(new CsvReadableStream({ asObject: true, trim: true })) + .on("data", rawRow => { + const row = rawRow as unknown as Ref; + if (row.id) { + eventIds.push(row.id); + } + }) + .on("error", msg => { + return reject(msg); + }) + .on("end", () => { + if (eventIds.length === 0) { + return reject(new Error("No event IDs found to process")); + } + return resolve(eventIds); + }); + }); +}