diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index c3e9c97..d2c3c96 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -28,7 +28,7 @@ The TypeScript library (`@correlaid/formtransform`), split into **format modules #### Format Modules -- **`src/xlsform/`** — load a workbook (`loader.ts`), parse its sheets (`parser.ts`), check it against the supported subset (`validate.ts`), sanitize names/codes (`sanitize.ts`). +- **`src/xlsform/`** — load a workbook and parse its sheets (`loader.ts`), check it against the supported subset (`validate.ts`), sanitize names/codes (`sanitize.ts`, `identifiers.ts`), row types (`types.ts`). - **`src/lstsv/`** — read (`parser.ts`) and write (`serialize.ts`) LimeSurvey structure TSV, plus the reverse-subset check (`validate.ts`). @@ -95,7 +95,7 @@ The library's own transformation tests are vitest under `tests/ts/` (`unit` / `i ## Pipeline Architecture -One module per supported direction. A pipeline owns everything cross-format; the format modules it draws on (`src/xlsform/`, `src/lstsv/`, `src/ddi/`) never import each other. +One module per supported direction. A pipeline owns everything cross-format; the format modules it draws on (`src/xlsform/`, `src/lstsv/`, `src/ddi/`) never import each other or a pipeline, and a pipeline never imports a sibling pipeline. Code both sides need lives in `src/conventions/`, `src/ddi/` (the `Variable` hub and its data CSV, `data.ts`), `src/diagnostics.ts` or `src/utils/`. ESLint enforces the rules (`no-restricted-imports`, plus `no-restricted-syntax` for dynamic `import()`); see the boundary block in `eslint.config.js`. ### DDI as the Hub diff --git a/eslint.config.js b/eslint.config.js index 598e7e3..a809538 100644 --- a/eslint.config.js +++ b/eslint.config.js @@ -1,3 +1,5 @@ +import { readdirSync } from 'node:fs'; + import { defineConfig } from 'eslint/config'; import globals from 'globals'; import js from '@eslint/js'; @@ -5,7 +7,71 @@ import tseslint from 'typescript-eslint'; const tsParser = tseslint.parser; +// Module boundaries (ARCHITECTURE.md): a format module never imports another +// format module or a pipeline, and a pipeline never imports a sibling pipeline. +// Shared code lives in src/conventions/, src/ddi/ (the Variable hub), +// src/diagnostics.ts or src/utils/. +const FORMATS = ['xlsform', 'lstsv', 'ddi']; +const PIPELINES = readdirSync(new URL('./src/pipelines/', import.meta.url), { + withFileTypes: true, +}) + .filter((d) => d.isDirectory()) + .map((d) => d.name); + +const boundaryRules = [ + ...FORMATS.map((format) => ({ + files: [`src/${format}/**/*.ts`], + rules: { + 'no-restricted-imports': [ + 'error', + { + patterns: [ + { + group: [ + ...FORMATS.filter((f) => f !== format).map((f) => `../${f}/*`), + '../pipelines/*', + ], + message: + 'A format module must not import another format module or a pipeline (ARCHITECTURE.md).', + }, + ], + }, + ], + // no-restricted-imports doesn't see dynamic import(); same rule for it. + 'no-restricted-syntax': [ + 'error', + { + selector: `ImportExpression[source.value=/^\\.\\.\\/(${[...FORMATS.filter((f) => f !== format), 'pipelines'].join('|')})\\//]`, + message: + 'A format module must not import another format module or a pipeline (ARCHITECTURE.md).', + }, + ], + }, + })), + ...PIPELINES.map((pipeline) => ({ + files: [`src/pipelines/${pipeline}/**/*.ts`], + rules: { + 'no-restricted-imports': [ + 'error', + { + patterns: [ + { + group: PIPELINES.filter((p) => p !== pipeline).map( + (p) => `../${p}/*`, + ), + message: + 'A pipeline must not import a sibling pipeline; move shared code to src/conventions/, src/ddi/ or src/utils/ (ARCHITECTURE.md).', + }, + ], + }, + ], + }, + })), +]; + export default defineConfig([ + ...boundaryRules, + // Global: fail on eslint-disable directives that no longer suppress anything, // so dead disables can't accumulate. { diff --git a/src/pipelines/xlsform2ddi/data.ts b/src/ddi/data.ts similarity index 95% rename from src/pipelines/xlsform2ddi/data.ts rename to src/ddi/data.ts index 46f1c2e..c8a6510 100644 --- a/src/pipelines/xlsform2ddi/data.ts +++ b/src/ddi/data.ts @@ -11,11 +11,11 @@ * codes (what Kobo and the LimeSurvey adapters both produce). */ -import { splitDataVars } from '../../ddi/codebook.js'; -import type { DataVarBuckets, OtherPattern } from '../../ddi/codebook.js'; -import { classifyNotes } from '../../ddi/notes.js'; -import { OTHER_CODE } from '../../conventions/other.js'; -import type { Variable } from '../../ddi/types.js'; +import { splitDataVars } from './codebook.js'; +import type { DataVarBuckets, OtherPattern } from './codebook.js'; +import { classifyNotes } from './notes.js'; +import { OTHER_CODE } from '../conventions/other.js'; +import type { Variable } from './types.js'; /** One raw response record, keyed by question name or `group/name` path. */ export type Submission = Record; diff --git a/src/diagnostics.ts b/src/diagnostics.ts new file mode 100644 index 0000000..c5efd35 --- /dev/null +++ b/src/diagnostics.ts @@ -0,0 +1,10 @@ +/** + * Findings the validators report. Shared by every format module, so it lives + * outside all of them. + */ + +/** One finding: an `error` blocks a lossless conversion, a `warning` doesn't. */ +export interface SubsetViolation { + severity: 'error' | 'warning'; + message: string; +} diff --git a/src/index.ts b/src/index.ts index 936ff31..0516b7f 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,6 +1,6 @@ // ── Format modules ───────────────────────────────────────────────────── export { XLSLoader } from './xlsform/loader.js'; -export { XLSFormParser } from './xlsform/parser.js'; +export { XLSFormParser } from './pipelines/xlsform2lstsv/xlsformParser.js'; export { XLSValidator } from './xlsform/validate.js'; export type { SubsetViolation, @@ -54,8 +54,8 @@ export { buildDataCsv, getDdiColumnNames, remapSubmissionsToDdi, -} from './pipelines/xlsform2ddi/data.js'; -export type { Submission } from './pipelines/xlsform2ddi/data.js'; +} from './ddi/data.js'; +export type { Submission } from './ddi/data.js'; export { parseResponses } from './responseFile.js'; export { parseVocabCsv } from './vocab.js'; diff --git a/src/lstsv/validate.ts b/src/lstsv/validate.ts index f5a17ba..e25e2b6 100644 --- a/src/lstsv/validate.ts +++ b/src/lstsv/validate.ts @@ -13,7 +13,7 @@ import { APPEARANCES } from '../generated/Appearances.js'; import { TYPE_MAPPINGS } from '../generated/TypeMappings.js'; -import type { SubsetViolation } from '../xlsform/validate.js'; +import type { SubsetViolation } from '../diagnostics.js'; // Supported LimeSurvey question-type codes: every code the registry maps a type // to, plus every appearance's `lsTypeOverride` (`T` for `multiline`, `!` for diff --git a/src/pipelines/README.md b/src/pipelines/README.md index 29a9915..9a257f5 100644 --- a/src/pipelines/README.md +++ b/src/pipelines/README.md @@ -17,7 +17,7 @@ hub, not any one format. ## The DDI data file -`xlsform2ddi/data.ts` emits the response-data CSV that the codebook describes +`src/ddi/data.ts` emits the response-data CSV that the codebook describes (`buildDataCsv`, plus `getDdiColumnNames` / `remapSubmissionsToDdi` for callers writing the file themselves). It is schema-side-agnostic in the same way the XML emitter is: it takes `Variable[]` and raw response rows, so either DDI pipeline diff --git a/src/pipelines/lstsv2ddi/data.ts b/src/pipelines/lstsv2ddi/data.ts index 4629b8a..dbbcf18 100644 --- a/src/pipelines/lstsv2ddi/data.ts +++ b/src/pipelines/lstsv2ddi/data.ts @@ -1,7 +1,7 @@ /** * LimeSurvey response export → submissions keyed by DDI variable name. * - * `buildDataCsv` (`xlsform2ddi/data.ts`) reads rows keyed by bare question name + * `buildDataCsv` (`ddi/data.ts`) reads rows keyed by bare question name * or `group/name`. A LimeSurvey response export (question-code headings, as the * RemoteControl `export_responses` call and the admin CSV export produce) is * keyed differently, so this adapter re-keys each row onto the variables @@ -23,7 +23,7 @@ */ import type { Variable } from '../../ddi/types.js'; -import type { Submission } from '../xlsform2ddi/data.js'; +import type { Submission } from '../../ddi/data.js'; import { OTHER_CODE, OTHER_SUFFIX } from '../../conventions/other.js'; diff --git a/src/pipelines/lstsv2ddi/index.ts b/src/pipelines/lstsv2ddi/index.ts index 3a4a746..ae2d0e2 100644 --- a/src/pipelines/lstsv2ddi/index.ts +++ b/src/pipelines/lstsv2ddi/index.ts @@ -12,8 +12,8 @@ import type { BuildDdiOptions } from '../../ddi/codebook.js'; import { parseLstsv } from '../../lstsv/parser.js'; import { validateLstsvSubset } from '../../lstsv/validate.js'; -import { buildDataCsv } from '../xlsform2ddi/data.js'; -import type { Submission } from '../xlsform2ddi/data.js'; +import { buildDataCsv } from '../../ddi/data.js'; +import type { Submission } from '../../ddi/data.js'; import { normalizeLimeSurveyResponses } from './data.js'; import type { NormalizeResponsesOptions } from './data.js'; import { lstsvToVariables } from './toVariables.js'; diff --git a/src/pipelines/lstsv2xlsform/index.ts b/src/pipelines/lstsv2xlsform/index.ts index d04205c..3d2f488 100644 --- a/src/pipelines/lstsv2xlsform/index.ts +++ b/src/pipelines/lstsv2xlsform/index.ts @@ -9,7 +9,7 @@ import { parseLstsv } from '../../lstsv/parser.js'; import { validateLstsvSubset } from '../../lstsv/validate.js'; -import type { SubsetViolation } from '../../xlsform/validate.js'; +import type { SubsetViolation } from '../../diagnostics.js'; import { lstsvRowsToXlsform } from './toXlsform.js'; import type { XlsformOutput } from './toXlsform.js'; diff --git a/src/pipelines/xlsform2ddi/index.ts b/src/pipelines/xlsform2ddi/index.ts index 5dc7157..7a28041 100644 --- a/src/pipelines/xlsform2ddi/index.ts +++ b/src/pipelines/xlsform2ddi/index.ts @@ -39,8 +39,8 @@ export { buildDataCsv, getDdiColumnNames, remapSubmissionsToDdi, -} from './data.js'; -export type { Submission } from './data.js'; +} from '../../ddi/data.js'; +export type { Submission } from '../../ddi/data.js'; export { extractVariables, diff --git a/src/xlsform/parser.ts b/src/pipelines/xlsform2lstsv/xlsformParser.ts similarity index 77% rename from src/xlsform/parser.ts rename to src/pipelines/xlsform2lstsv/xlsformParser.ts index c6dfdff..66553d9 100644 --- a/src/xlsform/parser.ts +++ b/src/pipelines/xlsform2lstsv/xlsformParser.ts @@ -1,11 +1,14 @@ /** - * @file Main entrypoint of this library. + * Convenience wrapper: load an XLSForm workbook and convert it to LimeSurvey + * TSV in one call. Pipeline code, so it lives in the pipeline (a format module + * must not import one). */ -import { ConversionConfig } from '../config/ConfigManager.js'; -import type { ChoiceRow } from './types.js'; +import type { LstsvConfig } from '../../config/types.js'; +import type { ChoiceRow } from '../../xlsform/types.js'; +import { XLSLoader } from '../../xlsform/loader.js'; -import { XLSLoader } from './loader.js'; +import { XLSFormToTSVConverter } from './index.js'; export class XLSFormParser { /** @@ -18,12 +21,9 @@ export class XLSFormParser { */ static async convertXLSFileToTSV( filePath: string, - config?: Partial, + config?: Partial, fileChoices?: Record, ): Promise { - const { XLSFormToTSVConverter } = - await import('../pipelines/xlsform2lstsv/index.js'); - // Load data (validation is included by default) const { surveyData, choicesData, settingsData } = XLSLoader.parseXLSFile(filePath); @@ -47,12 +47,9 @@ export class XLSFormParser { */ static async convertXLSDataToTSV( data: Buffer | ArrayBuffer, - config?: Partial, + config?: Partial, fileChoices?: Record, ): Promise { - const { XLSFormToTSVConverter } = - await import('../pipelines/xlsform2lstsv/index.js'); - // Load data (validation is included by default) const { surveyData, choicesData, settingsData } = XLSLoader.parseXLSData(data); diff --git a/src/responseFile.ts b/src/responseFile.ts index ab11fe4..4445024 100644 --- a/src/responseFile.ts +++ b/src/responseFile.ts @@ -14,7 +14,7 @@ * quotes wins. */ -import type { Submission } from './pipelines/xlsform2ddi/data.js'; +import type { Submission } from './ddi/data.js'; type Format = 'json' | 'csv'; diff --git a/src/xlsform/validate.ts b/src/xlsform/validate.ts index 4e0747c..a7df6a2 100644 --- a/src/xlsform/validate.ts +++ b/src/xlsform/validate.ts @@ -1,4 +1,5 @@ import conventions from '../generated/conventions.js'; +import type { SubsetViolation } from '../diagnostics.js'; import { APPEARANCES } from '../generated/Appearances.js'; import { TYPE_MAPPINGS } from '../generated/TypeMappings.js'; @@ -34,10 +35,7 @@ const STRUCTURAL = new Set([ const METADATA_TYPES = new Set(METADATA_ROW_TYPES); /** A single subset-validation finding. */ -export interface SubsetViolation { - severity: 'error' | 'warning'; - message: string; -} +export type { SubsetViolation } from '../diagnostics.js'; /** Options for {@link XLSValidator.validateSubset}. */ export interface SubsetOptions { diff --git a/tests/ts/unit/pipelines/xlsform2ddi/data.test.ts b/tests/ts/unit/ddi/data.test.ts similarity index 97% rename from tests/ts/unit/pipelines/xlsform2ddi/data.test.ts rename to tests/ts/unit/ddi/data.test.ts index 26065e3..8b46b18 100644 --- a/tests/ts/unit/pipelines/xlsform2ddi/data.test.ts +++ b/tests/ts/unit/ddi/data.test.ts @@ -5,13 +5,13 @@ import { buildDataCsv, getDdiColumnNames, remapSubmissionsToDdi, -} from '../../../../../src/pipelines/xlsform2ddi/data.js'; +} from '../../../../src/ddi/data.js'; import { buildDdiXml, extractVariables, choicesByListFromRows, -} from '../../../../../src/pipelines/xlsform2ddi/index.js'; -import type { Variable } from '../../../../../src/ddi/types.js'; +} from '../../../../src/pipelines/xlsform2ddi/index.js'; +import type { Variable } from '../../../../src/ddi/types.js'; type Row = Record;