Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions ARCHITECTURE.md
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ The TypeScript library (`@correlaid/formtransform`), split into **format modules

#### Format Modules

- **`src/xlsform/`** — load a workbook (`loader.ts`), parse its sheets (`parser.ts`), check it against the supported subset (`validate.ts`), sanitize names/codes (`sanitize.ts`).
- **`src/xlsform/`** — load a workbook and parse its sheets (`loader.ts`), check it against the supported subset (`validate.ts`), sanitize names/codes (`sanitize.ts`, `identifiers.ts`), row types (`types.ts`).

- **`src/lstsv/`** — read (`parser.ts`) and write (`serialize.ts`) LimeSurvey structure TSV, plus the reverse-subset check (`validate.ts`).

Expand Down Expand Up @@ -95,7 +95,7 @@ The library's own transformation tests are vitest under `tests/ts/` (`unit` / `i

## Pipeline Architecture

One module per supported direction. A pipeline owns everything cross-format; the format modules it draws on (`src/xlsform/`, `src/lstsv/`, `src/ddi/`) never import each other.
One module per supported direction. A pipeline owns everything cross-format; the format modules it draws on (`src/xlsform/`, `src/lstsv/`, `src/ddi/`) never import each other or a pipeline, and a pipeline never imports a sibling pipeline. Code both sides need lives in `src/conventions/`, `src/ddi/` (the `Variable` hub and its data CSV, `data.ts`), `src/diagnostics.ts` or `src/utils/`. ESLint enforces the rules (`no-restricted-imports`, plus `no-restricted-syntax` for dynamic `import()`); see the boundary block in `eslint.config.js`.

### DDI as the Hub

Expand Down
66 changes: 66 additions & 0 deletions eslint.config.js
Original file line number Diff line number Diff line change
@@ -1,11 +1,77 @@
import { readdirSync } from 'node:fs';

import { defineConfig } from 'eslint/config';
import globals from 'globals';
import js from '@eslint/js';
import tseslint from 'typescript-eslint';

const tsParser = tseslint.parser;

// Module boundaries (ARCHITECTURE.md): a format module never imports another
// format module or a pipeline, and a pipeline never imports a sibling pipeline.
// Shared code lives in src/conventions/, src/ddi/ (the Variable hub),
// src/diagnostics.ts or src/utils/.
const FORMATS = ['xlsform', 'lstsv', 'ddi'];
const PIPELINES = readdirSync(new URL('./src/pipelines/', import.meta.url), {
withFileTypes: true,
})
.filter((d) => d.isDirectory())
.map((d) => d.name);

const boundaryRules = [
...FORMATS.map((format) => ({
files: [`src/${format}/**/*.ts`],
rules: {
'no-restricted-imports': [
'error',
{
patterns: [
{
group: [
...FORMATS.filter((f) => f !== format).map((f) => `../${f}/*`),
'../pipelines/*',
],
message:
'A format module must not import another format module or a pipeline (ARCHITECTURE.md).',
},
],
},
],
// no-restricted-imports doesn't see dynamic import(); same rule for it.
'no-restricted-syntax': [
'error',
{
selector: `ImportExpression[source.value=/^\\.\\.\\/(${[...FORMATS.filter((f) => f !== format), 'pipelines'].join('|')})\\//]`,
message:
'A format module must not import another format module or a pipeline (ARCHITECTURE.md).',
},
],
},
})),
...PIPELINES.map((pipeline) => ({
files: [`src/pipelines/${pipeline}/**/*.ts`],
rules: {
'no-restricted-imports': [
'error',
{
patterns: [
{
group: PIPELINES.filter((p) => p !== pipeline).map(
(p) => `../${p}/*`,
),
message:
'A pipeline must not import a sibling pipeline; move shared code to src/conventions/, src/ddi/ or src/utils/ (ARCHITECTURE.md).',
},
],
},
],
},
})),
];

export default defineConfig([
...boundaryRules,

// Global: fail on eslint-disable directives that no longer suppress anything,
// so dead disables can't accumulate.
{
Expand Down
10 changes: 5 additions & 5 deletions src/pipelines/xlsform2ddi/data.ts → src/ddi/data.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,11 +11,11 @@
* codes (what Kobo and the LimeSurvey adapters both produce).
*/

import { splitDataVars } from '../../ddi/codebook.js';
import type { DataVarBuckets, OtherPattern } from '../../ddi/codebook.js';
import { classifyNotes } from '../../ddi/notes.js';
import { OTHER_CODE } from '../../conventions/other.js';
import type { Variable } from '../../ddi/types.js';
import { splitDataVars } from './codebook.js';
import type { DataVarBuckets, OtherPattern } from './codebook.js';
import { classifyNotes } from './notes.js';
import { OTHER_CODE } from '../conventions/other.js';
import type { Variable } from './types.js';

/** One raw response record, keyed by question name or `group/name` path. */
export type Submission = Record<string, unknown>;
Expand Down
10 changes: 10 additions & 0 deletions src/diagnostics.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
/**
* Findings the validators report. Shared by every format module, so it lives
* outside all of them.
*/

/** One finding: an `error` blocks a lossless conversion, a `warning` doesn't. */
export interface SubsetViolation {
severity: 'error' | 'warning';
message: string;
}
6 changes: 3 additions & 3 deletions src/index.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
// ── Format modules ─────────────────────────────────────────────────────
export { XLSLoader } from './xlsform/loader.js';
export { XLSFormParser } from './xlsform/parser.js';
export { XLSFormParser } from './pipelines/xlsform2lstsv/xlsformParser.js';
export { XLSValidator } from './xlsform/validate.js';
export type {
SubsetViolation,
Expand Down Expand Up @@ -54,8 +54,8 @@ export {
buildDataCsv,
getDdiColumnNames,
remapSubmissionsToDdi,
} from './pipelines/xlsform2ddi/data.js';
export type { Submission } from './pipelines/xlsform2ddi/data.js';
} from './ddi/data.js';
export type { Submission } from './ddi/data.js';
export { parseResponses } from './responseFile.js';
export { parseVocabCsv } from './vocab.js';

Expand Down
2 changes: 1 addition & 1 deletion src/lstsv/validate.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@

import { APPEARANCES } from '../generated/Appearances.js';
import { TYPE_MAPPINGS } from '../generated/TypeMappings.js';
import type { SubsetViolation } from '../xlsform/validate.js';
import type { SubsetViolation } from '../diagnostics.js';

// Supported LimeSurvey question-type codes: every code the registry maps a type
// to, plus every appearance's `lsTypeOverride` (`T` for `multiline`, `!` for
Expand Down
2 changes: 1 addition & 1 deletion src/pipelines/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ hub, not any one format.

## The DDI data file

`xlsform2ddi/data.ts` emits the response-data CSV that the codebook describes
`src/ddi/data.ts` emits the response-data CSV that the codebook describes
(`buildDataCsv`, plus `getDdiColumnNames` / `remapSubmissionsToDdi` for callers
writing the file themselves). It is schema-side-agnostic in the same way the XML
emitter is: it takes `Variable[]` and raw response rows, so either DDI pipeline
Expand Down
4 changes: 2 additions & 2 deletions src/pipelines/lstsv2ddi/data.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
/**
* LimeSurvey response export → submissions keyed by DDI variable name.
*
* `buildDataCsv` (`xlsform2ddi/data.ts`) reads rows keyed by bare question name
* `buildDataCsv` (`ddi/data.ts`) reads rows keyed by bare question name
* or `group/name`. A LimeSurvey response export (question-code headings, as the
* RemoteControl `export_responses` call and the admin CSV export produce) is
* keyed differently, so this adapter re-keys each row onto the variables
Expand All @@ -23,7 +23,7 @@
*/

import type { Variable } from '../../ddi/types.js';
import type { Submission } from '../xlsform2ddi/data.js';
import type { Submission } from '../../ddi/data.js';

import { OTHER_CODE, OTHER_SUFFIX } from '../../conventions/other.js';

Expand Down
4 changes: 2 additions & 2 deletions src/pipelines/lstsv2ddi/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,8 @@ import type { BuildDdiOptions } from '../../ddi/codebook.js';

import { parseLstsv } from '../../lstsv/parser.js';
import { validateLstsvSubset } from '../../lstsv/validate.js';
import { buildDataCsv } from '../xlsform2ddi/data.js';
import type { Submission } from '../xlsform2ddi/data.js';
import { buildDataCsv } from '../../ddi/data.js';
import type { Submission } from '../../ddi/data.js';
import { normalizeLimeSurveyResponses } from './data.js';
import type { NormalizeResponsesOptions } from './data.js';
import { lstsvToVariables } from './toVariables.js';
Expand Down
2 changes: 1 addition & 1 deletion src/pipelines/lstsv2xlsform/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@

import { parseLstsv } from '../../lstsv/parser.js';
import { validateLstsvSubset } from '../../lstsv/validate.js';
import type { SubsetViolation } from '../../xlsform/validate.js';
import type { SubsetViolation } from '../../diagnostics.js';

import { lstsvRowsToXlsform } from './toXlsform.js';
import type { XlsformOutput } from './toXlsform.js';
Expand Down
4 changes: 2 additions & 2 deletions src/pipelines/xlsform2ddi/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -39,8 +39,8 @@ export {
buildDataCsv,
getDdiColumnNames,
remapSubmissionsToDdi,
} from './data.js';
export type { Submission } from './data.js';
} from '../../ddi/data.js';
export type { Submission } from '../../ddi/data.js';

export {
extractVariables,
Expand Down
Original file line number Diff line number Diff line change
@@ -1,11 +1,14 @@
/**
* @file Main entrypoint of this library.
* Convenience wrapper: load an XLSForm workbook and convert it to LimeSurvey
* TSV in one call. Pipeline code, so it lives in the pipeline (a format module
* must not import one).
*/

import { ConversionConfig } from '../config/ConfigManager.js';
import type { ChoiceRow } from './types.js';
import type { LstsvConfig } from '../../config/types.js';
import type { ChoiceRow } from '../../xlsform/types.js';
import { XLSLoader } from '../../xlsform/loader.js';

import { XLSLoader } from './loader.js';
import { XLSFormToTSVConverter } from './index.js';

export class XLSFormParser {
/**
Expand All @@ -18,12 +21,9 @@ export class XLSFormParser {
*/
static async convertXLSFileToTSV(
filePath: string,
config?: Partial<ConversionConfig>,
config?: Partial<LstsvConfig>,
fileChoices?: Record<string, ChoiceRow[]>,
): Promise<string> {
const { XLSFormToTSVConverter } =
await import('../pipelines/xlsform2lstsv/index.js');

// Load data (validation is included by default)
const { surveyData, choicesData, settingsData } =
XLSLoader.parseXLSFile(filePath);
Expand All @@ -47,12 +47,9 @@ export class XLSFormParser {
*/
static async convertXLSDataToTSV(
data: Buffer | ArrayBuffer,
config?: Partial<ConversionConfig>,
config?: Partial<LstsvConfig>,
fileChoices?: Record<string, ChoiceRow[]>,
): Promise<string> {
const { XLSFormToTSVConverter } =
await import('../pipelines/xlsform2lstsv/index.js');

// Load data (validation is included by default)
const { surveyData, choicesData, settingsData } =
XLSLoader.parseXLSData(data);
Expand Down
2 changes: 1 addition & 1 deletion src/responseFile.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
* quotes wins.
*/

import type { Submission } from './pipelines/xlsform2ddi/data.js';
import type { Submission } from './ddi/data.js';

type Format = 'json' | 'csv';

Expand Down
6 changes: 2 additions & 4 deletions src/xlsform/validate.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import conventions from '../generated/conventions.js';
import type { SubsetViolation } from '../diagnostics.js';
import { APPEARANCES } from '../generated/Appearances.js';
import { TYPE_MAPPINGS } from '../generated/TypeMappings.js';

Expand Down Expand Up @@ -34,10 +35,7 @@ const STRUCTURAL = new Set([
const METADATA_TYPES = new Set<string>(METADATA_ROW_TYPES);

/** A single subset-validation finding. */
export interface SubsetViolation {
severity: 'error' | 'warning';
message: string;
}
export type { SubsetViolation } from '../diagnostics.js';

/** Options for {@link XLSValidator.validateSubset}. */
export interface SubsetOptions {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,13 @@ import {
buildDataCsv,
getDdiColumnNames,
remapSubmissionsToDdi,
} from '../../../../../src/pipelines/xlsform2ddi/data.js';
} from '../../../../src/ddi/data.js';
import {
buildDdiXml,
extractVariables,
choicesByListFromRows,
} from '../../../../../src/pipelines/xlsform2ddi/index.js';
import type { Variable } from '../../../../../src/ddi/types.js';
} from '../../../../src/pipelines/xlsform2ddi/index.js';
import type { Variable } from '../../../../src/ddi/types.js';

type Row = Record<string, unknown>;

Expand Down
Loading