refactor: one module per registry convention under src/conventions/

ARCHITECTURE.md

@@ -34,6 +34,8 @@ The TypeScript library (`@correlaid/formtransform`), split into **format modules
 
 - **`src/ddi/`** — the DDI-Codebook 2.5 emitter (`codebook.ts`, `xml.ts`, `notes.ts`) over the canonical `Variable[]` model (`types.ts`). This model is the hub: both DDI pipelines produce `Variable[]`, then one writer emits the XML.
 
+- **`src/conventions/`** — one module per registry convention (`other.ts`, `fromFile.ts`, `exclusive.ts`, `grid.ts`, `metadata.ts`). Each reads its values from `src/generated/conventions` and exposes helpers; every format module and pipeline imports them from here. No convention value (suffix, choice code, label, prefix) is written out anywhere else, which `tests/ts/unit/conventionLiterals.test.ts` enforces.
+
 #### Pipelines
 
 - **`src/pipelines/<source>2<target>/`** — One module per supported direction:

registry/conventions/externalCodeList.jsonld

@@ -34,6 +34,7 @@
         "ddiVocabFromFilename": "stripExtension",
         "ddiEmission": "concept[@vocab='<vocab>'] in place of inline catgry; catgry MUST NOT be emitted",
         "limesurveyEmission": "base select (L/M) with the vocabulary's options inlined as A/SQ rows + cssclass='cdlvocab-<vocab>' carrying provenance (a registered LimeSurvey question attribute, so it survives import)",
+        "limesurveyCssClassPrefix": "cdlvocab-",
         "structuralNote": "select_multiple_from_file emits a flat <var> with concept/@vocab, NOT a varGrp[@type='multipleResp'] with binary children. This differs from select_multiple.",
         "appliesTo": [
           "select_one_from_file",

registry/entities/grid/definition.jsonld

@@ -21,7 +21,8 @@
         }
       ],
       "trigger": {
-        "xlsformPattern": "begin_group row with appearance=table-list (or name contains 'grid', or label contains 'matrix')",
+        "appearance": "table-list",
+        "xlsformPattern": "begin_group row with appearance=table-list",
         "applyTo": "begin_group + all rows until matching end_group"
       },
       "input": {

registry/schema.jsonld

@@ -528,6 +528,15 @@
               "type": "object",
               "required": true,
               "fields": {
+                "appearance": {
+                  "type": "string",
+                  "required": true,
+                  "consumedBy": [
+                    "ls-transformer",
+                    "ddi-emitter"
+                  ],
+                  "description": "The begin_group appearance that makes a group this composite (machine-readable form of xlsformPattern)."
+                },
                 "xlsformPattern": {
                   "type": "string",
                   "required": true,

src/xlsform/exclusive.ts → src/conventions/exclusive.ts

@@ -5,7 +5,7 @@
  * `exclude_all_others` question attribute.
  */
 import conventions from '../generated/conventions.js';
-import type { ChoiceRow } from './types.js';
+import type { ChoiceRow } from '../xlsform/types.js';
 
 export const EXCLUSIVE_RULE = conventions.conventions.exclusiveChoice;
 

src/conventions/fromFile.ts

@@ -0,0 +1,47 @@
+/**
+ * `convention:externalCodeList`: `select_one_from_file <vocab>.csv` and
+ * `select_multiple_from_file`. LimeSurvey gets the base select with the
+ * vocabulary inlined and `cssclass="cdlvocab-<vocab>"` as provenance; DDI gets
+ * `<concept vocab>`.
+ */
+import conventions from '../generated/conventions.js';
+
+const RULE = conventions.conventions.externalCodeList;
+
+const SUFFIX = '_from_file';
+
+/** `select_*_from_file` type → the base select it is emitted as. */
+export const FROM_FILE_BASE: Readonly<Record<string, string>> =
+  Object.fromEntries(
+    RULE.appliesTo.map((t: string) => [t, t.slice(0, -SUFFIX.length)]),
+  );
+
+/** Whether an XLSForm base type is a `select_*_from_file`. */
+export function isFromFileType(baseType: string): boolean {
+  return baseType in FROM_FILE_BASE;
+}
+
+/** The `select_*_from_file` type for a base select (`select_one` → `select_one_from_file`). */
+export function fromFileTypeFor(baseSelect: string): string {
+  return baseSelect + SUFFIX;
+}
+
+/** Vocabulary id from a filename (`iso_3166_1.csv` → `iso_3166_1`). */
+export function vocabFromFilename(filename: string): string {
+  return filename.replace(/\.csv$/i, '');
+}
+
+/** Prefix of the LimeSurvey `cssclass` value carrying vocabulary provenance. */
+export const CDLVOCAB_PREFIX: string = RULE.limesurveyCssClassPrefix;
+
+/** The `cssclass` value recording a vocabulary. */
+export function cssClassForVocab(vocab: string): string {
+  return CDLVOCAB_PREFIX + vocab;
+}
+
+/** Vocabulary id from a `cssclass` value, or `''` when it carries none. */
+export function vocabFromCssClass(cssclass: string): string {
+  return cssclass.startsWith(CDLVOCAB_PREFIX)
+    ? cssclass.slice(CDLVOCAB_PREFIX.length)
+    : '';
+}

src/conventions/grid.ts

@@ -0,0 +1,17 @@
+/**
+ * The `grid` composite: a `begin_group` whose appearance marks it as a grid
+ * (one shared choice list, one row per question). LimeSurvey: array `F`;
+ * DDI: `<varGrp type="grid">`.
+ */
+import conventions from '../generated/conventions.js';
+
+const GRID = conventions.composites.find((c) => c.id === 'grid');
+if (!GRID) throw new Error('registry has no grid composite');
+
+/** The `begin_group` appearance that makes a group a grid. */
+export const GRID_APPEARANCE: string = GRID.trigger.appearance;
+
+/** Whether a group appearance (possibly several, space-separated) marks a grid. */
+export function isGridAppearance(appearance: string): boolean {
+  return appearance.split(/\s+/).includes(GRID_APPEARANCE);
+}

src/conventions/metadata.ts

@@ -0,0 +1,8 @@
+/**
+ * `convention:unregisteredRows`: XLSForm metadata rows (`start`, `end`,
+ * `today`, …) carry no question and are skipped by every emitter.
+ */
+import conventions from '../generated/conventions.js';
+
+export const METADATA_ROW_TYPES: readonly string[] =
+  conventions.conventions.unregisteredRows.metadataRowTypes;

src/conventions/other.ts

@@ -0,0 +1,35 @@
+/**
+ * `convention:other`: the semi-open "Other" answer. XLSForm writes it as a
+ * choice with code `other` plus a `<question>_other` text companion;
+ * LimeSurvey as `other=Y`; DDI as `<varGrp type="other">`.
+ */
+import conventions from '../generated/conventions.js';
+
+const RULE = conventions.conventions.other;
+
+/** Choice code of the "other" answer. */
+export const OTHER_CODE: string = RULE.choiceCode;
+
+/** Suffix of the free-text companion question (`<question>_other`). */
+export const OTHER_SUFFIX: string = RULE.companionSuffix;
+
+/** Type of the free-text companion question. */
+export const OTHER_COMPANION_TYPE: string = RULE.companionType;
+
+/** Select types that can carry an "other" answer. */
+export const OTHER_APPLIES_TO: readonly string[] = RULE.appliesTo;
+
+/** Canonical "other" label per language. */
+export const OTHER_LABELS: Readonly<Record<string, string>> = RULE.labels;
+
+/** Canonical "other" label for a language, falling back to English. */
+export function otherLabelFor(lang: string): string {
+  return OTHER_LABELS[lang] ?? OTHER_LABELS['en'];
+}
+
+/** The base question name of an `<base>_other` companion, or `null`. */
+export function otherCompanionBase(name: string): string | null {
+  return name.endsWith(OTHER_SUFFIX) && name.length > OTHER_SUFFIX.length
+    ? name.slice(0, -OTHER_SUFFIX.length)
+    : null;
+}

src/ddi/codebook.ts

@@ -12,6 +12,12 @@
 
 import { DDI_TYPE_MAP, RESPONSE_DOMAIN_MAP } from '../generated/DdiMappings.js';
 
+import {
+  OTHER_APPLIES_TO,
+  OTHER_CODE,
+  otherCompanionBase,
+} from '../conventions/other.js';
+import { isGridAppearance } from '../conventions/grid.js';
 import { XmlElement } from './xml.js';
 import { classifyNotes } from './notes.js';
 import { Choice, Variable } from './types.js';
@@ -38,7 +44,7 @@ function makeGrpId(name: string): string {
 function isGridGroup(variables: Variable[], groupName: string): boolean {
   for (const v of variables) {
     if (v.group === groupName && v.groupAppearance) {
-      return v.groupAppearance.includes('table-list');
+      return isGridAppearance(v.groupAppearance);
     }
   }
   return false;
@@ -141,15 +147,11 @@ function detectOtherPatterns(variables: Variable[]): Map<string, OtherPattern> {
   const byName = new Map(variables.map((v) => [v.name, v]));
   const patterns = new Map<string, OtherPattern>();
   for (const v of variables) {
-    if (v.type !== 'text' || !v.name.endsWith('_other')) continue;
-    const baseName = v.name.slice(0, -'_other'.length);
+    const baseName = v.type === 'text' ? otherCompanionBase(v.name) : null;
+    if (!baseName) continue;
     const base = byName.get(baseName);
-    if (
-      !base ||
-      (base.type !== 'select_one' && base.type !== 'select_multiple')
-    )
-      continue;
-    if (!base.choices.some((c) => c.name === 'other')) continue;
+    if (!base || !OTHER_APPLIES_TO.includes(base.type)) continue;
+    if (!base.choices.some((c) => c.name === OTHER_CODE)) continue;
     patterns.set(baseName, {
       base,
       otherVar: v,
@@ -166,7 +168,7 @@ function emitOtherPattern(dataDscr: XmlElement, p: OtherPattern): void {
   const baseName = base.name;
 
   if (p.isMulti) {
-    const nonOther = base.choices.filter((c) => c.name !== 'other');
+    const nonOther = base.choices.filter((c) => c.name !== OTHER_CODE);
     const childName = `${baseName}_choices`;
     const childId = makeGrpId(childName);
     const childMembers = nonOther
@@ -210,7 +212,7 @@ function emitOtherPatternVars(dataDscr: XmlElement, p: OtherPattern): void {
 
   if (p.isMulti) {
     for (const choice of base.choices) {
-      if (choice.name === 'other') continue;
+      if (choice.name === OTHER_CODE) continue;
       addBinaryVar(
         dataDscr,
         makeVarId(`${baseName}_${choice.name}`),

src/generated/conventions.json

@@ -23,6 +23,7 @@
       "ddiVocabFromFilename": "stripExtension",
       "ddiEmission": "concept[@vocab='<vocab>'] in place of inline catgry; catgry MUST NOT be emitted",
       "limesurveyEmission": "base select (L/M) with the vocabulary's options inlined as A/SQ rows + cssclass='cdlvocab-<vocab>' carrying provenance (a registered LimeSurvey question attribute, so it survives import)",
+      "limesurveyCssClassPrefix": "cdlvocab-",
       "structuralNote": "select_multiple_from_file emits a flat <var> with concept/@vocab, NOT a varGrp[@type='multipleResp'] with binary children. This differs from select_multiple.",
       "appliesTo": [
         "select_one_from_file",
@@ -304,7 +305,8 @@
         }
       ],
       "trigger": {
-        "xlsformPattern": "begin_group row with appearance=table-list (or name contains 'grid', or label contains 'matrix')",
+        "appearance": "table-list",
+        "xlsformPattern": "begin_group row with appearance=table-list",
         "applyTo": "begin_group + all rows until matching end_group"
       },
       "input": {

src/generated/conventions.ts

@@ -32,6 +32,7 @@ const conventions = {
       "ddiVocabFromFilename": "stripExtension",
       "ddiEmission": "concept[@vocab='<vocab>'] in place of inline catgry; catgry MUST NOT be emitted",
       "limesurveyEmission": "base select (L/M) with the vocabulary's options inlined as A/SQ rows + cssclass='cdlvocab-<vocab>' carrying provenance (a registered LimeSurvey question attribute, so it survives import)",
+      "limesurveyCssClassPrefix": "cdlvocab-",
       "structuralNote": "select_multiple_from_file emits a flat <var> with concept/@vocab, NOT a varGrp[@type='multipleResp'] with binary children. This differs from select_multiple.",
       "appliesTo": [
         "select_one_from_file",
@@ -313,7 +314,8 @@ const conventions = {
         }
       ],
       "trigger": {
-        "xlsformPattern": "begin_group row with appearance=table-list (or name contains 'grid', or label contains 'matrix')",
+        "appearance": "table-list",
+        "xlsformPattern": "begin_group row with appearance=table-list",
         "applyTo": "begin_group + all rows until matching end_group"
       },
       "input": {

src/pipelines/lstsv2ddi/data.ts

@@ -25,7 +25,7 @@
 import type { Variable } from '../../ddi/types.js';
 import type { Submission } from '../xlsform2ddi/data.js';
 
-import { OTHER_CODE, OTHER_SUFFIX } from './toVariables.js';
+import { OTHER_CODE, OTHER_SUFFIX } from '../../conventions/other.js';
 
 /** LimeSurvey's stored value for the "other" option of a list question. */
 const LS_OTHER_VALUE = '-oth-';

src/pipelines/lstsv2ddi/toVariables.ts

@@ -16,30 +16,18 @@
  *   - `F` (array)          → grid group of `select_one` variables
  */
 
-import conventions from '../../generated/conventions.js';
-
 import { Choice, Variable } from '../../ddi/types.js';
 import { APPEARANCES } from '../../generated/Appearances.js';
-
-// Semi-open "other" convention: LimeSurvey carries it as a native `other=Y`
-// flag (no code/label in the TSV), so the reverse path re-adds the `other`
-// category from these per-language labels and rebuilds the `<base>_other`
-// companion so the DDI emitter detects the pattern.
-const OTHER = conventions.conventions.other as {
-  choiceCode?: string;
-  companionSuffix?: string;
-  labels?: Record<string, string>;
-};
-/** LS answer code for the collapsed "other" choice (`convention:other`). */
-export const OTHER_CODE = OTHER.choiceCode ?? 'other';
-/** XLSForm companion-question suffix for the "other" free-text follow-up. */
-export const OTHER_SUFFIX = OTHER.companionSuffix ?? '_other';
-const OTHER_LABELS = OTHER.labels ?? {};
-
-/** Canonical "other" category label for a survey language (falls back to en). */
-export function otherLabelFor(lang: string): string {
-  return OTHER_LABELS[lang] ?? OTHER_LABELS['en'] ?? 'Other';
-}
+import {
+  OTHER_CODE,
+  OTHER_SUFFIX,
+  otherLabelFor,
+} from '../../conventions/other.js';
+import {
+  fromFileTypeFor,
+  vocabFromCssClass,
+} from '../../conventions/fromFile.js';
+import { GRID_APPEARANCE } from '../../conventions/grid.js';
 
 /** LimeSurvey question-type code → canonical standardized type slug. */
 const LS_TO_STD: Record<string, string> = {
@@ -69,26 +57,10 @@ for (const spec of Object.values(APPEARANCES)) {
 
 type Row = Record<string, string>;
 
-/** Prefix of the `cssclass` value carrying external-vocab provenance. */
-export const CDLVOCAB_PREFIX = 'cdlvocab-';
-
 function cell(row: Row, key: string): string {
   return (row[key] ?? '').trim();
 }
 
-/**
- * Extract the vocabulary id from a Q-row's `cssclass` attribute, or `''`.
- *
- * `select_*_from_file` questions inline their options as A/SQ rows and record
- * provenance as `cssclass="cdlvocab-<id>"` (a registered LimeSurvey attribute
- * that survives import) — see `xlsformConverter`.
- */
-export function vocabFromCssClass(cssclass: string): string {
-  return cssclass.startsWith(CDLVOCAB_PREFIX)
-    ? cssclass.slice(CDLVOCAB_PREFIX.length)
-    : '';
-}
-
 /**
  * Build a non-array question {@link Variable} from its Q-row fields. A non-empty
  * `cdlVocab` marks a `select_*_from_file` question: its type gains the
@@ -108,8 +80,9 @@ function buildQuestionVar(
 
   if (cdlVocab) {
     vocab = cdlVocab;
-    if (type === 'select_one') type = 'select_one_from_file';
-    else if (type === 'select_multiple') type = 'select_multiple_from_file';
+    if (type === 'select_one' || type === 'select_multiple') {
+      type = fromFileTypeFor(type);
+    }
   } else if (type === 'select_one' || type === 'select_multiple') {
     // Synthetic list keyed by the question; choices fill from A/SQ rows.
     listName = name;
@@ -276,7 +249,7 @@ function drainArray(variables: Variable[], array: ArrayAccumulator): void {
       label: sq.label,
       group: array.group,
       groupLabel: array.label,
-      groupAppearance: 'table-list',
+      groupAppearance: GRID_APPEARANCE,
       listName: array.name,
       vocab: '',
       choices: array.answers.map((a) => ({ ...a })),

src/pipelines/lstsv2xlsform/toXlsform.ts

@@ -28,13 +28,17 @@ import { defaultConfig } from '../../config/types.js';
 import type { SurveyRow, ChoiceRow, SettingsRow } from '../../xlsform/types.js';
 import { APPEARANCES } from '../../generated/Appearances.js';
 import { TYPE_MAPPINGS } from '../../generated/TypeMappings.js';
-import { EXCLUSIVE_RULE } from '../../xlsform/exclusive.js';
+import { EXCLUSIVE_RULE } from '../../conventions/exclusive.js';
 import {
   OTHER_CODE,
   OTHER_SUFFIX,
   otherLabelFor,
+} from '../../conventions/other.js';
+import {
+  fromFileTypeFor,
   vocabFromCssClass,
-} from '../lstsv2ddi/toVariables.js';
+} from '../../conventions/fromFile.js';
+import { GRID_APPEARANCE } from '../../conventions/grid.js';
 
 import { formatDefaultLanguage } from './languageNames.js';
 import { htmlToMarkdown } from '../../utils/markdownRenderer.js';
@@ -474,7 +478,7 @@ function emitBucketOpen(ctx: BucketOpenCtx): {
   let groupAppearance: string | undefined;
   if (items.length === 1 && arrayItem) {
     groupName = arrayItem.name;
-    groupAppearance = 'table-list';
+    groupAppearance = GRID_APPEARANCE;
   } else {
     groupName = slugifyGroupName(groupLabelText);
   }
@@ -619,10 +623,7 @@ function composeTypeWithList(
 ): { type: string; emittedChoices: boolean } {
   const vocab = vocabFromCssClass(item.cssclass);
   if (vocab) {
-    const fromFile =
-      base === 'select_one'
-        ? 'select_one_from_file'
-        : 'select_multiple_from_file';
+    const fromFile = fromFileTypeFor(base);
     return { type: `${fromFile} ${vocab}.csv`, emittedChoices: false };
   }
   if (base === 'select_one' || base === 'select_multiple') {