Legacy conversion revamp

.github/workflows/run_unit_tests.yml

@@ -38,7 +38,7 @@ jobs:
         pip install ${{ matrix.dependencies }}
     - name: Lint with flake8
       run: |
-        flake8 --exclude='bin,build,.eggs,src/highdicom/_*'
+        flake8 --exclude='bin,build,.eggs'
     - name: Test with pytest
       run: |
         pytest --cov=highdicom --cov-fail-under=80

bin/create_iods_modules.py

@@ -12,11 +12,12 @@
 logger = logging.getLogger(__name__)
 
 
-PGK_PATH = os.path.join(
+PKG_JSON_PATH = os.path.join(
     os.path.dirname(__file__),
     '..',
     'src',
-    'highdicom'
+    'highdicom',
+    '_standard',
 )
 
 
@@ -25,10 +26,6 @@ def _load_json_from_file(filename):
         return json.load(f)
 
 
-def _dump_json(data):
-    return json.dumps(data, indent=4, sort_keys=True)
-
-
 def _create_sop_to_iods(directory):
     filename = os.path.join(directory, 'sops.json')
     sops = _load_json_from_file(filename)
@@ -97,14 +94,16 @@ def _create_modules(directory):
     return modules
 
 
-if __name__ == '__main__':
-
+if __name__ == "__main__":
     logging.basicConfig()
     logger.setLevel(logging.DEBUG)
 
     # Positional argument is path to directory containing JSON files generated
     # using the dicom-standard Python package, see
     # https://github.com/innolitics/dicom-standard/tree/master/standard
+    # If generating yourself using the instructions in that repo, be sure to
+    # use "make all" instead of just "make", otherwise some of the required
+    # files are skipped
     try:
         directory = sys.argv[1]
     except IndexError as e:
@@ -119,37 +118,16 @@ def _create_modules(directory):
     current_time = datetime.datetime.time(now).strftime('%H:%M:%S')
 
     iods = _create_iods(directory)
-    iods_docstr = '\n'.join([
-        '"""DICOM Information Object Definitions (IODs)',
-        f'auto-generated on {current_date} at {current_time}.',
-        '"""',
-        'from typing import Dict, List'
-    ])
+    iods_filename = os.path.join(PKG_JSON_PATH, 'iods.json')
+    with open(iods_filename, 'w') as jf:
+        json.dump(iods, jf, indent=2)
+
     sop_to_iods = _create_sop_to_iods(directory)
-    iods_filename = os.path.join(PGK_PATH, '_iods.py')
-    with open(iods_filename, 'w') as fp:
-        fp.write(iods_docstr)
-        fp.write('\n\n')
-        iods_formatted = _dump_json(iods).replace('null', 'None')
-        fp.write(
-            f'IOD_MODULE_MAP: Dict[str, List[Dict[str, str]]] = {iods_formatted}'
-        )
-        fp.write('\n\n')
-        sop_to_iods_formatted = _dump_json(sop_to_iods).replace('null', 'None')
-        fp.write(f'SOP_CLASS_UID_IOD_KEY_MAP = {sop_to_iods_formatted}')
+    sop_to_iods_filename = os.path.join(PKG_JSON_PATH, 'sop_class_to_iod.json')
+    with open(sop_to_iods_filename, 'w') as jf:
+        json.dump(sop_to_iods, jf, indent=2)
 
     modules = _create_modules(directory)
-    modules_docstr = '\n'.join([
-        '"""DICOM modules'
-        f'auto-generated on {current_date} at {current_time}.'
-        '"""',
-        'from typing import Dict, List, Sequence, Union'
-    ])
-    modules_filename = os.path.join(PGK_PATH, '_modules.py')
-    with open(modules_filename, 'w') as fp:
-        fp.write(modules_docstr)
-        fp.write('\n\n')
-        modules_formatted = _dump_json(modules).replace('null', 'None')
-        fp.write(
-            f'MODULE_ATTRIBUTE_MAP: Dict[str, List[Dict[str, Union[str, Sequence[str]]]]] = {modules_formatted}'  # noqa: E501
-        )
+    modules_filename = os.path.join(PKG_JSON_PATH, 'modules.json')
+    with open(modules_filename, 'w') as jf:
+        json.dump(modules, jf, indent=2)

docs/legacy.rst

@@ -3,4 +3,229 @@
 Legacy Converted Enhanced Images
 ================================
 
-This page is under construction, and more detail will be added soon.
+Many medical images, including CT, MRI, and PET, consist of sets of 2D frames.
+When the DICOM standard was originally developed, such series were typically
+stored with each individual 2D frame in a separate DICOM instance (and
+therefore a separate DICOM file) to reduce file size. Many users will be
+familiar with MRI, PET, and CT images stored in this way using the CT Image
+("1.2.840.10008.5.1.4.1.1.2"), MR Image ("1.2.840.10008.5.1.4.1.1.4"), and
+Positron Emission Tomography Image ("1.2.840.10008.5.1.4.1.1.128") SOP Classes,
+often with tens or hundreds of separate files needed to store a single series.
+
+More recently, new additions to the standard prefer a "multiframe" arrangement
+where multiple 2D images are stored as individual frames within a single DICOM
+instance. Newer modalities, such as optical coherence tomography, whole slide
+microscopy images, or digital breast tomosynthesis (DBT) images, were
+implemented using these "multiframe" formats from the outset. Furthermore,
+"enhanced" multiframe versions of the older modalities (PET, CT, MRI) were also
+defined to allow storage of these modalities in a multiframe format, leading to
+the Enhanced PET Image, Enhanced CT Image, and Enhanced MR Image SOP Classes.
+However, adoption of these Enhanced SOP Classes by manufacturers for
+established modalities has been slow: most PET, CT, and MR images you see still
+use the original "legacy" SOP Classes.
+
+Because the multiframe formats are generally easier to store and work with, a
+common requirement is to convert old legacy (single-frame) series of images to
+the corresponding new multiframe/enhanced format. Unfortunately, this is not
+typically possible. The new enhanced formats made a number of other changes in
+addition to the number of frames in an instance, and there is not usually
+enough information in the legacy instances to populate all the required
+attributes correctly.
+
+To get around this problem, a further set of "Legacy Converted Enhanced" SOP
+Classes was created. These share many characteristics with the new Enhanced
+images, but are designed such that it is possible to convert existing legacy
+series to them. They also provide a mechanism to explicitly encode references
+to the original source images.
+
+Highdicom provides Python classes for these legacy converted enhanced images,
+enabling their construction from existing legacy (single-frame) instances:
+
+- :class:`highdicom.legacy.LegacyConvertedEnhancedMRImage` for MR images stored
+  in the legacy "MR Image Storage" SOP Class.
+- :class:`highdicom.legacy.LegacyConvertedEnhancedCTImage` for CT images stored
+  in the legacy "CT Image Storage" SOPClass.
+- :class:`highdicom.legacy.LegacyConvertedEnhancedPETImage` for PET images
+  stored in the legacy "Positron Emission Tomography" SOPClass.
+
+
+Basic Conversion
+----------------
+
+Basic conversion of legacy to enhanced format is very straightforward. You just
+pass the legacy instances as a list (order is unimportant using the default
+parameters) and specify UIDs, an instance number, and a series number for the
+new instance. Generally it is recommended to choose a series description too,
+but if you don't, the original series description of the legacy series will be
+used with "(enhanced conversion)" added as a suffix unless adding that suffix
+would go beyond the character limit for series descriptions (64 characters).
+
+
+.. code-block:: python
+
+  import highdicom as hd
+  from pydicom import dcmread
+  from pydicom.data import get_testdata_file
+
+
+  # Use this series of files from the pydicom test data
+  legacy_ct_files = [
+      get_testdata_file('dicomdirtests/77654033/CT2/17136'),
+      get_testdata_file('dicomdirtests/77654033/CT2/17196'),
+      get_testdata_file('dicomdirtests/77654033/CT2/17166'),
+  ]
+
+  # Read in the files
+  ct_series = [dcmread(f) for f in legacy_ct_files]
+
+  # Use the class constructor to perform the conversion
+  multiframe = hd.legacy.LegacyConvertedEnhancedCTImage(
+      ct_series,
+      series_number=1,
+      instance_number=1,
+      series_instance_uid=hd.UID(),
+      sop_instance_uid=hd.UID(),
+      series_description="Enhanced Test Files",
+  )
+
+  # Save out the new multiframe conversion
+  multiframe.save_as("legacy_converted_ct.dcm")
+
+
+Encoding, Decoding, and Transcoding
+-----------------------------------
+
+By default, the new instance will keep the transfer syntax of the legacy
+datasets. If the legacy datasets are compressed, highdicom will simply re-use
+the existing compressed pixel data without decoding it and combine the
+compressed frames together. This is both more efficient and avoids any possible
+further information loss due to re-encoding.
+
+Alternatively, you can opt to choose a new transfer syntax for the new
+instance, in which case highdicom will decode and/or (re-)encode the pixel data
+(as necessary) from the legacy instances. Since this can be slow, you can
+optionally specify a non-zero number of sub-processes to perform this operation
+using the ``workers`` parameter. When using multiple workers, you must place
+your code within a ``if __name__ == "__main__":`` guard (this is a requirement
+wherever you use multiprocessing in Python).
+
+.. code-block:: python
+
+  import highdicom as hd
+  from pydicom import dcmread
+  from pydicom.uid import RLELossless
+  from pydicom.data import get_testdata_file
+
+
+  if __name__ == "__main__":
+      # Use this series of files from the pydicom test data
+      legacy_ct_files = [
+          get_testdata_file('dicomdirtests/77654033/CT2/17136'),
+          get_testdata_file('dicomdirtests/77654033/CT2/17196'),
+          get_testdata_file('dicomdirtests/77654033/CT2/17166'),
+      ]
+
+      # Read in the files
+      ct_series = [dcmread(f) for f in legacy_ct_files]
+
+      # Create multiframe instance using lossless RLE compression
+      multiframe = hd.legacy.LegacyConvertedEnhancedCTImage(
+          ct_series,
+          series_number=1,
+          instance_number=1,
+          series_instance_uid=hd.UID(),
+          sop_instance_uid=hd.UID(),
+          series_description="Enhanced Test Files",
+          transfer_syntax_uid=RLELossless,
+          workers=8,  # 8 conversion sub-processes
+      )
+
+      # Save out the new multiframe conversion
+      multiframe.save_as("legacy_converted_ct.dcm")
+
+
+Including Multiple Series
+-------------------------
+
+Although a Legacy Converted Enhanced image most often consists of legacy
+datasets from a single series, this is not actually a limitation. In some
+situations, you may wish to include images from multiple series in one
+multiframe file. This is allowed if certain conditions are met, such as images
+having the same size, pixel representation, photometric interpretation, etc.
+
+Sorting Frames and Dimension Indices
+------------------------------------
+
+In multiframe objects such as Legacy Converted Enhanced images, an attribute
+called the Dimension Index Sequence describes how the frames are sorted along
+one or more dimensions.
+
+Under the default behavior, highdicom attempts to choose suitable sorting
+dimensions automatically based on the legacy images. The current logic is as
+follows (we may generalize in the future to more complex schemes):
+
+1. Attempt to sort frames spatially as a regularly-spaced volume. This may fail
+   because the spacings are not regular, orientations do not match, and/or there
+   are multiple frames at each image position.
+2. If step 1 fails, attempt to sort by Instance Number of the legacy series,
+   or, if there are multiple series, by Series Number then by Instance Number
+   as two dimension indices.
+3. If this fails (because series and/or instance numbers are missing), store
+   frames in the order they were passed and do not include a Dimension Index 
+   Sequence (it is optional).
+
+The ``include_dimension_index`` parameter allows you to control this process.
+The default value is ``None`` and results in the above behavior. If instead,
+you pass ``include_dimension_index=True``, an exception will be raised if steps
+1 and 2 fail rather than skipping the Dimension Index Sequence. A value of
+``False`` means that no attempt will be made to sort the frames, they will
+simply be stored in the order you passed them. Your alternative sorting
+scheme will *not* be stored in the Dimension Index Sequence (this capability
+may be added in the future).
+
+For example, to sort by ``KVP`` and then ``SliceLocation``:
+
+.. code-block:: python
+
+  import highdicom as hd
+  from pydicom import dcmread
+  from pydicom.data import get_testdata_file
+
+
+  # Use this series of files from the pydicom test data
+  legacy_ct_files = [
+      get_testdata_file('dicomdirtests/77654033/CT2/17136'),
+      get_testdata_file('dicomdirtests/77654033/CT2/17196'),
+      get_testdata_file('dicomdirtests/77654033/CT2/17166'),
+  ]
+
+  # Read in the files
+  ct_series = [dcmread(f) for f in legacy_ct_files]
+
+  # Sort using KVP and slice location
+  ct_series = sorted(
+      ct_series, key=lambda dcm: (dcm.KVP, dcm.SliceLocation)
+  )
+
+  # Create multiframe instance with custom sort key
+  multiframe = hd.legacy.LegacyConvertedEnhancedCTImage(
+      ct_series,
+      series_number=1,
+      instance_number=1,
+      series_instance_uid=hd.UID(),
+      sop_instance_uid=hd.UID(),
+      series_description="Enhanced Test Files",
+      include_dimension_index=False,  # disable default sorting
+  )
+
+  # Save out the new multiframe conversion
+  multiframe.save_as("legacy_converted_ct.dcm")
+
+Requiring Volumes
+-----------------
+
+The ``require_volume`` option allows you to specify that highdicom should only
+convert series that consist of parallel, regularly-spaced frames (i.e. those
+that could be used to form a :class:`highdicom.Volume`). This is optional,
+Legacy Converted Enhanced images that are not volumes are still entirely valid
+according to the DICOM standard.

docs/quickstart.rst

@@ -1029,11 +1029,49 @@ preferred for storing annotations for clinical or research purposes.
     gsps.save_as('gsps.dcm')
 
 
-.. .. _creation-legacy:
+.. _creation-legacy:
 
-.. Creating Legacy Converted Enhanced Images
-.. -----------------------------------------
+Creating Legacy Converted Enhanced Images
+-----------------------------------------
+
+Highdicom provides the following Python classes for converting legacy
+single-frame images to multi-frame Legacy Converted Enhanced images:
+
+- :class:`highdicom.legacy.LegacyConvertedEnhancedMRImage` for MR images stored
+  in the legacy "MR Image Storage" SOP Class.
+- :class:`highdicom.legacy.LegacyConvertedEnhancedCTImage` for CT images stored
+  in the legacy "CT Image Storage" SOPClass.
+- :class:`highdicom.legacy.LegacyConvertedEnhancedPETImage` for PET images
+  stored in the legacy "Positron Emission Tomography" SOPClass.
 
-.. .. code-block:: python
+Here is a simple example using CT. See :ref:`legacy` for more detail.
+
+.. code-block:: python
 
-..     from highdicom.legacy.sop import LegacyConvertedEnhancedCTImage
+  import highdicom as hd
+  from pydicom import dcmread
+  from pydicom.data import get_testdata_file
+
+
+  # Use this series of files from the pydicom test data
+  legacy_ct_files = [
+      get_testdata_file('dicomdirtests/77654033/CT2/17136'),
+      get_testdata_file('dicomdirtests/77654033/CT2/17196'),
+      get_testdata_file('dicomdirtests/77654033/CT2/17166'),
+  ]
+
+  # Read in the files
+  ct_series = [dcmread(f) for f in legacy_ct_files]
+
+  # Use the class constructor to perform the conversion
+  multiframe = hd.legacy.LegacyConvertedEnhancedCTImage(
+      ct_series,
+      series_number=1,
+      instance_number=1,
+      series_instance_uid=hd.UID(),
+      sop_instance_uid=hd.UID(),
+      series_description="Enhanced Test Files",
+  )
+
+  # Save out the new multiframe conversion
+  multiframe.save_as("legacy_converted_ct.dcm")

docs/volume.rst

@@ -227,7 +227,7 @@ volume.
     ]
     ct_series = [pydicom.dcmread(f) for f in ct_files]
 
-    vol = get_volume_from_series(ct_series)
+    vol = hd.get_volume_from_series(ct_series)
 
 Array Manipulation
 ------------------