PERF: Minimal, correct Python wrapping DEPENDS under lazy VTK loading

CMake/SlicerPrunePythonModuleDepends.py

@@ -0,0 +1,154 @@
+#!/usr/bin/env python3
+"""Prune a wrapped module's Python Init.data DEPENDS to the modules it truly needs.
+
+Background
+----------
+Slicer wraps its C++ classes for Python through vtkMacroKitPythonWrap, which
+seeds each module's wrapping dependencies from the whole VTK link line
+(``${VTK_LIBRARIES}``). The generated ``<Module>Init.data`` therefore lists a
+``DEPENDS`` on every VTK python module -- 125 of them -- even though a module
+such as vtkAddon only subclasses a handful of VTK base classes.
+
+That over-listing was harmless while ``import vtk`` eagerly loaded all of VTK.
+With lazy VTK loading it is not: a wrapped subclass whose direct base class has
+not been imported is built without that base and silently loses the inherited
+methods (the vtkOrientedGridTransform.SetDisplacementGridData regression). The
+fix restores the base-class imports -- but importing all 125 VTK modules at
+startup defeats the point of lazy loading.
+
+A wrapped subclass only needs its **direct base class** registered first; each
+VTK module in turn imports its own bases transitively. So the minimal, correct
+set of VTK DEPENDS is exactly the modules that own the direct base classes of
+the classes this module wraps.
+
+What this does
+--------------
+Reads the class list and DEPENDS from an Init.data file and the merged wrapping
+hierarchy (``Class : Super ; header ; owning_module``). Keeps every
+``vtkmodules.X`` line whose X owns a direct base class of a wrapped class, drops
+the rest, and passes all other DEPENDS lines (cross-module Slicer dependencies
+such as MRMLCorePython) through unchanged. Conservative by construction: only
+spurious VTK modules are removed.
+"""
+import argparse
+import sys
+
+
+def normalize(name):
+    """Reduce a hierarchy class token to a plain lookup key.
+
+    Drops template parameters (``Foo<T>`` -> ``Foo``) and rejects typedef-style
+    supers (``= something``) that are not real base classes.
+    """
+    name = name.strip()
+    lt = name.find('<')
+    if lt != -1:
+        name = name[:lt]
+    if name.startswith('='):
+        return ''
+    return name.strip()
+
+
+def parse_hierarchy(path):
+    """Return (super_of, module_of) mapping over every class in the file."""
+    super_of = {}
+    module_of = {}
+    with open(path) as handle:
+        for line in handle:
+            line = line.strip()
+            if not line:
+                continue
+            fields = [part.strip() for part in line.split(';')]
+            if len(fields) < 3:
+                continue
+            decl, _header, module = fields[0], fields[1], fields[2]
+            if ' : ' in decl:
+                cls, sup = decl.split(' : ', 1)
+            else:
+                cls, sup = decl, None
+            cls = normalize(cls)
+            if not cls:
+                continue
+            module_of[cls] = module
+            if sup:
+                super_of[cls] = normalize(sup)
+    return super_of, module_of
+
+
+def read_init_data(path):
+    """Split an Init.data file into (target, class_list, depends_list)."""
+    with open(path) as handle:
+        lines = [line.rstrip('\n') for line in handle]
+    target = lines[0] if lines else ''
+    classes = []
+    depends = []
+    in_depends = False
+    for line in lines[1:]:
+        if line.strip() == 'DEPENDS':
+            in_depends = True
+            continue
+        if in_depends:
+            if line.strip():
+                depends.append(line.strip())
+        else:
+            if line.strip():
+                classes.append(line.strip())
+    return target, classes, depends
+
+
+def base_owner_modules(classes, super_of, module_of, own_target):
+    """Owning modules of the direct base classes of the wrapped classes."""
+    owners = set()
+    for cls in classes:
+        sup = super_of.get(cls)
+        if not sup:
+            continue
+        owner = module_of.get(sup)
+        if owner and owner != own_target:
+            owners.add(owner)
+    return owners
+
+
+def prune(init_data_path, hierarchy_path, output_path):
+    target, classes, depends = read_init_data(init_data_path)
+    super_of, module_of = parse_hierarchy(hierarchy_path)
+    owners = base_owner_modules(classes, super_of, module_of, target)
+
+    kept = []
+    dropped = []
+    for dep in depends:
+        if dep.startswith('vtkmodules.'):
+            module = dep[len('vtkmodules.'):]
+            if module in owners:
+                kept.append(dep)
+            else:
+                dropped.append(dep)
+        else:
+            # Cross-module Slicer dependency (e.g. MRMLCorePython): keep as-is.
+            kept.append(dep)
+
+    with open(output_path, 'w') as handle:
+        handle.write(target + '\n')
+        for cls in classes:
+            handle.write(cls + '\n')
+        if kept:
+            handle.write('DEPENDS\n')
+            for dep in kept:
+                handle.write(dep + '\n')
+
+    sys.stderr.write(
+        '[prune-depends] %s: kept %d, dropped %d VTK module(s)\n'
+        % (target, len(kept), len(dropped)))
+
+
+def main(argv=None):
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('init_data', help='input <Module>Init.data (full DEPENDS)')
+    parser.add_argument('hierarchy', help='merged <Module>Hierarchy.txt')
+    parser.add_argument('output', help='pruned Init.data to write')
+    args = parser.parse_args(argv)
+    prune(args.init_data, args.hierarchy, args.output)
+
+
+if __name__ == '__main__':
+    main()

CMake/vtkWrapHierarchy.cmake

@@ -1,6 +1,16 @@
 #
 # a cmake macro to generate a text file with the class hierarchy
 #
+
+# Add a build-order dependency between two hierarchy targets if both exist.
+# Used as a deferred call (see VTK_WRAP_HIERARCHY) so the dependency can be wired
+# even when the dependee module is configured after the module that wraps it.
+function(_vtkAddon_wrap_hierarchy_add_dependency target dependency)
+  if(TARGET "${target}" AND TARGET "${dependency}")
+    add_dependencies("${target}" "${dependency}")
+  endif()
+endfunction()
+
 macro(VTK_WRAP_HIERARCHY module_name OUTPUT_DIR SOURCES)
   if(NOT VTK_WRAP_HIERARCHY_EXE)
     if(TARGET vtkWrapHierarchy)
@@ -169,4 +179,24 @@ $<$<BOOL:$<TARGET_PROPERTY:${module_name},INCLUDE_DIRECTORIES>>:
     DEPENDS
       ${CMAKE_CURRENT_BINARY_DIR}/${module_name}Hierarchy.stamp.txt)
 
+  # Wire the build-order dependency on the hierarchy targets this module wraps
+  # against. When a dependency is configured *after* this module (a forward
+  # reference in the subdirectory order), its "${dep}Hierarchy" target does not
+  # exist yet when OTHER_HIERARCHY_TARGETS is built above, so on the Makefiles
+  # generator that build-order edge is silently lost and parallel builds can
+  # race, failing with "vtkWrapHierarchy: couldn't open file ...Hierarchy.txt".
+  # Defer the wiring to the end of the top-level directory, where every module
+  # has been configured and all hierarchy targets exist, so forward and backward
+  # references are handled uniformly (this makes per-project workarounds that
+  # re-add known forward dependencies unnecessary).
+  foreach(dep ${${module_name}_WRAP_DEPENDS})
+    if(NOT "${module_name}" STREQUAL "${dep}")
+      if(NOT ${dep}_EXCLUDE_FROM_WRAPPING)
+        cmake_language(DEFER DIRECTORY "${CMAKE_SOURCE_DIR}"
+          CALL _vtkAddon_wrap_hierarchy_add_dependency
+               "${module_name}Hierarchy" "${dep}Hierarchy")
+      endif()
+    endif()
+  endforeach()
+
 endmacro()

CMake/vtkWrapPython.cmake

@@ -138,7 +138,31 @@ $<$<BOOL:$<TARGET_PROPERTY:${TARGET},INCLUDE_DIRECTORIES>>:
         if(NOT ${dep}_KIT STREQUAL kit_basename)
           list(APPEND _python_module_depends ${${dep}_KIT}KitPython)
         endif()
-      elseif(TARGET ${dep}Python AND NOT ${dep} MATCHES "VTK::")
+      elseif(dep MATCHES "^VTK::")
+        # Import the Python module of each VTK dependency so that base classes
+        # defined in VTK (for example vtkGridTransform in vtkFiltersHybrid) are
+        # registered before this library's wrapped subclasses are built. The
+        # eager "import vtk" used to load every VTK module up front; with lazy
+        # VTK loading it does not, so a wrapped subclass such as
+        # vtkOrientedGridTransform would otherwise be built without its VTK base
+        # class and silently lose the inherited methods. Follow VTK's own module
+        # wrapper (vtkModuleWrapPython): skip modules excluded from wrapping and
+        # name the rest "<python_package>.<library_name>".
+        _vtk_module_get_module_property("${dep}"
+          PROPERTY  "exclude_wrap"
+          VARIABLE  _dep_exclude_wrap)
+        if(NOT _dep_exclude_wrap)
+          _vtk_module_get_module_property("${dep}"
+            PROPERTY  "python_package"
+            VARIABLE  _dep_python_package)
+          _vtk_module_get_module_property("${dep}"
+            PROPERTY  "library_name"
+            VARIABLE  _dep_library_name)
+          if(_dep_python_package AND _dep_library_name)
+            list(APPEND _python_module_depends "${_dep_python_package}.${_dep_library_name}")
+          endif()
+        endif()
+      elseif(TARGET ${dep}Python)
         list(APPEND _python_module_depends ${dep}Python)
       endif()
     endforeach()
@@ -159,6 +183,32 @@ $<$<BOOL:$<TARGET_PROPERTY:${TARGET},INCLUDE_DIRECTORIES>>:
     @ONLY
     )
 
+  # The DEPENDS just written lists every VTK python module on the link line,
+  # because the wrapping dependencies are seeded from ${VTK_LIBRARIES}. Only the
+  # modules that own a direct base class of a wrapped class actually need to be
+  # imported first (each VTK module imports its own bases transitively). Prune
+  # the DEPENDS to that minimal set so lazy VTK loading is not defeated by
+  # importing all of VTK when this module is imported. Cross-module Slicer
+  # dependencies are preserved unchanged. Requires the merged hierarchy file and
+  # a Python interpreter; fall back to the unpruned data file when unavailable.
+  set(_init_data_file "${CMAKE_CURRENT_BINARY_DIR}/${TARGET}Init.data")
+  if(KIT_HIERARCHY_FILE AND PYTHON_EXECUTABLE AND ${VTK_VERSION} VERSION_GREATER_EQUAL "8.90")
+    add_custom_command(
+      OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${TARGET}Init.pruned.data
+      DEPENDS ${vtkAddon_CMAKE_DIR}/SlicerPrunePythonModuleDepends.py
+              ${CMAKE_CURRENT_BINARY_DIR}/${TARGET}Init.data
+              ${KIT_HIERARCHY_FILE}
+      COMMAND ${PYTHON_EXECUTABLE}
+              ${vtkAddon_CMAKE_DIR}/SlicerPrunePythonModuleDepends.py
+              ${CMAKE_CURRENT_BINARY_DIR}/${TARGET}Init.data
+              ${KIT_HIERARCHY_FILE}
+              ${CMAKE_CURRENT_BINARY_DIR}/${TARGET}Init.pruned.data
+      COMMENT "Pruning Python module dependencies for ${TARGET}"
+      VERBATIM
+      )
+    set(_init_data_file "${CMAKE_CURRENT_BINARY_DIR}/${TARGET}Init.pruned.data")
+  endif()
+
   set(_init_impl_src "")
   if(${VTK_VERSION} VERSION_LESS "9.5.0")
     set(_init_impl_src "${CMAKE_CURRENT_BINARY_DIR}/${TARGET}InitImpl.cxx")
@@ -167,9 +217,9 @@ $<$<BOOL:$<TARGET_PROPERTY:${TARGET},INCLUDE_DIRECTORIES>>:
     OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${TARGET}Init.cxx
            ${_init_impl_src}
     DEPENDS ${VTK_WRAP_PYTHON_INIT_EXE}
-            ${CMAKE_CURRENT_BINARY_DIR}/${TARGET}Init.data
+            ${_init_data_file}
     COMMAND ${VTK_WRAP_PYTHON_INIT_EXE}
-            ${CMAKE_CURRENT_BINARY_DIR}/${TARGET}Init.data
+            ${_init_data_file}
             ${CMAKE_CURRENT_BINARY_DIR}/${TARGET}Init.cxx
             ${_init_impl_src}
     COMMENT "Generating the Python module initialization sources for ${TARGET}"