fix[next]: Add build configuration to the compilation fingerprint

docs/user/next/advanced/HackTheToolchain.md

@@ -59,7 +59,7 @@ class PureCpp2WorkflowFactory(gtx.program_processors.runners.gtfn.GTFNCompileWor
     translation: workflow.Workflow[
         gtx.otf.definitions.CompilableProgramDef, gtx.otf.stages.ProgramSource
     ] = MyCodeGen()
-    bindings: workflow.Workflow[gtx.otf.stages.ProgramSource, gtx.otf.stages.CompilableProject] = (
+    bindings: workflow.Workflow[gtx.otf.stages.ProgramSource, gtx.otf.stages.ExtensionSource] = (
         Cpp2BindingsGen()
     )
 

src/gt4py/next/config.py

@@ -104,21 +104,18 @@ def _get_build_cache_version_id() -> str:
 
 #: Build type to be used when CMake is used to compile generated code.
 #: Might have no effect when CMake is not used as part of the toolchain.
-# FIXME[#2447](egparedes): compile-time setting, should be included in the build cache key.
 CMAKE_BUILD_TYPE: CMakeBuildType = CMakeBuildType[
     os.environ.get("GT4PY_CMAKE_BUILD_TYPE", "debug" if DEBUG else "release").upper()
 ]
 
 
 #: Experimental, use at your own risk: assume horizontal dimension has stride 1
-# FIXME[#2447](egparedes): compile-time setting, should be included in the build cache key.
 UNSTRUCTURED_HORIZONTAL_HAS_UNIT_STRIDE: bool = env_flag_to_bool(
     "GT4PY_UNSTRUCTURED_HORIZONTAL_HAS_UNIT_STRIDE", default=False
 )
 
 
 #: Add GPU trace markers (NVTX, ROC-TX) to the generated code, at compile time.
-# FIXME[#2447](egparedes): compile-time setting, should be included in the build cache key.
 ADD_GPU_TRACE_MARKERS: bool = env_flag_to_bool("GT4PY_ADD_GPU_TRACE_MARKERS", default=False)
 
 

src/gt4py/next/otf/binding/nanobind.py

@@ -10,6 +10,7 @@
 
 from __future__ import annotations
 
+import dataclasses
 from collections.abc import Collection
 from typing import Any, Optional, Sequence, TypeVar, Union
 
@@ -200,7 +201,9 @@ def _tuple_get(index: int, var: str) -> str:
     return f"gridtools::tuple_util::get<{index}>({var})"
 
 
-def make_argument(name: str, type_: ts.TypeSpec) -> str | BufferSID | Tuple:
+def make_argument(
+    name: str, type_: ts.TypeSpec, unstructured_horizontal_has_unit_stride: bool
+) -> str | BufferSID | Tuple:
     if isinstance(type_, ts.FieldType):
         return BufferSID(
             source_buffer=name,
@@ -209,7 +212,7 @@ def make_argument(name: str, type_: ts.TypeSpec) -> str | BufferSID | Tuple:
                     name=dim.value,
                     static_stride=1
                     if (
-                        config.UNSTRUCTURED_HORIZONTAL_HAS_UNIT_STRIDE
+                        unstructured_horizontal_has_unit_stride
                         and dim.kind == common.DimensionKind.HORIZONTAL
                     )
                     else None,
@@ -219,7 +222,10 @@ def make_argument(name: str, type_: ts.TypeSpec) -> str | BufferSID | Tuple:
             scalar_type=type_.dtype,
         )
     elif isinstance(type_, ts.TupleType):
-        elements = [make_argument(_tuple_get(i, name), t) for i, t in enumerate(type_.types)]
+        elements = [
+            make_argument(_tuple_get(i, name), t, unstructured_horizontal_has_unit_stride)
+            for i, t in enumerate(type_.types)
+        ]
         return Tuple(elems=elements)
     elif isinstance(type_, ts.ScalarType):
         return name
@@ -228,7 +234,7 @@ def make_argument(name: str, type_: ts.TypeSpec) -> str | BufferSID | Tuple:
 
 
 def create_bindings(
-    program_source: stages.ProgramSource[CodeSpecT],
+    program_source: stages.ProgramSource[CodeSpecT], unstructured_horizontal_has_unit_stride: bool
 ) -> stages.BindingSource[CodeSpecT, code_specs.PythonCodeSpec]:
     """
     Generate Python bindings through which a C++ function can be called.
@@ -274,7 +280,9 @@ def create_bindings(
                 expr=FunctionCall(
                     target=program_source.entry_point,
                     args=[
-                        make_argument(param.name, param.type_)
+                        make_argument(
+                            param.name, param.type_, unstructured_horizontal_has_unit_stride
+                        )
                         for param in program_source.entry_point.parameters
                     ],
                 )
@@ -305,7 +313,18 @@ def create_bindings(
     return stages.BindingSource(src, (interface.LibraryDependency("nanobind", "2.0.0"),))
 
 
-def bind_source(
-    inp: stages.ProgramSource[CodeSpecT],
-) -> stages.CompilableProject[CodeSpecT, code_specs.PythonCodeSpec]:
-    return stages.CompilableProject(program_source=inp, binding_source=create_bindings(inp))
+@dataclasses.dataclass(frozen=True)
+class ExtensionGenerator:
+    """
+    Generate a Python extension module that contains the bindings for a C++ function.
+    """
+
+    unstructured_horizontal_has_unit_stride: bool = config.UNSTRUCTURED_HORIZONTAL_HAS_UNIT_STRIDE
+
+    def __call__(
+        self, program_source: stages.ProgramSource[CodeSpecT]
+    ) -> stages.ExtensionSource[CodeSpecT, code_specs.PythonCodeSpec]:
+        binding_source = create_bindings(
+            program_source, self.unstructured_horizontal_has_unit_stride
+        )
+        return stages.ExtensionSource(program_source=program_source, binding_source=binding_source)

src/gt4py/next/otf/compilation/build_systems/cmake.py

@@ -66,15 +66,15 @@ def get_cmake_device_arch_option() -> str:
 class CMakeFactory(
     compiler.BuildSystemProjectGenerator[CPPLikeCodeSpecT, code_specs.PythonCodeSpec]
 ):
-    """Create a CMakeProject from a ``CompilableSource`` stage object with given CMake settings."""
+    """Create a CMakeProject from an ``ExtensionSource`` stage object with given CMake settings."""
 
     cmake_generator_name: str = "Ninja"
     cmake_build_type: config.CMakeBuildType = config.CMakeBuildType.DEBUG
     cmake_extra_flags: list[str] = dataclasses.field(default_factory=list)
 
     def __call__(
         self,
-        source: stages.CompilableProject[CPPLikeCodeSpecT, code_specs.PythonCodeSpec],
+        source: stages.ExtensionSource[CPPLikeCodeSpecT, code_specs.PythonCodeSpec],
         cache_lifetime: config.BuildCacheLifetime,
     ) -> CMakeProject:
         if not source.binding_source:

src/gt4py/next/otf/compilation/build_systems/compiledb.py

@@ -17,7 +17,7 @@
 from typing import Optional, TypeVar
 
 from gt4py._core import locking
-from gt4py.next import config, errors
+from gt4py.next import config, errors, fingerprinting
 from gt4py.next.otf import code_specs, stages
 from gt4py.next.otf.binding import interface
 from gt4py.next.otf.compilation import build_data, cache, compiler
@@ -32,7 +32,7 @@ class CompiledbFactory(
     compiler.BuildSystemProjectGenerator[CPPLikeCodeSpecT, code_specs.PythonCodeSpec]
 ):
     """
-    Create a CompiledbProject from a ``CompilableSource`` stage object with given CMake settings.
+    Create a CompiledbProject from an ``ExtensionSource`` stage object with given CMake settings.
 
     Use CMake to generate a compiledb with the required sequence of build commands.
     Generate a compiledb only if there isn't one for the given combination of cmake configuration
@@ -45,7 +45,7 @@ class CompiledbFactory(
 
     def __call__(
         self,
-        source: stages.CompilableProject[CPPLikeCodeSpecT, code_specs.PythonCodeSpec],
+        source: stages.ExtensionSource[CPPLikeCodeSpecT, code_specs.PythonCodeSpec],
         cache_lifetime: config.BuildCacheLifetime,
     ) -> CompiledbProject:
         if not source.binding_source:
@@ -72,7 +72,11 @@ def __call__(
         )
 
         return CompiledbProject(
-            root_path=cache.get_cache_folder(source, cache_lifetime),
+            root_path=cache.get_cache_folder(
+                source,
+                cache_lifetime,
+                build_context_id=fingerprinting.strict_fingerprinter(self),
+            ),
             program_name=name,
             source_files={
                 header_name: source.program_source.source_code,
@@ -264,16 +268,19 @@ def _cc_get_compiledb(
     cmake_flags: list[str],
     cache_lifetime: config.BuildCacheLifetime,
 ) -> pathlib.Path:
-    cache_path = cache.get_cache_folder(
-        stages.CompilableProject(prototype_program_source, None), cache_lifetime
+    # Use the same prototype source (with empty bindings) for both locating and creating the
+    # compiledb, so `get_cache_folder` names the same folder in either path.
+    prototype_source: stages.ExtensionSource = stages.ExtensionSource(
+        prototype_program_source, stages.BindingSource(source_code="", library_deps=())
     )
+    cache_path = cache.get_cache_folder(prototype_source, cache_lifetime)
 
     # In a multi-threaded environment, multiple threads may try to create the compiledb at the same time
     # leading to compilation errors.
     with locking.lock(cache_path):
         if renew_compiledb or not (compiled_db := _cc_find_compiledb(path=cache_path)):
             compiled_db = _cc_create_compiledb(
-                prototype_program_source=prototype_program_source,
+                prototype_source=prototype_source,
                 build_type=build_type,
                 cmake_flags=cmake_flags,
                 cache_lifetime=cache_lifetime,
@@ -292,7 +299,7 @@ def _cc_find_compiledb(path: pathlib.Path) -> Optional[pathlib.Path]:
 
 
 def _cc_create_compiledb(
-    prototype_program_source: stages.ProgramSource,
+    prototype_source: stages.ExtensionSource,
     build_type: config.CMakeBuildType,
     cmake_flags: list[str],
     cache_lifetime: config.BuildCacheLifetime,
@@ -302,18 +309,17 @@ def _cc_create_compiledb(
         cmake_build_type=build_type,
         cmake_extra_flags=cmake_flags,
     )(
-        stages.CompilableProject(
-            prototype_program_source, stages.BindingSource(source_code="", library_deps=())
-        ),
+        prototype_source,
         cache_lifetime,
     )
 
     path = prototype_project.root_path
     name = prototype_project.program_name
+    file_extension = prototype_source.program_source.code_spec.file_extension
     binding_src_name = next(
         name
         for name in prototype_project.source_files.keys()
-        if name.endswith(f"_bindings.{prototype_program_source.code_spec.file_extension}")
+        if name.endswith(f"_bindings.{file_extension}")
     )
 
     prototype_project.build()

src/gt4py/next/otf/compilation/cache.py

@@ -8,47 +8,18 @@
 
 """Caching for compiled backend artifacts."""
 
-import hashlib
 import pathlib
 import tempfile
 
-from gt4py.next import config
+from gt4py.next import config, fingerprinting
 from gt4py.next.otf import stages
-from gt4py.next.otf.binding import interface
 
 
 _session_cache_dir = tempfile.TemporaryDirectory(prefix="gt4py_session_")
 
 _session_cache_dir_path = pathlib.Path(_session_cache_dir.name)
 
 
-def _serialize_param(parameter: interface.Parameter) -> str:
-    return f"{parameter.name}: {parameter.type_!s}"
-
-
-def _serialize_library_dependency(dependency: interface.LibraryDependency) -> str:
-    return f"{dependency.name}/{dependency.version}"
-
-
-def _serialize_source(source: stages.ProgramSource) -> str:
-    parameters = [_serialize_param(param) for param in source.entry_point.parameters]
-    dependencies = [_serialize_library_dependency(dep) for dep in source.library_deps]
-    return f"""\
-    language: {source.code_spec}
-    name: {source.entry_point.name}
-    params: {", ".join(parameters)}
-    deps: {", ".join(dependencies)}
-    src: {source.source_code}
-    """
-
-
-def _cache_folder_name(source: stages.ProgramSource) -> str:
-    serialized = _serialize_source(source)
-    fingerprint = hashlib.sha256(serialized.encode(encoding="utf-8"))
-    fingerprint_hex_str = fingerprint.hexdigest()
-    return source.entry_point.name + "_" + fingerprint_hex_str
-
-
 def get_cache_base_path(lifetime: config.BuildCacheLifetime) -> pathlib.Path:
     """Return the base directory for cached artifacts with the given lifetime."""
     match lifetime:
@@ -61,15 +32,24 @@ def get_cache_base_path(lifetime: config.BuildCacheLifetime) -> pathlib.Path:
 
 
 def get_cache_folder(
-    compilable_source: stages.CompilableProject, lifetime: config.BuildCacheLifetime
+    ext_source: stages.ExtensionSource,
+    lifetime: config.BuildCacheLifetime,
+    build_context_id: str = "",
 ) -> pathlib.Path:
     """
-    Construct the path to where the build system project artifact of a compilable source should be cached.
+    Construct the path to where the build system project artifact of an extension source should be cached.
 
+    An optional ``build_context_id`` can be provided to distinguish between different contexts
+    that may produce different artifacts for the same extension source.
     The returned path points to an existing folder in all cases.
     """
-    # TODO(ricoh): make dependent on binding source too or add alternative that depends on bindings
-    folder_name = _cache_folder_name(compilable_source.program_source)
+    fingerprinter = fingerprinting.strict_fingerprinter
+    slug = ext_source.program_source.entry_point.name
+    if ext_source.binding_source:
+        slug = f"{slug}_pyext"
+    folder_name = f"{slug}_{fingerprinter(ext_source)}"
+    if build_context_id:
+        folder_name = f"{folder_name}_{build_context_id}"
 
     base_path = get_cache_base_path(lifetime)
     base_path.mkdir(exist_ok=True)

src/gt4py/next/otf/compilation/compiler.py

@@ -15,7 +15,7 @@
 import factory
 
 from gt4py._core import locking
-from gt4py.next import config
+from gt4py.next import config, fingerprinting
 from gt4py.next.otf import code_specs, definitions, stages, workflow
 from gt4py.next.otf.compilation import build_data, cache, importer
 
@@ -39,19 +39,19 @@ def module_exists(data: build_data.BuildData, src_dir: pathlib.Path) -> bool:
 class BuildSystemProjectGenerator(Protocol[CodeSpecT, TargetCodeSpecT]):
     def __call__(
         self,
-        source: stages.CompilableProject[CodeSpecT, TargetCodeSpecT],
+        source: stages.ExtensionSource[CodeSpecT, TargetCodeSpecT],
         cache_lifetime: config.BuildCacheLifetime,
     ) -> stages.BuildSystemProject[CodeSpecT, TargetCodeSpecT]: ...
 
 
 @dataclasses.dataclass(frozen=True)
 class Compiler(
     workflow.ChainableWorkflowMixin[
-        stages.CompilableProject[CPPLikeCodeSpecT, code_specs.PythonCodeSpec],
+        stages.ExtensionSource[CPPLikeCodeSpecT, code_specs.PythonCodeSpec],
         stages.ExecutableProgram,
     ],
     workflow.ReplaceEnabledWorkflowMixin[
-        stages.CompilableProject[CPPLikeCodeSpecT, code_specs.PythonCodeSpec],
+        stages.ExtensionSource[CPPLikeCodeSpecT, code_specs.PythonCodeSpec],
         stages.ExecutableProgram,
     ],
     definitions.CompilationStep[CPPLikeCodeSpecT, code_specs.PythonCodeSpec],
@@ -60,13 +60,19 @@ class Compiler(
 
     cache_lifetime: config.BuildCacheLifetime
     builder_factory: BuildSystemProjectGenerator[CPPLikeCodeSpecT, code_specs.PythonCodeSpec]
+    fingerprint_builder_factory: bool = True
     force_recompile: bool = False
 
     def __call__(
         self,
-        inp: stages.CompilableProject[CPPLikeCodeSpecT, code_specs.PythonCodeSpec],
+        inp: stages.ExtensionSource[CPPLikeCodeSpecT, code_specs.PythonCodeSpec],
     ) -> stages.ExecutableProgram:
-        src_dir = cache.get_cache_folder(inp, self.cache_lifetime)
+        build_context_id = (
+            fingerprinting.strict_fingerprinter(self.builder_factory)
+            if self.fingerprint_builder_factory
+            else ""
+        )
+        src_dir = cache.get_cache_folder(inp, self.cache_lifetime, build_context_id)
 
         # If we are compiling the same program at the same time (e.g. multiple MPI ranks),
         # we need to make sure that only one of them accesses the same build directory for compilation.

src/gt4py/next/otf/definitions.py

@@ -44,25 +44,23 @@ class TranslationStep(
 
 class BindingStep(Protocol[CodeSpecT, TargetCodeSpecT]):
     """
-    Generate Bindings for program source and package both together (ProgramSource -> CompilableSource).
+    Generate Bindings for program source and package both together (ProgramSource -> ExtensionSource).
 
     In the special cases where bindings are not required, such a step could also simply construct
-    a ``CompilableSource`` from the ``ProgramSource`` with bindings set to ``None``.
+    an ``ExtensionSource`` from the ``ProgramSource`` with bindings set to ``None``.
     """
 
     def __call__(
         self, program_source: stages.ProgramSource[CodeSpecT]
-    ) -> stages.CompilableProject[CodeSpecT, TargetCodeSpecT]: ...
+    ) -> stages.ExtensionSource[CodeSpecT, TargetCodeSpecT]: ...
 
 
 class CompilationStep(
-    workflow.Workflow[
-        stages.CompilableProject[CodeSpecT, TargetCodeSpecT], stages.ExecutableProgram
-    ],
+    workflow.Workflow[stages.ExtensionSource[CodeSpecT, TargetCodeSpecT], stages.ExecutableProgram],
     Protocol[CodeSpecT, TargetCodeSpecT],
 ):
-    """Compile program source code and bindings into a python callable (CompilableSource -> CompiledProgram)."""
+    """Compile program source code and bindings into a python callable (ExtensionSource -> CompiledProgram)."""
 
     def __call__(
-        self, source: stages.CompilableProject[CodeSpecT, TargetCodeSpecT]
+        self, source: stages.ExtensionSource[CodeSpecT, TargetCodeSpecT]
     ) -> stages.ExecutableProgram: ...

src/gt4py/next/otf/recipes.py

@@ -18,6 +18,6 @@ class OTFCompileWorkflow(workflow.NamedStepSequence):
     """The typical compiled backend steps composed into a workflow."""
 
     translation: definitions.TranslationStep
-    bindings: workflow.Workflow[stages.ProgramSource, stages.CompilableProject]
-    compilation: workflow.Workflow[stages.CompilableProject, stages.ExecutableProgram]
+    bindings: workflow.Workflow[stages.ProgramSource, stages.ExtensionSource]
+    compilation: workflow.Workflow[stages.ExtensionSource, stages.ExecutableProgram]
     decoration: workflow.Workflow[stages.ExecutableProgram, stages.ExecutableProgram]