[WIP] Detect EDM schema versions during reading and ensure consistency

include/podio/DatamodelRegistry.h

@@ -1,6 +1,7 @@
 #ifndef PODIO_DATAMODELREGISTRY_H
 #define PODIO_DATAMODELREGISTRY_H
 
+#include "podio/SchemaEvolution.h"
 #include "podio/podioVersion.h"
 
 #include <optional>
@@ -100,8 +101,25 @@ class DatamodelRegistry {
   /// @returns The name of the datamodel
   const std::string& getDatamodelName(size_t index) const;
 
+  /// Get the version of the datamodel with the given name
+  ///
+  /// @note This will return the version of the datamodel library **not the
+  /// schema version of the datamodel**.
+  ///
+  /// @param name The name of the datamodel
+  ///
+  /// @returns The version of the datamodel if the datamodel is known to the
+  ///          registry or an empty optional otherwise
   std::optional<podio::version::Version> getDatamodelVersion(const std::string& name) const;
 
+  /// Get the schema version of the datamodel with the given name
+  ///
+  /// @param name The name of the datamodel
+  ///
+  /// @returns The schema version of the datamodel if the datamodel is known to the
+  ///          registry or an empty optional otherwise
+  std::optional<podio::SchemaVersionT> getSchemaVersion(const std::string& name) const;
+
   /// Register a datamodel and return its index in the registry.
   ///
   /// This is the hook that is called during dynamic loading of an EDM to
@@ -133,6 +151,8 @@ class DatamodelRegistry {
   std::unordered_map<std::string_view, RelationNames> m_relations{};
 
   std::unordered_map<std::string, podio::version::Version> m_datamodelVersions{};
+
+  std::unordered_map<std::string, podio::SchemaVersionT> m_schemaVersions{};
 };
 } // namespace podio
 

include/podio/utilities/DatamodelRegistryIOHelpers.h

@@ -6,10 +6,18 @@
 
 #include <set>
 #include <string>
+#include <string_view>
 #include <tuple>
 #include <vector>
 
 namespace podio {
+namespace detail {
+  /// Extract schema version from a JSON datamodel definition
+  ///
+  /// @param definition The JSON definition string
+  /// @returns The schema version found in the definition, or 0 if not found
+  podio::SchemaVersionT extractSchemaVersion(const std::string_view definition);
+} // namespace detail
 
 /// Helper class to collect the datamodel (JSON) definitions that should be
 /// written.
@@ -63,6 +71,13 @@ class DatamodelDefinitionHolder {
 
   std::optional<podio::version::Version> getDatamodelVersion(const std::string& name) const;
 
+  /// Get the schema version for the given datamodel name by extracting it from
+  /// the stored datamodel definition.
+  ///
+  /// @param name The name of the datamodel
+  /// @returns The schema version if the datamodel is available, or std::nullopt otherwise
+  std::optional<podio::SchemaVersionT> getSchemaVersion(const std::string& name) const;
+
 protected:
   MapType m_availEDMDefs{};
   VersionList m_edmVersions{};

src/DatamodelRegistry.cc

@@ -1,4 +1,5 @@
 #include "podio/DatamodelRegistry.h"
+#include "podio/utilities/DatamodelRegistryIOHelpers.h"
 
 #include <algorithm>
 #include <iostream>
@@ -21,6 +22,7 @@ size_t DatamodelRegistry::registerDatamodel(std::string name, std::string_view d
 
   if (it == m_definitions.cend()) {
     int index = m_definitions.size();
+    m_schemaVersions.emplace(name, detail::extractSchemaVersion(definition));
     m_definitions.emplace_back(std::move(name), definition);
 
     for (const auto& [typeName, relations, vectorMembers] : relationNames) {
@@ -104,4 +106,11 @@ std::optional<podio::version::Version> DatamodelRegistry::getDatamodelVersion(co
   return std::nullopt;
 }
 
+std::optional<podio::SchemaVersionT> DatamodelRegistry::getSchemaVersion(const std::string& name) const {
+  if (const auto it = m_schemaVersions.find(name); it != m_schemaVersions.end()) {
+    return it->second;
+  }
+  return std::nullopt;
+}
+
 } // namespace podio

src/DatamodelRegistryIOHelpers.cc

@@ -1,8 +1,40 @@
 #include "podio/utilities/DatamodelRegistryIOHelpers.h"
+
 #include <algorithm>
+#include <charconv>
 #include <iterator>
 
 namespace podio {
+namespace detail {
+  podio::SchemaVersionT extractSchemaVersion(const std::string_view definition) {
+    // Extract schema_version from JSON definition without full parsing
+    // Look for "schema_version": followed by a number
+    constexpr std::string_view schemaVersionKey = "\"schema_version\":";
+    if (auto pos = definition.find(schemaVersionKey); pos != std::string_view::npos) {
+      pos += schemaVersionKey.length();
+      // Skip whitespace
+      while (pos < definition.length() && std::isspace(definition[pos])) {
+        ++pos;
+      }
+      // Extract the number
+      auto start = pos;
+      while (pos < definition.length() && std::isdigit(definition[pos])) {
+        ++pos;
+      }
+      if (pos > start) {
+        // Convert substring to integer using std::from_chars for better error handling
+        podio::SchemaVersionT schemaVersion = 0;
+        auto result = std::from_chars(definition.data() + start, definition.data() + pos, schemaVersion);
+        if (result.ec == std::errc{}) {
+          return schemaVersion;
+        }
+      }
+    }
+
+    // Return 0 if no valid schema version found
+    return 0;
+  }
+} // namespace detail
 
 void DatamodelDefinitionCollector::registerDatamodelDefinition(const podio::CollectionBase* coll,
                                                                const std::string& name) {
@@ -57,4 +89,12 @@ std::optional<podio::version::Version> DatamodelDefinitionHolder::getDatamodelVe
   return std::nullopt;
 }
 
+std::optional<podio::SchemaVersionT> DatamodelDefinitionHolder::getSchemaVersion(const std::string& name) const {
+  const auto definition = getDatamodelDefinition(name);
+  if (definition != "{}") {
+    return detail::extractSchemaVersion(definition);
+  }
+  return std::nullopt;
+}
+
 } // namespace podio

src/RNTupleReader.cc

@@ -4,6 +4,8 @@
 #include "podio/DatamodelRegistry.h"
 #include "podio/GenericParameters.h"
 #include "podio/utilities/RootHelpers.h"
+
+#include "ioUtils.h"
 #include "rootUtils.h"
 
 #include <ROOT/RError.hxx>
@@ -94,6 +96,10 @@ void RNTupleReader::openFiles(const std::vector<std::string>& filenames) {
   }
   m_datamodelHolder = DatamodelDefinitionHolder(std::move(edm), std::move(edmVersions));
 
+  for (const auto& warning : io_utils::checkEDMVersionsReadable(m_datamodelHolder)) {
+    std::cerr << "WARNING: " << warning << std::endl;
+  }
+
   auto availableCategoriesField = m_metadata->GetView<std::vector<std::string>>(root_utils::availableCategories);
   m_availableCategories = availableCategoriesField(0);
 

src/ROOTReader.cc

@@ -7,6 +7,8 @@
 #include "podio/GenericParameters.h"
 #include "podio/podioVersion.h"
 #include "podio/utilities/RootHelpers.h"
+
+#include "ioUtils.h"
 #include "rootUtils.h"
 
 // ROOT specific includes
@@ -158,6 +160,8 @@ std::optional<podio::CollectionReadBuffers> ROOTReader::getCollectionBuffers(ROO
   auto maybeBuffers = bufferFactory.createBuffers(collType, schemaVersion, isSubsetColl);
 
   if (!maybeBuffers) {
+    std::cerr << "WARNING: Buffers couldn't be created for collection " << name << " of type " << collType
+              << " and schema version " << schemaVersion << std::endl;
     return std::nullopt;
   }
 
@@ -326,6 +330,10 @@ void ROOTReader::openFiles(const std::vector<std::string>& filenames) {
     }
 
     m_datamodelHolder = DatamodelDefinitionHolder(std::move(datamodelDefs), std::move(edmVersions));
+
+    for (const auto& warning : io_utils::checkEDMVersionsReadable(m_datamodelHolder)) {
+      std::cerr << "WARNING: " << warning << std::endl;
+    }
   }
 
   // Do some work up front for setting up categories and setup all the chains

src/SIOReader.cc

@@ -1,5 +1,6 @@
 #include "podio/SIOReader.h"
 
+#include "ioUtils.h"
 #include "sioUtils.h"
 
 #include <sio/api.h>
@@ -129,6 +130,10 @@ void SIOReader::readEDMDefinitions() {
   auto datamodelDefs = static_cast<SIOMapBlock<std::string, std::string>*>(blocks[0].get());
   auto edmVersions = static_cast<SIOMapBlock<std::string, podio::version::Version>*>(blocks[1].get());
   m_datamodelHolder = DatamodelDefinitionHolder(std::move(datamodelDefs->mapData), std::move(edmVersions->mapData));
+
+  for (const auto& warning : io_utils::checkEDMVersionsReadable(m_datamodelHolder)) {
+    std::cerr << "WARNING: " << warning << std::endl;
+  }
 }
 
 } // namespace podio

src/ioUtils.h

@@ -0,0 +1,42 @@
+#ifndef PODIO_IO_UTILS_H // NOLINT(llvm-header-guard): internal headers confuse clang-tidy
+#define PODIO_IO_UTILS_H // NOLINT(llvm-header-guard): internal headers confuse clang-tidy
+
+#include "podio/DatamodelRegistry.h"
+#include "podio/utilities/DatamodelRegistryIOHelpers.h"
+
+#include <string>
+#include <vector>
+
+namespace podio::io_utils {
+
+/// Check and verify that all EDM (verions) that have been read from file are
+/// readable
+///
+/// Effectively this boils down to making sure that the EDM versions on file are
+/// not newer than the ones that have been loaded dynamically.
+///
+/// Returns all warnings that have been found
+inline std::vector<std::string> checkEDMVersionsReadable(const podio::DatamodelDefinitionHolder& fileEdms) {
+  std::vector<std::string> warnings{};
+  for (const auto& edmName : fileEdms.getAvailableDatamodels()) {
+    // There is no way we get an empty optional here
+    const auto fileSchemaVersion = fileEdms.getSchemaVersion(edmName).value();
+    const auto envSchemaVersion = podio::DatamodelRegistry::instance().getSchemaVersion(edmName);
+
+    if (!envSchemaVersion) {
+      warnings.emplace_back("EDM '" + edmName +
+                            "' exists in file but does not seem to be loaded from the environment by podio");
+      continue;
+    }
+    if (envSchemaVersion.value() < fileSchemaVersion) {
+      warnings.emplace_back("EDM '" + edmName + "' exists in file with schema version " +
+                            std::to_string(fileSchemaVersion) + " but podio loaded schema version " +
+                            std::to_string(envSchemaVersion.value()) + " from the environment");
+    }
+  }
+
+  return warnings;
+}
+} // namespace podio::io_utils
+
+#endif

tests/schema_evolution/code_gen/CMakeLists.txt

@@ -62,3 +62,49 @@ endif()
 #   ADD_SCHEMA_EVOLUTION_TEST(components_rename_member WITH_EVOLUTION RNTUPLE NO_GENERATE_MODELS)
 #   ADD_SCHEMA_EVOLUTION_TEST(multi_schema_components_rename_member OLD_MODELS old oldest WITH_EVOLUTION RNTUPLE NO_GENERATE_MODELS)
 # endif()
+
+
+# The following are tests that check whether trying to read files that have a
+# **newer** version of the datamodel than is available from the environment.
+# First we have to write the file with a new version. We will simply use one of
+# the check files of the schema evolution checks above
+add_executable(write_file_newer_model components_new_member/check.cpp)
+target_link_libraries(write_file_newer_model PRIVATE
+  components_new_member_newModel
+  podio::podioIO
+)
+target_include_directories(write_file_newer_model PRIVATE
+  ${CMAKE_CURRENT_SOURCE_DIR}
+)
+target_compile_definitions(write_file_newer_model PRIVATE
+  PODIO_SCHEMA_EVOLUTION_TEST_WRITE
+  TEST_CASE="write_file_newer_model"
+)
+# Write a file with a newer version
+add_test(NAME write_file_newer_model COMMAND write_file_newer_model)
+set_property(TEST write_file_newer_model
+  PROPERTY
+  ENVIRONMENT
+    ROOT_LIBRARY_PATH=${CMAKE_CURRENT_BINARY_DIR}/components_new_member/new_model
+    LD_LIBRARY_PATH=${PROJECT_BINARY_DIR}/src:$<TARGET_FILE_DIR:ROOT::Tree>:$<$<TARGET_EXISTS:SIO::sio>:$<TARGET_FILE_DIR:SIO::sio>>:$ENV{LD_LIBRARY_PATH}
+
+)
+set_tests_properties(write_file_newer_model
+  PROPERTIES
+  WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/components_new_member
+  FIXTURES_SETUP write_file_newer_model_setup
+)
+# Try to read back this file with an older version
+add_test(NAME read_file_older_env_edm_version COMMAND podio-dump-tool write_file_newer_model.root)
+set_property(TEST read_file_older_env_edm_version
+  PROPERTY
+  ENVIRONMENT
+    ROOT_LIBRARY_PATH=${CMAKE_CURRENT_BINARY_DIR}/components_new_member/old_model
+    LD_LIBRARY_PATH=${PROJECT_BINARY_DIR}/src:$<TARGET_FILE_DIR:ROOT::Tree>:$<$<TARGET_EXISTS:SIO::sio>:$<TARGET_FILE_DIR:SIO::sio>>:$ENV{LD_LIBRARY_PATH}
+)
+set_tests_properties(read_file_older_env_edm_version
+  PROPERTIES
+  WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/components_new_member
+  FIXTURES_REQUIRED write_file_newer_model_setup
+  PASS_REGULAR_EXPRESSION "WARNING: EDM 'datamodel' exists in file with schema version 2 but podio loaded schema version 1 from the environment"
+)

tools/src/podio-dump-tool.cpp

@@ -12,6 +12,7 @@
 #include <fmt/ranges.h>
 
 #include <algorithm>
+#include <functional>
 #include <iterator>
 #include <numeric>
 #include <ranges>