From ee8145749dc9e1516c42c62783004620e857a793 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Wed, 10 Jun 2026 10:56:30 +0300 Subject: [PATCH 01/14] poc single collection --- tests/unittests/CMakeLists.txt | 2 +- tests/unittests/filtering.cpp | 193 +++++++++++++++++++++++++++++++++ 2 files changed, 194 insertions(+), 1 deletion(-) create mode 100644 tests/unittests/filtering.cpp diff --git a/tests/unittests/CMakeLists.txt b/tests/unittests/CMakeLists.txt index 71f13436e..092d33c5d 100644 --- a/tests/unittests/CMakeLists.txt +++ b/tests/unittests/CMakeLists.txt @@ -41,7 +41,7 @@ else() endif() find_package(Threads REQUIRED) -add_executable(unittest_podio unittest.cpp frame.cpp buffer_factory.cpp interface_types.cpp std_interoperability.cpp links.cpp test_json_output.cpp) +add_executable(unittest_podio unittest.cpp frame.cpp buffer_factory.cpp interface_types.cpp std_interoperability.cpp links.cpp test_json_output.cpp filtering.cpp) target_link_libraries(unittest_podio PUBLIC TestDataModel ExtensionDataModel InterfaceExtensionDataModel PRIVATE Catch2::Catch2WithMain Threads::Threads podio::podioRootIO) # Catch2 TEST_CASE/SECTION macros expand __COUNTER__ at the call site in user # code, so marking Catch2 headers as system includes is not sufficient to diff --git a/tests/unittests/filtering.cpp b/tests/unittests/filtering.cpp new file mode 100644 index 000000000..9a83012a8 --- /dev/null +++ b/tests/unittests/filtering.cpp @@ -0,0 +1,193 @@ +// Prototype: filtering objects out of a (self-referential) collection while +// handling the relations that point at the removed objects. This lives in +// user space on purpose - it is a worked example of what a filtering utility +// could do, built only on the public generated API (clone + relation adders) +// and podio::ObjectID. See ExampleMC (parents/daughters) as a stand-in for a +// sim-particle collection. + +#include "datamodel/ExampleMCCollection.h" + +#include "podio/ObjectID.h" + +#include + +#include +#include +#include + +namespace { + +/// What to do with a relation whose target is being removed. +enum class OnDangling { + LeaveUnconnected, ///< drop the edge, leave the surviving object unconnected + Cascade ///< keep the target too (removal of the target is forbidden) +}; + +/// Filter an ExampleMCCollection by an arbitrary predicate, rewriting the +/// self-referential parents/daughters relations of the survivors. +/// +/// The policy is chosen per relation: a Cascade relation forces the referenced +/// object to survive as well (computed as a closure), whereas a LeaveUnconnected +/// relation simply drops edges that point at removed objects. +ExampleMCCollection filterMC(const ExampleMCCollection& input, + const std::function& keep, + OnDangling parentsPolicy, OnDangling daughtersPolicy) { + // Phase 1: survivor set, seeded by the predicate and then closed under the + // relations that are configured to cascade. The fixpoint loop is safe in the + // presence of cycles (e.g. a particle that is its own ancestor). + std::unordered_set survive; + for (auto particle : input) { + if (keep(particle)) { + survive.insert(particle.getObjectID()); + } + } + + for (bool changed = true; changed;) { + changed = false; + for (auto particle : input) { + if (!survive.count(particle.getObjectID())) { + continue; + } + if (parentsPolicy == OnDangling::Cascade) { + for (auto rel : particle.parents()) { + changed |= survive.insert(rel.getObjectID()).second; + } + } + if (daughtersPolicy == OnDangling::Cascade) { + for (auto rel : particle.daughters()) { + changed |= survive.insert(rel.getObjectID()).second; + } + } + } + } + + // Phase 2: clone the survivors without relations (data members only) and + // record a mapping from the old ObjectID to the new, mutable handle. + ExampleMCCollection output; + std::unordered_map remap; + for (auto particle : input) { + if (!survive.count(particle.getObjectID())) { + continue; + } + auto cloned = particle.clone(/*cloneRelations=*/false); + remap.emplace(particle.getObjectID(), cloned); + output.push_back(cloned); + } + + // Phase 3: rewire the relations, keeping only the edges whose target also + // survived. For Cascade relations every target is guaranteed to be present + // (it was pulled into the survivor set above), so no edge is ever dropped. + for (auto particle : input) { + const auto self = remap.find(particle.getObjectID()); + if (self == remap.end()) { + continue; + } + auto newParticle = self->second; + for (auto rel : particle.parents()) { + if (const auto target = remap.find(rel.getObjectID()); target != remap.end()) { + newParticle.addparents(target->second); + } + } + for (auto rel : particle.daughters()) { + if (const auto target = remap.find(rel.getObjectID()); target != remap.end()) { + newParticle.adddaughters(target->second); + } + } + } + + return output; +} + +/// Build a small particle tree to filter: +/// +/// 0 (E=10) ─┬─ 1 (E=5) ── 3 (E=0.5) +/// └─ 2 (E=1) +ExampleMCCollection makeTree() { + ExampleMCCollection mcs; + auto p0 = mcs.create(); + p0.energy(10.0); + auto p1 = mcs.create(); + p1.energy(5.0); + auto p2 = mcs.create(); + p2.energy(1.0); + auto p3 = mcs.create(); + p3.energy(0.5); + + p0.adddaughters(p1); + p0.adddaughters(p2); + p1.adddaughters(p3); + + p1.addparents(p0); + p2.addparents(p0); + p3.addparents(p1); + + return mcs; +} + +} // namespace + +TEST_CASE("filter leaves dangling relations unconnected", "[filtering]") { + const auto input = makeTree(); + + // Keep everything with energy >= 1, i.e. drop the leaf particle (E=0.5). + const auto output = filterMC( + input, [](const ExampleMC& p) { return p.energy() >= 1.0; }, OnDangling::LeaveUnconnected, + OnDangling::LeaveUnconnected); + + // Survivors keep their input order: 0, 1, 2. + REQUIRE(output.size() == 3); + const auto p0 = output[0]; + const auto p1 = output[1]; + const auto p2 = output[2]; + + // p0 still points at both of its daughters, which now live in the new collection. + REQUIRE(p0.daughters().size() == 2); + REQUIRE(p0.daughters()[0] == p1); + REQUIRE(p0.daughters()[1] == p2); + + // p1's daughter (the removed leaf) is gone, leaving p1 unconnected downward, + // but its parent edge to the surviving p0 is intact. + REQUIRE(p1.daughters().empty()); + REQUIRE(p1.parents().size() == 1); + REQUIRE(p1.parents()[0] == p0); + REQUIRE(p2.parents()[0] == p0); +} + +TEST_CASE("filter cascades along daughters", "[filtering]") { + const auto input = makeTree(); + + // Seed with only the root (E=10); cascading daughters must pull in the whole + // subtree below it. + const auto output = filterMC( + input, [](const ExampleMC& p) { return p.energy() >= 6.0; }, OnDangling::LeaveUnconnected, + OnDangling::Cascade); + + REQUIRE(output.size() == 4); + // The transitive daughter (E=0.5) survived and is still attached to its parent. + REQUIRE(output[1].daughters().size() == 1); + REQUIRE(output[1].daughters()[0] == output[3]); +} + +TEST_CASE("filter cascades along parents while dropping other daughters", "[filtering]") { + const auto input = makeTree(); + + // Seed with only the leaf (E=0.5); cascading parents pulls in its ancestor + // chain (1 then 0) but not the sibling (2). + const auto output = filterMC( + input, [](const ExampleMC& p) { return p.energy() <= 0.6; }, OnDangling::Cascade, + OnDangling::LeaveUnconnected); + + // Survivors in input order: 0, 1, 3 (2 is removed). + REQUIRE(output.size() == 3); + const auto p0 = output[0]; + const auto p1 = output[1]; + const auto p3 = output[2]; + + // p0 used to have daughters {1, 2}; 2 was removed, so only the edge to 1 remains. + REQUIRE(p0.daughters().size() == 1); + REQUIRE(p0.daughters()[0] == p1); + // The cascaded parent chain stays connected all the way down to the leaf. + REQUIRE(p1.daughters().size() == 1); + REQUIRE(p1.daughters()[0] == p3); + REQUIRE(p3.parents()[0] == p1); +} From 7d4990ffa3441bf40539df5c5c2efed02e08b298 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Wed, 10 Jun 2026 11:26:20 +0300 Subject: [PATCH 02/14] poc one to many --- tests/unittests/filtering.cpp | 153 ++++++++++++++++++++++++++++++++++ 1 file changed, 153 insertions(+) diff --git a/tests/unittests/filtering.cpp b/tests/unittests/filtering.cpp index 9a83012a8..84145b46b 100644 --- a/tests/unittests/filtering.cpp +++ b/tests/unittests/filtering.cpp @@ -5,6 +5,8 @@ // and podio::ObjectID. See ExampleMC (parents/daughters) as a stand-in for a // sim-particle collection. +#include "datamodel/ExampleClusterCollection.h" +#include "datamodel/ExampleHitCollection.h" #include "datamodel/ExampleMCCollection.h" #include "podio/ObjectID.h" @@ -191,3 +193,154 @@ TEST_CASE("filter cascades along parents while dropping other daughters", "[filt REQUIRE(p1.daughters()[0] == p3); REQUIRE(p3.parents()[0] == p1); } + +// --------------------------------------------------------------------------- +// Cross-collection filtering: filter one collection and rewire the relations of +// *another* collection that references it. The key is that every collection gets +// its own ObjectID-keyed remap, and rewiring a relation simply looks the target +// up in the remap of the collection that owns the target type - so the same +// pattern works whether the relation points within a collection (parents/ +// daughters above) or across collections (cluster -> hit here). + +namespace { + +template +using Remap = std::unordered_map; + +struct FilteredHits { + ExampleHitCollection hits; + ExampleClusterCollection clusters; +}; + +/// Filter ExampleHits by a predicate and rebuild the ExampleClusters that +/// reference them through their `Hits` relation. +/// +/// The per-relation policy applies to the cluster -> hit edge: LeaveUnconnected +/// drops edges to removed hits and keeps the cluster; Cascade instead removes any +/// cluster that referenced a removed hit. +FilteredHits filterHits(const ExampleHitCollection& inHits, const ExampleClusterCollection& inClusters, + const std::function& keepHit, OnDangling clusterHitsPolicy) { + // Phase 1: survivor sets. Hits are decided by the predicate alone (they own no + // relations here). A cluster only fails to survive when the policy cascades and + // one of its hits was removed. + std::unordered_set hitSurvive; + for (auto hit : inHits) { + if (keepHit(hit)) { + hitSurvive.insert(hit.getObjectID()); + } + } + + std::unordered_set clusterSurvive; + for (auto cluster : inClusters) { + bool survives = true; + if (clusterHitsPolicy == OnDangling::Cascade) { + for (auto hit : cluster.Hits()) { + if (!hitSurvive.count(hit.getObjectID())) { + survives = false; + break; + } + } + } + if (survives) { + clusterSurvive.insert(cluster.getObjectID()); + } + } + + // Phase 2: clone survivors of each collection, building one remap per collection. + FilteredHits out; + Remap hitRemap; + for (auto hit : inHits) { + if (!hitSurvive.count(hit.getObjectID())) { + continue; + } + auto cloned = hit.clone(/*cloneRelations=*/false); + hitRemap.emplace(hit.getObjectID(), cloned); + out.hits.push_back(cloned); + } + Remap clusterRemap; + for (auto cluster : inClusters) { + if (!clusterSurvive.count(cluster.getObjectID())) { + continue; + } + auto cloned = cluster.clone(/*cloneRelations=*/false); + clusterRemap.emplace(cluster.getObjectID(), cloned); + out.clusters.push_back(cloned); + } + + // Phase 3: rewire the cluster -> hit edges, resolving each hit through the hit + // remap (a *different* collection's table). Edges to removed hits are dropped; + // under Cascade the owning cluster is already gone, so none are. + for (auto cluster : inClusters) { + const auto self = clusterRemap.find(cluster.getObjectID()); + if (self == clusterRemap.end()) { + continue; + } + auto newCluster = self->second; + for (auto hit : cluster.Hits()) { + if (const auto target = hitRemap.find(hit.getObjectID()); target != hitRemap.end()) { + newCluster.addHits(target->second); + } + } + } + + return out; +} + +/// Build two clusters over four hits: +/// cluster 0 -> { hit0 (E=1), hit1 (E=2) } +/// cluster 1 -> { hit2 (E=3), hit3 (E=0.5) } +std::pair makeHitsAndClusters() { + ExampleHitCollection hits; + auto h0 = hits.create(); + h0.energy(1.0); + auto h1 = hits.create(); + h1.energy(2.0); + auto h2 = hits.create(); + h2.energy(3.0); + auto h3 = hits.create(); + h3.energy(0.5); + + ExampleClusterCollection clusters; + auto c0 = clusters.create(); + c0.addHits(h0); + c0.addHits(h1); + auto c1 = clusters.create(); + c1.addHits(h2); + c1.addHits(h3); + + return {std::move(hits), std::move(clusters)}; +} + +} // namespace + +TEST_CASE("cross-collection filter leaves dangling hit edges unconnected", "[filtering]") { + const auto [inHits, inClusters] = makeHitsAndClusters(); + + // Drop the low-energy hit (E=0.5); clusters survive but lose the dropped edge. + const auto out = filterHits( + inHits, inClusters, [](const ExampleHit& h) { return h.energy() >= 1.0; }, + OnDangling::LeaveUnconnected); + + REQUIRE(out.hits.size() == 3); + REQUIRE(out.clusters.size() == 2); + + // cluster 0 keeps both of its hits; they now live in the new hit collection. + REQUIRE(out.clusters[0].Hits().size() == 2); + REQUIRE(out.clusters[0].Hits()[0] == out.hits[0]); + // cluster 1 referenced the removed hit, so only the surviving one (E=3) remains. + REQUIRE(out.clusters[1].Hits().size() == 1); + REQUIRE(out.clusters[1].Hits()[0].energy() == 3.0); +} + +TEST_CASE("cross-collection filter cascades from hit to cluster", "[filtering]") { + const auto [inHits, inClusters] = makeHitsAndClusters(); + + // Same hit removed, but now removing it cascades to the cluster that used it. + const auto out = filterHits( + inHits, inClusters, [](const ExampleHit& h) { return h.energy() >= 1.0; }, OnDangling::Cascade); + + REQUIRE(out.hits.size() == 3); + // cluster 1 referenced the removed hit and is gone; cluster 0 is intact. + REQUIRE(out.clusters.size() == 1); + REQUIRE(out.clusters[0].Hits().size() == 2); +} From 49185020b256f1e4c5b244eb154506e45fb6916c Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Wed, 10 Jun 2026 11:31:56 +0300 Subject: [PATCH 03/14] poc multi --- tests/unittests/filtering.cpp | 225 ++++++++++++++++++++++++++++++++++ 1 file changed, 225 insertions(+) diff --git a/tests/unittests/filtering.cpp b/tests/unittests/filtering.cpp index 84145b46b..da12a2589 100644 --- a/tests/unittests/filtering.cpp +++ b/tests/unittests/filtering.cpp @@ -8,6 +8,7 @@ #include "datamodel/ExampleClusterCollection.h" #include "datamodel/ExampleHitCollection.h" #include "datamodel/ExampleMCCollection.h" +#include "datamodel/ExampleWithOneRelationCollection.h" #include "podio/ObjectID.h" @@ -344,3 +345,227 @@ TEST_CASE("cross-collection filter cascades from hit to cluster", "[filtering]") REQUIRE(out.clusters.size() == 1); REQUIRE(out.clusters[0].Hits().size() == 2); } + +// --------------------------------------------------------------------------- +// The full prototype: filter across three collections at once, covering every +// relation kind - a cross-collection OneToMany (cluster -> hit), a self +// OneToMany (cluster -> sub-cluster) and a OneToOne (ref -> cluster) - with a +// removal closure that propagates across collections (hit -> cluster -> ref). + +namespace { + +struct EventPolicies { + OnDangling clusterHits; ///< cluster -> hit (OneToMany, cross-collection) + OnDangling clusterClusters; ///< cluster -> sub-cluster (OneToMany, self) + OnDangling refCluster; ///< ref -> cluster (OneToOne, cross-collection) +}; + +struct FilteredEvent { + ExampleHitCollection hits; + ExampleClusterCollection clusters; + ExampleWithOneRelationCollection refs; +}; + +FilteredEvent filterEvent(const ExampleHitCollection& inHits, const ExampleClusterCollection& inClusters, + const ExampleWithOneRelationCollection& inRefs, + const std::function& keepHit, EventPolicies policy) { + using IdSet = std::unordered_set; + + // Phase 1: removal closure. The predicate seeds the removed hits; from there + // removal propagates along every relation configured to cascade, across + // collections, until the sets stop changing. + IdSet hitsRemoved; + for (auto hit : inHits) { + if (!keepHit(hit)) { + hitsRemoved.insert(hit.getObjectID()); + } + } + + IdSet clustersRemoved; + IdSet refsRemoved; + for (bool changed = true; changed;) { + changed = false; + + for (auto cluster : inClusters) { + if (clustersRemoved.count(cluster.getObjectID())) { + continue; + } + bool remove = false; + if (policy.clusterHits == OnDangling::Cascade) { + for (auto hit : cluster.Hits()) { + remove |= static_cast(hitsRemoved.count(hit.getObjectID())); + } + } + if (policy.clusterClusters == OnDangling::Cascade) { + for (auto sub : cluster.Clusters()) { + remove |= static_cast(clustersRemoved.count(sub.getObjectID())); + } + } + if (remove) { + changed |= clustersRemoved.insert(cluster.getObjectID()).second; + } + } + + for (auto ref : inRefs) { + if (refsRemoved.count(ref.getObjectID()) || policy.refCluster != OnDangling::Cascade) { + continue; + } + const auto cluster = ref.cluster(); + if (cluster.isAvailable() && clustersRemoved.count(cluster.getObjectID())) { + changed |= refsRemoved.insert(ref.getObjectID()).second; + } + } + } + + // Phase 2: clone survivors of every collection, one remap per collection. + FilteredEvent out; + Remap hitRemap; + for (auto hit : inHits) { + if (hitsRemoved.count(hit.getObjectID())) { + continue; + } + auto cloned = hit.clone(/*cloneRelations=*/false); + hitRemap.emplace(hit.getObjectID(), cloned); + out.hits.push_back(cloned); + } + Remap clusterRemap; + for (auto cluster : inClusters) { + if (clustersRemoved.count(cluster.getObjectID())) { + continue; + } + auto cloned = cluster.clone(/*cloneRelations=*/false); + clusterRemap.emplace(cluster.getObjectID(), cloned); + out.clusters.push_back(cloned); + } + Remap refRemap; + for (auto ref : inRefs) { + if (refsRemoved.count(ref.getObjectID())) { + continue; + } + auto cloned = ref.clone(/*cloneRelations=*/false); + refRemap.emplace(ref.getObjectID(), cloned); + out.refs.push_back(cloned); + } + + // Phase 3: rewire every relation, resolving each target through the remap of + // the collection that owns it. OneToMany edges to removed targets are simply + // skipped; a OneToOne to a removed target is left unset. + for (auto cluster : inClusters) { + const auto self = clusterRemap.find(cluster.getObjectID()); + if (self == clusterRemap.end()) { + continue; + } + auto newCluster = self->second; + for (auto hit : cluster.Hits()) { + if (const auto target = hitRemap.find(hit.getObjectID()); target != hitRemap.end()) { + newCluster.addHits(target->second); + } + } + for (auto sub : cluster.Clusters()) { + if (const auto target = clusterRemap.find(sub.getObjectID()); target != clusterRemap.end()) { + newCluster.addClusters(target->second); + } + } + } + for (auto ref : inRefs) { + const auto self = refRemap.find(ref.getObjectID()); + if (self == refRemap.end()) { + continue; + } + const auto cluster = ref.cluster(); + if (cluster.isAvailable()) { + if (const auto target = clusterRemap.find(cluster.getObjectID()); target != clusterRemap.end()) { + self->second.cluster(target->second); + } + } + } + + return out; +} + +/// Build a three-level event to filter: +/// hits: h0 (E=1) h1 (E=2) h2 (E=0.5) +/// clusters: c0 -> {h0, h1} c1 -> {h2}, c0 also has sub-cluster c1 +/// refs: r0 -> c0 r1 -> c1 +FilteredEvent makeEvent() { + FilteredEvent ev; + auto h0 = ev.hits.create(); + h0.energy(1.0); + auto h1 = ev.hits.create(); + h1.energy(2.0); + auto h2 = ev.hits.create(); + h2.energy(0.5); + + auto c0 = ev.clusters.create(); + c0.addHits(h0); + c0.addHits(h1); + auto c1 = ev.clusters.create(); + c1.addHits(h2); + c0.addClusters(c1); + + auto r0 = ev.refs.create(); + r0.cluster(c0); + auto r1 = ev.refs.create(); + r1.cluster(c1); + + return ev; +} + +} // namespace + +TEST_CASE("full filter leaves all dangling relations unconnected", "[filtering]") { + const auto in = makeEvent(); + + // Drop h2 (E=0.5). With every policy LeaveUnconnected, nothing else is removed; + // only the edges to h2 disappear. + const auto out = filterEvent( + in.hits, in.clusters, in.refs, [](const ExampleHit& h) { return h.energy() >= 1.0; }, + {OnDangling::LeaveUnconnected, OnDangling::LeaveUnconnected, OnDangling::LeaveUnconnected}); + + REQUIRE(out.hits.size() == 2); + REQUIRE(out.clusters.size() == 2); + REQUIRE(out.refs.size() == 2); + + // c0 keeps both hits and its sub-cluster; c1 lost its only hit but survives. + REQUIRE(out.clusters[0].Hits().size() == 2); + REQUIRE(out.clusters[0].Clusters().size() == 1); + REQUIRE(out.clusters[0].Clusters()[0] == out.clusters[1]); + REQUIRE(out.clusters[1].Hits().empty()); + // OneToOne edges still point at the surviving clusters. + REQUIRE(out.refs[1].cluster() == out.clusters[1]); +} + +TEST_CASE("full filter cascades across all three collections", "[filtering]") { + const auto in = makeEvent(); + + // Drop h2 (E=0.5) and cascade every relation: removing h2 removes c1 (it used + // h2), removing c1 removes both c0 (its sub-cluster is gone) and r1 (its + // cluster is gone), and removing c0 removes r0. + const auto out = filterEvent( + in.hits, in.clusters, in.refs, [](const ExampleHit& h) { return h.energy() >= 1.0; }, + {OnDangling::Cascade, OnDangling::Cascade, OnDangling::Cascade}); + + REQUIRE(out.hits.size() == 2); + REQUIRE(out.clusters.empty()); + REQUIRE(out.refs.empty()); +} + +TEST_CASE("full filter cascades to cluster but leaves its ref unconnected", "[filtering]") { + const auto in = makeEvent(); + + // Cascade hit -> cluster, but leave ref -> cluster unconnected: c1 is removed, + // and r1 survives with its OneToOne relation left unset. + const auto out = filterEvent( + in.hits, in.clusters, in.refs, [](const ExampleHit& h) { return h.energy() >= 1.0; }, + {OnDangling::Cascade, OnDangling::LeaveUnconnected, OnDangling::LeaveUnconnected}); + + // c1 (used h2) is gone; c0's sub-cluster edge to it is dropped but c0 survives. + REQUIRE(out.clusters.size() == 1); + REQUIRE(out.clusters[0].Hits().size() == 2); + REQUIRE(out.clusters[0].Clusters().empty()); + + // Both refs survive; r1's cluster was removed so its relation is now unset. + REQUIRE(out.refs.size() == 2); + REQUIRE(out.refs[0].cluster() == out.clusters[0]); + REQUIRE_FALSE(out.refs[1].cluster().isAvailable()); +} From 6011228413f3ff9c6a3c474c60b25c11841c5099 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Wed, 10 Jun 2026 11:32:46 +0300 Subject: [PATCH 04/14] poc links --- tests/unittests/filtering.cpp | 79 +++++++++++++++++++++++++++++++---- 1 file changed, 72 insertions(+), 7 deletions(-) diff --git a/tests/unittests/filtering.cpp b/tests/unittests/filtering.cpp index da12a2589..69a4046bd 100644 --- a/tests/unittests/filtering.cpp +++ b/tests/unittests/filtering.cpp @@ -9,6 +9,7 @@ #include "datamodel/ExampleHitCollection.h" #include "datamodel/ExampleMCCollection.h" #include "datamodel/ExampleWithOneRelationCollection.h" +#include "datamodel/TestLinkCollection.h" #include "podio/ObjectID.h" @@ -358,16 +359,19 @@ struct EventPolicies { OnDangling clusterHits; ///< cluster -> hit (OneToMany, cross-collection) OnDangling clusterClusters; ///< cluster -> sub-cluster (OneToMany, self) OnDangling refCluster; ///< ref -> cluster (OneToOne, cross-collection) + OnDangling linkFrom; ///< link -> hit (Link "From" endpoint) + OnDangling linkTo; ///< link -> cluster (Link "To" endpoint) }; struct FilteredEvent { ExampleHitCollection hits; ExampleClusterCollection clusters; ExampleWithOneRelationCollection refs; + TestLinkCollection links; }; FilteredEvent filterEvent(const ExampleHitCollection& inHits, const ExampleClusterCollection& inClusters, - const ExampleWithOneRelationCollection& inRefs, + const ExampleWithOneRelationCollection& inRefs, const TestLinkCollection& inLinks, const std::function& keepHit, EventPolicies policy) { using IdSet = std::unordered_set; @@ -480,6 +484,34 @@ FilteredEvent filterEvent(const ExampleHitCollection& inHits, const ExampleClust } } + // Links live in their own collection and reference objects in two others. A + // link is dropped when an endpoint was removed under a Cascade policy; + // otherwise it is kept, its weight preserved, and each surviving endpoint + // rewired through that endpoint's remap (a removed endpoint is left unset). + // Links are terminal - nothing references them - so they need no remap. + for (auto link : inLinks) { + const auto from = link.getFrom(); + const auto to = link.getTo(); + const bool fromRemoved = from.isAvailable() && hitsRemoved.count(from.getObjectID()); + const bool toRemoved = to.isAvailable() && clustersRemoved.count(to.getObjectID()); + if ((fromRemoved && policy.linkFrom == OnDangling::Cascade) || + (toRemoved && policy.linkTo == OnDangling::Cascade)) { + continue; + } + auto newLink = link.clone(/*cloneRelations=*/false); // copies the weight only + if (from.isAvailable()) { + if (const auto target = hitRemap.find(from.getObjectID()); target != hitRemap.end()) { + newLink.setFrom(target->second); + } + } + if (to.isAvailable()) { + if (const auto target = clusterRemap.find(to.getObjectID()); target != clusterRemap.end()) { + newLink.setTo(target->second); + } + } + out.links.push_back(newLink); + } + return out; } @@ -487,6 +519,7 @@ FilteredEvent filterEvent(const ExampleHitCollection& inHits, const ExampleClust /// hits: h0 (E=1) h1 (E=2) h2 (E=0.5) /// clusters: c0 -> {h0, h1} c1 -> {h2}, c0 also has sub-cluster c1 /// refs: r0 -> c0 r1 -> c1 +/// links: l0: h0 -> c0 l1: h2 -> c1 FilteredEvent makeEvent() { FilteredEvent ev; auto h0 = ev.hits.create(); @@ -508,6 +541,15 @@ FilteredEvent makeEvent() { auto r1 = ev.refs.create(); r1.cluster(c1); + auto l0 = ev.links.create(); + l0.setFrom(h0); + l0.setTo(c0); + l0.setWeight(1.0f); + auto l1 = ev.links.create(); + l1.setFrom(h2); + l1.setTo(c1); + l1.setWeight(2.0f); + return ev; } @@ -519,8 +561,9 @@ TEST_CASE("full filter leaves all dangling relations unconnected", "[filtering]" // Drop h2 (E=0.5). With every policy LeaveUnconnected, nothing else is removed; // only the edges to h2 disappear. const auto out = filterEvent( - in.hits, in.clusters, in.refs, [](const ExampleHit& h) { return h.energy() >= 1.0; }, - {OnDangling::LeaveUnconnected, OnDangling::LeaveUnconnected, OnDangling::LeaveUnconnected}); + in.hits, in.clusters, in.refs, in.links, [](const ExampleHit& h) { return h.energy() >= 1.0; }, + {OnDangling::LeaveUnconnected, OnDangling::LeaveUnconnected, OnDangling::LeaveUnconnected, + OnDangling::LeaveUnconnected, OnDangling::LeaveUnconnected}); REQUIRE(out.hits.size() == 2); REQUIRE(out.clusters.size() == 2); @@ -533,6 +576,15 @@ TEST_CASE("full filter leaves all dangling relations unconnected", "[filtering]" REQUIRE(out.clusters[1].Hits().empty()); // OneToOne edges still point at the surviving clusters. REQUIRE(out.refs[1].cluster() == out.clusters[1]); + + // Both links survive; l1's "From" hit was removed, so that endpoint is unset + // while its "To" cluster and the preserved weight remain intact. + REQUIRE(out.links.size() == 2); + REQUIRE(out.links[0].getFrom() == out.hits[0]); + REQUIRE(out.links[0].getTo() == out.clusters[0]); + REQUIRE_FALSE(out.links[1].getFrom().isAvailable()); + REQUIRE(out.links[1].getTo() == out.clusters[1]); + REQUIRE(out.links[1].getWeight() == 2.0f); } TEST_CASE("full filter cascades across all three collections", "[filtering]") { @@ -542,12 +594,16 @@ TEST_CASE("full filter cascades across all three collections", "[filtering]") { // h2), removing c1 removes both c0 (its sub-cluster is gone) and r1 (its // cluster is gone), and removing c0 removes r0. const auto out = filterEvent( - in.hits, in.clusters, in.refs, [](const ExampleHit& h) { return h.energy() >= 1.0; }, - {OnDangling::Cascade, OnDangling::Cascade, OnDangling::Cascade}); + in.hits, in.clusters, in.refs, in.links, [](const ExampleHit& h) { return h.energy() >= 1.0; }, + {OnDangling::Cascade, OnDangling::Cascade, OnDangling::Cascade, OnDangling::Cascade, + OnDangling::Cascade}); REQUIRE(out.hits.size() == 2); REQUIRE(out.clusters.empty()); REQUIRE(out.refs.empty()); + // l0's "To" cluster (c0) and l1's "From" hit (h2) were both removed under a + // cascading endpoint policy, so both links are dropped. + REQUIRE(out.links.empty()); } TEST_CASE("full filter cascades to cluster but leaves its ref unconnected", "[filtering]") { @@ -556,8 +612,9 @@ TEST_CASE("full filter cascades to cluster but leaves its ref unconnected", "[fi // Cascade hit -> cluster, but leave ref -> cluster unconnected: c1 is removed, // and r1 survives with its OneToOne relation left unset. const auto out = filterEvent( - in.hits, in.clusters, in.refs, [](const ExampleHit& h) { return h.energy() >= 1.0; }, - {OnDangling::Cascade, OnDangling::LeaveUnconnected, OnDangling::LeaveUnconnected}); + in.hits, in.clusters, in.refs, in.links, [](const ExampleHit& h) { return h.energy() >= 1.0; }, + {OnDangling::Cascade, OnDangling::LeaveUnconnected, OnDangling::LeaveUnconnected, + OnDangling::LeaveUnconnected, OnDangling::LeaveUnconnected}); // c1 (used h2) is gone; c0's sub-cluster edge to it is dropped but c0 survives. REQUIRE(out.clusters.size() == 1); @@ -568,4 +625,12 @@ TEST_CASE("full filter cascades to cluster but leaves its ref unconnected", "[fi REQUIRE(out.refs.size() == 2); REQUIRE(out.refs[0].cluster() == out.clusters[0]); REQUIRE_FALSE(out.refs[1].cluster().isAvailable()); + + // Both links survive under the leave-unconnected endpoint policies. l1 lost + // both endpoints (h2 removed, c1 cascaded away), so it is fully unset. + REQUIRE(out.links.size() == 2); + REQUIRE(out.links[0].getFrom() == out.hits[0]); + REQUIRE(out.links[0].getTo() == out.clusters[0]); + REQUIRE_FALSE(out.links[1].getFrom().isAvailable()); + REQUIRE_FALSE(out.links[1].getTo().isAvailable()); } From d34e07952d1a5c3862bb7493fde9795aba241127 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Wed, 10 Jun 2026 12:10:59 +0300 Subject: [PATCH 05/14] poc framefilter --- include/podio/FrameFilter.h | 238 +++++++++++++++++++++++++ include/podio/detail/FilterHooks.h | 108 +++++++++++ python/podio_gen/cpp_generator.py | 15 ++ python/podio_gen/generator_base.py | 2 + python/templates/FilterHooks.cc.jinja2 | 109 +++++++++++ tests/unittests/CMakeLists.txt | 2 +- tests/unittests/frame_filter.cpp | 177 ++++++++++++++++++ 7 files changed, 650 insertions(+), 1 deletion(-) create mode 100644 include/podio/FrameFilter.h create mode 100644 include/podio/detail/FilterHooks.h create mode 100644 python/templates/FilterHooks.cc.jinja2 create mode 100644 tests/unittests/frame_filter.cpp diff --git a/include/podio/FrameFilter.h b/include/podio/FrameFilter.h new file mode 100644 index 000000000..7bd89540e --- /dev/null +++ b/include/podio/FrameFilter.h @@ -0,0 +1,238 @@ +#ifndef PODIO_FRAMEFILTER_H +#define PODIO_FRAMEFILTER_H + +#include "podio/CollectionBase.h" +#include "podio/Frame.h" +#include "podio/ObjectID.h" +#include "podio/detail/FilterHooks.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace podio { + +namespace detail { + /// Deduce the (decayed) argument type of a unary predicate's operator(). + template + struct FilterPredArg; + template + struct FilterPredArg { + using type = std::decay_t; + }; + template + struct FilterPredArg { + using type = std::decay_t; + }; + template + using FilterPredArgT = typename FilterPredArg::operator())>::type; +} // namespace detail + +/// Filters the collections of a Frame into a new Frame, rewiring all relations. +/// +/// Two orthogonal axes control the result: +/// - Retention (per collection): keep-all (default), a predicate (`keep`), or +/// keep-if-referenced (`keepReferenced`, i.e. drop objects that no surviving +/// object points at). +/// - Edge policy (per relation): a dangling reference is left unset by default; +/// `cascade("Type.relation")` instead removes the object holding the relation +/// when its target is removed. +/// +/// All collections that participate in the relation graph are rebuilt so the +/// output Frame is internally consistent. +class FrameFilter { +public: + explicit FrameFilter(const podio::Frame& frame) : m_frame(frame) { + } + + /// Keep objects of the named collection for which @p predicate is true. + /// The collection type is deduced from the predicate's argument type. + template + FrameFilter& keep(const std::string& name, Pred predicate) { + using ValueT = detail::FilterPredArgT; + using CollT = typename ValueT::collection_type; + m_predicates[name] = [predicate = std::move(predicate)](const podio::CollectionBase& coll, std::size_t i) { + return predicate(static_cast(coll)[i]); + }; + return *this; + } + + /// Keep objects of the named collection only if a surviving object references + /// them (orphan sweep, shared-aware across collections). + FrameFilter& keepReferenced(const std::string& name) { + m_keepReferenced.insert(name); + return *this; + } + + /// Treat the named relation ("Type.relation") as mandatory: if its target is + /// removed, remove the object holding it (integrity cascade). + FrameFilter& cascade(const std::string& qualifiedRelation) { + m_cascade.insert(qualifiedRelation); + return *this; + } + + /// Run the filter and return a new, fully-rewired Frame. + podio::Frame run() const { + enum Mode { KeepAll, Predicate, Referenced }; + struct CollInfo { + std::string name; + const podio::CollectionBase* coll; + const detail::FilterHooks* hooks; + std::string type; + uint32_t id; + std::size_t size; + Mode mode; + std::vector killed; + std::vector marked; + }; + + std::vector colls; + std::unordered_map byId; + for (const auto& name : m_frame.getAvailableCollections()) { + const auto* coll = m_frame.get(name); + const auto type = std::string(coll->getValueTypeName()); + const auto* hooks = detail::getFilterHooks(type); + if (!hooks) { + throw std::runtime_error("podio::FrameFilter: no filter hooks registered for type '" + type + + "' (collection '" + name + "')"); + } + Mode mode = KeepAll; + if (m_predicates.count(name)) { + mode = Predicate; + } else if (m_keepReferenced.count(name)) { + mode = Referenced; + } + const auto size = coll->size(); + byId[coll->getID()] = colls.size(); + colls.push_back({name, coll, hooks, type, coll->getID(), size, mode, + std::vector(size, 0), std::vector(size, 0)}); + } + + // Resolve a relation target to (collection slot, object index). + const auto locate = [&](podio::ObjectID target) -> std::pair { + if (target.index < 0) { + return {-1, 0}; + } + const auto it = byId.find(target.collectionID); + return it == byId.end() ? std::pair{-1, 0} + : std::pair{static_cast(it->second), + static_cast(target.index)}; + }; + + // Phase 0: seed removals from predicates. + for (auto& ci : colls) { + if (ci.mode == Predicate) { + const auto& pred = m_predicates.at(ci.name); + for (std::size_t i = 0; i < ci.size; ++i) { + ci.killed[i] = pred(*ci.coll, i) ? 0 : 1; + } + } + } + + // Phase 1: integrity-cascade closure (monotonic). If a cascade relation's + // target is killed, the object holding it is killed too. + for (bool changed = true; changed;) { + changed = false; + for (auto& ci : colls) { + for (std::size_t i = 0; i < ci.size; ++i) { + if (ci.killed[i]) { + continue; + } + bool kill = false; + ci.hooks->edges(*ci.coll, i, [&](std::string_view rel, podio::ObjectID target) { + if (kill || !m_cascade.count(ci.type + "." + std::string(rel))) { + return; + } + const auto [k, idx] = locate(target); + if (k >= 0 && colls[k].killed[idx]) { + kill = true; + } + }); + if (kill) { + ci.killed[i] = 1; + changed = true; + } + } + } + } + + // Phase 2: reachability mark (monotonic). Roots are surviving objects in + // non-referenced collections; marking flows forward and decides the fate of + // keep-referenced collections. + for (bool changed = true; changed;) { + changed = false; + for (std::size_t k = 0; k < colls.size(); ++k) { + auto& ci = colls[k]; + for (std::size_t i = 0; i < ci.size; ++i) { + if (ci.killed[i] || (ci.mode == Referenced && !ci.marked[i])) { + continue; // not a surviving source (yet) + } + ci.hooks->edges(*ci.coll, i, [&](std::string_view, podio::ObjectID target) { + const auto [tk, tidx] = locate(target); + if (tk >= 0 && colls[tk].mode == Referenced && !colls[tk].killed[tidx] && !colls[tk].marked[tidx]) { + colls[tk].marked[tidx] = 1; + changed = true; + } + }); + } + } + } + + // Phase 3: sweep unreferenced objects in keep-referenced collections. + for (auto& ci : colls) { + if (ci.mode == Referenced) { + for (std::size_t i = 0; i < ci.size; ++i) { + if (!ci.marked[i]) { + ci.killed[i] = 1; + } + } + } + } + + // Phase 4: clone survivors, build the global remap, then rewire. + std::vector> survivors(colls.size()); + std::vector> outColls(colls.size()); + detail::FilterRemap remap; + for (std::size_t k = 0; k < colls.size(); ++k) { + auto& ci = colls[k]; + for (std::size_t i = 0; i < ci.size; ++i) { + if (!ci.killed[i]) { + survivors[k].push_back(i); + } + } + outColls[k] = ci.hooks->cloneSurvivors(*ci.coll, survivors[k]); + for (std::size_t j = 0; j < survivors[k].size(); ++j) { + podio::ObjectID original; + original.index = static_cast(survivors[k][j]); + original.collectionID = ci.id; + remap.add(original, outColls[k].get(), j); + } + } + for (std::size_t k = 0; k < colls.size(); ++k) { + colls[k].hooks->rewire(*colls[k].coll, survivors[k], *outColls[k], remap); + } + + // Phase 5: assemble the output Frame. + podio::Frame out; + for (std::size_t k = 0; k < colls.size(); ++k) { + out.put(std::move(outColls[k]), colls[k].name); + } + return out; + } + +private: + const podio::Frame& m_frame; + std::unordered_map> m_predicates{}; + std::unordered_set m_keepReferenced{}; + std::unordered_set m_cascade{}; +}; + +} // namespace podio + +#endif // PODIO_FRAMEFILTER_H diff --git a/include/podio/detail/FilterHooks.h b/include/podio/detail/FilterHooks.h new file mode 100644 index 000000000..4cee63b5c --- /dev/null +++ b/include/podio/detail/FilterHooks.h @@ -0,0 +1,108 @@ +#ifndef PODIO_DETAIL_FILTERHOOKS_H +#define PODIO_DETAIL_FILTERHOOKS_H + +#include "podio/ObjectID.h" + +#include +#include +#include +#include +#include +#include +#include + +namespace podio { +class CollectionBase; + +namespace detail { + + /// Where an object ended up in the filtered output: the new collection and the + /// index of the object within it. + struct RemapTarget { + podio::CollectionBase* collection{nullptr}; + std::size_t index{0}; + }; + + /// Maps an object's original ObjectID to its location in the filtered output. + /// A missing entry means the object was removed by the filter. + class FilterRemap { + public: + void add(podio::ObjectID original, podio::CollectionBase* collection, std::size_t index) { + m_map[original] = {collection, index}; + } + + /// @returns the new location of the object, or nullptr if it was removed + const RemapTarget* find(podio::ObjectID original) const { + const auto it = m_map.find(original); + return it == m_map.end() ? nullptr : &it->second; + } + + private: + std::unordered_map m_map{}; + }; + + /// Resolve a relation target to the const handle it now has in the filtered + /// output collection of type @p CollT, or std::nullopt if it was removed. Used + /// by the generated rewire hooks. + template + std::optional resolveTarget(const FilterRemap& remap, podio::ObjectID target) { + const auto* loc = remap.find(target); + if (!loc) { + return std::nullopt; + } + return static_cast(*loc->collection)[loc->index]; + } + + /// Callback used to report a single outgoing relation edge of an object during + /// the cascade/reachability passes: the relation name and the target ObjectID. + using EdgeSink = std::function; + + /// Type-erased, per-datatype operations the FrameFilter needs but which require + /// concrete type knowledge (so they are generated, one implementation per type, + /// and registered by type name). The FrameFilter only ever sees CollectionBase. + struct FilterHooks { + virtual ~FilterHooks() = default; + + /// Report the outgoing relation edges of object @p index in @p collection. + virtual void edges(const podio::CollectionBase& collection, std::size_t index, + const EdgeSink& sink) const = 0; + + /// Clone the objects at @p survivors (in order) into a freshly allocated + /// collection of the same type, copying data members but no relations. + virtual std::unique_ptr + cloneSurvivors(const podio::CollectionBase& collection, const std::vector& survivors) const = 0; + + /// Rewire the relations of the cloned survivors in @p output, resolving each + /// relation target through @p remap. @p input is the original collection and + /// @p survivors the same index list passed to cloneSurvivors (so output[j] + /// corresponds to input[survivors[j]]). + virtual void rewire(const podio::CollectionBase& input, const std::vector& survivors, + podio::CollectionBase& output, const FilterRemap& remap) const = 0; + }; + + /// The process-wide registry of filter hooks, keyed by datatype name (as + /// returned by CollectionBase::getTypeName()). Header-only so that generated + /// datamodel code can register into it without a link-time dependency. + inline std::unordered_map>& filterHookRegistry() { + static std::unordered_map> registry; + return registry; + } + + /// Register the hooks for a datatype. Returns true so it can be used to + /// initialize a namespace-scope bool in generated code. + inline bool registerFilterHooks(std::string typeName, std::unique_ptr hooks) { + filterHookRegistry()[std::move(typeName)] = std::move(hooks); + return true; + } + + /// @returns the hooks registered for @p typeName, or nullptr if none. + inline const FilterHooks* getFilterHooks(std::string_view typeName) { + auto& registry = filterHookRegistry(); + const auto it = registry.find(std::string(typeName)); + return it == registry.end() ? nullptr : it->second.get(); + } + +} // namespace detail +} // namespace podio + +#endif // PODIO_DETAIL_FILTERHOOKS_H diff --git a/python/podio_gen/cpp_generator.py b/python/podio_gen/cpp_generator.py index 417a0c463..685e9dec4 100644 --- a/python/podio_gen/cpp_generator.py +++ b/python/podio_gen/cpp_generator.py @@ -148,6 +148,16 @@ def do_process_datatype(self, name, datatype): self._fill_templates("Collection", datatype) self._fill_templates("CollectionData", datatype) + # Relations to interface types are polymorphic and have no single + # collection_type, so the filter hooks skip them for now. + datatype["filter_single_rel"] = [ + r for r in datatype["OneToOneRelations"] if not self._is_in(r.full_type, "interfaces") + ] + datatype["filter_multi_rel"] = [ + r for r in datatype["OneToManyRelations"] if not self._is_in(r.full_type, "interfaces") + ] + self._fill_templates("FilterHooks", datatype) + if "SIO" in self.io_handlers: self._fill_templates("SIOBlock", datatype) @@ -250,6 +260,11 @@ def do_process_link(self, _, link): ) ) self._fill_templates("LinkCollection", link) + # Interface endpoints are polymorphic and have no single collection_type, + # so the filter hooks skip them for now. + link["filter_from"] = not self._is_in(link["From"].full_type, "interfaces") + link["filter_to"] = not self._is_in(link["To"].full_type, "interfaces") + self._fill_templates("FilterHooks", link) return link def print_report(self): diff --git a/python/podio_gen/generator_base.py b/python/podio_gen/generator_base.py index e70b5fc61..9ab5cb123 100644 --- a/python/podio_gen/generator_base.py +++ b/python/podio_gen/generator_base.py @@ -257,6 +257,7 @@ def get_fn_format(tmpl): "CollectionData": "CollectionData", "MutableStruct": "Struct", "LinkCollection": "Collection", + "FilterHooks": "FilterHooks", } return f"{prefix.get(tmpl, '')}{{name}}{postfix.get(tmpl, '')}.{{end}}" @@ -269,6 +270,7 @@ def get_fn_format(tmpl): "ParentModule": ("jl",), "LinkCollection": ("h",), "DatamodelLinks": ("cc",), + "FilterHooks": ("cc",), }.get(template_base, ("h", "cc")) fn_templates = [] diff --git a/python/templates/FilterHooks.cc.jinja2 b/python/templates/FilterHooks.cc.jinja2 new file mode 100644 index 000000000..78ff5aac5 --- /dev/null +++ b/python/templates/FilterHooks.cc.jinja2 @@ -0,0 +1,109 @@ +// AUTOMATICALLY GENERATED FILE - DO NOT EDIT + +#include "{{ incfolder }}{{ class.bare_type }}Collection.h" +{% if From is defined %} +{% for include in include_types %} +{{ include }} +{% endfor %} +{% else %} +{% for include in includes_coll_cc %} +{{ include }} +{% endfor %} +{% endif %} + +#include + +#include +#include +#include + +{% set coll_type = class.full_type + 'Collection' %} +namespace { + +struct {{ class.bare_type }}FilterHooks : podio::detail::FilterHooks { + void edges([[maybe_unused]] const podio::CollectionBase& coll, [[maybe_unused]] std::size_t index, + [[maybe_unused]] const podio::detail::EdgeSink& sink) const override { + [[maybe_unused]] const auto obj = static_cast(coll)[index]; +{% if From is defined %} +{% if filter_from %} + if (obj.getFrom().isAvailable()) { + sink("from", obj.getFrom().getObjectID()); + } +{% endif %} +{% if filter_to %} + if (obj.getTo().isAvailable()) { + sink("to", obj.getTo().getObjectID()); + } +{% endif %} +{% else %} +{% for relation in filter_single_rel %} + if (obj.{{ relation.getter_name(use_get_syntax) }}().isAvailable()) { + sink("{{ relation.name }}", obj.{{ relation.getter_name(use_get_syntax) }}().getObjectID()); + } +{% endfor %} +{% for relation in filter_multi_rel %} + for (const auto& rel : obj.{{ relation.getter_name(use_get_syntax) }}()) { + sink("{{ relation.name }}", rel.getObjectID()); + } +{% endfor %} +{% endif %} + } + + std::unique_ptr + cloneSurvivors(const podio::CollectionBase& coll, const std::vector& survivors) const override { + const auto& in = static_cast(coll); + auto out = std::make_unique<{{ coll_type }}>(); + for (auto idx : survivors) { + out->push_back(in[idx].clone(false)); + } + return out; + } + + void rewire([[maybe_unused]] const podio::CollectionBase& input, + [[maybe_unused]] const std::vector& survivors, + [[maybe_unused]] podio::CollectionBase& output, + [[maybe_unused]] const podio::detail::FilterRemap& remap) const override { + [[maybe_unused]] const auto& in = static_cast(input); + [[maybe_unused]] auto& out = static_cast<{{ coll_type }}&>(output); + for (std::size_t j = 0; j < survivors.size(); ++j) { + [[maybe_unused]] auto obj = out[j]; + [[maybe_unused]] const auto src = in[survivors[j]]; +{% if From is defined %} +{% if filter_from %} + if (src.getFrom().isAvailable()) { + if (auto t = podio::detail::resolveTarget<{{ From.full_type }}::collection_type>(remap, src.getFrom().getObjectID())) { + obj.setFrom(*t); + } + } +{% endif %} +{% if filter_to %} + if (src.getTo().isAvailable()) { + if (auto t = podio::detail::resolveTarget<{{ To.full_type }}::collection_type>(remap, src.getTo().getObjectID())) { + obj.setTo(*t); + } + } +{% endif %} +{% else %} +{% for relation in filter_single_rel %} + if (src.{{ relation.getter_name(use_get_syntax) }}().isAvailable()) { + if (auto t = podio::detail::resolveTarget<{{ relation.full_type }}::collection_type>(remap, src.{{ relation.getter_name(use_get_syntax) }}().getObjectID())) { + obj.{{ relation.setter_name(use_get_syntax) }}(*t); + } + } +{% endfor %} +{% for relation in filter_multi_rel %} + for (const auto& rel : src.{{ relation.getter_name(use_get_syntax) }}()) { + if (auto t = podio::detail::resolveTarget<{{ relation.full_type }}::collection_type>(remap, rel.getObjectID())) { + obj.{{ relation.setter_name(use_get_syntax, is_relation=True) }}(*t); + } + } +{% endfor %} +{% endif %} + } + } +}; + +const bool {{ class.bare_type }}_filter_registered = podio::detail::registerFilterHooks( + std::string({{ coll_type }}::valueTypeName), std::make_unique<{{ class.bare_type }}FilterHooks>()); + +} // namespace diff --git a/tests/unittests/CMakeLists.txt b/tests/unittests/CMakeLists.txt index 092d33c5d..d9fc3af4e 100644 --- a/tests/unittests/CMakeLists.txt +++ b/tests/unittests/CMakeLists.txt @@ -41,7 +41,7 @@ else() endif() find_package(Threads REQUIRED) -add_executable(unittest_podio unittest.cpp frame.cpp buffer_factory.cpp interface_types.cpp std_interoperability.cpp links.cpp test_json_output.cpp filtering.cpp) +add_executable(unittest_podio unittest.cpp frame.cpp buffer_factory.cpp interface_types.cpp std_interoperability.cpp links.cpp test_json_output.cpp filtering.cpp frame_filter.cpp) target_link_libraries(unittest_podio PUBLIC TestDataModel ExtensionDataModel InterfaceExtensionDataModel PRIVATE Catch2::Catch2WithMain Threads::Threads podio::podioRootIO) # Catch2 TEST_CASE/SECTION macros expand __COUNTER__ at the call site in user # code, so marking Catch2 headers as system includes is not sufficient to diff --git a/tests/unittests/frame_filter.cpp b/tests/unittests/frame_filter.cpp new file mode 100644 index 000000000..75121fc2a --- /dev/null +++ b/tests/unittests/frame_filter.cpp @@ -0,0 +1,177 @@ +// Exercises podio::FrameFilter (driven by the registered per-type FilterHooks) +// over Frames built from the test datamodel. Mirrors the hand-written prototype +// scenarios in filtering.cpp, plus the keep-referenced / orphan-sweep capability +// that only the Frame-level filter provides. + +#include "datamodel/ExampleClusterCollection.h" +#include "datamodel/ExampleHitCollection.h" +#include "datamodel/ExampleMCCollection.h" +#include "datamodel/ExampleWithOneRelationCollection.h" +#include "datamodel/TestLinkCollection.h" + +#include "podio/Frame.h" +#include "podio/FrameFilter.h" + +#include + +#include + +namespace { + +/// MC tree: 0 (E=10) -> {1 (E=5) -> 3 (E=0.5), 2 (E=1)} +podio::Frame makeMCFrame() { + ExampleMCCollection mcs; + auto p0 = mcs.create(); + p0.energy(10.0); + auto p1 = mcs.create(); + p1.energy(5.0); + auto p2 = mcs.create(); + p2.energy(1.0); + auto p3 = mcs.create(); + p3.energy(0.5); + p0.adddaughters(p1); + p0.adddaughters(p2); + p1.adddaughters(p3); + p1.addparents(p0); + p2.addparents(p0); + p3.addparents(p1); + + podio::Frame frame; + frame.put(std::move(mcs), "mcparticles"); + return frame; +} + +/// hits h0(1) h1(2) h2(0.5); clusters c0->{h0,h1} (+sub-cluster c1), c1->{h2}; +/// refs r0->c0, r1->c1; links l0: h0->c0, l1: h2->c1 +podio::Frame makeEventFrame() { + ExampleHitCollection hits; + auto h0 = hits.create(); + h0.energy(1.0); + auto h1 = hits.create(); + h1.energy(2.0); + auto h2 = hits.create(); + h2.energy(0.5); + + ExampleClusterCollection clusters; + auto c0 = clusters.create(); + c0.addHits(h0); + c0.addHits(h1); + auto c1 = clusters.create(); + c1.addHits(h2); + c0.addClusters(c1); + + ExampleWithOneRelationCollection refs; + auto r0 = refs.create(); + r0.cluster(c0); + auto r1 = refs.create(); + r1.cluster(c1); + + TestLinkCollection links; + auto l0 = links.create(); + l0.setFrom(h0); + l0.setTo(c0); + auto l1 = links.create(); + l1.setFrom(h2); + l1.setTo(c1); + + podio::Frame frame; + frame.put(std::move(hits), "hits"); + frame.put(std::move(clusters), "clusters"); + frame.put(std::move(refs), "refs"); + frame.put(std::move(links), "links"); + return frame; +} + +const std::string linkType = std::string(TestLinkCollection::valueTypeName); + +} // namespace + +TEST_CASE("FrameFilter: self-relations, leave unconnected", "[framefilter]") { + const auto in = makeMCFrame(); + const auto out = podio::FrameFilter{in} + .keep("mcparticles", [](const ExampleMC& p) { return p.energy() >= 1.0; }) + .run(); + + const auto& mcs = out.get("mcparticles"); + REQUIRE(mcs.size() == 3); + REQUIRE(mcs[0].daughters().size() == 2); // p0 keeps both surviving daughters + REQUIRE(mcs[1].daughters().empty()); // p1's removed leaf leaves it unconnected + REQUIRE(mcs[1].parents()[0] == mcs[0]); +} + +TEST_CASE("FrameFilter: whole event, leave unconnected", "[framefilter]") { + const auto in = makeEventFrame(); + const auto out = podio::FrameFilter{in} + .keep("hits", [](const ExampleHit& h) { return h.energy() >= 1.0; }) + .run(); + + const auto& hits = out.get("hits"); + const auto& clusters = out.get("clusters"); + const auto& links = out.get("links"); + REQUIRE(hits.size() == 2); + REQUIRE(clusters.size() == 2); + // c0 keeps both hits and its sub-cluster; c1 lost its only hit but survives. + REQUIRE(clusters[0].Hits().size() == 2); + REQUIRE(clusters[0].Clusters().size() == 1); + REQUIRE(clusters[1].Hits().empty()); + // l1's "From" hit was removed -> unset; weight and "To" intact. + REQUIRE(links.size() == 2); + REQUIRE_FALSE(links[1].getFrom().isAvailable()); + REQUIRE(links[1].getTo() == clusters[1]); +} + +TEST_CASE("FrameFilter: integrity cascade across the whole graph", "[framefilter]") { + const auto in = makeEventFrame(); + const auto out = podio::FrameFilter{in} + .keep("hits", [](const ExampleHit& h) { return h.energy() >= 1.0; }) + .cascade("ExampleCluster.Hits") + .cascade("ExampleCluster.Clusters") + .cascade("ExampleWithOneRelation.cluster") + .cascade(linkType + ".from") + .cascade(linkType + ".to") + .run(); + + // h2 removed -> c1 (used h2) -> c0 (sub-cluster c1) -> both refs and both links. + REQUIRE(out.get("hits").size() == 2); + REQUIRE(out.get("clusters").empty()); + REQUIRE(out.get("refs").empty()); + REQUIRE(out.get("links").empty()); +} + +TEST_CASE("FrameFilter: orphan sweep keeps shared, drops truly-orphaned", "[framefilter]") { + // Two cluster collections sharing one hit collection; h1 is shared. + ExampleHitCollection hits; + auto h0 = hits.create(); + h0.energy(1.0); + auto h1 = hits.create(); + h1.energy(2.0); + auto h2 = hits.create(); + h2.energy(3.0); + + ExampleClusterCollection clustersA; + auto ca = clustersA.create(); + ca.addHits(h0); + ca.addHits(h1); + ExampleClusterCollection clustersB; + auto cb = clustersB.create(); + cb.addHits(h1); + cb.addHits(h2); + + podio::Frame in; + in.put(std::move(hits), "hits"); + in.put(std::move(clustersA), "clustersA"); + in.put(std::move(clustersB), "clustersB"); + + // Keep clustersA (default), drop everything in clustersB, sweep orphan hits. + const auto out = podio::FrameFilter{in} + .keep("clustersB", [](const ExampleCluster&) { return false; }) + .keepReferenced("hits") + .run(); + + const auto& hitsOut = out.get("hits"); + // h0 and h1 are referenced by the surviving clustersA; h2 was referenced only + // by the dropped clustersB, so it is the only truly orphaned hit. + REQUIRE(hitsOut.size() == 2); + REQUIRE(out.get("clustersB").empty()); + REQUIRE(out.get("clustersA")[0].Hits().size() == 2); +} From 2361a4ce77b29faf474499f5f6daf2f953666c8b Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Wed, 10 Jun 2026 13:34:56 +0300 Subject: [PATCH 06/14] copy vector members --- python/templates/FilterHooks.cc.jinja2 | 12 ++++++++++++ tests/unittests/frame_filter.cpp | 25 +++++++++++++++++++++++++ 2 files changed, 37 insertions(+) diff --git a/python/templates/FilterHooks.cc.jinja2 b/python/templates/FilterHooks.cc.jinja2 index 78ff5aac5..24e2c09a9 100644 --- a/python/templates/FilterHooks.cc.jinja2 +++ b/python/templates/FilterHooks.cc.jinja2 @@ -54,7 +54,19 @@ struct {{ class.bare_type }}FilterHooks : podio::detail::FilterHooks { const auto& in = static_cast(coll); auto out = std::make_unique<{{ coll_type }}>(); for (auto idx : survivors) { +{% if VectorMembers %} + // clone(false) drops vector-member data along with the relations, so the + // vector members are copied back explicitly here. + auto cloned = in[idx].clone(false); +{% for member in VectorMembers %} + for (const auto& value : in[idx].{{ member.getter_name(use_get_syntax) }}()) { + cloned.{{ member.setter_name(use_get_syntax, is_relation=True) }}(value); + } +{% endfor %} + out->push_back(cloned); +{% else %} out->push_back(in[idx].clone(false)); +{% endif %} } return out; } diff --git a/tests/unittests/frame_filter.cpp b/tests/unittests/frame_filter.cpp index 75121fc2a..b421daefe 100644 --- a/tests/unittests/frame_filter.cpp +++ b/tests/unittests/frame_filter.cpp @@ -7,6 +7,7 @@ #include "datamodel/ExampleHitCollection.h" #include "datamodel/ExampleMCCollection.h" #include "datamodel/ExampleWithOneRelationCollection.h" +#include "datamodel/ExampleWithVectorMemberCollection.h" #include "datamodel/TestLinkCollection.h" #include "podio/Frame.h" @@ -175,3 +176,27 @@ TEST_CASE("FrameFilter: orphan sweep keeps shared, drops truly-orphaned", "[fram REQUIRE(out.get("clustersB").empty()); REQUIRE(out.get("clustersA")[0].Hits().size() == 2); } + +TEST_CASE("FrameFilter: vector-member data survives filtering", "[framefilter]") { + ExampleWithVectorMemberCollection vecs; + auto v0 = vecs.create(); + v0.addcount(1); + v0.addcount(2); + v0.addcount(3); + auto v1 = vecs.create(); + v1.addcount(7); + + podio::Frame in; + in.put(std::move(vecs), "vecs"); + + const auto out = podio::FrameFilter{in} + .keep("vecs", [](const ExampleWithVectorMember& v) { return v.count_size() >= 2; }) + .run(); + + const auto& res = out.get("vecs"); + REQUIRE(res.size() == 1); + REQUIRE(res[0].count_size() == 3); + REQUIRE(res[0].count(0) == 1); + REQUIRE(res[0].count(1) == 2); + REQUIRE(res[0].count(2) == 3); +} From be3f04b7b30b3bf873dea632b79aeb00013f3516 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Wed, 10 Jun 2026 13:38:27 +0300 Subject: [PATCH 07/14] centralize filter hook registry --- include/podio/FrameFilter.h | 3 +- include/podio/detail/FilterHookRegistry.h | 54 +++++++++++++++++++++++ include/podio/detail/FilterHooks.h | 22 --------- python/templates/FilterHooks.cc.jinja2 | 1 + src/CMakeLists.txt | 1 + src/FilterHookRegistry.cc | 25 +++++++++++ 6 files changed, 83 insertions(+), 23 deletions(-) create mode 100644 include/podio/detail/FilterHookRegistry.h create mode 100644 src/FilterHookRegistry.cc diff --git a/include/podio/FrameFilter.h b/include/podio/FrameFilter.h index 7bd89540e..b9017c323 100644 --- a/include/podio/FrameFilter.h +++ b/include/podio/FrameFilter.h @@ -4,6 +4,7 @@ #include "podio/CollectionBase.h" #include "podio/Frame.h" #include "podio/ObjectID.h" +#include "podio/detail/FilterHookRegistry.h" #include "podio/detail/FilterHooks.h" #include @@ -97,7 +98,7 @@ class FrameFilter { for (const auto& name : m_frame.getAvailableCollections()) { const auto* coll = m_frame.get(name); const auto type = std::string(coll->getValueTypeName()); - const auto* hooks = detail::getFilterHooks(type); + const auto* hooks = detail::FilterHookRegistry::instance().getHooks(type); if (!hooks) { throw std::runtime_error("podio::FrameFilter: no filter hooks registered for type '" + type + "' (collection '" + name + "')"); diff --git a/include/podio/detail/FilterHookRegistry.h b/include/podio/detail/FilterHookRegistry.h new file mode 100644 index 000000000..0f69b9c55 --- /dev/null +++ b/include/podio/detail/FilterHookRegistry.h @@ -0,0 +1,54 @@ +#ifndef PODIO_DETAIL_FILTERHOOKREGISTRY_H +#define PODIO_DETAIL_FILTERHOOKREGISTRY_H + +#include "podio/detail/FilterHooks.h" + +#include +#include +#include +#include + +namespace podio { +namespace detail { + + /// Process-wide registry of per-datatype FilterHooks, keyed by datatype name + /// (CollectionBase::getValueTypeName()). Mirrors CollectionBufferFactory: it is + /// a singleton living in the core podio library and is populated when a + /// datamodel library is loaded, so registration is unambiguous across shared + /// libraries. Registration happens single-threaded at load time; lookups are + /// read-only afterwards. + class FilterHookRegistry { + public: + FilterHookRegistry(const FilterHookRegistry&) = delete; + FilterHookRegistry& operator=(const FilterHookRegistry&) = delete; + FilterHookRegistry(FilterHookRegistry&&) = delete; + FilterHookRegistry& operator=(FilterHookRegistry&&) = delete; + ~FilterHookRegistry() = default; + + /// Mutable instance, used for registration during library loading. + static FilterHookRegistry& mutInstance(); + /// Const instance, used to look up hooks at filter time. + static const FilterHookRegistry& instance(); + + /// Register the hooks for a datatype. + void registerHooks(const std::string& typeName, std::unique_ptr hooks); + + /// @returns the hooks for @p typeName, or nullptr if none are registered. + const FilterHooks* getHooks(std::string_view typeName) const; + + private: + FilterHookRegistry() = default; + std::unordered_map> m_hooks{}; + }; + + /// Convenience used by generated registration code. Returns true so it can + /// initialize a namespace-scope bool. + inline bool registerFilterHooks(const std::string& typeName, std::unique_ptr hooks) { + FilterHookRegistry::mutInstance().registerHooks(typeName, std::move(hooks)); + return true; + } + +} // namespace detail +} // namespace podio + +#endif // PODIO_DETAIL_FILTERHOOKREGISTRY_H diff --git a/include/podio/detail/FilterHooks.h b/include/podio/detail/FilterHooks.h index 4cee63b5c..dcf21c469 100644 --- a/include/podio/detail/FilterHooks.h +++ b/include/podio/detail/FilterHooks.h @@ -80,28 +80,6 @@ namespace detail { podio::CollectionBase& output, const FilterRemap& remap) const = 0; }; - /// The process-wide registry of filter hooks, keyed by datatype name (as - /// returned by CollectionBase::getTypeName()). Header-only so that generated - /// datamodel code can register into it without a link-time dependency. - inline std::unordered_map>& filterHookRegistry() { - static std::unordered_map> registry; - return registry; - } - - /// Register the hooks for a datatype. Returns true so it can be used to - /// initialize a namespace-scope bool in generated code. - inline bool registerFilterHooks(std::string typeName, std::unique_ptr hooks) { - filterHookRegistry()[std::move(typeName)] = std::move(hooks); - return true; - } - - /// @returns the hooks registered for @p typeName, or nullptr if none. - inline const FilterHooks* getFilterHooks(std::string_view typeName) { - auto& registry = filterHookRegistry(); - const auto it = registry.find(std::string(typeName)); - return it == registry.end() ? nullptr : it->second.get(); - } - } // namespace detail } // namespace podio diff --git a/python/templates/FilterHooks.cc.jinja2 b/python/templates/FilterHooks.cc.jinja2 index 24e2c09a9..46a75094e 100644 --- a/python/templates/FilterHooks.cc.jinja2 +++ b/python/templates/FilterHooks.cc.jinja2 @@ -11,6 +11,7 @@ {% endfor %} {% endif %} +#include #include #include diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index e5fecaa38..f6b33e9fb 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -55,6 +55,7 @@ SET(core_sources DatamodelRegistryIOHelpers.cc UserDataCollection.cc CollectionBufferFactory.cc + FilterHookRegistry.cc MurmurHash3.cpp SchemaEvolution.cc Glob.cc diff --git a/src/FilterHookRegistry.cc b/src/FilterHookRegistry.cc new file mode 100644 index 000000000..fd7a4b031 --- /dev/null +++ b/src/FilterHookRegistry.cc @@ -0,0 +1,25 @@ +#include "podio/detail/FilterHookRegistry.h" + +namespace podio { +namespace detail { + +FilterHookRegistry& FilterHookRegistry::mutInstance() { + static FilterHookRegistry registry; + return registry; +} + +const FilterHookRegistry& FilterHookRegistry::instance() { + return mutInstance(); +} + +void FilterHookRegistry::registerHooks(const std::string& typeName, std::unique_ptr hooks) { + m_hooks[typeName] = std::move(hooks); +} + +const FilterHooks* FilterHookRegistry::getHooks(std::string_view typeName) const { + const auto it = m_hooks.find(std::string(typeName)); + return it == m_hooks.end() ? nullptr : it->second.get(); +} + +} // namespace detail +} // namespace podio From 6851b092452a4f2f7f15662514309076547f47f9 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Wed, 10 Jun 2026 13:52:43 +0300 Subject: [PATCH 08/14] support for interfaces --- python/podio_gen/cpp_generator.py | 30 +++++--- python/podio_gen/generator_base.py | 2 + python/templates/FilterHooks.cc.jinja2 | 24 ++---- python/templates/Interface.h.jinja2 | 9 +++ .../InterfaceFilterResolver.cc.jinja2 | 29 +++++++ tests/unittests/frame_filter.cpp | 76 +++++++++++++++++++ 6 files changed, 142 insertions(+), 28 deletions(-) create mode 100644 python/templates/InterfaceFilterResolver.cc.jinja2 diff --git a/python/podio_gen/cpp_generator.py b/python/podio_gen/cpp_generator.py index 685e9dec4..4260001ca 100644 --- a/python/podio_gen/cpp_generator.py +++ b/python/podio_gen/cpp_generator.py @@ -148,14 +148,12 @@ def do_process_datatype(self, name, datatype): self._fill_templates("Collection", datatype) self._fill_templates("CollectionData", datatype) - # Relations to interface types are polymorphic and have no single - # collection_type, so the filter hooks skip them for now. - datatype["filter_single_rel"] = [ - r for r in datatype["OneToOneRelations"] if not self._is_in(r.full_type, "interfaces") - ] - datatype["filter_multi_rel"] = [ - r for r in datatype["OneToManyRelations"] if not self._is_in(r.full_type, "interfaces") - ] + # Each relation gets a resolver expression: a concrete relation resolves + # via its collection_type, an interface relation via the interface's + # generated resolver. This keeps the filter hooks free of any + # concrete/interface branching. + for relation in datatype["OneToOneRelations"] + datatype["OneToManyRelations"]: + relation.filter_resolver = self._filter_resolver_expr(relation.full_type) self._fill_templates("FilterHooks", datatype) if "SIO" in self.io_handlers: @@ -243,6 +241,7 @@ def do_process_interface(self, _, interface): ] self._fill_templates("Interface", interface) + self._fill_templates("InterfaceFilterResolver", interface) return interface def do_process_link(self, _, link): @@ -260,13 +259,20 @@ def do_process_link(self, _, link): ) ) self._fill_templates("LinkCollection", link) - # Interface endpoints are polymorphic and have no single collection_type, - # so the filter hooks skip them for now. - link["filter_from"] = not self._is_in(link["From"].full_type, "interfaces") - link["filter_to"] = not self._is_in(link["To"].full_type, "interfaces") + # Each endpoint resolves either via its collection_type (concrete) or via + # the interface's generated resolver (interface). + link["from_resolver"] = self._filter_resolver_expr(link["From"].full_type) + link["to_resolver"] = self._filter_resolver_expr(link["To"].full_type) self._fill_templates("FilterHooks", link) return link + def _filter_resolver_expr(self, full_type): + """The callable used by the filter hooks to resolve a relation target of + the given type to its handle in the filtered output.""" + if self._is_in(full_type, "interfaces"): + return f"{full_type}::resolveFilteredTarget" + return f"podio::detail::resolveTarget<{full_type}::collection_type>" + def print_report(self): """Print a summary report about the generated code""" if not self.verbose: diff --git a/python/podio_gen/generator_base.py b/python/podio_gen/generator_base.py index 9ab5cb123..39bda27b8 100644 --- a/python/podio_gen/generator_base.py +++ b/python/podio_gen/generator_base.py @@ -258,6 +258,7 @@ def get_fn_format(tmpl): "MutableStruct": "Struct", "LinkCollection": "Collection", "FilterHooks": "FilterHooks", + "InterfaceFilterResolver": "FilterResolver", } return f"{prefix.get(tmpl, '')}{{name}}{postfix.get(tmpl, '')}.{{end}}" @@ -271,6 +272,7 @@ def get_fn_format(tmpl): "LinkCollection": ("h",), "DatamodelLinks": ("cc",), "FilterHooks": ("cc",), + "InterfaceFilterResolver": ("cc",), }.get(template_base, ("h", "cc")) fn_templates = [] diff --git a/python/templates/FilterHooks.cc.jinja2 b/python/templates/FilterHooks.cc.jinja2 index 46a75094e..937398433 100644 --- a/python/templates/FilterHooks.cc.jinja2 +++ b/python/templates/FilterHooks.cc.jinja2 @@ -26,23 +26,19 @@ struct {{ class.bare_type }}FilterHooks : podio::detail::FilterHooks { [[maybe_unused]] const podio::detail::EdgeSink& sink) const override { [[maybe_unused]] const auto obj = static_cast(coll)[index]; {% if From is defined %} -{% if filter_from %} if (obj.getFrom().isAvailable()) { sink("from", obj.getFrom().getObjectID()); } -{% endif %} -{% if filter_to %} if (obj.getTo().isAvailable()) { sink("to", obj.getTo().getObjectID()); } -{% endif %} {% else %} -{% for relation in filter_single_rel %} +{% for relation in OneToOneRelations %} if (obj.{{ relation.getter_name(use_get_syntax) }}().isAvailable()) { sink("{{ relation.name }}", obj.{{ relation.getter_name(use_get_syntax) }}().getObjectID()); } {% endfor %} -{% for relation in filter_multi_rel %} +{% for relation in OneToManyRelations %} for (const auto& rel : obj.{{ relation.getter_name(use_get_syntax) }}()) { sink("{{ relation.name }}", rel.getObjectID()); } @@ -82,31 +78,27 @@ struct {{ class.bare_type }}FilterHooks : podio::detail::FilterHooks { [[maybe_unused]] auto obj = out[j]; [[maybe_unused]] const auto src = in[survivors[j]]; {% if From is defined %} -{% if filter_from %} if (src.getFrom().isAvailable()) { - if (auto t = podio::detail::resolveTarget<{{ From.full_type }}::collection_type>(remap, src.getFrom().getObjectID())) { + if (auto t = {{ from_resolver }}(remap, src.getFrom().getObjectID())) { obj.setFrom(*t); } } -{% endif %} -{% if filter_to %} if (src.getTo().isAvailable()) { - if (auto t = podio::detail::resolveTarget<{{ To.full_type }}::collection_type>(remap, src.getTo().getObjectID())) { + if (auto t = {{ to_resolver }}(remap, src.getTo().getObjectID())) { obj.setTo(*t); } } -{% endif %} {% else %} -{% for relation in filter_single_rel %} +{% for relation in OneToOneRelations %} if (src.{{ relation.getter_name(use_get_syntax) }}().isAvailable()) { - if (auto t = podio::detail::resolveTarget<{{ relation.full_type }}::collection_type>(remap, src.{{ relation.getter_name(use_get_syntax) }}().getObjectID())) { + if (auto t = {{ relation.filter_resolver }}(remap, src.{{ relation.getter_name(use_get_syntax) }}().getObjectID())) { obj.{{ relation.setter_name(use_get_syntax) }}(*t); } } {% endfor %} -{% for relation in filter_multi_rel %} +{% for relation in OneToManyRelations %} for (const auto& rel : src.{{ relation.getter_name(use_get_syntax) }}()) { - if (auto t = podio::detail::resolveTarget<{{ relation.full_type }}::collection_type>(remap, rel.getObjectID())) { + if (auto t = {{ relation.filter_resolver }}(remap, rel.getObjectID())) { obj.{{ relation.setter_name(use_get_syntax, is_relation=True) }}(*t); } } diff --git a/python/templates/Interface.h.jinja2 b/python/templates/Interface.h.jinja2 index 569f13125..5bbb555c2 100644 --- a/python/templates/Interface.h.jinja2 +++ b/python/templates/Interface.h.jinja2 @@ -12,9 +12,11 @@ #include "podio/ObjectID.h" #include "podio/utilities/TypeHelpers.h" +#include "podio/detail/FilterHooks.h" #include "podio/detail/OrderKey.h" #include +#include #include #include @@ -140,6 +142,13 @@ public: podio::ObjectID id() const { return getObjectID(); } podio::ObjectID getObjectID() const { return m_self->getObjectID(); } + /// Resolve a relation target of this interface type to its concrete handle in + /// a filtered output collection, used by podio::FrameFilter. Returns + /// std::nullopt if the target was removed by the filter. Defined in the + /// generated {{ class.bare_type }}FilterResolver.cc. + static std::optional<{{ class.bare_type }}> resolveFilteredTarget(const podio::detail::FilterRemap& remap, + podio::ObjectID id); + /// Check if the object currently holds a value of the requested type template bool isA() const { diff --git a/python/templates/InterfaceFilterResolver.cc.jinja2 b/python/templates/InterfaceFilterResolver.cc.jinja2 new file mode 100644 index 000000000..08f37cd22 --- /dev/null +++ b/python/templates/InterfaceFilterResolver.cc.jinja2 @@ -0,0 +1,29 @@ +// AUTOMATICALLY GENERATED FILE - DO NOT EDIT + +#include "{{ incfolder }}{{ class.bare_type }}.h" +{% for include in include_types %} +{{ include }} +{% endfor %} + +#include + +#include +#include + +// Resolve an interface-typed relation target to its concrete handle in the +// filtered output. The interface stores only ObjectIDs, so the concrete type is +// recovered at runtime from the target collection's value type name. +std::optional<{{ class.full_type }}> +{{ class.full_type }}::resolveFilteredTarget(const podio::detail::FilterRemap& remap, podio::ObjectID id) { + const auto* loc = remap.find(id); + if (!loc) { + return std::nullopt; + } + const auto valueType = loc->collection->getValueTypeName(); +{% for member in Types %} + if (valueType == {{ member.full_type }}::typeName) { + return {{ class.full_type }}(static_cast(*loc->collection)[loc->index]); + } +{% endfor %} + return std::nullopt; +} diff --git a/tests/unittests/frame_filter.cpp b/tests/unittests/frame_filter.cpp index b421daefe..13d8bfbed 100644 --- a/tests/unittests/frame_filter.cpp +++ b/tests/unittests/frame_filter.cpp @@ -8,7 +8,12 @@ #include "datamodel/ExampleMCCollection.h" #include "datamodel/ExampleWithOneRelationCollection.h" #include "datamodel/ExampleWithVectorMemberCollection.h" +#include "datamodel/TestInterfaceLinkCollection.h" #include "datamodel/TestLinkCollection.h" +#include "datamodel/TypeWithEnergy.h" + +#include "interface_extension_model/EnergyInterface.h" +#include "interface_extension_model/ExampleWithInterfaceRelationCollection.h" #include "podio/Frame.h" #include "podio/FrameFilter.h" @@ -200,3 +205,74 @@ TEST_CASE("FrameFilter: vector-member data survives filtering", "[framefilter]") REQUIRE(res[0].count(1) == 2); REQUIRE(res[0].count(2) == 3); } + +TEST_CASE("FrameFilter: interface relations are rewired polymorphically", "[framefilter]") { + // ExampleWithInterfaceRelation references ExampleHits through the polymorphic + // EnergyInterface; filtering the hits must rewire those interface relations. + ExampleHitCollection hits; + auto h0 = hits.create(); + h0.energy(5.0); + auto h1 = hits.create(); + h1.energy(0.5); + + iextension::ExampleWithInterfaceRelationCollection withIface; + auto w0 = withIface.create(); + w0.singleEnergy(iextension::EnergyInterface(h0)); + w0.addmanyEnergies(iextension::EnergyInterface(h0)); + w0.addmanyEnergies(iextension::EnergyInterface(h1)); + + podio::Frame in; + in.put(std::move(hits), "hits"); + in.put(std::move(withIface), "withIface"); + + const auto out = podio::FrameFilter{in} + .keep("hits", [](const ExampleHit& h) { return h.energy() >= 1.0; }) + .run(); + + const auto& res = out.get("withIface"); + REQUIRE(res.size() == 1); + // singleEnergy still resolves to the surviving hit (energy 5). + REQUIRE(res[0].singleEnergy().isAvailable()); + REQUIRE(res[0].singleEnergy().energy() == 5.0); + // the removed hit (energy 0.5) is dropped from the many-relation. + REQUIRE(res[0].manyEnergies().size() == 1); + REQUIRE(res[0].manyEnergies()[0].energy() == 5.0); +} + +TEST_CASE("FrameFilter: interface link endpoints are rewired and dropped", "[framefilter]") { + // TestInterfaceLink: From ExampleCluster (concrete), To TypeWithEnergy (interface). + ExampleHitCollection hits; + auto h0 = hits.create(); + h0.energy(5.0); + ExampleClusterCollection clusters; + auto c0 = clusters.create(); + c0.energy(1.0); + TestInterfaceLinkCollection ilinks; + auto l0 = ilinks.create(); + l0.setFrom(c0); + l0.setTo(TypeWithEnergy(h0)); + + podio::Frame in; + in.put(std::move(hits), "hits"); + in.put(std::move(clusters), "clusters"); + in.put(std::move(ilinks), "ilinks"); + + SECTION("keep everything: both endpoints resolve") { + const auto out = podio::FrameFilter{in}.run(); + const auto& links = out.get("ilinks"); + REQUIRE(links.size() == 1); + REQUIRE(links[0].getFrom() == out.get("clusters")[0]); + REQUIRE(links[0].getTo().isAvailable()); + REQUIRE(links[0].getTo().energy() == 5.0); // interface "To" resolved to the hit + } + + SECTION("drop the interface target: 'To' is left unset") { + const auto out = podio::FrameFilter{in} + .keep("hits", [](const ExampleHit&) { return false; }) + .run(); + const auto& links = out.get("ilinks"); + REQUIRE(links.size() == 1); + REQUIRE(links[0].getFrom().isAvailable()); + REQUIRE_FALSE(links[0].getTo().isAvailable()); + } +} From 3902d08b5e53c68ee6ba375af8d3340c715fdaf0 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Wed, 10 Jun 2026 14:10:10 +0300 Subject: [PATCH 09/14] subset handling --- include/podio/FrameFilter.h | 53 ++++++++++++++++++++++---- python/templates/FilterHooks.cc.jinja2 | 27 ++++++++++++- tests/unittests/frame_filter.cpp | 49 ++++++++++++++++++++++++ 3 files changed, 121 insertions(+), 8 deletions(-) diff --git a/include/podio/FrameFilter.h b/include/podio/FrameFilter.h index b9017c323..d1ec4721e 100644 --- a/include/podio/FrameFilter.h +++ b/include/podio/FrameFilter.h @@ -89,6 +89,7 @@ class FrameFilter { uint32_t id; std::size_t size; Mode mode; + bool isSubset; std::vector killed; std::vector marked; }; @@ -111,7 +112,7 @@ class FrameFilter { } const auto size = coll->size(); byId[coll->getID()] = colls.size(); - colls.push_back({name, coll, hooks, type, coll->getID(), size, mode, + colls.push_back({name, coll, hooks, type, coll->getID(), size, mode, coll->isSubsetCollection(), std::vector(size, 0), std::vector(size, 0)}); } @@ -196,6 +197,26 @@ class FrameFilter { } } + // Phase 3b: a subset collection only references objects owned elsewhere, so + // drop any entry whose referenced object was removed (a reference cannot + // dangle). + for (auto& ci : colls) { + if (!ci.isSubset) { + continue; + } + for (std::size_t i = 0; i < ci.size; ++i) { + if (ci.killed[i]) { + continue; + } + ci.hooks->edges(*ci.coll, i, [&](std::string_view, podio::ObjectID target) { + const auto [k, idx] = locate(target); + if (k < 0 || colls[k].killed[idx]) { + ci.killed[i] = 1; + } + }); + } + } + // Phase 4: clone survivors, build the global remap, then rewire. std::vector> survivors(colls.size()); std::vector> outColls(colls.size()); @@ -208,22 +229,40 @@ class FrameFilter { } } outColls[k] = ci.hooks->cloneSurvivors(*ci.coll, survivors[k]); - for (std::size_t j = 0; j < survivors[k].size(); ++j) { - podio::ObjectID original; - original.index = static_cast(survivors[k][j]); - original.collectionID = ci.id; - remap.add(original, outColls[k].get(), j); + // Subset entries are references to objects owned elsewhere; they have no + // identity of their own, so they are not added to the remap. + if (!ci.isSubset) { + for (std::size_t j = 0; j < survivors[k].size(); ++j) { + podio::ObjectID original; + original.index = static_cast(survivors[k][j]); + original.collectionID = ci.id; + remap.add(original, outColls[k].get(), j); + } } } for (std::size_t k = 0; k < colls.size(); ++k) { colls[k].hooks->rewire(*colls[k].coll, survivors[k], *outColls[k], remap); } - // Phase 5: assemble the output Frame. + // Phase 5: assemble the output Frame and copy the parameters verbatim. podio::Frame out; for (std::size_t k = 0; k < colls.size(); ++k) { out.put(std::move(outColls[k]), colls[k].name); } + + const auto& params = m_frame.getParameters(); + auto copyParams = [&](auto typeTag) { + using T = decltype(typeTag); + auto [keys, values] = params.getKeysAndValues(); + for (std::size_t i = 0; i < keys.size(); ++i) { + out.putParameter>(keys[i], std::move(values[i])); + } + }; + copyParams(int{}); + copyParams(float{}); + copyParams(double{}); + copyParams(std::string{}); + return out; } diff --git a/python/templates/FilterHooks.cc.jinja2 b/python/templates/FilterHooks.cc.jinja2 index 937398433..c5b58e41f 100644 --- a/python/templates/FilterHooks.cc.jinja2 +++ b/python/templates/FilterHooks.cc.jinja2 @@ -33,6 +33,13 @@ struct {{ class.bare_type }}FilterHooks : podio::detail::FilterHooks { sink("to", obj.getTo().getObjectID()); } {% else %} + if (coll.isSubsetCollection()) { + // a subset entry's only edge is to the object it references + if (obj.isAvailable()) { + sink("", obj.getObjectID()); + } + return; + } {% for relation in OneToOneRelations %} if (obj.{{ relation.getter_name(use_get_syntax) }}().isAvailable()) { sink("{{ relation.name }}", obj.{{ relation.getter_name(use_get_syntax) }}().getObjectID()); @@ -48,8 +55,15 @@ struct {{ class.bare_type }}FilterHooks : podio::detail::FilterHooks { std::unique_ptr cloneSurvivors(const podio::CollectionBase& coll, const std::vector& survivors) const override { - const auto& in = static_cast(coll); auto out = std::make_unique<{{ coll_type }}>(); +{% if From is not defined %} + if (coll.isSubsetCollection()) { + // a subset owns no data; its surviving references are filled in during rewire + out->setSubsetCollection(); + return out; + } +{% endif %} + const auto& in = static_cast(coll); for (auto idx : survivors) { {% if VectorMembers %} // clone(false) drops vector-member data along with the relations, so the @@ -74,6 +88,17 @@ struct {{ class.bare_type }}FilterHooks : podio::detail::FilterHooks { [[maybe_unused]] const podio::detail::FilterRemap& remap) const override { [[maybe_unused]] const auto& in = static_cast(input); [[maybe_unused]] auto& out = static_cast<{{ coll_type }}&>(output); +{% if From is not defined %} + if (output.isSubsetCollection()) { + // re-reference the surviving targets in the filtered output + for (auto idx : survivors) { + if (auto t = podio::detail::resolveTarget<{{ coll_type }}>(remap, in[idx].getObjectID())) { + out.push_back(*t); + } + } + return; + } +{% endif %} for (std::size_t j = 0; j < survivors.size(); ++j) { [[maybe_unused]] auto obj = out[j]; [[maybe_unused]] const auto src = in[survivors[j]]; diff --git a/tests/unittests/frame_filter.cpp b/tests/unittests/frame_filter.cpp index 13d8bfbed..9dc55c81d 100644 --- a/tests/unittests/frame_filter.cpp +++ b/tests/unittests/frame_filter.cpp @@ -239,6 +239,55 @@ TEST_CASE("FrameFilter: interface relations are rewired polymorphically", "[fram REQUIRE(res[0].manyEnergies()[0].energy() == 5.0); } +TEST_CASE("FrameFilter: subset collections re-reference surviving objects", "[framefilter]") { + ExampleHitCollection hits; + auto h0 = hits.create(); + h0.energy(5.0); + auto h1 = hits.create(); + h1.energy(0.5); + auto h2 = hits.create(); + h2.energy(3.0); + + ExampleHitCollection selected; // subset referencing all three + selected.setSubsetCollection(); + selected.push_back(h0); + selected.push_back(h1); + selected.push_back(h2); + + podio::Frame in; + in.put(std::move(hits), "hits"); + in.put(std::move(selected), "selected"); + + const auto out = podio::FrameFilter{in} + .keep("hits", [](const ExampleHit& h) { return h.energy() >= 1.0; }) + .run(); + + const auto& outHits = out.get("hits"); + const auto& outSel = out.get("selected"); + REQUIRE(outHits.size() == 2); // h0, h2 + // the subset stays a subset and drops the reference to the removed h1, with the + // survivors now pointing at the objects in the filtered owner collection. + REQUIRE(outSel.isSubsetCollection()); + REQUIRE(outSel.size() == 2); + REQUIRE(outSel[0] == outHits[0]); + REQUIRE(outSel[1] == outHits[1]); +} + +TEST_CASE("FrameFilter: frame parameters are copied to the output", "[framefilter]") { + auto in = makeMCFrame(); + in.putParameter("run", 42); + in.putParameter("tag", std::string("nominal")); + in.putParameter("weights", std::vector{1.0, 2.5, 3.0}); + + const auto out = podio::FrameFilter{in} + .keep("mcparticles", [](const ExampleMC& p) { return p.energy() >= 1.0; }) + .run(); + + REQUIRE(out.getParameter("run") == 42); + REQUIRE(out.getParameter("tag") == "nominal"); + REQUIRE(out.getParameter>("weights").value().size() == 3); +} + TEST_CASE("FrameFilter: interface link endpoints are rewired and dropped", "[framefilter]") { // TestInterfaceLink: From ExampleCluster (concrete), To TypeWithEnergy (interface). ExampleHitCollection hits; From 7c70af1dd83927975819166afd52120468bb981b Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Wed, 10 Jun 2026 14:14:50 +0300 Subject: [PATCH 10/14] doc --- doc/filtering.md | 144 +++++++++++++++++++++++++++++++++++++++++++++++ doc/index.rst | 1 + 2 files changed, 145 insertions(+) create mode 100644 doc/filtering.md diff --git a/doc/filtering.md b/doc/filtering.md new file mode 100644 index 000000000..3b394c65a --- /dev/null +++ b/doc/filtering.md @@ -0,0 +1,144 @@ +# Filtering Frames + +`podio::FrameFilter` produces a new `Frame` from an existing one by selecting a +subset of the objects in its collections and **rewiring all relations** so that +the result is internally consistent. It is the right tool when you want to drop +some objects (e.g. low-energy hits, or MC particles outside a region of +interest) while keeping the surviving relation graph valid. + +Filtering is inherently a *Frame-level* operation: because relations cross +collections, dropping an object in one collection can leave dangling references +in another. `FrameFilter` therefore rebuilds every collection of the input +`Frame` and returns a new, fully-rewired `Frame`. + +## Basic usage + +```cpp +#include "podio/FrameFilter.h" + +podio::Frame out = podio::FrameFilter{inFrame} + .keep("MCParticles", [](const ExampleMC& p) { return p.energy() > 1.0; }) + .keepReferenced("hits") + .cascade("ExampleCluster.Hits") + .run(); +``` + +- `keep(name, predicate)` — keep only the objects of the named collection for + which `predicate` returns `true`. The collection type is deduced from the + predicate's argument type. +- `keepReferenced(name)` — keep an object of the named collection only if some + surviving object still references it (an *orphan sweep*). +- `cascade("Type.relation")` — mark a relation as mandatory (see below). +- `run()` — execute and return the new `Frame`. + +Collections of the input `Frame` that are not mentioned are kept in full, but +are still rebuilt so their relations into filtered collections are corrected. +Frame parameters are copied over verbatim. + +## The two axes + +The behaviour is controlled by two orthogonal axes. + +### Retention mode (per collection) + +How an object earns its place in the output: + +- **keep-all** (default) — every object survives. +- **predicate** (`keep`) — an object survives iff the predicate holds. +- **keep-if-referenced** (`keepReferenced`) — an object survives iff a surviving + object references it. This is an orphan sweep, computed as a mark-and-sweep + reachability from the surviving *roots* (the predicate/keep-all survivors). + +Orphan sweep is shared-aware: if two collections reference the same `hit` and +only one of the referrers survives, the `hit` is kept. That is precisely why +`keepReferenced` is preferred over a "containment cascade" (deleting an object +deletes everything it points at) — the latter would wrongly drop objects that +are still shared with a survivor. + +### Edge policy (per relation) + +What happens to a single relation whose target was removed: + +- **leave-unconnected** (default) — drop the dangling edge. A + `OneToManyRelation` simply omits the removed target; a `OneToOneRelation` is + left unset. +- **cascade** (`cascade("Type.relation")`) — treat the relation as mandatory: + if the target is removed, remove the object that holds the relation as well + (*integrity cascade*). This propagates transitively across collections. + +A relation is named `.`, e.g. +`ExampleCluster.Hits` or `ExampleMC.daughters`. For links the value type name is +the templated `podio::Link` name, so a link endpoint is cascaded as +e.g. `cascade(std::string(MyLinkCollection::valueTypeName) + ".to")`. + +## How `run()` works + +The orchestration is type-agnostic and proceeds in phases: + +1. **Seed** — apply the predicates to mark the initially-removed objects. +2. **Cascade closure** — for every relation flagged `cascade`, if its target is + removed, remove the holder; iterate to a fixpoint (cycle-safe). +3. **Mark** — forward reachability from the surviving roots, marking the objects + that keep-referenced collections should retain. +4. **Sweep** — remove unmarked objects from keep-referenced collections. +5. **Subset fix-up** — a subset collection only references objects owned + elsewhere, so drop any entry whose target was removed. +6. **Clone & remap** — clone the surviving objects (data members only) into new + collections and record an `ObjectID → new location` map. +7. **Rewire** — walk every relation of every survivor and re-point it through + the remap; targets that were removed are dropped (or already cascaded away). +8. **Assemble** — put the rebuilt collections into a new `Frame` and copy the + parameters. + +## Design: a generic core plus generated hooks + +The orchestration above is entirely type-erased — `FrameFilter` only ever sees +`podio::CollectionBase`. The parts that genuinely need concrete type knowledge +are generated once per datatype and registered, mirroring the +`CollectionBufferFactory` pattern: + +- `podio::detail::FilterHooks` (in `podio/detail/FilterHooks.h`) is the + per-datatype interface with three operations: + - `edges` — report an object's outgoing relation targets as `ObjectID`s + (used by the cascade and mark phases). + - `cloneSurvivors` — clone the surviving objects into a fresh collection, + copying data members and vector members but no relations. + - `rewire` — re-point the cloned survivors' relations through the remap. +- `podio::detail::FilterHookRegistry` is the singleton (one per process, living + in the core library) that maps a datatype name to its hooks. Generated code + registers into it at library-load time. +- `podio::detail::resolveTarget` looks a remapped target up and returns + the concrete handle to hand to a setter/adder. + +For each datatype and link the generator emits a `FilterHooks.cc` that +implements and registers these hooks. The implementation is uniform: each +relation carries a *resolver expression*, so the generated code does not branch +on whether a relation is concrete or an interface. + +### Interface relations + +A relation to an interface type is polymorphic — its target may live in any of +the interface's member collections. Because the remap is keyed by `ObjectID` +(type-agnostic), the target can always be located; recovering the concrete type +to build the interface handle is done by a generated resolver, one per +interface: + +```cpp +std::optional +EnergyInterface::resolveFilteredTarget(const podio::detail::FilterRemap& remap, podio::ObjectID id); +``` + +It dispatches on the target collection's `getValueTypeName()` over the +interface's member types and constructs the interface from the concrete handle. +It is declared on the interface (in the generated interface header) and defined +once in a generated `FilterResolver.cc`, so every relation that uses +the interface shares a single resolver. + +## Limitations + +- Interface relations are resolved by a linear match over the interface's member + types (one comparison per candidate concrete type). +- Subset *link* collections are not handled; link collections are assumed to own + their endpoints. +- A subset collection's synthetic membership edge has no relation name, so it + cannot be targeted by `cascade(...)`. diff --git a/doc/index.rst b/doc/index.rst index 1b54ffc05..723f9a222 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -17,6 +17,7 @@ Welcome to PODIO's documentation! frame.md userdata.md links.md + filtering.md storage_details.md tools.md cmake.md From f3bf1fac53ead0ff5ad7d4fc1e3a517e404619bc Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Wed, 10 Jun 2026 14:14:50 +0300 Subject: [PATCH 11/14] drop helper --- doc/filtering.md | 5 +++++ include/podio/FrameFilter.h | 22 ++++++++++++++++++---- tests/unittests/frame_filter.cpp | 32 ++++++++++++++++++++++++++++++++ 3 files changed, 55 insertions(+), 4 deletions(-) diff --git a/doc/filtering.md b/doc/filtering.md index 3b394c65a..52ad22639 100644 --- a/doc/filtering.md +++ b/doc/filtering.md @@ -28,6 +28,9 @@ podio::Frame out = podio::FrameFilter{inFrame} predicate's argument type. - `keepReferenced(name)` — keep an object of the named collection only if some surviving object still references it (an *orphan sweep*). +- `drop(name)` — remove every object of the named collection. The (now empty) + collection is still present in the output. Unlike `keep`, this needs only the + name, not the collection's type. - `cascade("Type.relation")` — mark a relation as mandatory (see below). - `run()` — execute and return the new `Frame`. @@ -45,6 +48,8 @@ How an object earns its place in the output: - **keep-all** (default) — every object survives. - **predicate** (`keep`) — an object survives iff the predicate holds. +- **drop-all** (`drop`) — no object survives. Equivalent to an always-false + predicate, but type-free since it never inspects an object. - **keep-if-referenced** (`keepReferenced`) — an object survives iff a surviving object references it. This is an orphan sweep, computed as a mark-and-sweep reachability from the surviving *roots* (the predicate/keep-all survivors). diff --git a/include/podio/FrameFilter.h b/include/podio/FrameFilter.h index d1ec4721e..a9f321ec3 100644 --- a/include/podio/FrameFilter.h +++ b/include/podio/FrameFilter.h @@ -7,6 +7,7 @@ #include "podio/detail/FilterHookRegistry.h" #include "podio/detail/FilterHooks.h" +#include #include #include #include @@ -71,6 +72,14 @@ class FrameFilter { return *this; } + /// Drop every object of the named collection. The (now empty) collection is + /// still present in the output. Unlike `keep`, this needs only the name, not + /// the collection's type, and composes with `cascade` like any other removal. + FrameFilter& drop(const std::string& name) { + m_drop.insert(name); + return *this; + } + /// Treat the named relation ("Type.relation") as mandatory: if its target is /// removed, remove the object holding it (integrity cascade). FrameFilter& cascade(const std::string& qualifiedRelation) { @@ -80,7 +89,7 @@ class FrameFilter { /// Run the filter and return a new, fully-rewired Frame. podio::Frame run() const { - enum Mode { KeepAll, Predicate, Referenced }; + enum Mode { KeepAll, Predicate, Referenced, DropAll }; struct CollInfo { std::string name; const podio::CollectionBase* coll; @@ -105,7 +114,9 @@ class FrameFilter { "' (collection '" + name + "')"); } Mode mode = KeepAll; - if (m_predicates.count(name)) { + if (m_drop.count(name)) { + mode = DropAll; + } else if (m_predicates.count(name)) { mode = Predicate; } else if (m_keepReferenced.count(name)) { mode = Referenced; @@ -127,9 +138,11 @@ class FrameFilter { static_cast(target.index)}; }; - // Phase 0: seed removals from predicates. + // Phase 0: seed removals from predicates and whole-collection drops. for (auto& ci : colls) { - if (ci.mode == Predicate) { + if (ci.mode == DropAll) { + std::fill(ci.killed.begin(), ci.killed.end(), 1); + } else if (ci.mode == Predicate) { const auto& pred = m_predicates.at(ci.name); for (std::size_t i = 0; i < ci.size; ++i) { ci.killed[i] = pred(*ci.coll, i) ? 0 : 1; @@ -270,6 +283,7 @@ class FrameFilter { const podio::Frame& m_frame; std::unordered_map> m_predicates{}; std::unordered_set m_keepReferenced{}; + std::unordered_set m_drop{}; std::unordered_set m_cascade{}; }; diff --git a/tests/unittests/frame_filter.cpp b/tests/unittests/frame_filter.cpp index 9dc55c81d..a865cb509 100644 --- a/tests/unittests/frame_filter.cpp +++ b/tests/unittests/frame_filter.cpp @@ -182,6 +182,38 @@ TEST_CASE("FrameFilter: orphan sweep keeps shared, drops truly-orphaned", "[fram REQUIRE(out.get("clustersA")[0].Hits().size() == 2); } +TEST_CASE("FrameFilter: drop empties a collection without naming its type", "[framefilter]") { + const auto in = makeEventFrame(); + // drop() needs only the name, not the collection's C++ type, and composes + // with the other axes. Here it removes every cluster and an orphan sweep then + // prunes hits that nothing else references. + const auto out = podio::FrameFilter{in} + .drop("clusters") + .keepReferenced("hits") + .run(); + + // The dropped collection is still present, but empty. + REQUIRE(out.get("clusters").empty()); + + // refs pointed only at clusters; the dangling cluster relation is left unset. + const auto& refs = out.get("refs"); + REQUIRE(refs.size() == 2); + REQUIRE_FALSE(refs[0].cluster().isAvailable()); + + // h0 and h2 are still referenced by the surviving links' "From"; h1 was + // referenced only by the dropped clusters, so the sweep removes it. + const auto& hits = out.get("hits"); + REQUIRE(hits.size() == 2); + REQUIRE(hits[0].energy() == 1.0); // h0 + REQUIRE(hits[1].energy() == 0.5); // h2 + + // links keep their hit "From" but lose the dropped cluster "To". + const auto& links = out.get("links"); + REQUIRE(links.size() == 2); + REQUIRE(links[0].getFrom() == hits[0]); + REQUIRE_FALSE(links[0].getTo().isAvailable()); +} + TEST_CASE("FrameFilter: vector-member data survives filtering", "[framefilter]") { ExampleWithVectorMemberCollection vecs; auto v0 = vecs.create(); From 9b5e605d826d83adcff9ebf2c1d790d174738124 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Wed, 10 Jun 2026 14:31:15 +0300 Subject: [PATCH 12/14] span over vector --- include/podio/detail/FilterHooks.h | 6 +++--- python/templates/FilterHooks.cc.jinja2 | 5 +++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/include/podio/detail/FilterHooks.h b/include/podio/detail/FilterHooks.h index dcf21c469..a23803490 100644 --- a/include/podio/detail/FilterHooks.h +++ b/include/podio/detail/FilterHooks.h @@ -6,10 +6,10 @@ #include #include #include +#include #include #include #include -#include namespace podio { class CollectionBase; @@ -70,13 +70,13 @@ namespace detail { /// Clone the objects at @p survivors (in order) into a freshly allocated /// collection of the same type, copying data members but no relations. virtual std::unique_ptr - cloneSurvivors(const podio::CollectionBase& collection, const std::vector& survivors) const = 0; + cloneSurvivors(const podio::CollectionBase& collection, std::span survivors) const = 0; /// Rewire the relations of the cloned survivors in @p output, resolving each /// relation target through @p remap. @p input is the original collection and /// @p survivors the same index list passed to cloneSurvivors (so output[j] /// corresponds to input[survivors[j]]). - virtual void rewire(const podio::CollectionBase& input, const std::vector& survivors, + virtual void rewire(const podio::CollectionBase& input, std::span survivors, podio::CollectionBase& output, const FilterRemap& remap) const = 0; }; diff --git a/python/templates/FilterHooks.cc.jinja2 b/python/templates/FilterHooks.cc.jinja2 index c5b58e41f..14f182cba 100644 --- a/python/templates/FilterHooks.cc.jinja2 +++ b/python/templates/FilterHooks.cc.jinja2 @@ -16,6 +16,7 @@ #include #include +#include #include {% set coll_type = class.full_type + 'Collection' %} @@ -54,7 +55,7 @@ struct {{ class.bare_type }}FilterHooks : podio::detail::FilterHooks { } std::unique_ptr - cloneSurvivors(const podio::CollectionBase& coll, const std::vector& survivors) const override { + cloneSurvivors(const podio::CollectionBase& coll, std::span survivors) const override { auto out = std::make_unique<{{ coll_type }}>(); {% if From is not defined %} if (coll.isSubsetCollection()) { @@ -83,7 +84,7 @@ struct {{ class.bare_type }}FilterHooks : podio::detail::FilterHooks { } void rewire([[maybe_unused]] const podio::CollectionBase& input, - [[maybe_unused]] const std::vector& survivors, + [[maybe_unused]] std::span survivors, [[maybe_unused]] podio::CollectionBase& output, [[maybe_unused]] const podio::detail::FilterRemap& remap) const override { [[maybe_unused]] const auto& in = static_cast(input); From ffefa07c5f4bf06240a8189a68822e33855cf87e Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Wed, 10 Jun 2026 14:45:22 +0300 Subject: [PATCH 13/14] drop omits collection --- doc/filtering.md | 15 +++++++++------ include/podio/FrameFilter.h | 17 ++++++++++++++--- tests/unittests/frame_filter.cpp | 13 ++++++++----- 3 files changed, 31 insertions(+), 14 deletions(-) diff --git a/doc/filtering.md b/doc/filtering.md index 52ad22639..7530d9099 100644 --- a/doc/filtering.md +++ b/doc/filtering.md @@ -28,9 +28,10 @@ podio::Frame out = podio::FrameFilter{inFrame} predicate's argument type. - `keepReferenced(name)` — keep an object of the named collection only if some surviving object still references it (an *orphan sweep*). -- `drop(name)` — remove every object of the named collection. The (now empty) - collection is still present in the output. Unlike `keep`, this needs only the - name, not the collection's type. +- `drop(name)` — remove the named collection entirely; it is omitted from the + output Frame. Unlike `keep`, this needs only the name, not the collection's + type. (To keep an empty collection instead, use an always-false `keep` + predicate.) - `cascade("Type.relation")` — mark a relation as mandatory (see below). - `run()` — execute and return the new `Frame`. @@ -48,8 +49,9 @@ How an object earns its place in the output: - **keep-all** (default) — every object survives. - **predicate** (`keep`) — an object survives iff the predicate holds. -- **drop-all** (`drop`) — no object survives. Equivalent to an always-false - predicate, but type-free since it never inspects an object. +- **drop-all** (`drop`) — no object survives *and* the collection is omitted from + the output Frame. Type-free, since it never inspects an object. (An always-false + `keep` predicate is the variant that leaves an empty collection in place.) - **keep-if-referenced** (`keepReferenced`) — an object survives iff a surviving object references it. This is an orphan sweep, computed as a mark-and-sweep reachability from the surviving *roots* (the predicate/keep-all survivors). @@ -93,7 +95,8 @@ The orchestration is type-agnostic and proceeds in phases: 7. **Rewire** — walk every relation of every survivor and re-point it through the remap; targets that were removed are dropped (or already cascaded away). 8. **Assemble** — put the rebuilt collections into a new `Frame` and copy the - parameters. + parameters. Collections marked `drop` are skipped here, so they do not appear + in the output `Frame` at all. ## Design: a generic core plus generated hooks diff --git a/include/podio/FrameFilter.h b/include/podio/FrameFilter.h index a9f321ec3..2a97e54f8 100644 --- a/include/podio/FrameFilter.h +++ b/include/podio/FrameFilter.h @@ -72,9 +72,11 @@ class FrameFilter { return *this; } - /// Drop every object of the named collection. The (now empty) collection is - /// still present in the output. Unlike `keep`, this needs only the name, not - /// the collection's type, and composes with `cascade` like any other removal. + /// Drop the named collection entirely: it is omitted from the output Frame. + /// Unlike `keep`, this needs only the name, not the collection's type, and + /// composes with `cascade` like any other removal (objects elsewhere that + /// cascade on a dropped target are removed too). To instead keep an empty + /// collection in the output, use an always-false `keep` predicate. FrameFilter& drop(const std::string& name) { m_drop.insert(name); return *this; @@ -236,6 +238,9 @@ class FrameFilter { detail::FilterRemap remap; for (std::size_t k = 0; k < colls.size(); ++k) { auto& ci = colls[k]; + if (ci.mode == DropAll) { + continue; // omitted from the output entirely; outColls[k] stays null + } for (std::size_t i = 0; i < ci.size; ++i) { if (!ci.killed[i]) { survivors[k].push_back(i); @@ -254,12 +259,18 @@ class FrameFilter { } } for (std::size_t k = 0; k < colls.size(); ++k) { + if (colls[k].mode == DropAll) { + continue; + } colls[k].hooks->rewire(*colls[k].coll, survivors[k], *outColls[k], remap); } // Phase 5: assemble the output Frame and copy the parameters verbatim. podio::Frame out; for (std::size_t k = 0; k < colls.size(); ++k) { + if (colls[k].mode == DropAll) { + continue; // dropped collections are not put into the output Frame + } out.put(std::move(outColls[k]), colls[k].name); } diff --git a/tests/unittests/frame_filter.cpp b/tests/unittests/frame_filter.cpp index a865cb509..533447b8b 100644 --- a/tests/unittests/frame_filter.cpp +++ b/tests/unittests/frame_filter.cpp @@ -20,6 +20,7 @@ #include +#include #include namespace { @@ -182,18 +183,20 @@ TEST_CASE("FrameFilter: orphan sweep keeps shared, drops truly-orphaned", "[fram REQUIRE(out.get("clustersA")[0].Hits().size() == 2); } -TEST_CASE("FrameFilter: drop empties a collection without naming its type", "[framefilter]") { +TEST_CASE("FrameFilter: drop removes a collection without naming its type", "[framefilter]") { const auto in = makeEventFrame(); // drop() needs only the name, not the collection's C++ type, and composes - // with the other axes. Here it removes every cluster and an orphan sweep then - // prunes hits that nothing else references. + // with the other axes. Here it removes the clusters collection entirely and an + // orphan sweep then prunes hits that nothing else references. const auto out = podio::FrameFilter{in} .drop("clusters") .keepReferenced("hits") .run(); - // The dropped collection is still present, but empty. - REQUIRE(out.get("clusters").empty()); + // The dropped collection is omitted from the output Frame entirely. + const auto avail = out.getAvailableCollections(); + REQUIRE(std::find(avail.begin(), avail.end(), "clusters") == avail.end()); + REQUIRE(out.get("clusters") == nullptr); // refs pointed only at clusters; the dangling cluster relation is left unset. const auto& refs = out.get("refs"); From 2179abeed6ac83c7c4e85dd15fc941a61600a33e Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Wed, 10 Jun 2026 14:46:47 +0300 Subject: [PATCH 14/14] drop unneeded unit test --- tests/unittests/CMakeLists.txt | 2 +- tests/unittests/filtering.cpp | 636 ------------------------------- tests/unittests/frame_filter.cpp | 7 +- 3 files changed, 5 insertions(+), 640 deletions(-) delete mode 100644 tests/unittests/filtering.cpp diff --git a/tests/unittests/CMakeLists.txt b/tests/unittests/CMakeLists.txt index d9fc3af4e..f2a702a4a 100644 --- a/tests/unittests/CMakeLists.txt +++ b/tests/unittests/CMakeLists.txt @@ -41,7 +41,7 @@ else() endif() find_package(Threads REQUIRED) -add_executable(unittest_podio unittest.cpp frame.cpp buffer_factory.cpp interface_types.cpp std_interoperability.cpp links.cpp test_json_output.cpp filtering.cpp frame_filter.cpp) +add_executable(unittest_podio unittest.cpp frame.cpp buffer_factory.cpp interface_types.cpp std_interoperability.cpp links.cpp test_json_output.cpp frame_filter.cpp) target_link_libraries(unittest_podio PUBLIC TestDataModel ExtensionDataModel InterfaceExtensionDataModel PRIVATE Catch2::Catch2WithMain Threads::Threads podio::podioRootIO) # Catch2 TEST_CASE/SECTION macros expand __COUNTER__ at the call site in user # code, so marking Catch2 headers as system includes is not sufficient to diff --git a/tests/unittests/filtering.cpp b/tests/unittests/filtering.cpp deleted file mode 100644 index 69a4046bd..000000000 --- a/tests/unittests/filtering.cpp +++ /dev/null @@ -1,636 +0,0 @@ -// Prototype: filtering objects out of a (self-referential) collection while -// handling the relations that point at the removed objects. This lives in -// user space on purpose - it is a worked example of what a filtering utility -// could do, built only on the public generated API (clone + relation adders) -// and podio::ObjectID. See ExampleMC (parents/daughters) as a stand-in for a -// sim-particle collection. - -#include "datamodel/ExampleClusterCollection.h" -#include "datamodel/ExampleHitCollection.h" -#include "datamodel/ExampleMCCollection.h" -#include "datamodel/ExampleWithOneRelationCollection.h" -#include "datamodel/TestLinkCollection.h" - -#include "podio/ObjectID.h" - -#include - -#include -#include -#include - -namespace { - -/// What to do with a relation whose target is being removed. -enum class OnDangling { - LeaveUnconnected, ///< drop the edge, leave the surviving object unconnected - Cascade ///< keep the target too (removal of the target is forbidden) -}; - -/// Filter an ExampleMCCollection by an arbitrary predicate, rewriting the -/// self-referential parents/daughters relations of the survivors. -/// -/// The policy is chosen per relation: a Cascade relation forces the referenced -/// object to survive as well (computed as a closure), whereas a LeaveUnconnected -/// relation simply drops edges that point at removed objects. -ExampleMCCollection filterMC(const ExampleMCCollection& input, - const std::function& keep, - OnDangling parentsPolicy, OnDangling daughtersPolicy) { - // Phase 1: survivor set, seeded by the predicate and then closed under the - // relations that are configured to cascade. The fixpoint loop is safe in the - // presence of cycles (e.g. a particle that is its own ancestor). - std::unordered_set survive; - for (auto particle : input) { - if (keep(particle)) { - survive.insert(particle.getObjectID()); - } - } - - for (bool changed = true; changed;) { - changed = false; - for (auto particle : input) { - if (!survive.count(particle.getObjectID())) { - continue; - } - if (parentsPolicy == OnDangling::Cascade) { - for (auto rel : particle.parents()) { - changed |= survive.insert(rel.getObjectID()).second; - } - } - if (daughtersPolicy == OnDangling::Cascade) { - for (auto rel : particle.daughters()) { - changed |= survive.insert(rel.getObjectID()).second; - } - } - } - } - - // Phase 2: clone the survivors without relations (data members only) and - // record a mapping from the old ObjectID to the new, mutable handle. - ExampleMCCollection output; - std::unordered_map remap; - for (auto particle : input) { - if (!survive.count(particle.getObjectID())) { - continue; - } - auto cloned = particle.clone(/*cloneRelations=*/false); - remap.emplace(particle.getObjectID(), cloned); - output.push_back(cloned); - } - - // Phase 3: rewire the relations, keeping only the edges whose target also - // survived. For Cascade relations every target is guaranteed to be present - // (it was pulled into the survivor set above), so no edge is ever dropped. - for (auto particle : input) { - const auto self = remap.find(particle.getObjectID()); - if (self == remap.end()) { - continue; - } - auto newParticle = self->second; - for (auto rel : particle.parents()) { - if (const auto target = remap.find(rel.getObjectID()); target != remap.end()) { - newParticle.addparents(target->second); - } - } - for (auto rel : particle.daughters()) { - if (const auto target = remap.find(rel.getObjectID()); target != remap.end()) { - newParticle.adddaughters(target->second); - } - } - } - - return output; -} - -/// Build a small particle tree to filter: -/// -/// 0 (E=10) ─┬─ 1 (E=5) ── 3 (E=0.5) -/// └─ 2 (E=1) -ExampleMCCollection makeTree() { - ExampleMCCollection mcs; - auto p0 = mcs.create(); - p0.energy(10.0); - auto p1 = mcs.create(); - p1.energy(5.0); - auto p2 = mcs.create(); - p2.energy(1.0); - auto p3 = mcs.create(); - p3.energy(0.5); - - p0.adddaughters(p1); - p0.adddaughters(p2); - p1.adddaughters(p3); - - p1.addparents(p0); - p2.addparents(p0); - p3.addparents(p1); - - return mcs; -} - -} // namespace - -TEST_CASE("filter leaves dangling relations unconnected", "[filtering]") { - const auto input = makeTree(); - - // Keep everything with energy >= 1, i.e. drop the leaf particle (E=0.5). - const auto output = filterMC( - input, [](const ExampleMC& p) { return p.energy() >= 1.0; }, OnDangling::LeaveUnconnected, - OnDangling::LeaveUnconnected); - - // Survivors keep their input order: 0, 1, 2. - REQUIRE(output.size() == 3); - const auto p0 = output[0]; - const auto p1 = output[1]; - const auto p2 = output[2]; - - // p0 still points at both of its daughters, which now live in the new collection. - REQUIRE(p0.daughters().size() == 2); - REQUIRE(p0.daughters()[0] == p1); - REQUIRE(p0.daughters()[1] == p2); - - // p1's daughter (the removed leaf) is gone, leaving p1 unconnected downward, - // but its parent edge to the surviving p0 is intact. - REQUIRE(p1.daughters().empty()); - REQUIRE(p1.parents().size() == 1); - REQUIRE(p1.parents()[0] == p0); - REQUIRE(p2.parents()[0] == p0); -} - -TEST_CASE("filter cascades along daughters", "[filtering]") { - const auto input = makeTree(); - - // Seed with only the root (E=10); cascading daughters must pull in the whole - // subtree below it. - const auto output = filterMC( - input, [](const ExampleMC& p) { return p.energy() >= 6.0; }, OnDangling::LeaveUnconnected, - OnDangling::Cascade); - - REQUIRE(output.size() == 4); - // The transitive daughter (E=0.5) survived and is still attached to its parent. - REQUIRE(output[1].daughters().size() == 1); - REQUIRE(output[1].daughters()[0] == output[3]); -} - -TEST_CASE("filter cascades along parents while dropping other daughters", "[filtering]") { - const auto input = makeTree(); - - // Seed with only the leaf (E=0.5); cascading parents pulls in its ancestor - // chain (1 then 0) but not the sibling (2). - const auto output = filterMC( - input, [](const ExampleMC& p) { return p.energy() <= 0.6; }, OnDangling::Cascade, - OnDangling::LeaveUnconnected); - - // Survivors in input order: 0, 1, 3 (2 is removed). - REQUIRE(output.size() == 3); - const auto p0 = output[0]; - const auto p1 = output[1]; - const auto p3 = output[2]; - - // p0 used to have daughters {1, 2}; 2 was removed, so only the edge to 1 remains. - REQUIRE(p0.daughters().size() == 1); - REQUIRE(p0.daughters()[0] == p1); - // The cascaded parent chain stays connected all the way down to the leaf. - REQUIRE(p1.daughters().size() == 1); - REQUIRE(p1.daughters()[0] == p3); - REQUIRE(p3.parents()[0] == p1); -} - -// --------------------------------------------------------------------------- -// Cross-collection filtering: filter one collection and rewire the relations of -// *another* collection that references it. The key is that every collection gets -// its own ObjectID-keyed remap, and rewiring a relation simply looks the target -// up in the remap of the collection that owns the target type - so the same -// pattern works whether the relation points within a collection (parents/ -// daughters above) or across collections (cluster -> hit here). - -namespace { - -template -using Remap = std::unordered_map; - -struct FilteredHits { - ExampleHitCollection hits; - ExampleClusterCollection clusters; -}; - -/// Filter ExampleHits by a predicate and rebuild the ExampleClusters that -/// reference them through their `Hits` relation. -/// -/// The per-relation policy applies to the cluster -> hit edge: LeaveUnconnected -/// drops edges to removed hits and keeps the cluster; Cascade instead removes any -/// cluster that referenced a removed hit. -FilteredHits filterHits(const ExampleHitCollection& inHits, const ExampleClusterCollection& inClusters, - const std::function& keepHit, OnDangling clusterHitsPolicy) { - // Phase 1: survivor sets. Hits are decided by the predicate alone (they own no - // relations here). A cluster only fails to survive when the policy cascades and - // one of its hits was removed. - std::unordered_set hitSurvive; - for (auto hit : inHits) { - if (keepHit(hit)) { - hitSurvive.insert(hit.getObjectID()); - } - } - - std::unordered_set clusterSurvive; - for (auto cluster : inClusters) { - bool survives = true; - if (clusterHitsPolicy == OnDangling::Cascade) { - for (auto hit : cluster.Hits()) { - if (!hitSurvive.count(hit.getObjectID())) { - survives = false; - break; - } - } - } - if (survives) { - clusterSurvive.insert(cluster.getObjectID()); - } - } - - // Phase 2: clone survivors of each collection, building one remap per collection. - FilteredHits out; - Remap hitRemap; - for (auto hit : inHits) { - if (!hitSurvive.count(hit.getObjectID())) { - continue; - } - auto cloned = hit.clone(/*cloneRelations=*/false); - hitRemap.emplace(hit.getObjectID(), cloned); - out.hits.push_back(cloned); - } - Remap clusterRemap; - for (auto cluster : inClusters) { - if (!clusterSurvive.count(cluster.getObjectID())) { - continue; - } - auto cloned = cluster.clone(/*cloneRelations=*/false); - clusterRemap.emplace(cluster.getObjectID(), cloned); - out.clusters.push_back(cloned); - } - - // Phase 3: rewire the cluster -> hit edges, resolving each hit through the hit - // remap (a *different* collection's table). Edges to removed hits are dropped; - // under Cascade the owning cluster is already gone, so none are. - for (auto cluster : inClusters) { - const auto self = clusterRemap.find(cluster.getObjectID()); - if (self == clusterRemap.end()) { - continue; - } - auto newCluster = self->second; - for (auto hit : cluster.Hits()) { - if (const auto target = hitRemap.find(hit.getObjectID()); target != hitRemap.end()) { - newCluster.addHits(target->second); - } - } - } - - return out; -} - -/// Build two clusters over four hits: -/// cluster 0 -> { hit0 (E=1), hit1 (E=2) } -/// cluster 1 -> { hit2 (E=3), hit3 (E=0.5) } -std::pair makeHitsAndClusters() { - ExampleHitCollection hits; - auto h0 = hits.create(); - h0.energy(1.0); - auto h1 = hits.create(); - h1.energy(2.0); - auto h2 = hits.create(); - h2.energy(3.0); - auto h3 = hits.create(); - h3.energy(0.5); - - ExampleClusterCollection clusters; - auto c0 = clusters.create(); - c0.addHits(h0); - c0.addHits(h1); - auto c1 = clusters.create(); - c1.addHits(h2); - c1.addHits(h3); - - return {std::move(hits), std::move(clusters)}; -} - -} // namespace - -TEST_CASE("cross-collection filter leaves dangling hit edges unconnected", "[filtering]") { - const auto [inHits, inClusters] = makeHitsAndClusters(); - - // Drop the low-energy hit (E=0.5); clusters survive but lose the dropped edge. - const auto out = filterHits( - inHits, inClusters, [](const ExampleHit& h) { return h.energy() >= 1.0; }, - OnDangling::LeaveUnconnected); - - REQUIRE(out.hits.size() == 3); - REQUIRE(out.clusters.size() == 2); - - // cluster 0 keeps both of its hits; they now live in the new hit collection. - REQUIRE(out.clusters[0].Hits().size() == 2); - REQUIRE(out.clusters[0].Hits()[0] == out.hits[0]); - // cluster 1 referenced the removed hit, so only the surviving one (E=3) remains. - REQUIRE(out.clusters[1].Hits().size() == 1); - REQUIRE(out.clusters[1].Hits()[0].energy() == 3.0); -} - -TEST_CASE("cross-collection filter cascades from hit to cluster", "[filtering]") { - const auto [inHits, inClusters] = makeHitsAndClusters(); - - // Same hit removed, but now removing it cascades to the cluster that used it. - const auto out = filterHits( - inHits, inClusters, [](const ExampleHit& h) { return h.energy() >= 1.0; }, OnDangling::Cascade); - - REQUIRE(out.hits.size() == 3); - // cluster 1 referenced the removed hit and is gone; cluster 0 is intact. - REQUIRE(out.clusters.size() == 1); - REQUIRE(out.clusters[0].Hits().size() == 2); -} - -// --------------------------------------------------------------------------- -// The full prototype: filter across three collections at once, covering every -// relation kind - a cross-collection OneToMany (cluster -> hit), a self -// OneToMany (cluster -> sub-cluster) and a OneToOne (ref -> cluster) - with a -// removal closure that propagates across collections (hit -> cluster -> ref). - -namespace { - -struct EventPolicies { - OnDangling clusterHits; ///< cluster -> hit (OneToMany, cross-collection) - OnDangling clusterClusters; ///< cluster -> sub-cluster (OneToMany, self) - OnDangling refCluster; ///< ref -> cluster (OneToOne, cross-collection) - OnDangling linkFrom; ///< link -> hit (Link "From" endpoint) - OnDangling linkTo; ///< link -> cluster (Link "To" endpoint) -}; - -struct FilteredEvent { - ExampleHitCollection hits; - ExampleClusterCollection clusters; - ExampleWithOneRelationCollection refs; - TestLinkCollection links; -}; - -FilteredEvent filterEvent(const ExampleHitCollection& inHits, const ExampleClusterCollection& inClusters, - const ExampleWithOneRelationCollection& inRefs, const TestLinkCollection& inLinks, - const std::function& keepHit, EventPolicies policy) { - using IdSet = std::unordered_set; - - // Phase 1: removal closure. The predicate seeds the removed hits; from there - // removal propagates along every relation configured to cascade, across - // collections, until the sets stop changing. - IdSet hitsRemoved; - for (auto hit : inHits) { - if (!keepHit(hit)) { - hitsRemoved.insert(hit.getObjectID()); - } - } - - IdSet clustersRemoved; - IdSet refsRemoved; - for (bool changed = true; changed;) { - changed = false; - - for (auto cluster : inClusters) { - if (clustersRemoved.count(cluster.getObjectID())) { - continue; - } - bool remove = false; - if (policy.clusterHits == OnDangling::Cascade) { - for (auto hit : cluster.Hits()) { - remove |= static_cast(hitsRemoved.count(hit.getObjectID())); - } - } - if (policy.clusterClusters == OnDangling::Cascade) { - for (auto sub : cluster.Clusters()) { - remove |= static_cast(clustersRemoved.count(sub.getObjectID())); - } - } - if (remove) { - changed |= clustersRemoved.insert(cluster.getObjectID()).second; - } - } - - for (auto ref : inRefs) { - if (refsRemoved.count(ref.getObjectID()) || policy.refCluster != OnDangling::Cascade) { - continue; - } - const auto cluster = ref.cluster(); - if (cluster.isAvailable() && clustersRemoved.count(cluster.getObjectID())) { - changed |= refsRemoved.insert(ref.getObjectID()).second; - } - } - } - - // Phase 2: clone survivors of every collection, one remap per collection. - FilteredEvent out; - Remap hitRemap; - for (auto hit : inHits) { - if (hitsRemoved.count(hit.getObjectID())) { - continue; - } - auto cloned = hit.clone(/*cloneRelations=*/false); - hitRemap.emplace(hit.getObjectID(), cloned); - out.hits.push_back(cloned); - } - Remap clusterRemap; - for (auto cluster : inClusters) { - if (clustersRemoved.count(cluster.getObjectID())) { - continue; - } - auto cloned = cluster.clone(/*cloneRelations=*/false); - clusterRemap.emplace(cluster.getObjectID(), cloned); - out.clusters.push_back(cloned); - } - Remap refRemap; - for (auto ref : inRefs) { - if (refsRemoved.count(ref.getObjectID())) { - continue; - } - auto cloned = ref.clone(/*cloneRelations=*/false); - refRemap.emplace(ref.getObjectID(), cloned); - out.refs.push_back(cloned); - } - - // Phase 3: rewire every relation, resolving each target through the remap of - // the collection that owns it. OneToMany edges to removed targets are simply - // skipped; a OneToOne to a removed target is left unset. - for (auto cluster : inClusters) { - const auto self = clusterRemap.find(cluster.getObjectID()); - if (self == clusterRemap.end()) { - continue; - } - auto newCluster = self->second; - for (auto hit : cluster.Hits()) { - if (const auto target = hitRemap.find(hit.getObjectID()); target != hitRemap.end()) { - newCluster.addHits(target->second); - } - } - for (auto sub : cluster.Clusters()) { - if (const auto target = clusterRemap.find(sub.getObjectID()); target != clusterRemap.end()) { - newCluster.addClusters(target->second); - } - } - } - for (auto ref : inRefs) { - const auto self = refRemap.find(ref.getObjectID()); - if (self == refRemap.end()) { - continue; - } - const auto cluster = ref.cluster(); - if (cluster.isAvailable()) { - if (const auto target = clusterRemap.find(cluster.getObjectID()); target != clusterRemap.end()) { - self->second.cluster(target->second); - } - } - } - - // Links live in their own collection and reference objects in two others. A - // link is dropped when an endpoint was removed under a Cascade policy; - // otherwise it is kept, its weight preserved, and each surviving endpoint - // rewired through that endpoint's remap (a removed endpoint is left unset). - // Links are terminal - nothing references them - so they need no remap. - for (auto link : inLinks) { - const auto from = link.getFrom(); - const auto to = link.getTo(); - const bool fromRemoved = from.isAvailable() && hitsRemoved.count(from.getObjectID()); - const bool toRemoved = to.isAvailable() && clustersRemoved.count(to.getObjectID()); - if ((fromRemoved && policy.linkFrom == OnDangling::Cascade) || - (toRemoved && policy.linkTo == OnDangling::Cascade)) { - continue; - } - auto newLink = link.clone(/*cloneRelations=*/false); // copies the weight only - if (from.isAvailable()) { - if (const auto target = hitRemap.find(from.getObjectID()); target != hitRemap.end()) { - newLink.setFrom(target->second); - } - } - if (to.isAvailable()) { - if (const auto target = clusterRemap.find(to.getObjectID()); target != clusterRemap.end()) { - newLink.setTo(target->second); - } - } - out.links.push_back(newLink); - } - - return out; -} - -/// Build a three-level event to filter: -/// hits: h0 (E=1) h1 (E=2) h2 (E=0.5) -/// clusters: c0 -> {h0, h1} c1 -> {h2}, c0 also has sub-cluster c1 -/// refs: r0 -> c0 r1 -> c1 -/// links: l0: h0 -> c0 l1: h2 -> c1 -FilteredEvent makeEvent() { - FilteredEvent ev; - auto h0 = ev.hits.create(); - h0.energy(1.0); - auto h1 = ev.hits.create(); - h1.energy(2.0); - auto h2 = ev.hits.create(); - h2.energy(0.5); - - auto c0 = ev.clusters.create(); - c0.addHits(h0); - c0.addHits(h1); - auto c1 = ev.clusters.create(); - c1.addHits(h2); - c0.addClusters(c1); - - auto r0 = ev.refs.create(); - r0.cluster(c0); - auto r1 = ev.refs.create(); - r1.cluster(c1); - - auto l0 = ev.links.create(); - l0.setFrom(h0); - l0.setTo(c0); - l0.setWeight(1.0f); - auto l1 = ev.links.create(); - l1.setFrom(h2); - l1.setTo(c1); - l1.setWeight(2.0f); - - return ev; -} - -} // namespace - -TEST_CASE("full filter leaves all dangling relations unconnected", "[filtering]") { - const auto in = makeEvent(); - - // Drop h2 (E=0.5). With every policy LeaveUnconnected, nothing else is removed; - // only the edges to h2 disappear. - const auto out = filterEvent( - in.hits, in.clusters, in.refs, in.links, [](const ExampleHit& h) { return h.energy() >= 1.0; }, - {OnDangling::LeaveUnconnected, OnDangling::LeaveUnconnected, OnDangling::LeaveUnconnected, - OnDangling::LeaveUnconnected, OnDangling::LeaveUnconnected}); - - REQUIRE(out.hits.size() == 2); - REQUIRE(out.clusters.size() == 2); - REQUIRE(out.refs.size() == 2); - - // c0 keeps both hits and its sub-cluster; c1 lost its only hit but survives. - REQUIRE(out.clusters[0].Hits().size() == 2); - REQUIRE(out.clusters[0].Clusters().size() == 1); - REQUIRE(out.clusters[0].Clusters()[0] == out.clusters[1]); - REQUIRE(out.clusters[1].Hits().empty()); - // OneToOne edges still point at the surviving clusters. - REQUIRE(out.refs[1].cluster() == out.clusters[1]); - - // Both links survive; l1's "From" hit was removed, so that endpoint is unset - // while its "To" cluster and the preserved weight remain intact. - REQUIRE(out.links.size() == 2); - REQUIRE(out.links[0].getFrom() == out.hits[0]); - REQUIRE(out.links[0].getTo() == out.clusters[0]); - REQUIRE_FALSE(out.links[1].getFrom().isAvailable()); - REQUIRE(out.links[1].getTo() == out.clusters[1]); - REQUIRE(out.links[1].getWeight() == 2.0f); -} - -TEST_CASE("full filter cascades across all three collections", "[filtering]") { - const auto in = makeEvent(); - - // Drop h2 (E=0.5) and cascade every relation: removing h2 removes c1 (it used - // h2), removing c1 removes both c0 (its sub-cluster is gone) and r1 (its - // cluster is gone), and removing c0 removes r0. - const auto out = filterEvent( - in.hits, in.clusters, in.refs, in.links, [](const ExampleHit& h) { return h.energy() >= 1.0; }, - {OnDangling::Cascade, OnDangling::Cascade, OnDangling::Cascade, OnDangling::Cascade, - OnDangling::Cascade}); - - REQUIRE(out.hits.size() == 2); - REQUIRE(out.clusters.empty()); - REQUIRE(out.refs.empty()); - // l0's "To" cluster (c0) and l1's "From" hit (h2) were both removed under a - // cascading endpoint policy, so both links are dropped. - REQUIRE(out.links.empty()); -} - -TEST_CASE("full filter cascades to cluster but leaves its ref unconnected", "[filtering]") { - const auto in = makeEvent(); - - // Cascade hit -> cluster, but leave ref -> cluster unconnected: c1 is removed, - // and r1 survives with its OneToOne relation left unset. - const auto out = filterEvent( - in.hits, in.clusters, in.refs, in.links, [](const ExampleHit& h) { return h.energy() >= 1.0; }, - {OnDangling::Cascade, OnDangling::LeaveUnconnected, OnDangling::LeaveUnconnected, - OnDangling::LeaveUnconnected, OnDangling::LeaveUnconnected}); - - // c1 (used h2) is gone; c0's sub-cluster edge to it is dropped but c0 survives. - REQUIRE(out.clusters.size() == 1); - REQUIRE(out.clusters[0].Hits().size() == 2); - REQUIRE(out.clusters[0].Clusters().empty()); - - // Both refs survive; r1's cluster was removed so its relation is now unset. - REQUIRE(out.refs.size() == 2); - REQUIRE(out.refs[0].cluster() == out.clusters[0]); - REQUIRE_FALSE(out.refs[1].cluster().isAvailable()); - - // Both links survive under the leave-unconnected endpoint policies. l1 lost - // both endpoints (h2 removed, c1 cascaded away), so it is fully unset. - REQUIRE(out.links.size() == 2); - REQUIRE(out.links[0].getFrom() == out.hits[0]); - REQUIRE(out.links[0].getTo() == out.clusters[0]); - REQUIRE_FALSE(out.links[1].getFrom().isAvailable()); - REQUIRE_FALSE(out.links[1].getTo().isAvailable()); -} diff --git a/tests/unittests/frame_filter.cpp b/tests/unittests/frame_filter.cpp index 533447b8b..d55b7d57f 100644 --- a/tests/unittests/frame_filter.cpp +++ b/tests/unittests/frame_filter.cpp @@ -1,7 +1,8 @@ // Exercises podio::FrameFilter (driven by the registered per-type FilterHooks) -// over Frames built from the test datamodel. Mirrors the hand-written prototype -// scenarios in filtering.cpp, plus the keep-referenced / orphan-sweep capability -// that only the Frame-level filter provides. +// over Frames built from the test datamodel: leave-unconnected and integrity +// cascade edge policies, the keep-referenced orphan sweep, whole-collection +// drops, vector members, interface relations/links, subset collections and +// frame parameters. #include "datamodel/ExampleClusterCollection.h" #include "datamodel/ExampleHitCollection.h"