Skip to content
This repository was archived by the owner on Oct 6, 2025. It is now read-only.

Commit 5f8c810

Browse files
hhundhhund
committed
Merge remote-tracking branch
'origin/issues/72_Validator_Generates_False-Positives' into develop
2 parents 42d7aef + 881fea4 commit 5f8c810

13 files changed

Lines changed: 369 additions & 125 deletions

File tree

‎codex-process-data-transfer/pom.xml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
<parent>
77
<groupId>de.netzwerk-universitaetsmedizin.codex</groupId>
88
<artifactId>codex-processes-ap1</artifactId>
9-
<version>0.5.1-SNAPSHOT</version>
9+
<version>0.6.0-SNAPSHOT</version>
1010
</parent>
1111

1212
<properties>

‎codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/spring/config/ValidationConfig.java‎

Lines changed: 31 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,9 @@
5555
import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.ValueSetExpansionClient;
5656
import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.ValueSetExpansionClientJersey;
5757
import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.ValueSetExpansionClientWithFileSystemCache;
58+
import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.ValueSetExpansionClientWithModifiers;
5859
import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.structure_definition.StructureDefinitionModifier;
60+
import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.value_set.ValueSetModifier;
5961
import de.rwh.utils.crypto.CertificateHelper;
6062
import de.rwh.utils.crypto.io.CertificateReader;
6163
import de.rwh.utils.crypto.io.PemIo;
@@ -198,10 +200,16 @@ public static enum TerminologyServerConnectionTestStatus
198200
@Value("${de.netzwerk.universitaetsmedizin.codex.gecco.validation.valueset.expansion.client.verbose:false}")
199201
private boolean valueSetExpansionClientVerbose;
200202

203+
@ProcessDocumentation(description = "List of ValueSet modifier classes, modifiers are executed before atempting to expand a ValueSet and after", processNames = "wwwnetzwerk-universitaetsmedizinde_dataSend")
204+
@Value("#{'${de.netzwerk.universitaetsmedizin.codex.gecco.validation.valueset.expansion.modifierClasses:"
205+
+ "de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.value_set.MissingEntriesIncluder"
206+
+ "}'.trim().split('(,[ ]?)|(\\n)')}")
207+
private List<String> valueSetModifierClasses;
208+
201209
@ProcessDocumentation(description = "List of StructureDefinition modifier classes, modifiers are executed before atempting to generate a StructureDefinition snapshot", processNames = "wwwnetzwerk-universitaetsmedizinde_dataSend")
202210
@Value("#{'${de.netzwerk.universitaetsmedizin.codex.gecco.validation.structuredefinition.modifierClasses:"
203211
+ "de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.structure_definition.ClosedTypeSlicingRemover,"
204-
+ "de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.structure_definition.ObservationIdentifierRemover,"
212+
+ "de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.structure_definition.IdentifierRemover,"
205213
+ "de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.structure_definition.SliceMinFixer"
206214
+ "}'.trim().split('(,[ ]?)|(\\n)')}")
207215
private List<String> structureDefinitionModifierClasses;
@@ -410,8 +418,29 @@ private ValidationPackageClientJersey validationPackageClientJersey()
410418
@Bean
411419
public ValueSetExpansionClient valueSetExpansionClient()
412420
{
421+
List<ValueSetModifier> modifiers = valueSetModifierClasses.stream().map(this::createValueSetModifier)
422+
.collect(Collectors.toList());
423+
413424
return new ValueSetExpansionClientWithFileSystemCache(valueSetCacheFolder(), fhirContext,
414-
valueSetExpansionClientJersey());
425+
new ValueSetExpansionClientWithModifiers(valueSetExpansionClientJersey(), modifiers));
426+
}
427+
428+
private ValueSetModifier createValueSetModifier(String className)
429+
{
430+
try
431+
{
432+
Class<?> modifierClass = Class.forName(className);
433+
if (ValueSetModifier.class.isAssignableFrom(modifierClass))
434+
return (ValueSetModifier) modifierClass.getConstructor().newInstance();
435+
else
436+
throw new IllegalArgumentException(
437+
"Class " + className + " not compatible with " + ValueSetModifier.class.getName());
438+
}
439+
catch (ClassNotFoundException | InstantiationException | IllegalAccessException | InvocationTargetException
440+
| NoSuchMethodException | SecurityException e)
441+
{
442+
throw new RuntimeException(e);
443+
}
415444
}
416445

417446
@Bean

‎codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/PluginSnapshotGeneratorWithModifiers.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -11,22 +11,22 @@
1111
import org.springframework.beans.factory.InitializingBean;
1212

1313
import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.structure_definition.ClosedTypeSlicingRemover;
14-
import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.structure_definition.ObservationIdentifierRemover;
14+
import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.structure_definition.IdentifierRemover;
1515
import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.structure_definition.SliceMinFixer;
1616
import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.structure_definition.StructureDefinitionModifier;
1717

1818
public class PluginSnapshotGeneratorWithModifiers implements SnapshotGenerator, InitializingBean
1919
{
2020
public static final StructureDefinitionModifier CLOSED_TYPE_SLICING_REMOVER = new ClosedTypeSlicingRemover();
21-
public static final StructureDefinitionModifier OBSERVATION_IDENTIFIER_REMOVER = new ObservationIdentifierRemover();
21+
public static final StructureDefinitionModifier IDENTIFIER_REMOVER = new IdentifierRemover();
2222
public static final StructureDefinitionModifier SLICE_MIN_FIXER = new SliceMinFixer();
2323

2424
private final SnapshotGenerator delegate;
2525
private final List<StructureDefinitionModifier> structureDefinitionModifiers = new ArrayList<>();
2626

2727
public PluginSnapshotGeneratorWithModifiers(SnapshotGenerator delegate)
2828
{
29-
this(delegate, Arrays.asList(CLOSED_TYPE_SLICING_REMOVER, OBSERVATION_IDENTIFIER_REMOVER, SLICE_MIN_FIXER));
29+
this(delegate, Arrays.asList(CLOSED_TYPE_SLICING_REMOVER, IDENTIFIER_REMOVER, SLICE_MIN_FIXER));
3030
}
3131

3232
public PluginSnapshotGeneratorWithModifiers(SnapshotGenerator delegate,

‎codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageManagerImpl.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -202,7 +202,7 @@ private List<ValueSet> withExpandedValueSets(ValidationPackageWithDepedencies pa
202202
ValueSetExpander expander = internalValueSetExpanderFactory.apply(fhirContext,
203203
createSupportChain(fhirContext, packageWithDependencies, Collections.emptyList(), expandedValueSets));
204204

205-
packageWithDependencies.getValueSetsIncludingDependencies(valueSetBindingStrengths).forEach(v ->
205+
packageWithDependencies.getValueSetsIncludingDependencies(valueSetBindingStrengths, fhirContext).forEach(v ->
206206
{
207207
logger.debug("Expanding ValueSet {}|{}", v.getUrl(), v.getVersion());
208208

‎codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageWithDepedencies.java‎

Lines changed: 32 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,11 +23,16 @@
2323
import org.hl7.fhir.r4.model.StructureDefinition.StructureDefinitionDifferentialComponent;
2424
import org.hl7.fhir.r4.model.UriType;
2525
import org.hl7.fhir.r4.model.ValueSet;
26+
import org.slf4j.Logger;
27+
import org.slf4j.LoggerFactory;
2628

2729
import ca.uhn.fhir.context.FhirContext;
30+
import ca.uhn.fhir.context.support.IValidationSupport;
2831

2932
public class ValidationPackageWithDepedencies extends ValidationPackage
3033
{
34+
private static final Logger logger = LoggerFactory.getLogger(ValidationPackageWithDepedencies.class);
35+
3136
public static ValidationPackageWithDepedencies from(
3237
Map<ValidationPackageIdentifier, ValidationPackage> packagesByNameAndVersion,
3338
ValidationPackageIdentifier rootPackageIdentifier)
@@ -216,17 +221,42 @@ private Set<String> findValueSetsWithBindingStrength(Stream<StructureDefinition>
216221
.map(StructureDefinitionDifferentialComponent::getElement).flatMap(List::stream)
217222
.filter(ElementDefinition::hasBinding).map(ElementDefinition::getBinding)
218223
.filter(b -> bindingStrengths.contains(b.getStrength()))
224+
.filter(ElementDefinitionBindingComponent::hasValueSet)
219225
.map(ElementDefinitionBindingComponent::getValueSet).collect(Collectors.toSet());
220226
}
221227

222-
public List<ValueSet> getValueSetsIncludingDependencies(EnumSet<BindingStrength> bindingStrengths)
228+
public List<ValueSet> getValueSetsIncludingDependencies(EnumSet<BindingStrength> bindingStrengths,
229+
FhirContext fhirContext)
223230
{
224231
Stream<StructureDefinition> sds = getValidationSupportResources().getStructureDefinitions().stream()
225232
.flatMap(sd -> Stream.concat(Stream.of(sd), getStructureDefinitionDependencies(sd).stream()))
226233
.distinct();
227234

228235
Set<String> neededValueSets = findValueSetsWithBindingStrength(sds, bindingStrengths);
229-
return getAllValueSets().stream().filter(vs -> neededValueSets.contains(vs.getUrl())
236+
List<ValueSet> foundValueSets = getAllValueSets().stream().filter(vs -> neededValueSets.contains(vs.getUrl())
230237
|| neededValueSets.contains(vs.getUrl() + "|" + vs.getVersion())).collect(Collectors.toList());
238+
239+
logMissingValueSets(neededValueSets, foundValueSets, fhirContext);
240+
241+
return foundValueSets;
242+
}
243+
244+
private void logMissingValueSets(Set<String> neededValueSets, List<ValueSet> foundValueSets,
245+
FhirContext fhirContext)
246+
{
247+
Set<String> foundValueSetUrls = foundValueSets.stream()
248+
.flatMap(vs -> Stream.of(vs.getUrl(), vs.getUrl() + "|" + vs.getVersion())).collect(Collectors.toSet());
249+
250+
IValidationSupport validationSupport = fhirContext.getValidationSupport();
251+
String missingValueSets = neededValueSets.stream()
252+
.filter(e -> !foundValueSetUrls.contains(e) && validationSupport.fetchValueSet(e) == null).distinct()
253+
.sorted().collect(Collectors.joining(", ", "[", "]"));
254+
255+
if (!missingValueSets.isEmpty())
256+
{
257+
logger.warn(
258+
"The following ValueSet are required for validation but could not be found in validation package {}|{} or its dependencies: {}",
259+
getName(), getVersion(), missingValueSets);
260+
}
231261
}
232262
}

‎codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValueSetExpansionClientWithModifiers.java‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
package de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation;
2+
3+
import java.io.IOException;
4+
import java.util.ArrayList;
5+
import java.util.Arrays;
6+
import java.util.Collection;
7+
import java.util.List;
8+
import java.util.Objects;
9+
10+
import javax.ws.rs.WebApplicationException;
11+
12+
import org.hl7.fhir.r4.model.CapabilityStatement;
13+
import org.hl7.fhir.r4.model.ValueSet;
14+
import org.springframework.beans.factory.InitializingBean;
15+
16+
import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.value_set.MissingEntriesIncluder;
17+
import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.value_set.ValueSetModifier;
18+
19+
public class ValueSetExpansionClientWithModifiers implements ValueSetExpansionClient, InitializingBean
20+
{
21+
public static final ValueSetModifier MISSING_ENTRIES_INCLUDER = new MissingEntriesIncluder();
22+
23+
private final ValueSetExpansionClient delegate;
24+
private final List<ValueSetModifier> valueSetModifiers = new ArrayList<>();
25+
26+
public ValueSetExpansionClientWithModifiers(ValueSetExpansionClient delegate)
27+
{
28+
this(delegate, Arrays.asList(MISSING_ENTRIES_INCLUDER));
29+
}
30+
31+
public ValueSetExpansionClientWithModifiers(ValueSetExpansionClient delegate,
32+
Collection<? extends ValueSetModifier> valueSetModifiers)
33+
{
34+
this.delegate = delegate;
35+
36+
if (valueSetModifiers != null)
37+
this.valueSetModifiers.addAll(valueSetModifiers);
38+
}
39+
40+
@Override
41+
public void afterPropertiesSet() throws Exception
42+
{
43+
Objects.requireNonNull(delegate, "delegate");
44+
}
45+
46+
@Override
47+
public ValueSet expand(ValueSet valueSet) throws IOException, WebApplicationException
48+
{
49+
if (valueSet == null)
50+
return null;
51+
52+
for (ValueSetModifier modifier : valueSetModifiers)
53+
valueSet = modifier.modifyPreExpansion(valueSet);
54+
55+
ValueSet expandedValueSet = delegate.expand(valueSet);
56+
57+
for (ValueSetModifier modifier : valueSetModifiers)
58+
expandedValueSet = modifier.modifyPostExpansion(valueSet, expandedValueSet);
59+
60+
return expandedValueSet;
61+
}
62+
63+
@Override
64+
public CapabilityStatement getMetadata() throws WebApplicationException
65+
{
66+
return delegate.getMetadata();
67+
}
68+
}

‎codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/structure_definition/IdentifierRemover.java‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
package de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.structure_definition;
2+
3+
import java.util.List;
4+
import java.util.function.Predicate;
5+
import java.util.stream.Collectors;
6+
7+
import org.hl7.fhir.r4.model.ElementDefinition;
8+
import org.hl7.fhir.r4.model.ResourceType;
9+
import org.hl7.fhir.r4.model.StructureDefinition;
10+
import org.hl7.fhir.r4.model.StructureDefinition.StructureDefinitionKind;
11+
import org.slf4j.Logger;
12+
import org.slf4j.LoggerFactory;
13+
14+
/**
15+
* Mandatory identifier on resources other then Patient not compatible with data protection rules and current
16+
* pseudonymization implementation.
17+
*/
18+
public class IdentifierRemover implements StructureDefinitionModifier
19+
{
20+
private static final Logger logger = LoggerFactory.getLogger(IdentifierRemover.class);
21+
22+
@Override
23+
public StructureDefinition modify(StructureDefinition sd)
24+
{
25+
String type = sd.getType();
26+
27+
if (StructureDefinitionKind.RESOURCE.equals(sd.getKind()) && type != null
28+
&& !ResourceType.Patient.name().equals(type))
29+
{
30+
Predicate<? super ElementDefinition> toRemove = e -> e.hasPath()
31+
&& e.getPath().startsWith(type + ".identifier");
32+
33+
List<ElementDefinition> filteredRules = sd.getDifferential().getElement().stream().filter(toRemove.negate())
34+
.collect(Collectors.toList());
35+
36+
if (filteredRules.size() < sd.getDifferential().getElement().size())
37+
{
38+
logger.warn("Removing validation rules with ids {} from StructureDefinition {}|{}",
39+
sd.getDifferential().getElement().stream().filter(toRemove).map(ElementDefinition::getId)
40+
.collect(Collectors.joining(", ", "[", "]")),
41+
sd.getUrl(), sd.getVersion());
42+
43+
sd.getDifferential().setElement(filteredRules);
44+
}
45+
}
46+
47+
return sd;
48+
}
49+
}

‎codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/structure_definition/ObservationIdentifierRemover.java‎

Lines changed: 0 additions & 44 deletions
This file was deleted.

‎codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/value_set/MissingEntriesIncluder.java‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
package de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.value_set;
2+
3+
import java.util.Set;
4+
import java.util.stream.Collectors;
5+
6+
import org.hl7.fhir.r4.model.ValueSet;
7+
import org.hl7.fhir.r4.model.ValueSet.ConceptSetComponent;
8+
import org.hl7.fhir.r4.model.ValueSet.ValueSetExpansionContainsComponent;
9+
import org.slf4j.Logger;
10+
import org.slf4j.LoggerFactory;
11+
12+
public class MissingEntriesIncluder implements ValueSetModifier
13+
{
14+
private static final Logger logger = LoggerFactory.getLogger(MissingEntriesIncluder.class);
15+
16+
@Override
17+
public ValueSet modifyPostExpansion(ValueSet vsWithComposition, ValueSet vsWithExpansion)
18+
{
19+
if (vsWithExpansion == null)
20+
return null;
21+
if (!vsWithExpansion.hasExpansion())
22+
return vsWithExpansion;
23+
24+
if (vsWithComposition.hasCompose()
25+
&& vsWithComposition.getCompose().getInclude().stream().anyMatch(ConceptSetComponent::hasConcept))
26+
{
27+
Set<String> expandedEntries = vsWithExpansion.getExpansion().getContains().stream()
28+
.map(c -> toEntry(c.getSystem(), c.getCode())).distinct().collect(Collectors.toSet());
29+
30+
vsWithComposition.getCompose().getInclude().stream().filter(ConceptSetComponent::hasConcept)
31+
.forEach(include ->
32+
{
33+
String system = include.getSystem();
34+
String version = include.getVersion();
35+
36+
include.getConcept().forEach(concept ->
37+
{
38+
if (!expandedEntries.contains(toEntry(system, concept.getCode()))
39+
&& !expandedEntries.contains(toEntry(system, concept.getCode())))
40+
{
41+
logger.warn(
42+
"Adding missing concept to ValueSet {}|{}: system: '{}', version: '{}', code: '{}', display: '{}'",
43+
vsWithExpansion.getUrl(), vsWithExpansion.getVersion(), system, version,
44+
concept.getCode(), concept.getDisplay());
45+
46+
vsWithExpansion.getExpansion()
47+
.addContains(new ValueSetExpansionContainsComponent().setSystem(system)
48+
.setVersion(version).setCode(concept.getCode())
49+
.setDisplay(concept.getDisplay()));
50+
}
51+
});
52+
});
53+
}
54+
55+
return vsWithExpansion;
56+
}
57+
58+
private String toEntry(String system, String code)
59+
{
60+
return system + code;
61+
}
62+
}

0 commit comments

Comments
 (0)