Commit 5be7c236 authored by Georg Fette's avatar Georg Fette
Browse files

added type checking

parent 7e5bdf95
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>QueryMapper</groupId>
......@@ -10,8 +11,8 @@
<maven.compiler.target>1.8</maven.compiler.target>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
<log4j.version>2.11.1</log4j.version>
<slf4j.version>1.7.25</slf4j.version>
<log4j.version>2.11.1</log4j.version>
<slf4j.version>1.7.25</slf4j.version>
</properties>
<repositories>
<repository>
......@@ -45,6 +46,11 @@
<artifactId>hapi-fhir-client</artifactId>
<version>3.7.0</version>
</dependency>
<dependency>
<groupId>ca.uhn.hapi.fhir</groupId>
<artifactId>hapi-fhir-structures-dstu3</artifactId>
<version>3.6.0</version>
</dependency>
<dependency>
<groupId>org.jdom</groupId>
<artifactId>jdom</artifactId>
......@@ -55,18 +61,18 @@
<artifactId>cql-to-elm</artifactId>
<version>1.2.20</version>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.8.2</version>
<scope>test</scope>
</dependency>
<!-- Needed by junit -->
<dependency>
<groupId>org.hamcrest</groupId>
<artifactId>hamcrest-all</artifactId>
<version>1.3</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.8.2</version>
<scope>test</scope>
</dependency>
<!-- Needed by junit -->
<dependency>
<groupId>org.hamcrest</groupId>
<artifactId>hamcrest-all</artifactId>
<version>1.3</version>
<scope>test</scope>
</dependency>
</dependencies>
</project>
\ No newline at end of file
package de.uniwue.model;
import java.util.ArrayList;
import java.util.List;
public class Concept {
public static String intTermCodes;
public String name;
public List<String> termCodes;
public List<ConceptField> fields;
public void addField(String fieldName, List<String> fieldTypes, boolean isArrayFlag) {
if (fields == null) {
fields = new ArrayList<ConceptField>();
}
ConceptField field = new ConceptField();
field.name = fieldName;
field.isArray = isArrayFlag;
field.types = fieldTypes;
fields.add(field);
}
}
package de.uniwue.model;
import java.util.List;
public class ConceptField {
public String name;
public List<String> types;
public boolean isArray;
}
package de.uniwue.model;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import com.fasterxml.jackson.annotation.JsonInclude.Include;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.SerializationFeature;
public class Model {
public List<Concept> concepts = new ArrayList<Concept>();
public Map<String, Concept> conceptMap = new HashMap<String, Concept>();
public static Model fromJSON(String jsonString) throws IOException {
ObjectMapper objectMapper = new ObjectMapper();
Object model = objectMapper.readValue(jsonString, Model.class);
return (Model) model;
}
public String toJSON() throws JsonProcessingException {
ObjectMapper objectMapper = new ObjectMapper();
objectMapper.setSerializationInclusion(Include.NON_NULL);
objectMapper.enable(SerializationFeature.INDENT_OUTPUT);
String writeValueAsString = objectMapper.writeValueAsString(this);
return writeValueAsString;
}
}
package de.uniwue.model;
import java.io.File;
import java.util.ArrayList;
import java.util.List;
import org.hl7.fhir.dstu3.model.Bundle;
import org.hl7.fhir.dstu3.model.Bundle.BundleEntryComponent;
import org.hl7.fhir.dstu3.model.ElementDefinition;
import org.hl7.fhir.dstu3.model.ElementDefinition.TypeRefComponent;
import org.hl7.fhir.dstu3.model.Resource;
import org.hl7.fhir.dstu3.model.StructureDefinition;
import ca.uhn.fhir.context.FhirContext;
import ca.uhn.fhir.parser.IParser;
import de.uniwue.misc.util.FileUtilsUniWue;
import de.uniwue.misc.util.ResourceUtil;
public class StructureDef2ModelConverter {
private FhirContext ctx;
private Model model = new Model();
private StructureDefinition currentStrucDef;
private List<String> primitiveTypes = new ArrayList<String>();
private List<String> baseTypes = new ArrayList<String>();
public StructureDef2ModelConverter(FhirContext ctx)
throws NoSuchMethodException, SecurityException {
this.ctx = ctx;
}
public void translateStructureDefs2HAPIJavaClasses(File strucDefsOutputDir, File outDirHAPI)
throws Exception {
File structureDefDir = new File(strucDefsOutputDir, "xml");
IParser parser = ctx.newXmlParser();
// parser = ctx.newJsonParser();
for (File aFile : structureDefDir.listFiles()) {
System.out.println(aFile.getName());
String structureDefString = FileUtilsUniWue.file2String(aFile);
currentStrucDef = (StructureDefinition) parser.parseResource(structureDefString);
convertStrucDefToConcept();
}
}
public static void main(String[] args) throws Exception {
FhirContext ctx = FhirContext.forDstu3();
StructureDef2ModelConverter converter = new StructureDef2ModelConverter(ctx);
converter.work();
}
private void loadStructureDefsBundle(String path) throws Exception {
String dataElementsString = ResourceUtil.loadFileAsString(path);
Bundle bundle = (Bundle) ctx.newJsonParser().parseResource(dataElementsString);
List<BundleEntryComponent> entries = bundle.getEntry();
fillTypeCheckLists(entries);
for (BundleEntryComponent anEntry : entries) {
Resource resource = anEntry.getResource();
currentStrucDef = (StructureDefinition) resource;
convertStrucDefToConcept();
System.out.print(".");
}
}
public void work() throws Exception {
String profileTypesPath = "classpath:v3/profiles-types.json";
String profileResourcesPath = "classpath:v3/profiles-resources.json";
loadStructureDefsBundle(profileTypesPath);
loadStructureDefsBundle(profileResourcesPath);
String json = model.toJSON();
FileUtilsUniWue.saveString2File(json, new File("D:\\tmp\\model.json"));
int x = 0;
}
private void fillTypeCheckLists(List<BundleEntryComponent> entries) {
for (BundleEntryComponent anEntry : entries) {
Resource resource = anEntry.getResource();
String id = resource.getId();
String dataElementName = id.substring(id.lastIndexOf("/") + 1, id.length());
if (dataElementName.matches("^[a-z].*")) {
primitiveTypes.add(dataElementName);
} else {
baseTypes.add(dataElementName);
}
}
}
private List<String> getTypes(ElementDefinition anElem) {
List<String> result = new ArrayList<String>();
for (TypeRefComponent aType : anElem.getType()) {
String code = aType.getCode();
result.add(code);
}
return result;
}
private List<ElementDefinition> getElements(String basePath, int backboneOrNonBackboneTypes) {
List<ElementDefinition> element = currentStrucDef.getSnapshot().getElement();
List<ElementDefinition> result = new ArrayList<ElementDefinition>();
for (ElementDefinition anElem : element) {
String path = anElem.getPath();
if (!path.startsWith(basePath + ".")) {
// element has to begin with the basePath
continue;
}
String subPath = path.substring(basePath.length() + 1);
if (subPath.contains(".")) {
// element has to be a direct child and not a child of another child
continue;
}
List<String> types = getTypes(anElem);
// when it is am "Element", there is only one type
if (types.size() == 0) {
System.out.println("strange, no types for '" + basePath + "'");
continue;
}
boolean isBackboneElement = types.get(0).equals("BackboneElement");
if ((backboneOrNonBackboneTypes == 0) && isBackboneElement) {
if (UnwantedStuff.unwantedTypes.contains(types.get(0))) {
continue;
}
result.add(anElem);
}
if ((backboneOrNonBackboneTypes == 1) && !isBackboneElement) {
result.add(anElem);
}
}
return result;
}
private void addBackboneElement(ElementDefinition anElem, Concept parentConcept) {
String path = anElem.getPath();
String name = path.substring(path.lastIndexOf(".") + 1);
boolean isList = isList(anElem);
List<String> fieldTypes = new ArrayList<String>();
fieldTypes.add(path);
parentConcept.addField(name, fieldTypes, isList);
if (fieldTypes.contains("BackboneElement")) {
Concept nestedConcept = new Concept();
model.concepts.add(nestedConcept);
nestedConcept.name = path;
addBackboneComponents(path, nestedConcept);
addNonBackboneMembers(path, nestedConcept);
}
}
private void addBackboneComponents(String path, Concept concept) {
List<ElementDefinition> elements = getElements(path, 0);
for (ElementDefinition anElem : elements) {
addBackboneElement(anElem, concept);
}
}
private void addNonBackboneMembers(String parentName, Concept parentConcept) {
List<ElementDefinition> elements = getElements(parentName, 1);
for (ElementDefinition anElem : elements) {
addNonBackboneElement(anElem, parentConcept);
}
}
private boolean isList(ElementDefinition anElem) {
return (anElem.getMax() != null) && !anElem.getMax().matches("1");
}
private void addNonBackboneElement(ElementDefinition anElem, Concept parentConcept) {
String path = anElem.getPath();
String name = path.substring(path.lastIndexOf(".") + 1);
name = name.replaceAll("\\.", "");
name = name.replaceAll("\\[.*\\]", "");
List<String> fhirTypes = getTypes(anElem);
boolean isList = isList(anElem);
parentConcept.addField(name, fhirTypes, isList);
}
public void convertStrucDefToConcept() throws Exception {
Concept concept = new Concept();
String name = currentStrucDef.getName();
concept.name = name;
if (!primitiveTypes.contains(name)) {
addBackboneComponents(concept.name, concept);
addNonBackboneMembers(concept.name, concept);
}
model.concepts.add(concept);
}
}
package de.uniwue.model;
import java.util.Arrays;
import java.util.List;
public class UnwantedStuff {
public static List<String> unwantedTypes = Arrays.asList(new String[] {
// TODO: were are the profiles for these things ? "Metadatatypes" !?
"DataRequirement", "TriggerDefinition", "RelatedArtifact", "ParameterDefinition",
"Expression", "UsageContext", "Contributor",
// TODO: those appear complicated. But we'll have to see...
"Extension",
// As every resource has a meta object the export can be reduced a lot by omitting this
// object. It could be indexed nonetheless
"Meta",
// don't index Resources and Elements, as they are untyped data and therefore complicated
"Resource", "Element",
// "Binary" is often very large. It could be allowed but it bloats the exports in the current
// early development
"Binary",
// Bundle can contain arbitrary resources and is thus totally unpredictable
"Bundle",
// this is only human readable stuff, so it does not need to be indexed
"Narrative",
// "canonical" may be a Resource and those are untyped data and therefore complicated
"canonical",
// these types are strangely defined by elements starting with "Quantity" instead of
// "MoneyQuantity". I do not yet know how to handle this
"MoneyQuantity", "SimpleQuantity",
// these are very technical types that do not contain medical data, better skip them
"CapabilityStatement", "OperationDefinition", "SearchParameter", "StructureDefinition",
"Library", "StructureMap", "Meta", "CodeSystem",
// these contains many 10.000 items per instance, so better skip it
"ValueSet", "ConceptMap"
// TODO: this does not work on http://test.fhir.org/r3/
// "MedicationRequest"
//
// the HAPI test server contains some strange resources with presumably broken lastUpdated
// timestamps, so better skip the whole profile
// "CodeSystem",
});
public static List<String> unwantedFields = Arrays.asList(new String[] {
// this is unnecessary as we know the type of the object
"resourceType",
// "fhir_comments" does not exists anymore since R2
"fhir_comments",
// as the type has not been loaded by the ProfileHandlerBuilder, it has to be filtered by name
"extension",
// this can be everything (untyped), so don't index it
"contained",
// The dosage is somehow defined as a special type. Therefore better exclude it because it
// appears to be too complicated
"dosage", "dosageInstruction" });
public static List<String> problematicPaths = Arrays.asList(new String[] {
// these paths are defined via a concept-reference, which I do not know how to handle
"CodeSystem.concept.concept", "Questionnaire.item.item", "QuestionnaireResponse.item.item",
"QuestionnaireResponse.item.answer.item", "StructureMap.group.rule.rule",
"GraphDefinition.link.target.link", "ImplementationGuide.page.page",
"Library.dataRequirement.codeFilter.valueCode", "PlanDefinition.action.action",
"RequestGroup.action.action",
// there seems to be something strange with the signatures on this HAPI server as they contain
// a field named "whoReference" that does not exist in the Signature-Profile
"Provenance.signature",
// TODO: prevent indexiation of ANY-types automatically without having them to be explicitly
// listed
// don't index 'ANY' types
"Composition.section.entry", "Condition.evidence.detail", "Provenance.target",
"Appointment.supportingInformation", "CarePlan.supportingInfo",
"CarePlan.activity.outcomeReference", "Composition.event.detail", "Linkage.item.resource",
// These have "value[x]" members that are not specified
"Task.output", "Task.input",
// TODO:
"Observation.component.referenceRange",
"StructureDefinition.snapshot.element.binding.valueSetReference",
"StructureDefinition.differential.element.binding.valueSetReference", "Bundle.entry.resource",
"Parameters.parameter.resource", "CapabilityStatement.rest.searchParam",
"CapabilityStatement.rest.operation", "TestReport.test.action", "TestReport.teardown.action",
"TestScript.test.action", "TestScript.teardown.action" });
}
......@@ -82,17 +82,15 @@ import org.cqframework.cql.gen.cqlParser.WhereClauseContext;
import org.cqframework.cql.gen.cqlParser.WithClauseContext;
import de.uniwue.misc.util.RegexUtil;
import de.uniwue.model.Model;
import de.uniwue.query.OperatorType;
import de.uniwue.query.Primitive;
import de.uniwue.query.Primitive.PrimitiveDataType;
import de.uniwue.query.PrimitiveDateTime;
import de.uniwue.query.QueryOperator;
import de.uniwue.query.System_2_Graph_Mapper;
public class CQL_2_Graph_Mapper extends System_2_Graph_Mapper {
private boolean parsingReturnRoot;
private boolean parsingRetrieveClause;
private List<String> availableAliasNames;
......@@ -107,7 +105,6 @@ public class CQL_2_Graph_Mapper extends System_2_Graph_Mapper {
protected void clear() {
super.clear();
parsingReturnRoot = false;
parsingRetrieveClause = false;
availableAliasNames = new ArrayList<String>(
Arrays.asList(new String[] { "A", "B", "C", "D", "E", "F", "G", "H", "I", "J" }));
......@@ -127,8 +124,9 @@ public class CQL_2_Graph_Mapper extends System_2_Graph_Mapper {
return expression;
}
public QueryOperator parse(String aQueryString) throws IOException {
public QueryOperator parse(String aQueryString, Model model) throws IOException {
clear();
query.setModel(model);
ExpressionContext e = parseWithAntlr(aQueryString);
currentOp = query;
parseExpression(e);
......@@ -217,7 +215,6 @@ public class CQL_2_Graph_Mapper extends System_2_Graph_Mapper {
String prettyPrint = prettyPrint(returnClause);
currentOp = query;
addOp(OperatorType.RETURN);
parsingReturnRoot = true;
TermExpressionContext te = (TermExpressionContext) returnClause.expression();
TermExpressionTermContext tet = (TermExpressionTermContext) te.expressionTerm();
TermContext term = tet.term();
......@@ -363,7 +360,6 @@ public class CQL_2_Graph_Mapper extends System_2_Graph_Mapper {
private QueryOperator parseExpression(ExpressionContext anExp) {
String prettyPrint = prettyPrint(anExp);
parsingReturnRoot = false;
QueryOperator createdOp;
if (anExp instanceof InequalityExpressionContext) {
InequalityExpressionContext ineq = (InequalityExpressionContext) anExp;
......@@ -895,8 +891,8 @@ public class CQL_2_Graph_Mapper extends System_2_Graph_Mapper {
} else if (aChild instanceof BooleanLiteralContext) {
BooleanLiteralContext btl = (BooleanLiteralContext) aChild;
String text = btl.getText();
Primitive parameter = currentOp.addParameter(text);
parameter.dataType = PrimitiveDataType.Boolean;
boolean valueOf = Boolean.valueOf(text);
Primitive parameter = currentOp.addParameter(valueOf);
} else {
throw new RuntimeException("unexpected");
}
......
......@@ -7,12 +7,12 @@ import java.util.HashSet;
import java.util.List;
import java.util.Set;
import de.uniwue.query.DataType;
import de.uniwue.query.Expression;
import de.uniwue.query.Graph_2_System_Mapper;
import de.uniwue.query.OperatorMap;
import de.uniwue.query.OperatorType;
import de.uniwue.query.Primitive;
import de.uniwue.query.Primitive.PrimitiveDataType;
import de.uniwue.query.QueryOperator;
public class Graph_2_CQL_Mapper extends Graph_2_System_Mapper {
......@@ -169,43 +169,43 @@ public class Graph_2_CQL_Mapper extends Graph_2_System_Mapper {
return alias;
}
// private String writeSelector(Expression aQuery) {
// String result = "";
// Set<QueryOperator> conceptsToBeAliased = getConceptsToBeAliased(aQuery);
// Set<QueryOperator> alreadyMentionedConcepts = new HashSet<QueryOperator>();
// boolean first = true;
// for (QueryOperator aConc : conceptsToBeAliased) {
// if (!first) {
// result += " with ";
// }
// result += "[" + aConc.getCode() + "] ";
// String alias = getNextAlias(aConc);
// result += alias + " ";
// String suchThat = "";
// List<QueryOperator> hullOfSuchThatOperators = getHullOfSuchThatOperators(aConc,
// new ArrayList<QueryOperator>(alreadyMentionedConcepts));
// boolean firstWithOps = true;
// for (QueryOperator anOp : hullOfSuchThatOperators) {
// if (firstWithOps) {
// firstWithOps = false;
// } else {
// suchThat += " and ";
// }
// suchThat += writeOp(anOp);
// alreadyHandledInWith.add(anOp);
// // TODO: what with multiple parents ?
// if ((anOp.parents.size() > 0) && (anOp.parents.get(0).type == OperatorType.EXISTS)) {
// alreadyHandledInWith.add(anOp.parents.get(0));
// }
// }
// if (!first && !suchThat.isEmpty()) {
// result += "such that " + suchThat;
// }
// alreadyMentionedConcepts.add(aConc);
// first = false;
// }
// return result;
// }
// private String writeSelector(Expression aQuery) {
// String result = "";
// Set<QueryOperator> conceptsToBeAliased = getConceptsToBeAliased(aQuery);
// Set<QueryOperator> alreadyMentionedConcepts = new HashSet<QueryOperator>();
// boolean first = true;
// for (QueryOperator aConc : conceptsToBeAliased) {
// if (!first) {
// result += " with ";
// }
// result += "[" + aConc.getCode() + "] ";
// String alias = getNextAlias(aConc);
// result += alias + " ";
// String suchThat = "";
// List<QueryOperator> hullOfSuchThatOperators = getHullOfSuchThatOperators(aConc,
// new ArrayList<QueryOperator>(alreadyMentionedConcepts));
// boolean firstWithOps = true;
// for (QueryOperator anOp : hullOfSuchThatOperators) {
// if (firstWithOps) {
// firstWithOps = false;
// } else {
// suchThat += " and ";
// }
// suchThat += writeOp(anOp);
// alreadyHandledInWith.add(anOp);
// // TODO: what with multiple parents ?
// if ((anOp.parents.size() > 0) && (anOp.parents.get(0).type == OperatorType.EXISTS)) {
// alreadyHandledInWith.add(anOp.parents.get(0));
// }
// }
// if (!first && !suchThat.isEmpty()) {
// result += "such that " + suchThat;
// }
// alreadyMentionedConcepts.add(aConc);
// first = false;
// }
// return result;
// }
private String getPath(QueryOperator aConc) {
if (aConc.type == OperatorType.ALIAS_REF) {
......@@ -236,8 +236,8 @@ public class Graph_2_CQL_Mapper extends Graph_2_System_Mapper {
private String writePrimitive(Primitive aPrim) {
String result = aPrim.value;
if (aPrim.dataType == PrimitiveDataType.DateTime) {
} else if (aPrim.dataType == PrimitiveDataType.String) {
if (aPrim.dataType == DataType.DATETIME) {
} else if (aPrim.dataType == DataType.STRING) {
if (nextPrimitiveIsValueSet) {
result = "\"" + result + "\"";
} else {
......