added type checking

QueryMapper/pom.xml

-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
 	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
 	<modelVersion>4.0.0</modelVersion>
 	<groupId>QueryMapper</groupId>
@@ -10,8 +11,8 @@
 		<maven.compiler.target>1.8</maven.compiler.target>
 		<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
 		<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
-        <log4j.version>2.11.1</log4j.version>
-        <slf4j.version>1.7.25</slf4j.version>
+		<log4j.version>2.11.1</log4j.version>
+		<slf4j.version>1.7.25</slf4j.version>
 	</properties>
 	<repositories>
 		<repository>
@@ -45,6 +46,11 @@
 			<artifactId>hapi-fhir-client</artifactId>
 			<version>3.7.0</version>
 		</dependency>
+		<dependency>
+			<groupId>ca.uhn.hapi.fhir</groupId>
+			<artifactId>hapi-fhir-structures-dstu3</artifactId>
+			<version>3.6.0</version>
+		</dependency>
 		<dependency>
 			<groupId>org.jdom</groupId>
 			<artifactId>jdom</artifactId>
@@ -55,18 +61,18 @@
 			<artifactId>cql-to-elm</artifactId>
 			<version>1.2.20</version>
 		</dependency>
-        <dependency>
-            <groupId>junit</groupId>
-            <artifactId>junit</artifactId>
-            <version>4.8.2</version>
-            <scope>test</scope>
-        </dependency>
-        <!-- Needed by junit -->
-        <dependency>
-            <groupId>org.hamcrest</groupId>
-            <artifactId>hamcrest-all</artifactId>
-            <version>1.3</version>
-            <scope>test</scope>
-        </dependency>		
+		<dependency>
+			<groupId>junit</groupId>
+			<artifactId>junit</artifactId>
+			<version>4.8.2</version>
+			<scope>test</scope>
+		</dependency>
+		<!-- Needed by junit -->
+		<dependency>
+			<groupId>org.hamcrest</groupId>
+			<artifactId>hamcrest-all</artifactId>
+			<version>1.3</version>
+			<scope>test</scope>
+		</dependency>
 	</dependencies>
 </project>
\ No newline at end of file

QueryMapper/src/main/java/de/uniwue/model/Concept.java

+package de.uniwue.model;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class Concept {
+
+  public static String intTermCodes;
+
+  public String name; 
+
+  public List<String> termCodes;
+
+  public List<ConceptField> fields;
+
+  public void addField(String fieldName, List<String> fieldTypes, boolean isArrayFlag) {
+    if (fields == null) {
+      fields = new ArrayList<ConceptField>();
+    }
+    ConceptField field = new ConceptField();
+    field.name = fieldName;
+    field.isArray = isArrayFlag;
+    field.types = fieldTypes;
+    fields.add(field);
+  }
+
+}

QueryMapper/src/main/java/de/uniwue/model/ConceptField.java

+package de.uniwue.model;
+
+import java.util.List;
+
+public class ConceptField {
+
+  public String name;
+  
+  public List<String> types;
+  
+  public boolean isArray;
+  
+}

QueryMapper/src/main/java/de/uniwue/model/Model.java

+package de.uniwue.model;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import com.fasterxml.jackson.annotation.JsonInclude.Include;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.SerializationFeature;
+
+public class Model {
+
+  public List<Concept> concepts = new ArrayList<Concept>();
+
+  public Map<String, Concept> conceptMap = new HashMap<String, Concept>();
+
+  public static Model fromJSON(String jsonString) throws IOException {
+    ObjectMapper objectMapper = new ObjectMapper();
+    Object model = objectMapper.readValue(jsonString, Model.class);    
+    return (Model) model;
+  }
+  
+  public String toJSON() throws JsonProcessingException {
+    ObjectMapper objectMapper = new ObjectMapper();
+    objectMapper.setSerializationInclusion(Include.NON_NULL);
+    objectMapper.enable(SerializationFeature.INDENT_OUTPUT);
+    String writeValueAsString = objectMapper.writeValueAsString(this);
+    return writeValueAsString;
+  }
+
+}

QueryMapper/src/main/java/de/uniwue/model/StructureDef2ModelConverter.java

+package de.uniwue.model;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.hl7.fhir.dstu3.model.Bundle;
+import org.hl7.fhir.dstu3.model.Bundle.BundleEntryComponent;
+import org.hl7.fhir.dstu3.model.ElementDefinition;
+import org.hl7.fhir.dstu3.model.ElementDefinition.TypeRefComponent;
+import org.hl7.fhir.dstu3.model.Resource;
+import org.hl7.fhir.dstu3.model.StructureDefinition;
+
+import ca.uhn.fhir.context.FhirContext;
+import ca.uhn.fhir.parser.IParser;
+import de.uniwue.misc.util.FileUtilsUniWue;
+import de.uniwue.misc.util.ResourceUtil;
+
+public class StructureDef2ModelConverter {
+
+  private FhirContext ctx;
+
+  private Model model = new Model();
+
+  private StructureDefinition currentStrucDef;
+
+  private List<String> primitiveTypes = new ArrayList<String>();
+
+  private List<String> baseTypes = new ArrayList<String>();
+
+  public StructureDef2ModelConverter(FhirContext ctx)
+          throws NoSuchMethodException, SecurityException {
+    this.ctx = ctx;
+  }
+
+  public void translateStructureDefs2HAPIJavaClasses(File strucDefsOutputDir, File outDirHAPI)
+          throws Exception {
+    File structureDefDir = new File(strucDefsOutputDir, "xml");
+    IParser parser = ctx.newXmlParser();
+    // parser = ctx.newJsonParser();
+    for (File aFile : structureDefDir.listFiles()) {
+      System.out.println(aFile.getName());
+      String structureDefString = FileUtilsUniWue.file2String(aFile);
+      currentStrucDef = (StructureDefinition) parser.parseResource(structureDefString);
+      convertStrucDefToConcept();
+    }
+  }
+
+  public static void main(String[] args) throws Exception {
+    FhirContext ctx = FhirContext.forDstu3();
+    StructureDef2ModelConverter converter = new StructureDef2ModelConverter(ctx);
+    converter.work();
+  }
+
+  private void loadStructureDefsBundle(String path) throws Exception {
+    String dataElementsString = ResourceUtil.loadFileAsString(path);
+    Bundle bundle = (Bundle) ctx.newJsonParser().parseResource(dataElementsString);
+    List<BundleEntryComponent> entries = bundle.getEntry();
+    fillTypeCheckLists(entries);
+    for (BundleEntryComponent anEntry : entries) {
+      Resource resource = anEntry.getResource();
+      currentStrucDef = (StructureDefinition) resource;
+      convertStrucDefToConcept();
+      System.out.print(".");
+    }
+  }
+
+  public void work() throws Exception {
+    String profileTypesPath = "classpath:v3/profiles-types.json";
+    String profileResourcesPath = "classpath:v3/profiles-resources.json";
+    loadStructureDefsBundle(profileTypesPath);
+    loadStructureDefsBundle(profileResourcesPath);
+    String json = model.toJSON();
+    FileUtilsUniWue.saveString2File(json, new File("D:\\tmp\\model.json"));
+    int x = 0;
+  }
+
+  private void fillTypeCheckLists(List<BundleEntryComponent> entries) {
+    for (BundleEntryComponent anEntry : entries) {
+      Resource resource = anEntry.getResource();
+      String id = resource.getId();
+      String dataElementName = id.substring(id.lastIndexOf("/") + 1, id.length());
+      if (dataElementName.matches("^[a-z].*")) {
+        primitiveTypes.add(dataElementName);
+      } else {
+        baseTypes.add(dataElementName);
+      }
+    }
+  }
+
+  private List<String> getTypes(ElementDefinition anElem) {
+    List<String> result = new ArrayList<String>();
+    for (TypeRefComponent aType : anElem.getType()) {
+      String code = aType.getCode();
+      result.add(code);
+    }
+    return result;
+  }
+
+  private List<ElementDefinition> getElements(String basePath, int backboneOrNonBackboneTypes) {
+    List<ElementDefinition> element = currentStrucDef.getSnapshot().getElement();
+    List<ElementDefinition> result = new ArrayList<ElementDefinition>();
+    for (ElementDefinition anElem : element) {
+      String path = anElem.getPath();
+      if (!path.startsWith(basePath + ".")) {
+        // element has to begin with the basePath
+        continue;
+      }
+      String subPath = path.substring(basePath.length() + 1);
+      if (subPath.contains(".")) {
+        // element has to be a direct child and not a child of another child
+        continue;
+      }
+      List<String> types = getTypes(anElem);
+      // when it is am "Element", there is only one type
+      if (types.size() == 0) {
+        System.out.println("strange, no types for '" + basePath + "'");
+        continue;
+      }
+      boolean isBackboneElement = types.get(0).equals("BackboneElement");
+      if ((backboneOrNonBackboneTypes == 0) && isBackboneElement) {
+        if (UnwantedStuff.unwantedTypes.contains(types.get(0))) {
+          continue;
+        }
+        result.add(anElem);
+      }
+      if ((backboneOrNonBackboneTypes == 1) && !isBackboneElement) {
+        result.add(anElem);
+      }
+    }
+    return result;
+  }
+
+  private void addBackboneElement(ElementDefinition anElem, Concept parentConcept) {
+    String path = anElem.getPath();
+    String name = path.substring(path.lastIndexOf(".") + 1);
+    boolean isList = isList(anElem);
+    List<String> fieldTypes = new ArrayList<String>();
+    fieldTypes.add(path);
+    parentConcept.addField(name, fieldTypes, isList);
+    if (fieldTypes.contains("BackboneElement")) {
+      Concept nestedConcept = new Concept();
+      model.concepts.add(nestedConcept);
+      nestedConcept.name = path;
+      addBackboneComponents(path, nestedConcept);
+      addNonBackboneMembers(path, nestedConcept);
+    }
+  }
+
+  private void addBackboneComponents(String path, Concept concept) {
+    List<ElementDefinition> elements = getElements(path, 0);
+    for (ElementDefinition anElem : elements) {
+      addBackboneElement(anElem, concept);
+    }
+  }
+
+  private void addNonBackboneMembers(String parentName, Concept parentConcept) {
+    List<ElementDefinition> elements = getElements(parentName, 1);
+    for (ElementDefinition anElem : elements) {
+      addNonBackboneElement(anElem, parentConcept);
+    }
+  }
+
+  private boolean isList(ElementDefinition anElem) {
+    return (anElem.getMax() != null) && !anElem.getMax().matches("1");
+  }
+
+  private void addNonBackboneElement(ElementDefinition anElem, Concept parentConcept) {
+    String path = anElem.getPath();
+    String name = path.substring(path.lastIndexOf(".") + 1);
+    name = name.replaceAll("\\.", "");
+    name = name.replaceAll("\\[.*\\]", "");
+    List<String> fhirTypes = getTypes(anElem);
+    boolean isList = isList(anElem);
+    parentConcept.addField(name, fhirTypes, isList);
+  }
+
+  public void convertStrucDefToConcept() throws Exception {
+    Concept concept = new Concept();
+    String name = currentStrucDef.getName();
+    concept.name = name;
+    if (!primitiveTypes.contains(name)) {
+      addBackboneComponents(concept.name, concept);
+      addNonBackboneMembers(concept.name, concept);
+    }
+    model.concepts.add(concept);
+  }
+}

QueryMapper/src/main/java/de/uniwue/model/UnwantedStuff.java

+package de.uniwue.model;
+
+import java.util.Arrays;
+import java.util.List;
+
+public class UnwantedStuff {
+
+  public static List<String> unwantedTypes = Arrays.asList(new String[] {
+      // TODO: were are the profiles for these things ? "Metadatatypes" !?
+      "DataRequirement", "TriggerDefinition", "RelatedArtifact", "ParameterDefinition",
+      "Expression", "UsageContext", "Contributor",
+      // TODO: those appear complicated. But we'll have to see...
+      "Extension",
+      // As every resource has a meta object the export can be reduced a lot by omitting this
+      // object. It could be indexed nonetheless
+      "Meta",
+      // don't index Resources and Elements, as they are untyped data and therefore complicated
+      "Resource", "Element",
+      // "Binary" is often very large. It could be allowed but it bloats the exports in the current
+      // early development
+      "Binary",
+      // Bundle can contain arbitrary resources and is thus totally unpredictable
+      "Bundle",
+      // this is only human readable stuff, so it does not need to be indexed
+      "Narrative",
+      // "canonical" may be a Resource and those are untyped data and therefore complicated
+      "canonical",
+      // these types are strangely defined by elements starting with "Quantity" instead of
+      // "MoneyQuantity". I do not yet know how to handle this
+      "MoneyQuantity", "SimpleQuantity",
+      // these are very technical types that do not contain medical data, better skip them
+      "CapabilityStatement", "OperationDefinition", "SearchParameter", "StructureDefinition",
+      "Library", "StructureMap", "Meta", "CodeSystem",
+      // these contains many 10.000 items per instance, so better skip it
+      "ValueSet", "ConceptMap"
+      // TODO: this does not work on http://test.fhir.org/r3/
+      // "MedicationRequest"
+      //
+      // the HAPI test server contains some strange resources with presumably broken lastUpdated
+      // timestamps, so better skip the whole profile
+      // "CodeSystem",
+  });
+
+  public static List<String> unwantedFields = Arrays.asList(new String[] {
+      // this is unnecessary as we know the type of the object
+      "resourceType",
+      // "fhir_comments" does not exists anymore since R2
+      "fhir_comments",
+      // as the type has not been loaded by the ProfileHandlerBuilder, it has to be filtered by name
+      "extension",
+      // this can be everything (untyped), so don't index it
+      "contained",
+      // The dosage is somehow defined as a special type. Therefore better exclude it because it
+      // appears to be too complicated
+      "dosage", "dosageInstruction" });
+
+  public static List<String> problematicPaths = Arrays.asList(new String[] {
+      // these paths are defined via a concept-reference, which I do not know how to handle
+      "CodeSystem.concept.concept", "Questionnaire.item.item", "QuestionnaireResponse.item.item",
+      "QuestionnaireResponse.item.answer.item", "StructureMap.group.rule.rule",
+      "GraphDefinition.link.target.link", "ImplementationGuide.page.page",
+      "Library.dataRequirement.codeFilter.valueCode", "PlanDefinition.action.action",
+      "RequestGroup.action.action",
+      // there seems to be something strange with the signatures on this HAPI server as they contain
+      // a field named "whoReference" that does not exist in the Signature-Profile
+      "Provenance.signature",
+      // TODO: prevent indexiation of ANY-types automatically without having them to be explicitly
+      // listed
+      // don't index 'ANY' types
+      "Composition.section.entry", "Condition.evidence.detail", "Provenance.target",
+      "Appointment.supportingInformation", "CarePlan.supportingInfo",
+      "CarePlan.activity.outcomeReference", "Composition.event.detail", "Linkage.item.resource",
+      // These have "value[x]" members that are not specified
+      "Task.output", "Task.input",
+      // TODO:
+      "Observation.component.referenceRange",
+      "StructureDefinition.snapshot.element.binding.valueSetReference",
+      "StructureDefinition.differential.element.binding.valueSetReference", "Bundle.entry.resource",
+      "Parameters.parameter.resource", "CapabilityStatement.rest.searchParam",
+      "CapabilityStatement.rest.operation", "TestReport.test.action", "TestReport.teardown.action",
+      "TestScript.test.action", "TestScript.teardown.action" });
+
+}

QueryMapper/src/main/java/de/uniwue/query/CQL/CQL_2_Graph_Mapper.java

@@ -82,17 +82,15 @@ import org.cqframework.cql.gen.cqlParser.WhereClauseContext;
 import org.cqframework.cql.gen.cqlParser.WithClauseContext;
 
 import de.uniwue.misc.util.RegexUtil;
+import de.uniwue.model.Model;
 import de.uniwue.query.OperatorType;
 import de.uniwue.query.Primitive;
-import de.uniwue.query.Primitive.PrimitiveDataType;
 import de.uniwue.query.PrimitiveDateTime;
 import de.uniwue.query.QueryOperator;
 import de.uniwue.query.System_2_Graph_Mapper;
 
 public class CQL_2_Graph_Mapper extends System_2_Graph_Mapper {
 
-  private boolean parsingReturnRoot;
-
   private boolean parsingRetrieveClause;
 
   private List<String> availableAliasNames;
@@ -107,7 +105,6 @@ public class CQL_2_Graph_Mapper extends System_2_Graph_Mapper {
 
   protected void clear() {
     super.clear();
-    parsingReturnRoot = false;
     parsingRetrieveClause = false;
     availableAliasNames = new ArrayList<String>(
             Arrays.asList(new String[] { "A", "B", "C", "D", "E", "F", "G", "H", "I", "J" }));
@@ -127,8 +124,9 @@ public class CQL_2_Graph_Mapper extends System_2_Graph_Mapper {
     return expression;
   }
 
-  public QueryOperator parse(String aQueryString) throws IOException {
+  public QueryOperator parse(String aQueryString, Model model) throws IOException {
     clear();
+    query.setModel(model);
     ExpressionContext e = parseWithAntlr(aQueryString);
     currentOp = query;
     parseExpression(e);
@@ -217,7 +215,6 @@ public class CQL_2_Graph_Mapper extends System_2_Graph_Mapper {
     String prettyPrint = prettyPrint(returnClause);
     currentOp = query;
     addOp(OperatorType.RETURN);
-    parsingReturnRoot = true;
     TermExpressionContext te = (TermExpressionContext) returnClause.expression();
     TermExpressionTermContext tet = (TermExpressionTermContext) te.expressionTerm();
     TermContext term = tet.term();
@@ -363,7 +360,6 @@ public class CQL_2_Graph_Mapper extends System_2_Graph_Mapper {
 
   private QueryOperator parseExpression(ExpressionContext anExp) {
     String prettyPrint = prettyPrint(anExp);
-    parsingReturnRoot = false;
     QueryOperator createdOp;
     if (anExp instanceof InequalityExpressionContext) {
       InequalityExpressionContext ineq = (InequalityExpressionContext) anExp;
@@ -895,8 +891,8 @@ public class CQL_2_Graph_Mapper extends System_2_Graph_Mapper {
       } else if (aChild instanceof BooleanLiteralContext) {
         BooleanLiteralContext btl = (BooleanLiteralContext) aChild;
         String text = btl.getText();
-        Primitive parameter = currentOp.addParameter(text);
-        parameter.dataType = PrimitiveDataType.Boolean;
+        boolean valueOf = Boolean.valueOf(text);
+        Primitive parameter = currentOp.addParameter(valueOf);
       } else {
         throw new RuntimeException("unexpected");
       }

QueryMapper/src/main/java/de/uniwue/query/CQL/Graph_2_CQL_Mapper.java

@@ -7,12 +7,12 @@ import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 
+import de.uniwue.query.DataType;
 import de.uniwue.query.Expression;
 import de.uniwue.query.Graph_2_System_Mapper;
 import de.uniwue.query.OperatorMap;
 import de.uniwue.query.OperatorType;
 import de.uniwue.query.Primitive;
-import de.uniwue.query.Primitive.PrimitiveDataType;
 import de.uniwue.query.QueryOperator;
 
 public class Graph_2_CQL_Mapper extends Graph_2_System_Mapper {
@@ -169,43 +169,43 @@ public class Graph_2_CQL_Mapper extends Graph_2_System_Mapper {
     return alias;
   }
 
-//  private String writeSelector(Expression aQuery) {
-//    String result = "";
-//    Set<QueryOperator> conceptsToBeAliased = getConceptsToBeAliased(aQuery);
-//    Set<QueryOperator> alreadyMentionedConcepts = new HashSet<QueryOperator>();
-//    boolean first = true;
-//    for (QueryOperator aConc : conceptsToBeAliased) {
-//      if (!first) {
-//        result += " with ";
-//      }
-//      result += "[" + aConc.getCode() + "] ";
-//      String alias = getNextAlias(aConc);
-//      result += alias + " ";
-//      String suchThat = "";
-//      List<QueryOperator> hullOfSuchThatOperators = getHullOfSuchThatOperators(aConc,
-//              new ArrayList<QueryOperator>(alreadyMentionedConcepts));
-//      boolean firstWithOps = true;
-//      for (QueryOperator anOp : hullOfSuchThatOperators) {
-//        if (firstWithOps) {
-//          firstWithOps = false;
-//        } else {
-//          suchThat += " and ";
-//        }
-//        suchThat += writeOp(anOp);
-//        alreadyHandledInWith.add(anOp);
-//        // TODO: what with multiple parents ?
-//        if ((anOp.parents.size() > 0) && (anOp.parents.get(0).type == OperatorType.EXISTS)) {
-//          alreadyHandledInWith.add(anOp.parents.get(0));
-//        }
-//      }
-//      if (!first && !suchThat.isEmpty()) {
-//        result += "such that " + suchThat;
-//      }
-//      alreadyMentionedConcepts.add(aConc);
-//      first = false;
-//    }
-//    return result;
-//  }
+  // private String writeSelector(Expression aQuery) {
+  // String result = "";
+  // Set<QueryOperator> conceptsToBeAliased = getConceptsToBeAliased(aQuery);
+  // Set<QueryOperator> alreadyMentionedConcepts = new HashSet<QueryOperator>();
+  // boolean first = true;
+  // for (QueryOperator aConc : conceptsToBeAliased) {
+  // if (!first) {
+  // result += " with ";
+  // }
+  // result += "[" + aConc.getCode() + "] ";
+  // String alias = getNextAlias(aConc);
+  // result += alias + " ";
+  // String suchThat = "";
+  // List<QueryOperator> hullOfSuchThatOperators = getHullOfSuchThatOperators(aConc,
+  // new ArrayList<QueryOperator>(alreadyMentionedConcepts));
+  // boolean firstWithOps = true;
+  // for (QueryOperator anOp : hullOfSuchThatOperators) {
+  // if (firstWithOps) {
+  // firstWithOps = false;
+  // } else {
+  // suchThat += " and ";
+  // }
+  // suchThat += writeOp(anOp);
+  // alreadyHandledInWith.add(anOp);
+  // // TODO: what with multiple parents ?
+  // if ((anOp.parents.size() > 0) && (anOp.parents.get(0).type == OperatorType.EXISTS)) {
+  // alreadyHandledInWith.add(anOp.parents.get(0));
+  // }
+  // }
+  // if (!first && !suchThat.isEmpty()) {
+  // result += "such that " + suchThat;
+  // }
+  // alreadyMentionedConcepts.add(aConc);
+  // first = false;
+  // }
+  // return result;
+  // }
 
   private String getPath(QueryOperator aConc) {
     if (aConc.type == OperatorType.ALIAS_REF) {
@@ -236,8 +236,8 @@ public class Graph_2_CQL_Mapper extends Graph_2_System_Mapper {
 
   private String writePrimitive(Primitive aPrim) {
     String result = aPrim.value;
-    if (aPrim.dataType == PrimitiveDataType.DateTime) {
-    } else if (aPrim.dataType == PrimitiveDataType.String) {
+    if (aPrim.dataType == DataType.DATETIME) {
+    } else if (aPrim.dataType == DataType.STRING) {
       if (nextPrimitiveIsValueSet) {
         result = "\"" + result + "\"";
       } else {