Commit aee18fc8 authored by Georg Fette's avatar Georg Fette
Browse files

- fixed an interface inconsistence with FILTER

- tried to write Fuzzer but again failed to really create useful queries
parent 116de98d
......@@ -129,9 +129,18 @@ public class CQL_2_Graph_Mapper extends System_2_Graph_Mapper {
currentOp = query;
parseExpression(e);
addLeafConceptOpsToReturn();
removeUnnecessaryFilters();
return query;
}
private void removeUnnecessaryFilters() {
for (QueryOperator aFilter : query.getOpsRecursiveWithType(OperatorType.FILTER)) {
if (aFilter.getParameters().size() == 1) {
aFilter.parent.replaceParameter(aFilter, aFilter.getParameters().get(0));
}
}
}
private void addLeafConceptOpsToReturn() {
if (query.getParameters().size() == 1) {
QueryOperator returnOp = addOp(OperatorType.RETURN);
......
......@@ -242,9 +242,7 @@ public class Graph_2_CQL_Mapper extends Graph_2_System_Mapper {
private String writeExpr(Expression anExp) {
String result;
if (anExp.isConcept()) {
result = getPath(anExp.asConcept());
} else if (anExp.isPrimitive()) {
if (anExp.isPrimitive()) {
result = writePrimitive(anExp.asPrimitive());
} else if (anExp.isOperator()) {
result = writeOp(anExp.asOperator());
......@@ -373,6 +371,12 @@ public class Graph_2_CQL_Mapper extends Graph_2_System_Mapper {
} else if (infixOps.contains(anOp.type)) {
String string = writeInfixOp(anOp);
result += string;
} else if ((anOp.type == OperatorType.ITERATE) || (anOp.type == OperatorType.FIELD)) {
String path = getPath(anOp);
result += path;
if (anOp.alias != null) {
result += " " + anOp.alias;
}
} else if (anOp.type == OperatorType.FILTER) {
String queryString = writeFilter(anOp);
result += queryString;
......@@ -410,10 +414,6 @@ public class Graph_2_CQL_Mapper extends Graph_2_System_Mapper {
result += anOp.getParameters().get(0).asPrimitive().value + " ";
String unitString = anOp.getParameters().get(1).asPrimitive().value;
result += unitString;
} else if (anOp.type == OperatorType.FIELD) {
String frstExp = writeExpr(anOp.getParameters().get(0));
String sndExp = anOp.getParameters().get(1).asConcept().getIteratorSelector();
result += frstExp + "." + sndExp;
} else if (anOp.type == OperatorType.RETURN) {
writeReturn(anOp);
} else if (anOp.getParameters().size() == 1) {
......
package de.uniwue.query;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Random;
import java.util.Map;
import de.uniwue.model.Concept;
import de.uniwue.model.ConceptField;
import de.uniwue.model.Model;
import de.uniwue.query.StructureProblem.ProblemType;
import de.uniwue.query.opInterfaces.OperatorType;
import de.uniwue.query.opInterfaces.OperatorType.InterfaceFlags;
......@@ -17,262 +15,210 @@ public class Fuzzer {
private Model model;
private Random rand;
private List<Expression> finishedOps = new ArrayList<Expression>();
private List<QueryOperator> opParts = new ArrayList<QueryOperator>();
private List<QueryOperator> unfinishedOps = new ArrayList<QueryOperator>();
private QueryOperator query, returnOp;
private List<Expression> newFinishedOps = new ArrayList<Expression>();
private int maxOpPartsCount = 50;
private List<QueryOperator> newUnfinishedOps = new ArrayList<QueryOperator>();
private double primitiveProb = 0.5;
private List<Expression> finishedOpsOfLastRound = new ArrayList<Expression>();
private List<QueryOperator> unfinishedOpsOfLastRound = new ArrayList<QueryOperator>();
private List<QueryOperator> queries = new ArrayList<QueryOperator>();
private List<QueryOperator> queriesWithProblems = new ArrayList<QueryOperator>();
private int aliasIndex = 1;
private Map<String, QueryOperator> aliases = new HashMap<String, QueryOperator>();
private int maxRecombinationDepth = 4;
public Fuzzer(Model model) {
this.model = model;
}
public QueryOperator createQuery() {
rand = new Random(1);
query = new QueryOperator(OperatorType.QUERY, model);
returnOp = new QueryOperator(OperatorType.RETURN, model);
opParts.add(query);
opParts.add(returnOp);
while (opParts.size() < maxOpPartsCount) {
createOp();
}
List<StructureProblem> checkProblems;
do {
combineOpsIfPossible();
if (opParts.size() == maxOpPartsCount) {
removeOneOpPart();
}
while (opParts.size() < maxOpPartsCount) {
createOp();
}
checkProblems = query.checkProblems();
} while (checkProblems.size() > 0);
return query;
}
protected void combineOpsIfPossible() {
List<QueryOperator> unfinishedOps = new ArrayList<QueryOperator>();
List<QueryOperator> finishedOps = new ArrayList<QueryOperator>();
for (QueryOperator anOp : opParts) {
if (isFinished(anOp)) {
finishedOps.add(anOp);
} else {
unfinishedOps.add(anOp);
public void createQueries() {
QueryOperator query = new QueryOperator(OperatorType.QUERY, model);
QueryOperator returnOp = new QueryOperator(OperatorType.RETURN, model);
unfinishedOpsOfLastRound.add(query);
unfinishedOpsOfLastRound.add(returnOp);
createOps();
createPrimitives();
for (int i = 0; i < maxRecombinationDepth; i++) {
int x = 0;
for (QueryOperator anUnfinishednOp : unfinishedOpsOfLastRound) {
for (Expression aFinishednOp : finishedOps) {
combinOp(anUnfinishednOp, aFinishednOp);
}
}
}
finishedOps.remove(query);
finishedOps.remove(returnOp);
Collections.shuffle(unfinishedOps);
for (QueryOperator anOp : unfinishedOps) {
if (anOp.type == OperatorType.ALIAS_REF) {
fillOpAliasRef(anOp);
} else if ((anOp.type == OperatorType.FIELD) && anOp.getParameters().size() == 1) {
fillOpField2ndParam(anOp);
} else if (anOp.type == OperatorType.QUERY) {
fillOpQuery(anOp, finishedOps);
} else {
combinOpGeneric(anOp, finishedOps);
for (QueryOperator anUnfinishednOp : unfinishedOps) {
for (Expression aFinishednOp : finishedOpsOfLastRound) {
combinOp(anUnfinishednOp, aFinishednOp);
}
}
if (isFinished(anOp)) {
return;
for (QueryOperator anUnfinishednOp : unfinishedOpsOfLastRound) {
for (Expression aFinishednOp : finishedOpsOfLastRound) {
combinOp(anUnfinishednOp, aFinishednOp);
}
}
finishedOps.addAll(finishedOpsOfLastRound);
unfinishedOps.addAll(unfinishedOpsOfLastRound);
finishedOpsOfLastRound = newFinishedOps;
unfinishedOpsOfLastRound = newUnfinishedOps;
newUnfinishedOps = new ArrayList<QueryOperator>();
newFinishedOps = new ArrayList<Expression>();
}
}
private void combinOpGeneric(QueryOperator anOp, List<QueryOperator> finishedOps) {
private void createAlias(QueryOperator anOp) {
String newAlias = "a" + aliasIndex++;
anOp.alias = newAlias;
QueryOperator newAliasRef = new QueryAlias(OperatorType.ALIAS_REF, model, anOp);
newAliasRef.addParameter(newAlias);
aliases.put(newAlias, anOp);
newFinishedOps.add(newAliasRef);
}
private void combinOp(QueryOperator anOp, Expression anExpToInsert) {
List<Concept> possibleDataTypesForParamIndex = getPossibleDataTypesForParamIndex(anOp,
anOp.getParameters().size());
boolean allowsPrimitiveTypes = allowsPrimitiveTypes(possibleDataTypesForParamIndex);
if (allowsPrimitiveTypes && (rand.nextDouble() > primitiveProb)) {
addRandomPrimitive(anOp, possibleDataTypesForParamIndex);
} else {
for (QueryOperator anOpToInsert : finishedOps) {
Concept dataType = anOpToInsert.getDataType();
for (Concept aPossibleType : possibleDataTypesForParamIndex) {
if ((dataType == aPossibleType) || dataType.inheritsFrom(aPossibleType)) {
if ((anOp == query) && (anOpToInsert.getDepth() > 2)) {
anOp.addParameter(anOpToInsert);
isFinished(anOp);
opParts.remove(anOpToInsert);
return;
if (!isCombinationValid(anOp, anExpToInsert)) {
return;
}
Concept dataType = getDataType(anExpToInsert);
for (Concept aPossibleType : possibleDataTypesForParamIndex) {
if ((dataType == aPossibleType) || dataType.inheritsFrom(aPossibleType)) {
QueryOperator newCombination = anOp.clone();
Expression anExpToInsertClone = anExpToInsert.clone();
newCombination.addParameter(anExpToInsertClone);
if (newCombination.type == OperatorType.FIELD) {
fillOpField2ndParam(newCombination);
return;
}
if (isFinished(newCombination)) {
if (newCombination.type == OperatorType.QUERY) {
if (newCombination.checkProblems().size() == 0) {
queries.add(newCombination);
} else {
queriesWithProblems.add(newCombination);
}
return;
} else if (newCombination.getDataType().isArray()
&& (newCombination.type != OperatorType.RETURN)) {
createAlias(newCombination);
}
newFinishedOps.add(newCombination);
} else {
newUnfinishedOps.add(newCombination);
return;
}
}
}
}
private boolean allowsPrimitiveTypes(List<Concept> possibleDataTypesForParamIndex) {
for (Concept aConc : possibleDataTypesForParamIndex) {
if (model.primitiveTypes.contains(aConc)) {
return true;
private boolean isCombinationValid(QueryOperator anOp, Expression anExpToInsert) {
if (anOp.type == OperatorType.QUERY) {
if ((anOp.getParameters().size() == 1) && (anExpToInsert.isPrimitive()
|| (anExpToInsert.asOperator().type != OperatorType.RETURN))) {
// the 2nd parameter of a QUERY has to be a RETURN
return false;
}
}
return false;
}
private boolean isFinished(QueryOperator anOp) {
List<StructureProblem> checkProblems = anOp.checkProblems();
if (checkProblems.size() > 0) {
for (StructureProblem aProblem : checkProblems) {
if (aProblem.type != ProblemType.ParamCountMismatch) {
throw new RuntimeException("unexpected");
}
if ((anOp.getParameters().size() == 0) && (anExpToInsert.isPrimitive()
|| (anExpToInsert.asOperator().type == OperatorType.RETURN))) {
// the 1st parameter of a QUERY may not be a RETURN
return false;
}
} else if (anExpToInsert.isOperator()
&& anExpToInsert.asOperator().type == OperatorType.RETURN) {
// a RETRUN may only be inserted in a QUERY
return false;
}
return true;
}
protected void createOp() {
if (opParts.size() == maxOpPartsCount) {
removeOneOpPart();
}
OperatorType choice = chooseRandomOpType();
QueryOperator newOp = new QueryOperator(choice, model);
if (newOp.type == OperatorType.ITERATE) {
fillOpIterate(newOp);
if (anOp.type.interfaceFlags.contains(InterfaceFlags.AllParamsNoLists)
&& anExpToInsert.getDataType().isArray()) {
return false;
}
opParts.add(newOp);
return true;
}
private void removeOneOpPart() {
QueryOperator opToRemove = null;
int minDepth = Integer.MAX_VALUE;
int minStructCount = Integer.MAX_VALUE;
for (QueryOperator anOp : opParts) {
if ((anOp == query) || (anOp == returnOp) || (anOp.type == OperatorType.FIELD)
|| (anOp.type == OperatorType.ITERATE) || (anOp.type == OperatorType.ALIAS_REF)) {
// don't remove the query or the return
continue;
}
int opDepth = anOp.getDepth();
if ((opDepth < minDepth)) {
minDepth = opDepth;
opToRemove = anOp;
}
int structureCountRecursive = anOp.getStructureCountRecursive();
if (structureCountRecursive < minStructCount) {
minStructCount = structureCountRecursive;
opToRemove = anOp;
private Concept getDataType(Expression anExp) {
if (anExp.isOperator()) {
QueryOperator anOpToInsert = anExp.asOperator();
if (anOpToInsert.type == OperatorType.ALIAS_REF) {
String value = anOpToInsert.getParameters().get(0).asPrimitive().value;
QueryOperator anAliasedOp = aliases.get(value);
return anAliasedOp.getDataType();
} else {
return anExp.getDataType();
}
}
opParts.remove(opToRemove);
}
private OperatorType chooseRandomOpType() {
OperatorType choice;
List<OperatorType> values = new ArrayList<OperatorType>(OperatorType.allOpTypes.values());
for (int i = 0; i < 5; i++) {
values.add(OperatorType.ITERATE);
}
for (int i = 0; i < 10; i++) {
values.add(OperatorType.FIELD);
}
for (int i = 0; i < 10; i++) {
values.add(OperatorType.ALIAS_REF);
}
do {
int choiceIndex = rand.nextInt(values.size());
choice = values.get(choiceIndex);
} while ((choice == OperatorType.QUERY) || (choice == OperatorType.RETURN)
|| (choice == OperatorType.FIELD_WITH_TYPE));
return choice;
}
public void fillOpQuery(QueryOperator anOp, List<QueryOperator> finishedOps) {
combinOpGeneric(anOp, finishedOps);
if (anOp.getParameters().size() > 0) {
anOp.addParameter(returnOp);
} else {
return anExp.getDataType();
}
}
public void fillOpIterate(QueryOperator anOp) {
List<Concept> concepts = anOp.getModel().getConcepts();
for (Concept aConc : concepts.toArray(new Concept[0])) {
if (!aConc.inheritsFrom(anOp.getModel().CONCEPT) || aConc.name.contains(".")) {
// no primitives and no backbone elements
concepts.remove(aConc);
}
}
int choice = rand.nextInt(concepts.size());
Concept aConcept = concepts.get(choice);
anOp.addParameter(aConcept.name);
anOp.alias = anOp.getNewAliasName();
private boolean isFinished(QueryOperator anOp) {
return anOp.getParameters().size() >= anOp.type.getMinParamCount();
}
public void fillOpField2ndParam(QueryOperator anOp) {
QueryOperator firstParam = anOp.getParameters().get(0).asOperator();
Concept dataType = firstParam.getDataType();
List<ConceptField> fields = dataType.getFields();
int choice = rand.nextInt(fields.size());
ConceptField aField = fields.get(choice);
anOp.addParameter(aField.name);
}
public void fillOpFieldWithType(QueryOperator anOp) {
throw new RuntimeException("TODO");
// createRandomFittingParamForParamIndex(anOp, 0);
// QueryOperator firstParam = anOp.getParameters().get(0).asOperator();
// Concept dataType = firstParam.getDataType();
// List<ConceptField> fields = dataType.getFields();
// int choice = rand.nextInt(fields.size());
// ConceptField aField = fields.get(choice);
// anOp.addParameter(aField.name);
for (ConceptField aField : fields) {
QueryOperator finishedField = anOp.clone();
finishedField.addParameter(aField.name);
newFinishedOps.add(finishedField);
}
}
protected void createOps() {
OperatorType[] selectedTypes = new OperatorType[] { OperatorType.AND, OperatorType.OR,
OperatorType.NOT, OperatorType.EQUALS, OperatorType.DURING, OperatorType.LESS,
OperatorType.BETWEEN, OperatorType.CONTAINS, OperatorType.AFTER, OperatorType.FILTER,
OperatorType.COLLECT, OperatorType.FIRST, OperatorType.SORT, OperatorType.EXISTS,
OperatorType.DEMOTE, OperatorType.END, OperatorType.COUNT, OperatorType.MIN,
OperatorType.ADD, OperatorType.FIELD, OperatorType.QUANTITY, OperatorType.DURATION };
for (OperatorType aType : selectedTypes) {
QueryOperator newOp = new QueryOperator(aType, model);
if (newOp.type.getMinParamCount() == 0) {
finishedOpsOfLastRound.add(newOp);
}
if (newOp.type.getMaxParamCount() > 0) {
unfinishedOpsOfLastRound.add(newOp);
}
}
fillOpIterate();
}
public void fillOpAliasRef(QueryOperator anOp) {
List<String> possibleAliases = anOp.type.getAllowedAliases(anOp);
if (possibleAliases.size() > 0) {
int randIndex = rand.nextInt(possibleAliases.size());
anOp.addParameter(possibleAliases.get(randIndex));
}
private void createPrimitives() {
finishedOpsOfLastRound.add(new Primitive(1, model));
finishedOpsOfLastRound.add(new Primitive(0.1, model));
finishedOpsOfLastRound.add(new Primitive(true, model));
finishedOpsOfLastRound.add(new Primitive("x", model));
finishedOpsOfLastRound
.add(new PrimitiveDateTime(2019, 1, 1, 0, 0, 0, 0, "2019-1-1 00:00:00.000", model));
}
private void addRandomPrimitive(QueryOperator anOp,
List<Concept> possibleDataTypesForParamIndex) {
List<Concept> possiblePrimitiveTypes = new ArrayList<Concept>();
if (possibleDataTypesForParamIndex.contains(anOp.getModel().INTEGER)) {
possiblePrimitiveTypes.add(anOp.getModel().INTEGER);
} else if (possibleDataTypesForParamIndex.contains(anOp.getModel().DOUBLE)
|| possibleDataTypesForParamIndex.contains(anOp.getModel().NUMBER)) {
possiblePrimitiveTypes.add(anOp.getModel().DOUBLE);
} else if (possibleDataTypesForParamIndex.contains(anOp.getModel().STRING)) {
possiblePrimitiveTypes.add(anOp.getModel().STRING);
} else if (possibleDataTypesForParamIndex.contains(anOp.getModel().BOOLEAN)) {
possiblePrimitiveTypes.add(anOp.getModel().BOOLEAN);
} else if (possibleDataTypesForParamIndex.contains(anOp.getModel().DATETIME)) {
possiblePrimitiveTypes.add(anOp.getModel().DATETIME);
}
int choiceIndex = rand.nextInt(possiblePrimitiveTypes.size());
Concept choice = possiblePrimitiveTypes.get(choiceIndex);
if (choice == anOp.getModel().INTEGER) {
int nextInt = rand.nextInt(20);
anOp.addParameter(nextInt);
} else if (choice == anOp.getModel().DOUBLE) {
double nextDouble = ((double) rand.nextInt(200)) / 10;
anOp.addParameter(nextDouble);
} else if (choice == anOp.getModel().STRING) {
byte[] array = new byte[7];
new Random().nextBytes(array);
String generatedString = new String(array, Charset.forName("UTF-8"));
anOp.addParameter(generatedString);
} else if (choice == anOp.getModel().BOOLEAN) {
boolean aValue = rand.nextBoolean();
anOp.addParameter(aValue);
} else if (choice == anOp.getModel().DATETIME) {
String aValue = "19860101";
anOp.addParameter(aValue, true);
}
public void fillOpIterate() {
QueryOperator newOp = new QueryOperator(OperatorType.ITERATE, model);
QueryOperator newOp2 = new QueryOperator(OperatorType.ITERATE, model);
newFinishedOps.add(newOp);
newFinishedOps.add(newOp2);
newOp.addParameter("Observation");
newOp2.addParameter("Patient");
createAlias(newOp);
createAlias(newOp2);
}
public List<Concept> getPossibleDataTypesForParamIndex(QueryOperator aContainerOp, int index) {
List<Concept> result = new ArrayList<Concept>();
if (aContainerOp.type.interfaceFlags.contains(InterfaceFlags.AllParamsTheSame)
if (aContainerOp.type.parameterTypes.size() > index) {
Concept concept = aContainerOp.type.parameterTypes.get(index);
result.add(concept);
} else if (aContainerOp.type.interfaceFlags.contains(InterfaceFlags.AllParamsTheSame)
|| aContainerOp.type.interfaceFlags.contains(InterfaceFlags.ParamsHaveToConform)
|| aContainerOp.type.interfaceFlags
.contains(InterfaceFlags.AllParamsMayBeAllQuantityTypes)) {
......@@ -287,12 +233,12 @@ public class Fuzzer {
result.add(model.QUANTITY);
result.add(model.DATETIME);
}
} else if (aContainerOp.type.interfaceFlags.contains(InterfaceFlags.AggregateOp)) {
result.add(model.LIST_OF_NUMBER);
result.add(model.LIST_OF_QUANTITY);
} else if (aContainerOp.type.interfaceFlags.contains(InterfaceFlags.UnlimitedParams)) {
Concept concept = aContainerOp.type.parameterTypes.get(0);
result.add(concept);
} else if (aContainerOp.type.parameterTypes.size() > index) {
Concept concept = aContainerOp.type.parameterTypes.get(index);
result.add(concept);
} else {
throw new RuntimeException("unexpected");
}
......
package de.uniwue.query;
import de.uniwue.model.Model;
import de.uniwue.query.opInterfaces.OperatorType;
public class QueryAlias extends QueryOperator {
private QueryOperator aliasedOp;
public QueryAlias(OperatorType opType, Model model, QueryOperator anOp) {
super(opType, model);
aliasedOp= anOp;
}
public QueryOperator resolveAlias() {
return aliasedOp;
}
@Override
public QueryOperator clone() {
QueryOperator result = new QueryAlias(type, getModel(), aliasedOp);
for (Expression anExp : getParameters()) {
result.addParameter(anExp.clone());
}
result.alias = alias;
result.outputName = outputName;
result.optional = optional;
return result;
}
}
......@@ -81,6 +81,9 @@ public class QueryOperator extends Expression {
for (Expression anExp : getParameters()) {
result.addParameter(anExp.clone());
}
result.alias = alias;
result.outputName = outputName;
result.optional = optional;
return result;
}
......@@ -183,6 +186,10 @@ public class QueryOperator extends Expression {
return result;
}
public static enum WhiteOrBlackList {
White, Black
}
public List<StructureProblem> checkProblems() {
List<StructureProblem> result = new ArrayList<StructureProblem>();
checkProblems(result);
......@@ -201,8 +208,7 @@ public class QueryOperator extends Expression {
private void checkParentChildSynchonicity(List<StructureProblem> result) {
if ((parent != null) && !parent.getParameters().contains(this)) {
result.add(new StructureProblem(ProblemType.ParentChildSynchronicity,
"parent does not contain me"));
addProblem(result, ProblemType.ParentChildSynchronicity, "parent does not contain me");
}