Commit 87bf23f2 authored by Thomas Michael Timmermanns's avatar Thomas Michael Timmermanns
Browse files

added first tests

parent 8283bd90
......@@ -2,9 +2,9 @@ package de.monticore.lang.monticar;
grammar CNNTrain extends de.monticore.lang.math.Math {
symbol scope CNNTrainCompilationUnit = "training" name:Name& "{" TrainingConfiguration "}";
symbol scope CNNTrainCompilationUnit = "Configuration" name:Name& "{" TrainingConfiguration "}";
TrainingConfiguration = (assignments:ParameterAssignment)+;
TrainingConfiguration = (assignments:ParameterAssignment)*;
ParameterAssignment = lhs:TrainingParameter "=" rhs:ParameterRhs;
enum TrainingParameter = DATA:"data"
......@@ -13,8 +13,8 @@ grammar CNNTrain extends de.monticore.lang.math.Math {
| BATCHSIZE:"batch_size"
| OPTIMIZER:"optimizer"
| LEARNINGRATE:"learning_rate";
//eval_metric
//validation_split
ParameterRhs = stringVal:String
| number:Number
......
......@@ -23,7 +23,8 @@ package de.monticore.lang.monticar.cnntrain._cocos;
public class CNNTrainCocos {
public static CNNTrainCoCoChecker createChecker() {
return new CNNTrainCoCoChecker();
return new CNNTrainCoCoChecker()
.addCoCo(new DuplicatedParameterCheck());
}
}
/**
*
* ******************************************************************************
* MontiCAR Modeling Family, www.se-rwth.de
* Copyright (c) 2017, Software Engineering Group at RWTH Aachen,
* All rights reserved.
*
* This project is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3.0 of the License, or (at your option) any later version.
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this project. If not, see <http://www.gnu.org/licenses/>.
* *******************************************************************************
*/
package de.monticore.lang.monticar.cnntrain._cocos;
import de.monticore.lang.monticar.cnntrain._ast.ASTParameterAssignment;
import de.monticore.lang.monticar.cnntrain._ast.ASTTrainingConfiguration;
import de.se_rwth.commons.logging.Log;
import java.util.HashSet;
import java.util.Set;
public class DuplicatedParameterCheck implements CNNTrainASTTrainingConfigurationCoCo {
@Override
public void check(ASTTrainingConfiguration node) {
Set<Enum> set = new HashSet<>();
for (ASTParameterAssignment assignment : node.getAssignments()) {
if (set.contains(assignment.getLhs())) {
Log.error("0x03201 Multiple assignments of the same parameter are not allowed",
assignment.get_SourcePositionStart());
}
else {
set.add(assignment.getLhs());
}
}
}
}
/**
*
* ******************************************************************************
* MontiCAR Modeling Family, www.se-rwth.de
* Copyright (c) 2017, Software Engineering Group at RWTH Aachen,
* All rights reserved.
*
* This project is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3.0 of the License, or (at your option) any later version.
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this project. If not, see <http://www.gnu.org/licenses/>.
* *******************************************************************************
*/
package de.monticore.lang.monticar.cnntrain;
import de.monticore.ModelingLanguageFamily;
import de.monticore.io.paths.ModelPath;
import de.monticore.lang.embeddedmontiarc.LogConfig;
import de.monticore.lang.monticar.cnntrain._symboltable.CNNTrainLanguage;
import de.monticore.symboltable.GlobalScope;
import de.monticore.symboltable.Scope;
import java.nio.file.Paths;
public class AbstractSymtabTest {
protected static Scope createSymTab(String... modelPath) {
ModelingLanguageFamily fam = new ModelingLanguageFamily();
fam.addModelingLanguage(new CNNTrainLanguage());
final ModelPath mp = new ModelPath();
for (String m : modelPath) {
mp.addEntry(Paths.get(m));
}
GlobalScope scope = new GlobalScope(mp, fam);
LogConfig.init();
return scope;
}
}
/**
*
* ******************************************************************************
* MontiCAR Modeling Family, www.se-rwth.de
* Copyright (c) 2017, Software Engineering Group at RWTH Aachen,
* All rights reserved.
*
* This project is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3.0 of the License, or (at your option) any later version.
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this project. If not, see <http://www.gnu.org/licenses/>.
* *******************************************************************************
*/
package de.monticore.lang.monticar.cnntrain;
import de.monticore.lang.monticar.cnntrain._ast.ASTCNNTrainCompilationUnit;
import de.monticore.lang.monticar.cnntrain._parser.CNNTrainParser;
import de.se_rwth.commons.logging.Log;
import org.junit.Before;
import org.junit.Test;
import java.io.IOException;
import java.nio.file.*;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Optional;
import java.util.stream.Collectors;
import static org.junit.Assert.assertTrue;
public class ParserTest {
public static final boolean ENABLE_FAIL_QUICK = false;
private static List<String> expectedParseErrorModels = Arrays.asList(
"src/test/resources/WrongParameterName.cnnt")
.stream().map(s -> Paths.get(s).toString())
.collect(Collectors.toList());
@Before
public void setUp() {
// ensure an empty log
Log.getFindings().clear();
Log.enableFailQuick(ENABLE_FAIL_QUICK);
}
@Test
public void testCNNTrain() throws Exception {
test("cnnt");
if (Log.getErrorCount() > 0) {
throw new Exception("Test Failed, found errors");
}
}
private void test(String fileEnding) throws IOException {
ParseTest parserTest = new ParseTest("." + fileEnding);
Files.walkFileTree(Paths.get("src/test/resources"), parserTest);
if (!parserTest.getModelsInError().isEmpty()) {
Log.debug("Models in error", "ParserTest");
for (String model : parserTest.getModelsInError()) {
Log.debug(" " + model, "ParserTest");
}
}
Log.info("Count of tested models: " + parserTest.getTestCount(), "ParserTest");
Log.info("Count of correctly parsed models: "
+ (parserTest.getTestCount() - parserTest.getModelsInError().size()), "ParserTest");
assertTrue("There were models that could not be parsed", parserTest.getModelsInError()
.isEmpty());
}
/**
* Visits files of the given file ending and checks whether they are parsable.
*
* @author Robert Heim
* @see Files#walkFileTree(Path, java.nio.file.FileVisitor)
*/
private static class ParseTest extends SimpleFileVisitor<Path> {
private String fileEnding;
private List<String> modelsInError = new ArrayList<>();
private int testCount = 0;
public ParseTest(String fileEnding) {
super();
this.fileEnding = fileEnding;
}
/**
* @return testCount
*/
public int getTestCount() {
return this.testCount;
}
/**
* @return modelsInError
*/
public List<String> getModelsInError() {
return this.modelsInError;
}
@Override
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs)
throws IOException {
if (file.toFile().isFile()
&& (file.toString().toLowerCase().endsWith(fileEnding))) {
Log.debug("Parsing file " + file.toString(), "ParserTest");
testCount++;
Optional<ASTCNNTrainCompilationUnit> model = Optional.empty();
boolean expectingError = ParserTest.expectedParseErrorModels.contains(file.toString());
CNNTrainParser parser = new CNNTrainParser();
try {
if (expectingError) {
Log.enableFailQuick(false);
}
model = parser.parse(file.toString());
}
catch (Exception e) {
if (!expectingError) {
Log.error("Exception during test", e);
}
}
if (!expectingError && (parser.hasErrors() || !model.isPresent())) {
modelsInError.add(file.toString());
Log.error("There were unexpected parser errors");
}
else {
Log.getFindings().clear();
}
Log.enableFailQuick(ParserTest.ENABLE_FAIL_QUICK);
}
return FileVisitResult.CONTINUE;
}
}
}
/**
*
* ******************************************************************************
* MontiCAR Modeling Family, www.se-rwth.de
* Copyright (c) 2017, Software Engineering Group at RWTH Aachen,
* All rights reserved.
*
* This project is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3.0 of the License, or (at your option) any later version.
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this project. If not, see <http://www.gnu.org/licenses/>.
* *******************************************************************************
*/
package de.monticore.lang.monticar.cnntrain;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import de.monticore.lang.monticar.cnntrain._parser.CNNTrainParser;
import de.monticore.lang.monticar.cnntrain._symboltable.CNNTrainCompilationUnitSymbol;
import de.monticore.symboltable.Scope;
import org.junit.Ignore;
import org.junit.Test;
public class SymtabTest extends AbstractSymtabTest {
@Test
public void testParsing() throws Exception {
CNNTrainParser parser = new CNNTrainParser();
assertTrue(parser.parse("src/test/resources/SimpleConfig1.cnnt").isPresent());
}
@Test
public void testAlexnet(){
Scope symTab = createSymTab("src/test/resources");
CNNTrainCompilationUnitSymbol a = symTab.<CNNTrainCompilationUnitSymbol>resolve(
"SimpleConfig2",
CNNTrainCompilationUnitSymbol.KIND).orElse(null);
assertNotNull(a);
}
}
/**
*
* ******************************************************************************
* MontiCAR Modeling Family, www.se-rwth.de
* Copyright (c) 2017, Software Engineering Group at RWTH Aachen,
* All rights reserved.
*
* This project is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3.0 of the License, or (at your option) any later version.
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this project. If not, see <http://www.gnu.org/licenses/>.
* *******************************************************************************
*/
package de.monticore.lang.monticar.cnntrain.cocos;
import de.monticore.lang.monticar.cnntrain.AbstractSymtabTest;
import de.monticore.lang.monticar.cnntrain._ast.ASTCNNTrainNode;
import de.monticore.lang.monticar.cnntrain._cocos.CNNTrainCoCoChecker;
import de.monticore.lang.monticar.cnntrain._cocos.CNNTrainCocos;
import de.monticore.lang.monticar.cnntrain._symboltable.CNNTrainCompilationUnitSymbol;
import de.monticore.symboltable.Scope;
import de.se_rwth.commons.logging.Finding;
import de.se_rwth.commons.logging.Log;
import java.util.Arrays;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.function.Predicate;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import static org.junit.Assert.*;
public class AbstractCoCoTest extends AbstractSymtabTest {
private static final String MODEL_PATH = "src/test/resources/";
protected static ASTCNNTrainNode getAstNode(String modelPath, String model) {
Scope symTab = createSymTab(MODEL_PATH + modelPath);
CNNTrainCompilationUnitSymbol comp = symTab.<CNNTrainCompilationUnitSymbol> resolve(
model, CNNTrainCompilationUnitSymbol.KIND).orElse(null);
assertNotNull("Could not resolve model " + model, comp);
return (ASTCNNTrainNode) comp.getAstNode().get();
}
/**
* Checks all cocos on the given model. Don't use for writing new test cases, use checkValid and
* checkInvalid instead.
*/
@Deprecated
protected static void runCheckerWithSymTab(String modelPath, String model) {
Log.getFindings().clear();
ASTCNNTrainNode node = getAstNode(modelPath, model);
CNNTrainCocos.createChecker().checkAll(node);
}
/**
* Checks all cocos on the given node, and checks for absence of errors. Use this for checking
* valid models.
*/
protected static void checkValid(String modelPath, String model) {
Log.getFindings().clear();
CNNTrainCocos.createChecker().checkAll(getAstNode(modelPath, model));
new ExpectedErrorInfo().checkOnlyExpectedPresent(Log.getFindings());
}
/**
* Runs coco checks on the model with two different coco sets: Once with all cocos, checking that
* the expected errors are present; once only with the given cocos, checking that no additional
* errors are present.
*/
protected static void checkInvalid(CNNTrainCoCoChecker cocos, ASTCNNTrainNode node,
ExpectedErrorInfo expectedErrors) {
// check whether all the expected errors are present when using all cocos
Log.getFindings().clear();
CNNTrainCocos.createChecker().checkAll(node);
expectedErrors.checkExpectedPresent(Log.getFindings(), "Got no findings when checking all "
+ "cocos. Did you forget to add the new coco to MontiArcCocos?");
// check whether only the expected errors are present when using only the given cocos
Log.getFindings().clear();
cocos.checkAll(node);
expectedErrors.checkOnlyExpectedPresent(Log.getFindings(), "Got no findings when checking only "
+ "the given coco. Did you pass an empty coco checker?");
}
protected static class ExpectedErrorInfo {
private static final Pattern ERROR_CODE_PATTERN = Pattern.compile("x[0-9A-F]{5}");
private int numExpectedFindings = 0;
private HashSet<String> expectedErrorCodes = new HashSet<>();
private Predicate<String> containsExpectedErrorCode;
/**
* Raises an error if the given error codes don't match the convention for error codes in test
* cases (no leading zero, capital hexadecimal digits)
*/
protected static void checkExpectedErrorCodes(String[] errorCodes) {
for (String errorCode : errorCodes) {
if (!ERROR_CODE_PATTERN.matcher(errorCode).matches()) {
Log.error(String.format(
"The given expected error code \"%s\" is not a valid error code (pattern: \"%s\")",
errorCode, ERROR_CODE_PATTERN.pattern()));
}
}
}
protected static Set<String> collectErrorCodes(String findings) {
Matcher matcher = ERROR_CODE_PATTERN.matcher(findings);
Set<String> errorCodes = new HashSet<>();
while (matcher.find()) {
errorCodes.add(matcher.group());
}
return errorCodes;
}
private void initContainsExpectedErrorCode() {
containsExpectedErrorCode = new Predicate<String>() {
@Override
public boolean test(String s) {
for (String errorCode : expectedErrorCodes) {
if (s.contains(errorCode)) {
return true;
}
}
return false;
}
};
}
public ExpectedErrorInfo() {
this(0);
}
public ExpectedErrorInfo(int numExpectedFindings, String... expectedErrorCodes) {
checkExpectedErrorCodes(expectedErrorCodes);
this.numExpectedFindings = numExpectedFindings;
this.expectedErrorCodes.addAll(Arrays.asList(expectedErrorCodes));
initContainsExpectedErrorCode();
}
private String concatenateFindings(List<Finding> findings) {
return findings.stream().map(f -> f.buildMsg())
.collect(Collectors.joining("\n"));
}
public void checkExpectedPresent(List<Finding> findings, String emptyFindingsHint) {
String findingsString = concatenateFindings(findings);
if (findingsString.isEmpty()) {
findingsString = emptyFindingsHint;
}
assertEquals(findingsString, numExpectedFindings,
findings.stream().map(f -> f.buildMsg()).filter(containsExpectedErrorCode).count());
assertTrue(collectErrorCodes(findingsString).containsAll(expectedErrorCodes));
}
public void checkOnlyExpectedPresent(List<Finding> findings) {
checkOnlyExpectedPresent(findings, "");
}
public void checkOnlyExpectedPresent(List<Finding> findings, String emptyFindingsHint) {
checkExpectedPresent(findings, emptyFindingsHint);
checkNoAdditionalErrorCodesPresent(concatenateFindings(findings));
}
private void checkNoAdditionalErrorCodesPresent(String findingsString) {
Set<String> actualErrorCodes = collectErrorCodes(findingsString);
// check whether there are unexpected error codes
Set<String> unexpectedErrorCodes = new HashSet<>(actualErrorCodes);
unexpectedErrorCodes.removeAll(expectedErrorCodes);
assertEquals(findingsString, 0, unexpectedErrorCodes.size());
}
}
}
/**
*
* ******************************************************************************
* MontiCAR Modeling Family, www.se-rwth.de
* Copyright (c) 2017, Software Engineering Group at RWTH Aachen,
* All rights reserved.
*
* This project is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3.0 of the License, or (at your option) any later version.
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this project. If not, see <http://www.gnu.org/licenses/>.
* *******************************************************************************
*/
package de.monticore.lang.monticar.cnntrain.cocos;
import de.monticore.lang.monticar.cnntrain._cocos.CNNTrainCocos;
import org.junit.Test;
import java.io.IOException;
import static org.junit.Assert.assertTrue;
public class AllCoCoTest extends AbstractCoCoTest{
String baseDir="src/test/resources";
@Test
public void testCoCosSimulator() throws IOException {
testModel("SimpleConfig1");
testModel("SimpleConfig2");
testModel("DataReferenceTest");
testInvalidModel("DuplicatedParameter",1,"x03201");
testInvalidModel("InvalidDataPath",1,"x03203");
testInvalidModel("InvalidOptimizer",1,"x0320B");