Commit d542c62c authored by Alexander David Hellwig's avatar Alexander David Hellwig
Browse files

Added ability to run multiple clustering algorithms on one component and...

Added ability to run multiple clustering algorithms on one component and compare the result, improved cli
parent 79d05e3f
Pipeline #102185 failed with stages
in 23 minutes and 55 seconds
...@@ -3,8 +3,12 @@ package de.monticore.lang.monticar.generator.middleware; ...@@ -3,8 +3,12 @@ package de.monticore.lang.monticar.generator.middleware;
import de.monticore.lang.embeddedmontiarc.embeddedmontiarc._symboltable.instanceStructure.EMAComponentInstanceSymbol; import de.monticore.lang.embeddedmontiarc.embeddedmontiarc._symboltable.instanceStructure.EMAComponentInstanceSymbol;
import de.monticore.lang.monticar.generator.FileContent; import de.monticore.lang.monticar.generator.FileContent;
import de.monticore.lang.monticar.generator.middleware.cli.ClusteringParameters; import de.monticore.lang.monticar.generator.middleware.cli.ClusteringParameters;
import de.monticore.lang.monticar.generator.middleware.clustering.ClusterFromTagsHelper; import de.monticore.lang.monticar.generator.middleware.cli.ResultChoosingStrategy;
import de.monticore.lang.monticar.generator.middleware.helpers.*; import de.monticore.lang.monticar.generator.middleware.clustering.*;
import de.monticore.lang.monticar.generator.middleware.helpers.FileHelper;
import de.monticore.lang.monticar.generator.middleware.helpers.NameHelper;
import de.monticore.lang.monticar.generator.middleware.helpers.RosHelper;
import de.monticore.lang.monticar.generator.middleware.helpers.TemplateHelper;
import de.monticore.lang.monticar.generator.middleware.impls.GeneratorImpl; import de.monticore.lang.monticar.generator.middleware.impls.GeneratorImpl;
import de.monticore.lang.monticar.generator.middleware.impls.MiddlewareTagGenImpl; import de.monticore.lang.monticar.generator.middleware.impls.MiddlewareTagGenImpl;
import de.monticore.lang.tagging._symboltable.TaggingResolver; import de.monticore.lang.tagging._symboltable.TaggingResolver;
...@@ -32,16 +36,24 @@ public class DistributedTargetGenerator extends CMakeGenerator { ...@@ -32,16 +36,24 @@ public class DistributedTargetGenerator extends CMakeGenerator {
public DistributedTargetGenerator() { public DistributedTargetGenerator() {
} }
public ClusteringParameters getClusteringParameters() {
return clusteringParameters;
}
public void setClusteringParameters(ClusteringParameters clusteringParameters) {
this.clusteringParameters = clusteringParameters;
}
@Override @Override
public void setGenerationTargetPath(String path) { public void setGenerationTargetPath(String path) {
super.setGenerationTargetPath(path); super.setGenerationTargetPath(path);
} }
@Override @Override
public List<File> generate(EMAComponentInstanceSymbol componentInstanceSymbol, TaggingResolver taggingResolver) throws IOException { public List<File> generate(EMAComponentInstanceSymbol genComp, TaggingResolver taggingResolver) throws IOException {
Map<EMAComponentInstanceSymbol, GeneratorImpl> generatorMap = new HashMap<>(); Map<EMAComponentInstanceSymbol, GeneratorImpl> generatorMap = new HashMap<>();
fixComponentInstance(componentInstanceSymbol); EMAComponentInstanceSymbol componentInstanceSymbol = preprocessing(genComp);
List<EMAComponentInstanceSymbol> clusterSubcomponents = ClusterFromTagsHelper.getClusterSubcomponents(componentInstanceSymbol); List<EMAComponentInstanceSymbol> clusterSubcomponents = ClusterFromTagsHelper.getClusterSubcomponents(componentInstanceSymbol);
if (clusterSubcomponents.size() > 0) { if (clusterSubcomponents.size() > 0) {
...@@ -78,6 +90,47 @@ public class DistributedTargetGenerator extends CMakeGenerator { ...@@ -78,6 +90,47 @@ public class DistributedTargetGenerator extends CMakeGenerator {
return files; return files;
} }
private EMAComponentInstanceSymbol preprocessing(EMAComponentInstanceSymbol genComp) {
EMAComponentInstanceSymbol componentInstanceSymbol = genComp;
if(clusteringParameters != null){
//Flatten
if(clusteringParameters.getFlatten()){
if(clusteringParameters.getFlattenLevel().isPresent()){
Integer level = clusteringParameters.getFlattenLevel().get();
componentInstanceSymbol = FlattenArchitecture.flattenArchitecture(genComp, new HashMap<>(), level);
}else {
componentInstanceSymbol = FlattenArchitecture.flattenArchitecture(genComp);
}
}
//Cluster
if(clusteringParameters.getAlgorithmParameters().size() > 0) {
ClusteringResultList clusteringResults = AutomaticClusteringHelper.executeClusteringFromParams(componentInstanceSymbol, clusteringParameters.getAlgorithmParameters());
Optional<Integer> nOpt = clusteringParameters.getNumberOfClusters();
for(ClusteringResult c : clusteringResults){
String prefix = nOpt.isPresent() && !c.hasNumberOfClusters(nOpt.get()) ? "[IGNORED]" : "";
System.out.println(prefix + "Score was " + c.getScore() + " for " + c.getParameters().toString());
}
Optional<ClusteringResult> clusteringOpt;
if(nOpt.isPresent() && clusteringParameters.getChooseBy().equals(ResultChoosingStrategy.bestWithFittingN)){
clusteringOpt = clusteringResults.getBestResultWithFittingN(nOpt.get());
}else{
clusteringOpt = clusteringResults.getBestResultOverall();
}
if(clusteringOpt.isPresent()){
ClusteringResult clusteringResult = clusteringOpt.get();
System.out.println("Best score was " + clusteringResult.getScore() + " for " + clusteringResult.getParameters().toString());
AutomaticClusteringHelper.annotateComponentWithRosTagsForClusters(componentInstanceSymbol, clusteringResult.getClustering());
}
}
}
fixComponentInstance(componentInstanceSymbol);
return componentInstanceSymbol;
}
private File generateRosMsgGen() throws IOException { private File generateRosMsgGen() throws IOException {
File file = new File(generationTargetPath + "rosMsg/CMakeLists.txt"); File file = new File(generationTargetPath + "rosMsg/CMakeLists.txt");
FileUtils.write(file, TemplateHelper.getStruct_msgsCmakeTemplate()); FileUtils.write(file, TemplateHelper.getStruct_msgsCmakeTemplate());
...@@ -103,7 +156,6 @@ public class DistributedTargetGenerator extends CMakeGenerator { ...@@ -103,7 +156,6 @@ public class DistributedTargetGenerator extends CMakeGenerator {
fileContent.setFileName("CMakeLists.txt"); fileContent.setFileName("CMakeLists.txt");
StringBuilder content = new StringBuilder(); StringBuilder content = new StringBuilder();
content.append("cmake_minimum_required(VERSION 3.5)\n"); content.append("cmake_minimum_required(VERSION 3.5)\n");
//TODO setProjectName?
content.append("project (default)\n"); content.append("project (default)\n");
content.append("set (CMAKE_CXX_STANDARD 11)\n"); content.append("set (CMAKE_CXX_STANDARD 11)\n");
......
...@@ -3,6 +3,7 @@ package de.monticore.lang.monticar.generator.middleware.cli; ...@@ -3,6 +3,7 @@ package de.monticore.lang.monticar.generator.middleware.cli;
import com.google.gson.*; import com.google.gson.*;
import com.google.gson.stream.JsonReader; import com.google.gson.stream.JsonReader;
import de.monticore.lang.monticar.generator.middleware.cli.algorithms.*; import de.monticore.lang.monticar.generator.middleware.cli.algorithms.*;
import de.se_rwth.commons.logging.Log;
import java.io.FileNotFoundException; import java.io.FileNotFoundException;
import java.io.FileReader; import java.io.FileReader;
...@@ -36,7 +37,10 @@ public class CliParametersLoader { ...@@ -36,7 +37,10 @@ public class CliParametersLoader {
case "dbscan": return DBScanCliParameters.class; case "dbscan": return DBScanCliParameters.class;
case "affinitypropagation": return AffinityPropagationCliParameters.class; case "affinitypropagation": return AffinityPropagationCliParameters.class;
case "markov": return MarkovCliParameters.class; case "markov": return MarkovCliParameters.class;
default: return UnknownAlgorithmCliParameters.class; default:{
Log.warn("Loaded config of unknown clustering algorithm: " + algoName);
return UnknownAlgorithmCliParameters.class;
}
} }
} }
} }
......
package de.monticore.lang.monticar.generator.middleware.cli; package de.monticore.lang.monticar.generator.middleware.cli;
import de.monticore.lang.monticar.generator.middleware.cli.algorithms.AlgorithmCliParameters; import de.monticore.lang.monticar.generator.middleware.cli.algorithms.AlgorithmCliParameters;
import de.monticore.lang.monticar.generator.middleware.cli.algorithms.SpectralClusteringCliParameters;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
...@@ -8,6 +9,8 @@ import java.util.Optional; ...@@ -8,6 +9,8 @@ import java.util.Optional;
public class ClusteringParameters { public class ClusteringParameters {
private Integer numberOfClusters; private Integer numberOfClusters;
private Boolean flatten;
private Integer flattenLevel;
private ResultChoosingStrategy chooseBy = ResultChoosingStrategy.bestWithFittingN; private ResultChoosingStrategy chooseBy = ResultChoosingStrategy.bestWithFittingN;
private List<AlgorithmCliParameters> algorithmParameters = new ArrayList<>(); private List<AlgorithmCliParameters> algorithmParameters = new ArrayList<>();
...@@ -23,6 +26,22 @@ public class ClusteringParameters { ...@@ -23,6 +26,22 @@ public class ClusteringParameters {
} }
public List<AlgorithmCliParameters> getAlgorithmParameters() { public List<AlgorithmCliParameters> getAlgorithmParameters() {
//Override numberOfClusters for all spectral clustering parameters
if(getNumberOfClusters().isPresent()){
Integer n = getNumberOfClusters().get();
algorithmParameters.stream()
.filter(a -> a.getName().equals(AlgorithmCliParameters.TYPE_SPECTRAL_CLUSTERING))
.forEach(a -> ((SpectralClusteringCliParameters)a).setNumberOfClusters(n));
}
return algorithmParameters; return algorithmParameters;
} }
public boolean getFlatten(){
return flatten == null ? false : flatten;
}
public Optional<Integer> getFlattenLevel() {
return Optional.ofNullable(flattenLevel);
}
} }
...@@ -154,6 +154,15 @@ public final class DistributedTargetGeneratorCli { ...@@ -154,6 +154,15 @@ public final class DistributedTargetGeneratorCli {
generator.add(new ODVGenImpl(), "odv"); generator.add(new ODVGenImpl(), "odv");
} }
if (cliParameters.getClusteringParameters().isPresent()) {
ClusteringParameters clusteringParameters = cliParameters.getClusteringParameters().get();
generator.setClusteringParameters(clusteringParameters);
clusteringParameters.getAlgorithmParameters().stream()
.filter(alg -> !alg.isValid())
.forEach(alg -> Log.error("Parameters for the algorithm " + alg.getName() + " are invalid!"));
}
try { try {
generator.generate(componentInstanceSymbol, taggingResolver); generator.generate(componentInstanceSymbol, taggingResolver);
} catch (IOException e) { } catch (IOException e) {
......
...@@ -3,26 +3,33 @@ package de.monticore.lang.monticar.generator.middleware.cli.algorithms; ...@@ -3,26 +3,33 @@ package de.monticore.lang.monticar.generator.middleware.cli.algorithms;
import de.monticore.lang.monticar.generator.middleware.clustering.ClusteringAlgorithm; import de.monticore.lang.monticar.generator.middleware.clustering.ClusteringAlgorithm;
import de.monticore.lang.monticar.generator.middleware.clustering.algorithms.AffinityPropagationAlgorithm; import de.monticore.lang.monticar.generator.middleware.clustering.algorithms.AffinityPropagationAlgorithm;
import java.util.ArrayList;
import java.util.List;
public class AffinityPropagationCliParameters extends AlgorithmCliParameters{ public class AffinityPropagationCliParameters extends AlgorithmCliParameters{
public AffinityPropagationCliParameters() { public AffinityPropagationCliParameters() {
} }
@Override
public String getName() {
return TYPE_AFFINITY_PROPAGATION;
}
@Override @Override
public ClusteringAlgorithm asClusteringAlgorithm() { public ClusteringAlgorithm asClusteringAlgorithm() {
return new AffinityPropagationAlgorithm(); return new AffinityPropagationAlgorithm();
} }
@Override @Override
public List<Object> asAlgorithmArgs(){ public Object[] asAlgorithmArgs(){
return new ArrayList<>(); return new Object[]{};
} }
@Override @Override
public boolean isValid() { public boolean isValid() {
return true; return true;
} }
@Override
public String toString() {
return "AffinityPropagation";
}
} }
\ No newline at end of file
...@@ -2,13 +2,12 @@ package de.monticore.lang.monticar.generator.middleware.cli.algorithms; ...@@ -2,13 +2,12 @@ package de.monticore.lang.monticar.generator.middleware.cli.algorithms;
import de.monticore.lang.monticar.generator.middleware.clustering.ClusteringAlgorithm; import de.monticore.lang.monticar.generator.middleware.clustering.ClusteringAlgorithm;
import java.util.List;
public abstract class AlgorithmCliParameters { public abstract class AlgorithmCliParameters {
public static final String TYPE_SPECTRAL_CLUSTERING = "SpectralClustering"; public static final String TYPE_SPECTRAL_CLUSTERING = "SpectralClustering";
public static final String TYPE_UNKOWN = "Unkown"; public static final String TYPE_UNKOWN = "Unkown";
public static final String TYPE_DBSCAN = "DBScan"; public static final String TYPE_DBSCAN = "DBScan";
public static final String TYPE_MARKOV = "Markov"; public static final String TYPE_MARKOV = "Markov";
public static final String TYPE_AFFINITY_PROPAGATION = "AffinityPropagation";
protected String name; protected String name;
public AlgorithmCliParameters() { public AlgorithmCliParameters() {
...@@ -20,7 +19,7 @@ public abstract class AlgorithmCliParameters { ...@@ -20,7 +19,7 @@ public abstract class AlgorithmCliParameters {
public abstract ClusteringAlgorithm asClusteringAlgorithm(); public abstract ClusteringAlgorithm asClusteringAlgorithm();
public abstract List<Object> asAlgorithmArgs(); public abstract Object[] asAlgorithmArgs();
public abstract boolean isValid(); public abstract boolean isValid();
} }
...@@ -30,11 +30,11 @@ public class DBScanCliParameters extends AlgorithmCliParameters { ...@@ -30,11 +30,11 @@ public class DBScanCliParameters extends AlgorithmCliParameters {
} }
@Override @Override
public List<Object> asAlgorithmArgs() { public Object[] asAlgorithmArgs() {
List<Object> res = new ArrayList<>(); List<Object> res = new ArrayList<>();
if(!isValid()){ if(!isValid()){
Log.error("DBScanCliParameters: The min_pts or radius parameters are mandatory but at least one is unset!"); Log.error("DBScanCliParameters: The min_pts or radius parameters are mandatory but at least one is unset!");
return res; return res.toArray();
} }
res.add(DBSCANClusteringBuilder.DBSCANParameters.DBSCAN_MIN_PTS); res.add(DBSCANClusteringBuilder.DBSCANParameters.DBSCAN_MIN_PTS);
...@@ -42,7 +42,7 @@ public class DBScanCliParameters extends AlgorithmCliParameters { ...@@ -42,7 +42,7 @@ public class DBScanCliParameters extends AlgorithmCliParameters {
res.add(DBSCANClusteringBuilder.DBSCANParameters.DBSCAN_RADIUS); res.add(DBSCANClusteringBuilder.DBSCANParameters.DBSCAN_RADIUS);
res.add(radius); res.add(radius);
return res; return res.toArray();
} }
@Override @Override
...@@ -65,4 +65,12 @@ public class DBScanCliParameters extends AlgorithmCliParameters { ...@@ -65,4 +65,12 @@ public class DBScanCliParameters extends AlgorithmCliParameters {
public void setRadius(Double radius) { public void setRadius(Double radius) {
this.radius = radius; this.radius = radius;
} }
@Override
public String toString() {
return "DBScan{" +
"min_pts=" + min_pts +
", radius=" + radius +
'}';
}
} }
...@@ -30,7 +30,7 @@ public class MarkovCliParameters extends AlgorithmCliParameters { ...@@ -30,7 +30,7 @@ public class MarkovCliParameters extends AlgorithmCliParameters {
} }
@Override @Override
public List<Object> asAlgorithmArgs() { public Object[] asAlgorithmArgs() {
List<Object> res = new ArrayList<>(); List<Object> res = new ArrayList<>();
if(max_residual != null){ if(max_residual != null){
res.add(MarkovClusteringBuilder.MarkovParameters.MARKOV_MAX_RESIDUAL); res.add(MarkovClusteringBuilder.MarkovParameters.MARKOV_MAX_RESIDUAL);
...@@ -48,7 +48,7 @@ public class MarkovCliParameters extends AlgorithmCliParameters { ...@@ -48,7 +48,7 @@ public class MarkovCliParameters extends AlgorithmCliParameters {
res.add(MarkovClusteringBuilder.MarkovParameters.MARKOV_ZERO_MAX); res.add(MarkovClusteringBuilder.MarkovParameters.MARKOV_ZERO_MAX);
res.add(zero_max); res.add(zero_max);
} }
return res; return res.toArray();
} }
@Override @Override
...@@ -87,4 +87,14 @@ public class MarkovCliParameters extends AlgorithmCliParameters { ...@@ -87,4 +87,14 @@ public class MarkovCliParameters extends AlgorithmCliParameters {
public void setZeroMax(Double zero_max) { public void setZeroMax(Double zero_max) {
this.zero_max = zero_max; this.zero_max = zero_max;
} }
@Override
public String toString() {
return "Markov{" +
"max_residual=" + max_residual +
", gamma_exp=" + gamma_exp +
", loop_gain=" + loop_gain +
", zero_max=" + zero_max +
'}';
}
} }
...@@ -6,7 +6,6 @@ import de.monticore.lang.monticar.generator.middleware.clustering.algorithms.Spe ...@@ -6,7 +6,6 @@ import de.monticore.lang.monticar.generator.middleware.clustering.algorithms.Spe
import de.se_rwth.commons.logging.Log; import de.se_rwth.commons.logging.Log;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List;
import java.util.Optional; import java.util.Optional;
public class SpectralClusteringCliParameters extends AlgorithmCliParameters { public class SpectralClusteringCliParameters extends AlgorithmCliParameters {
...@@ -30,12 +29,12 @@ public class SpectralClusteringCliParameters extends AlgorithmCliParameters { ...@@ -30,12 +29,12 @@ public class SpectralClusteringCliParameters extends AlgorithmCliParameters {
} }
@Override @Override
public List<Object> asAlgorithmArgs(){ public Object[] asAlgorithmArgs(){
ArrayList<Object> res = new ArrayList<>(); ArrayList<Object> res = new ArrayList<>();
if(!isValid()){ if(!isValid()){
Log.error("SpectralClusteringCliParameters: The numberOfClusters parameter is mandatory but unset!"); Log.error("SpectralClusteringCliParameters: The numberOfClusters parameter is mandatory but unset!");
return res; return res.toArray();
} }
res.add(SpectralClusteringBuilder.SpectralParameters.SPECTRAL_NUM_CLUSTERS); res.add(SpectralClusteringBuilder.SpectralParameters.SPECTRAL_NUM_CLUSTERS);
...@@ -51,7 +50,7 @@ public class SpectralClusteringCliParameters extends AlgorithmCliParameters { ...@@ -51,7 +50,7 @@ public class SpectralClusteringCliParameters extends AlgorithmCliParameters {
res.add(sigma); res.add(sigma);
} }
return res; return res.toArray();
} }
@Override @Override
...@@ -82,4 +81,13 @@ public class SpectralClusteringCliParameters extends AlgorithmCliParameters { ...@@ -82,4 +81,13 @@ public class SpectralClusteringCliParameters extends AlgorithmCliParameters {
public void setSigma(Double sigma) { public void setSigma(Double sigma) {
this.sigma = sigma; this.sigma = sigma;
} }
@Override
public String toString() {
return "SpectralClustering{" +
"numberOfClusters=" + numberOfClusters +
", l=" + l +
", sigma=" + sigma +
'}';
}
} }
...@@ -2,8 +2,6 @@ package de.monticore.lang.monticar.generator.middleware.cli.algorithms; ...@@ -2,8 +2,6 @@ package de.monticore.lang.monticar.generator.middleware.cli.algorithms;
import de.monticore.lang.monticar.generator.middleware.clustering.ClusteringAlgorithm; import de.monticore.lang.monticar.generator.middleware.clustering.ClusteringAlgorithm;
import java.util.List;
public class UnknownAlgorithmCliParameters extends AlgorithmCliParameters { public class UnknownAlgorithmCliParameters extends AlgorithmCliParameters {
public UnknownAlgorithmCliParameters() { public UnknownAlgorithmCliParameters() {
...@@ -20,7 +18,7 @@ public class UnknownAlgorithmCliParameters extends AlgorithmCliParameters { ...@@ -20,7 +18,7 @@ public class UnknownAlgorithmCliParameters extends AlgorithmCliParameters {
} }
@Override @Override
public List<Object> asAlgorithmArgs() { public Object[] asAlgorithmArgs() {
return null; return null;
} }
......
package de.monticore.lang.monticar.generator.middleware.clustering; package de.monticore.lang.monticar.generator.middleware.clustering;
import de.monticore.expressionsbasis._ast.ASTExpression; import de.monticore.expressionsbasis._ast.ASTExpression;
import de.monticore.lang.embeddedmontiarc.embeddedmontiarc._symboltable.*;
import de.monticore.lang.embeddedmontiarc.embeddedmontiarc._symboltable.cncModel.EMAPortSymbol; import de.monticore.lang.embeddedmontiarc.embeddedmontiarc._symboltable.cncModel.EMAPortSymbol;
import de.monticore.lang.embeddedmontiarc.embeddedmontiarc._symboltable.instanceStructure.EMAComponentInstanceSymbol; import de.monticore.lang.embeddedmontiarc.embeddedmontiarc._symboltable.instanceStructure.EMAComponentInstanceSymbol;
import de.monticore.lang.embeddedmontiarc.embeddedmontiarc._symboltable.instanceStructure.EMAConnectorInstanceSymbol; import de.monticore.lang.embeddedmontiarc.embeddedmontiarc._symboltable.instanceStructure.EMAConnectorInstanceSymbol;
import de.monticore.lang.embeddedmontiarc.tagging.middleware.ros.RosConnectionSymbol; import de.monticore.lang.embeddedmontiarc.tagging.middleware.ros.RosConnectionSymbol;
import de.monticore.lang.math._ast.ASTNumberExpression; import de.monticore.lang.math._ast.ASTNumberExpression;
import de.monticore.lang.monticar.common2._ast.ASTCommonMatrixType; import de.monticore.lang.monticar.common2._ast.ASTCommonMatrixType;
import de.monticore.lang.monticar.generator.middleware.cli.algorithms.AlgorithmCliParameters;
import de.monticore.lang.monticar.ts.MCTypeSymbol; import de.monticore.lang.monticar.ts.MCTypeSymbol;
import de.monticore.lang.monticar.ts.references.MCASTTypeSymbolReference; import de.monticore.lang.monticar.ts.references.MCASTTypeSymbolReference;
import de.monticore.lang.monticar.ts.references.MCTypeReference; import de.monticore.lang.monticar.ts.references.MCTypeReference;
import de.monticore.symboltable.CommonScope; import de.monticore.symboltable.CommonSymbol;
import de.monticore.symboltable.MutableScope;
import de.monticore.symboltable.Symbol;
import de.monticore.symboltable.resolving.ResolvingFilter;
import de.se_rwth.commons.logging.Log;
import java.util.*; import java.util.Collection;