Commit d542c62c authored by Alexander David Hellwig's avatar Alexander David Hellwig
Browse files

Added ability to run multiple clustering algorithms on one component and...

Added ability to run multiple clustering algorithms on one component and compare the result, improved cli
parent 79d05e3f
Pipeline #102185 failed with stages
in 23 minutes and 55 seconds
......@@ -3,8 +3,12 @@ package de.monticore.lang.monticar.generator.middleware;
import de.monticore.lang.embeddedmontiarc.embeddedmontiarc._symboltable.instanceStructure.EMAComponentInstanceSymbol;
import de.monticore.lang.monticar.generator.FileContent;
import de.monticore.lang.monticar.generator.middleware.cli.ClusteringParameters;
import de.monticore.lang.monticar.generator.middleware.clustering.ClusterFromTagsHelper;
import de.monticore.lang.monticar.generator.middleware.helpers.*;
import de.monticore.lang.monticar.generator.middleware.cli.ResultChoosingStrategy;
import de.monticore.lang.monticar.generator.middleware.clustering.*;
import de.monticore.lang.monticar.generator.middleware.helpers.FileHelper;
import de.monticore.lang.monticar.generator.middleware.helpers.NameHelper;
import de.monticore.lang.monticar.generator.middleware.helpers.RosHelper;
import de.monticore.lang.monticar.generator.middleware.helpers.TemplateHelper;
import de.monticore.lang.monticar.generator.middleware.impls.GeneratorImpl;
import de.monticore.lang.monticar.generator.middleware.impls.MiddlewareTagGenImpl;
import de.monticore.lang.tagging._symboltable.TaggingResolver;
......@@ -32,16 +36,24 @@ public class DistributedTargetGenerator extends CMakeGenerator {
public DistributedTargetGenerator() {
}
public ClusteringParameters getClusteringParameters() {
return clusteringParameters;
}
public void setClusteringParameters(ClusteringParameters clusteringParameters) {
this.clusteringParameters = clusteringParameters;
}
@Override
public void setGenerationTargetPath(String path) {
super.setGenerationTargetPath(path);
}
@Override
public List<File> generate(EMAComponentInstanceSymbol componentInstanceSymbol, TaggingResolver taggingResolver) throws IOException {
public List<File> generate(EMAComponentInstanceSymbol genComp, TaggingResolver taggingResolver) throws IOException {
Map<EMAComponentInstanceSymbol, GeneratorImpl> generatorMap = new HashMap<>();
fixComponentInstance(componentInstanceSymbol);
EMAComponentInstanceSymbol componentInstanceSymbol = preprocessing(genComp);
List<EMAComponentInstanceSymbol> clusterSubcomponents = ClusterFromTagsHelper.getClusterSubcomponents(componentInstanceSymbol);
if (clusterSubcomponents.size() > 0) {
......@@ -78,6 +90,47 @@ public class DistributedTargetGenerator extends CMakeGenerator {
return files;
}
private EMAComponentInstanceSymbol preprocessing(EMAComponentInstanceSymbol genComp) {
EMAComponentInstanceSymbol componentInstanceSymbol = genComp;
if(clusteringParameters != null){
//Flatten
if(clusteringParameters.getFlatten()){
if(clusteringParameters.getFlattenLevel().isPresent()){
Integer level = clusteringParameters.getFlattenLevel().get();
componentInstanceSymbol = FlattenArchitecture.flattenArchitecture(genComp, new HashMap<>(), level);
}else {
componentInstanceSymbol = FlattenArchitecture.flattenArchitecture(genComp);
}
}
//Cluster
if(clusteringParameters.getAlgorithmParameters().size() > 0) {
ClusteringResultList clusteringResults = AutomaticClusteringHelper.executeClusteringFromParams(componentInstanceSymbol, clusteringParameters.getAlgorithmParameters());
Optional<Integer> nOpt = clusteringParameters.getNumberOfClusters();
for(ClusteringResult c : clusteringResults){
String prefix = nOpt.isPresent() && !c.hasNumberOfClusters(nOpt.get()) ? "[IGNORED]" : "";
System.out.println(prefix + "Score was " + c.getScore() + " for " + c.getParameters().toString());
}
Optional<ClusteringResult> clusteringOpt;
if(nOpt.isPresent() && clusteringParameters.getChooseBy().equals(ResultChoosingStrategy.bestWithFittingN)){
clusteringOpt = clusteringResults.getBestResultWithFittingN(nOpt.get());
}else{
clusteringOpt = clusteringResults.getBestResultOverall();
}
if(clusteringOpt.isPresent()){
ClusteringResult clusteringResult = clusteringOpt.get();
System.out.println("Best score was " + clusteringResult.getScore() + " for " + clusteringResult.getParameters().toString());
AutomaticClusteringHelper.annotateComponentWithRosTagsForClusters(componentInstanceSymbol, clusteringResult.getClustering());
}
}
}
fixComponentInstance(componentInstanceSymbol);
return componentInstanceSymbol;
}
private File generateRosMsgGen() throws IOException {
File file = new File(generationTargetPath + "rosMsg/CMakeLists.txt");
FileUtils.write(file, TemplateHelper.getStruct_msgsCmakeTemplate());
......@@ -103,7 +156,6 @@ public class DistributedTargetGenerator extends CMakeGenerator {
fileContent.setFileName("CMakeLists.txt");
StringBuilder content = new StringBuilder();
content.append("cmake_minimum_required(VERSION 3.5)\n");
//TODO setProjectName?
content.append("project (default)\n");
content.append("set (CMAKE_CXX_STANDARD 11)\n");
......
......@@ -3,6 +3,7 @@ package de.monticore.lang.monticar.generator.middleware.cli;
import com.google.gson.*;
import com.google.gson.stream.JsonReader;
import de.monticore.lang.monticar.generator.middleware.cli.algorithms.*;
import de.se_rwth.commons.logging.Log;
import java.io.FileNotFoundException;
import java.io.FileReader;
......@@ -36,7 +37,10 @@ public class CliParametersLoader {
case "dbscan": return DBScanCliParameters.class;
case "affinitypropagation": return AffinityPropagationCliParameters.class;
case "markov": return MarkovCliParameters.class;
default: return UnknownAlgorithmCliParameters.class;
default:{
Log.warn("Loaded config of unknown clustering algorithm: " + algoName);
return UnknownAlgorithmCliParameters.class;
}
}
}
}
......
package de.monticore.lang.monticar.generator.middleware.cli;
import de.monticore.lang.monticar.generator.middleware.cli.algorithms.AlgorithmCliParameters;
import de.monticore.lang.monticar.generator.middleware.cli.algorithms.SpectralClusteringCliParameters;
import java.util.ArrayList;
import java.util.List;
......@@ -8,6 +9,8 @@ import java.util.Optional;
public class ClusteringParameters {
private Integer numberOfClusters;
private Boolean flatten;
private Integer flattenLevel;
private ResultChoosingStrategy chooseBy = ResultChoosingStrategy.bestWithFittingN;
private List<AlgorithmCliParameters> algorithmParameters = new ArrayList<>();
......@@ -23,6 +26,22 @@ public class ClusteringParameters {
}
public List<AlgorithmCliParameters> getAlgorithmParameters() {
//Override numberOfClusters for all spectral clustering parameters
if(getNumberOfClusters().isPresent()){
Integer n = getNumberOfClusters().get();
algorithmParameters.stream()
.filter(a -> a.getName().equals(AlgorithmCliParameters.TYPE_SPECTRAL_CLUSTERING))
.forEach(a -> ((SpectralClusteringCliParameters)a).setNumberOfClusters(n));
}
return algorithmParameters;
}
public boolean getFlatten(){
return flatten == null ? false : flatten;
}
public Optional<Integer> getFlattenLevel() {
return Optional.ofNullable(flattenLevel);
}
}
......@@ -154,6 +154,15 @@ public final class DistributedTargetGeneratorCli {
generator.add(new ODVGenImpl(), "odv");
}
if (cliParameters.getClusteringParameters().isPresent()) {
ClusteringParameters clusteringParameters = cliParameters.getClusteringParameters().get();
generator.setClusteringParameters(clusteringParameters);
clusteringParameters.getAlgorithmParameters().stream()
.filter(alg -> !alg.isValid())
.forEach(alg -> Log.error("Parameters for the algorithm " + alg.getName() + " are invalid!"));
}
try {
generator.generate(componentInstanceSymbol, taggingResolver);
} catch (IOException e) {
......
......@@ -3,26 +3,33 @@ package de.monticore.lang.monticar.generator.middleware.cli.algorithms;
import de.monticore.lang.monticar.generator.middleware.clustering.ClusteringAlgorithm;
import de.monticore.lang.monticar.generator.middleware.clustering.algorithms.AffinityPropagationAlgorithm;
import java.util.ArrayList;
import java.util.List;
public class AffinityPropagationCliParameters extends AlgorithmCliParameters{
public AffinityPropagationCliParameters() {
}
@Override
public String getName() {
return TYPE_AFFINITY_PROPAGATION;
}
@Override
public ClusteringAlgorithm asClusteringAlgorithm() {
return new AffinityPropagationAlgorithm();
}
@Override
public List<Object> asAlgorithmArgs(){
return new ArrayList<>();
public Object[] asAlgorithmArgs(){
return new Object[]{};
}
@Override
public boolean isValid() {
return true;
}
@Override
public String toString() {
return "AffinityPropagation";
}
}
\ No newline at end of file
......@@ -2,13 +2,12 @@ package de.monticore.lang.monticar.generator.middleware.cli.algorithms;
import de.monticore.lang.monticar.generator.middleware.clustering.ClusteringAlgorithm;
import java.util.List;
public abstract class AlgorithmCliParameters {
public static final String TYPE_SPECTRAL_CLUSTERING = "SpectralClustering";
public static final String TYPE_UNKOWN = "Unkown";
public static final String TYPE_DBSCAN = "DBScan";
public static final String TYPE_MARKOV = "Markov";
public static final String TYPE_AFFINITY_PROPAGATION = "AffinityPropagation";
protected String name;
public AlgorithmCliParameters() {
......@@ -20,7 +19,7 @@ public abstract class AlgorithmCliParameters {
public abstract ClusteringAlgorithm asClusteringAlgorithm();
public abstract List<Object> asAlgorithmArgs();
public abstract Object[] asAlgorithmArgs();
public abstract boolean isValid();
}
......@@ -30,11 +30,11 @@ public class DBScanCliParameters extends AlgorithmCliParameters {
}
@Override
public List<Object> asAlgorithmArgs() {
public Object[] asAlgorithmArgs() {
List<Object> res = new ArrayList<>();
if(!isValid()){
Log.error("DBScanCliParameters: The min_pts or radius parameters are mandatory but at least one is unset!");
return res;
return res.toArray();
}
res.add(DBSCANClusteringBuilder.DBSCANParameters.DBSCAN_MIN_PTS);
......@@ -42,7 +42,7 @@ public class DBScanCliParameters extends AlgorithmCliParameters {
res.add(DBSCANClusteringBuilder.DBSCANParameters.DBSCAN_RADIUS);
res.add(radius);
return res;
return res.toArray();
}
@Override
......@@ -65,4 +65,12 @@ public class DBScanCliParameters extends AlgorithmCliParameters {
public void setRadius(Double radius) {
this.radius = radius;
}
@Override
public String toString() {
return "DBScan{" +
"min_pts=" + min_pts +
", radius=" + radius +
'}';
}
}
......@@ -30,7 +30,7 @@ public class MarkovCliParameters extends AlgorithmCliParameters {
}
@Override
public List<Object> asAlgorithmArgs() {
public Object[] asAlgorithmArgs() {
List<Object> res = new ArrayList<>();
if(max_residual != null){
res.add(MarkovClusteringBuilder.MarkovParameters.MARKOV_MAX_RESIDUAL);
......@@ -48,7 +48,7 @@ public class MarkovCliParameters extends AlgorithmCliParameters {
res.add(MarkovClusteringBuilder.MarkovParameters.MARKOV_ZERO_MAX);
res.add(zero_max);
}
return res;
return res.toArray();
}
@Override
......@@ -87,4 +87,14 @@ public class MarkovCliParameters extends AlgorithmCliParameters {
public void setZeroMax(Double zero_max) {
this.zero_max = zero_max;
}
@Override
public String toString() {
return "Markov{" +
"max_residual=" + max_residual +
", gamma_exp=" + gamma_exp +
", loop_gain=" + loop_gain +
", zero_max=" + zero_max +
'}';
}
}
......@@ -6,7 +6,6 @@ import de.monticore.lang.monticar.generator.middleware.clustering.algorithms.Spe
import de.se_rwth.commons.logging.Log;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
public class SpectralClusteringCliParameters extends AlgorithmCliParameters {
......@@ -30,12 +29,12 @@ public class SpectralClusteringCliParameters extends AlgorithmCliParameters {
}
@Override
public List<Object> asAlgorithmArgs(){
public Object[] asAlgorithmArgs(){
ArrayList<Object> res = new ArrayList<>();
if(!isValid()){
Log.error("SpectralClusteringCliParameters: The numberOfClusters parameter is mandatory but unset!");
return res;
return res.toArray();
}
res.add(SpectralClusteringBuilder.SpectralParameters.SPECTRAL_NUM_CLUSTERS);
......@@ -51,7 +50,7 @@ public class SpectralClusteringCliParameters extends AlgorithmCliParameters {
res.add(sigma);
}
return res;
return res.toArray();
}
@Override
......@@ -82,4 +81,13 @@ public class SpectralClusteringCliParameters extends AlgorithmCliParameters {
public void setSigma(Double sigma) {
this.sigma = sigma;
}
@Override
public String toString() {
return "SpectralClustering{" +
"numberOfClusters=" + numberOfClusters +
", l=" + l +
", sigma=" + sigma +
'}';
}
}
......@@ -2,8 +2,6 @@ package de.monticore.lang.monticar.generator.middleware.cli.algorithms;
import de.monticore.lang.monticar.generator.middleware.clustering.ClusteringAlgorithm;
import java.util.List;
public class UnknownAlgorithmCliParameters extends AlgorithmCliParameters {
public UnknownAlgorithmCliParameters() {
......@@ -20,7 +18,7 @@ public class UnknownAlgorithmCliParameters extends AlgorithmCliParameters {
}
@Override
public List<Object> asAlgorithmArgs() {
public Object[] asAlgorithmArgs() {
return null;
}
......
package de.monticore.lang.monticar.generator.middleware.clustering;
import de.monticore.expressionsbasis._ast.ASTExpression;
import de.monticore.lang.embeddedmontiarc.embeddedmontiarc._symboltable.*;
import de.monticore.lang.embeddedmontiarc.embeddedmontiarc._symboltable.cncModel.EMAPortSymbol;
import de.monticore.lang.embeddedmontiarc.embeddedmontiarc._symboltable.instanceStructure.EMAComponentInstanceSymbol;
import de.monticore.lang.embeddedmontiarc.embeddedmontiarc._symboltable.instanceStructure.EMAConnectorInstanceSymbol;
import de.monticore.lang.embeddedmontiarc.tagging.middleware.ros.RosConnectionSymbol;
import de.monticore.lang.math._ast.ASTNumberExpression;
import de.monticore.lang.monticar.common2._ast.ASTCommonMatrixType;
import de.monticore.lang.monticar.generator.middleware.cli.algorithms.AlgorithmCliParameters;
import de.monticore.lang.monticar.ts.MCTypeSymbol;
import de.monticore.lang.monticar.ts.references.MCASTTypeSymbolReference;
import de.monticore.lang.monticar.ts.references.MCTypeReference;
import de.monticore.symboltable.CommonScope;
import de.monticore.symboltable.MutableScope;
import de.monticore.symboltable.Symbol;
import de.monticore.symboltable.resolving.ResolvingFilter;
import de.se_rwth.commons.logging.Log;
import de.monticore.symboltable.CommonSymbol;
import java.util.*;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
public class AutomaticClusteringHelper {
......@@ -133,6 +132,41 @@ public class AutomaticClusteringHelper {
}
public static double getTypeCostHeuristic(EMAComponentInstanceSymbol componentInstanceSymbol, List<Set<EMAComponentInstanceSymbol>> clustering){
List<Set<String>> clusteringAsNames = clustering.stream()
.map(s -> s.stream()
.map(CommonSymbol::getFullName)
.collect(Collectors.toSet()))
.collect(Collectors.toList());
List<EMAConnectorInstanceSymbol> interClusterConnectors = componentInstanceSymbol.getConnectorInstances().stream()
.filter(con -> {
EMAComponentInstanceSymbol sourceComp = con.getSourcePort().getComponentInstance();
EMAComponentInstanceSymbol targetComp = con.getTargetPort().getComponentInstance();
int sourceClusterIndex = -1;
int targetClusterIndex = -1;
for (int i = 0; i < clusteringAsNames.size(); i++) {
if (clusteringAsNames.get(i).contains(sourceComp.getFullName())) {
sourceClusterIndex = i;
}
if (clusteringAsNames.get(i).contains(targetComp.getFullName())) {
targetClusterIndex = i;
}
}
return sourceClusterIndex != targetClusterIndex;
})
.collect(Collectors.toList());
return interClusterConnectors.stream()
.map(EMAConnectorInstanceSymbol::getTargetPort)
.map(AutomaticClusteringHelper::getTypeCostHeuristic)
.mapToDouble(d -> d)
.sum();
}
public static double getTypeCostHeuristic(EMAPortSymbol port){
return getTypeCostHeuristic(port.getTypeReference());
}
......@@ -174,5 +208,13 @@ public class AutomaticClusteringHelper {
}
public static ClusteringResultList executeClusteringFromParams(EMAComponentInstanceSymbol emaComponentInstance, List<AlgorithmCliParameters> algoParams) {
ClusteringResultList res = new ClusteringResultList();
for (int i = 0; i < algoParams.size(); i++) {
System.out.println("Clustering with algorithm " + (i+1) + "/" + algoParams.size() + ": " +algoParams.get(i).toString());
res.add(ClusteringResult.fromParameters(emaComponentInstance, algoParams.get(i)));
}
return res;
}
}
......@@ -9,11 +9,14 @@ import java.util.Set;
public interface ClusteringAlgorithm {
List<Set<EMAComponentInstanceSymbol>> cluster(EMAComponentInstanceSymbol component, Object... args);
//TODO: add arguments as typed state of the algorithms
default List<Set<EMAComponentInstanceSymbol>> cluster(EMAComponentInstanceSymbol component){
return cluster(component ,getArgs());
//TODO: add arguments as typed state of the algorithms(instead of untyped)
default List<Set<EMAComponentInstanceSymbol>> clusterWithState(EMAComponentInstanceSymbol component){
Object[] args = getArgs();
return cluster(component, args);
}
List<Object> getArgs();
default Object[] getArgs(){
return null;
}
}
package de.monticore.lang.monticar.generator.middleware.clustering;
import de.monticore.lang.embeddedmontiarc.embeddedmontiarc._symboltable.instanceStructure.EMAComponentInstanceSymbol;
import de.monticore.lang.monticar.generator.middleware.cli.algorithms.AlgorithmCliParameters;
import java.util.List;
import java.util.Set;
public class ClusteringResult {
Double score = null;
private EMAComponentInstanceSymbol component;
private AlgorithmCliParameters parameters;
private List<Set<EMAComponentInstanceSymbol>> clustering;
private ClusteringResult(EMAComponentInstanceSymbol component, AlgorithmCliParameters parameters, List<Set<EMAComponentInstanceSymbol>> clustering) {
this.component = component;
this.parameters = parameters;
this.clustering = clustering;
}
public static ClusteringResult fromParameters(EMAComponentInstanceSymbol component, AlgorithmCliParameters parameters){
List<Set<EMAComponentInstanceSymbol>> res = parameters.asClusteringAlgorithm().clusterWithState(component);
return new ClusteringResult(component, parameters, res);
}
public double getScore(){
if(score == null){
score = AutomaticClusteringHelper.getTypeCostHeuristic(component, clustering);
}
return score;
}
public EMAComponentInstanceSymbol getComponent() {
return component;
}
public AlgorithmCliParameters getParameters() {
return parameters;
}
public List<Set<EMAComponentInstanceSymbol>> getClustering() {
return clustering;
}
public int getNumberOfClusters(){
return clustering.size();
}
public boolean hasNumberOfClusters(int n){
return getNumberOfClusters() == n;
}
}
package de.monticore.lang.monticar.generator.middleware.clustering;
import de.se_rwth.commons.logging.Log;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.Optional;
public class ClusteringResultList extends ArrayList<ClusteringResult> {
public ClusteringResultList() {
super();
}
public Optional<ClusteringResult> getBestResultWithFittingN(int n){
ClusteringResultList filteredList = new ClusteringResultList();
for(ClusteringResult c : this){
if(!c.hasNumberOfClusters(n)){
Log.warn("Not the right number of clusters! Ignoring the clustering from algorithm " + c.getParameters().toString());