Added ability to run multiple clustering algorithms on one component and...

Added ability to run multiple clustering algorithms on one component and compare the result, improved cli

src/main/java/de/monticore/lang/monticar/generator/middleware/DistributedTargetGenerator.java

@@ -3,8 +3,12 @@ package de.monticore.lang.monticar.generator.middleware;
 import de.monticore.lang.embeddedmontiarc.embeddedmontiarc._symboltable.instanceStructure.EMAComponentInstanceSymbol;
 import de.monticore.lang.monticar.generator.FileContent;
 import de.monticore.lang.monticar.generator.middleware.cli.ClusteringParameters;
-import de.monticore.lang.monticar.generator.middleware.clustering.ClusterFromTagsHelper;
-import de.monticore.lang.monticar.generator.middleware.helpers.*;
+import de.monticore.lang.monticar.generator.middleware.cli.ResultChoosingStrategy;
+import de.monticore.lang.monticar.generator.middleware.clustering.*;
+import de.monticore.lang.monticar.generator.middleware.helpers.FileHelper;
+import de.monticore.lang.monticar.generator.middleware.helpers.NameHelper;
+import de.monticore.lang.monticar.generator.middleware.helpers.RosHelper;
+import de.monticore.lang.monticar.generator.middleware.helpers.TemplateHelper;
 import de.monticore.lang.monticar.generator.middleware.impls.GeneratorImpl;
 import de.monticore.lang.monticar.generator.middleware.impls.MiddlewareTagGenImpl;
 import de.monticore.lang.tagging._symboltable.TaggingResolver;
@@ -32,16 +36,24 @@ public class DistributedTargetGenerator extends CMakeGenerator {
     public DistributedTargetGenerator() {
     }
 
+    public ClusteringParameters getClusteringParameters() {
+        return clusteringParameters;
+    }
+
+    public void setClusteringParameters(ClusteringParameters clusteringParameters) {
+        this.clusteringParameters = clusteringParameters;
+    }
+
     @Override
     public void setGenerationTargetPath(String path) {
         super.setGenerationTargetPath(path);
     }
 
     @Override
-    public List<File> generate(EMAComponentInstanceSymbol componentInstanceSymbol, TaggingResolver taggingResolver) throws IOException {
+    public List<File> generate(EMAComponentInstanceSymbol genComp, TaggingResolver taggingResolver) throws IOException {
         Map<EMAComponentInstanceSymbol, GeneratorImpl> generatorMap = new HashMap<>();
 
-        fixComponentInstance(componentInstanceSymbol);
+        EMAComponentInstanceSymbol componentInstanceSymbol = preprocessing(genComp);
 
         List<EMAComponentInstanceSymbol> clusterSubcomponents = ClusterFromTagsHelper.getClusterSubcomponents(componentInstanceSymbol);
         if (clusterSubcomponents.size() > 0) {
@@ -78,6 +90,47 @@ public class DistributedTargetGenerator extends CMakeGenerator {
         return files;
     }
 
+    private EMAComponentInstanceSymbol preprocessing(EMAComponentInstanceSymbol genComp) {
+        EMAComponentInstanceSymbol componentInstanceSymbol = genComp;
+        if(clusteringParameters != null){
+            //Flatten
+            if(clusteringParameters.getFlatten()){
+                if(clusteringParameters.getFlattenLevel().isPresent()){
+                    Integer level = clusteringParameters.getFlattenLevel().get();
+                    componentInstanceSymbol = FlattenArchitecture.flattenArchitecture(genComp, new HashMap<>(), level);
+                }else {
+                    componentInstanceSymbol = FlattenArchitecture.flattenArchitecture(genComp);
+                }
+            }
+
+            //Cluster
+            if(clusteringParameters.getAlgorithmParameters().size() > 0) {
+                ClusteringResultList clusteringResults = AutomaticClusteringHelper.executeClusteringFromParams(componentInstanceSymbol, clusteringParameters.getAlgorithmParameters());
+                Optional<Integer> nOpt = clusteringParameters.getNumberOfClusters();
+                for(ClusteringResult c : clusteringResults){
+                    String prefix = nOpt.isPresent() && !c.hasNumberOfClusters(nOpt.get()) ? "[IGNORED]" : "";
+                    System.out.println(prefix + "Score was " + c.getScore() + " for " + c.getParameters().toString());
+                }
+
+                Optional<ClusteringResult> clusteringOpt;
+                if(nOpt.isPresent() && clusteringParameters.getChooseBy().equals(ResultChoosingStrategy.bestWithFittingN)){
+                    clusteringOpt = clusteringResults.getBestResultWithFittingN(nOpt.get());
+                }else{
+                    clusteringOpt = clusteringResults.getBestResultOverall();
+                }
+
+                if(clusteringOpt.isPresent()){
+                    ClusteringResult clusteringResult = clusteringOpt.get();
+                    System.out.println("Best score was " + clusteringResult.getScore() + " for " + clusteringResult.getParameters().toString());
+                    AutomaticClusteringHelper.annotateComponentWithRosTagsForClusters(componentInstanceSymbol, clusteringResult.getClustering());
+                }
+            }
+        }
+
+        fixComponentInstance(componentInstanceSymbol);
+        return componentInstanceSymbol;
+    }
+
     private File generateRosMsgGen() throws IOException {
         File file = new File(generationTargetPath + "rosMsg/CMakeLists.txt");
         FileUtils.write(file, TemplateHelper.getStruct_msgsCmakeTemplate());
@@ -103,7 +156,6 @@ public class DistributedTargetGenerator extends CMakeGenerator {
         fileContent.setFileName("CMakeLists.txt");
         StringBuilder content = new StringBuilder();
         content.append("cmake_minimum_required(VERSION 3.5)\n");
-        //TODO setProjectName?
         content.append("project (default)\n");
         content.append("set (CMAKE_CXX_STANDARD 11)\n");
 

src/main/java/de/monticore/lang/monticar/generator/middleware/cli/CliParametersLoader.java

@@ -3,6 +3,7 @@ package de.monticore.lang.monticar.generator.middleware.cli;
 import com.google.gson.*;
 import com.google.gson.stream.JsonReader;
 import de.monticore.lang.monticar.generator.middleware.cli.algorithms.*;
+import de.se_rwth.commons.logging.Log;
 
 import java.io.FileNotFoundException;
 import java.io.FileReader;
@@ -36,7 +37,10 @@ public class CliParametersLoader {
                 case "dbscan": return DBScanCliParameters.class;
                 case "affinitypropagation": return AffinityPropagationCliParameters.class;
                 case "markov": return MarkovCliParameters.class;
-                default: return UnknownAlgorithmCliParameters.class;
+                default:{
+                    Log.warn("Loaded config of unknown clustering algorithm: " + algoName);
+                    return UnknownAlgorithmCliParameters.class;
+                }
             }
         }
     }

src/main/java/de/monticore/lang/monticar/generator/middleware/cli/ClusteringParameters.java

 package de.monticore.lang.monticar.generator.middleware.cli;
 
 import de.monticore.lang.monticar.generator.middleware.cli.algorithms.AlgorithmCliParameters;
+import de.monticore.lang.monticar.generator.middleware.cli.algorithms.SpectralClusteringCliParameters;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -8,6 +9,8 @@ import java.util.Optional;
 
 public class ClusteringParameters {
     private Integer numberOfClusters;
+    private Boolean flatten;
+    private Integer flattenLevel;
     private ResultChoosingStrategy chooseBy = ResultChoosingStrategy.bestWithFittingN;
     private List<AlgorithmCliParameters> algorithmParameters = new ArrayList<>();
 
@@ -23,6 +26,22 @@ public class ClusteringParameters {
     }
 
     public List<AlgorithmCliParameters> getAlgorithmParameters() {
+        //Override numberOfClusters for all spectral clustering parameters
+        if(getNumberOfClusters().isPresent()){
+            Integer n = getNumberOfClusters().get();
+            algorithmParameters.stream()
+                    .filter(a -> a.getName().equals(AlgorithmCliParameters.TYPE_SPECTRAL_CLUSTERING))
+                    .forEach(a -> ((SpectralClusteringCliParameters)a).setNumberOfClusters(n));
+        }
+
         return algorithmParameters;
     }
+
+    public boolean getFlatten(){
+        return flatten == null ? false : flatten;
+    }
+
+    public Optional<Integer> getFlattenLevel() {
+        return Optional.ofNullable(flattenLevel);
+    }
 }

src/main/java/de/monticore/lang/monticar/generator/middleware/cli/DistributedTargetGeneratorCli.java

@@ -154,6 +154,15 @@ public final class DistributedTargetGeneratorCli {
             generator.add(new ODVGenImpl(), "odv");
         }
 
+        if (cliParameters.getClusteringParameters().isPresent()) {
+            ClusteringParameters clusteringParameters = cliParameters.getClusteringParameters().get();
+            generator.setClusteringParameters(clusteringParameters);
+
+            clusteringParameters.getAlgorithmParameters().stream()
+                    .filter(alg -> !alg.isValid())
+                    .forEach(alg -> Log.error("Parameters for the algorithm " + alg.getName() + " are invalid!"));
+        }
+
         try {
             generator.generate(componentInstanceSymbol, taggingResolver);
         } catch (IOException e) {

src/main/java/de/monticore/lang/monticar/generator/middleware/cli/algorithms/AffinityPropagationCliParameters.java

@@ -3,26 +3,33 @@ package de.monticore.lang.monticar.generator.middleware.cli.algorithms;
 import de.monticore.lang.monticar.generator.middleware.clustering.ClusteringAlgorithm;
 import de.monticore.lang.monticar.generator.middleware.clustering.algorithms.AffinityPropagationAlgorithm;
 
-import java.util.ArrayList;
-import java.util.List;
-
 public class AffinityPropagationCliParameters extends AlgorithmCliParameters{
 
     public AffinityPropagationCliParameters() {
     }
 
+    @Override
+    public String getName() {
+        return TYPE_AFFINITY_PROPAGATION;
+    }
+
     @Override
     public ClusteringAlgorithm asClusteringAlgorithm() {
         return new AffinityPropagationAlgorithm();
     }
 
     @Override
-    public List<Object> asAlgorithmArgs(){
-        return new ArrayList<>();
+    public Object[] asAlgorithmArgs(){
+        return new Object[]{};
     }
 
     @Override
     public boolean isValid() {
         return true;
     }
+
+    @Override
+    public String toString() {
+        return "AffinityPropagation";
+    }
 }
\ No newline at end of file

src/main/java/de/monticore/lang/monticar/generator/middleware/cli/algorithms/AlgorithmCliParameters.java

@@ -2,13 +2,12 @@ package de.monticore.lang.monticar.generator.middleware.cli.algorithms;
 
 import de.monticore.lang.monticar.generator.middleware.clustering.ClusteringAlgorithm;
 
-import java.util.List;
-
 public abstract class AlgorithmCliParameters {
     public static final String TYPE_SPECTRAL_CLUSTERING = "SpectralClustering";
     public static final String TYPE_UNKOWN = "Unkown";
     public static final String TYPE_DBSCAN = "DBScan";
     public static final String TYPE_MARKOV = "Markov";
+    public static final String TYPE_AFFINITY_PROPAGATION = "AffinityPropagation";
     protected String name;
 
     public AlgorithmCliParameters() {
@@ -20,7 +19,7 @@ public abstract class AlgorithmCliParameters {
 
     public abstract ClusteringAlgorithm asClusteringAlgorithm();
 
-    public abstract List<Object> asAlgorithmArgs();
+    public abstract Object[] asAlgorithmArgs();
 
     public abstract boolean isValid();
 }

src/main/java/de/monticore/lang/monticar/generator/middleware/cli/algorithms/DBScanCliParameters.java

@@ -30,11 +30,11 @@ public class DBScanCliParameters extends AlgorithmCliParameters {
     }
 
     @Override
-    public List<Object> asAlgorithmArgs() {
+    public Object[] asAlgorithmArgs() {
         List<Object> res = new ArrayList<>();
         if(!isValid()){
             Log.error("DBScanCliParameters: The min_pts or radius parameters are mandatory but at least one is unset!");
-            return res;
+            return res.toArray();
         }
 
         res.add(DBSCANClusteringBuilder.DBSCANParameters.DBSCAN_MIN_PTS);
@@ -42,7 +42,7 @@ public class DBScanCliParameters extends AlgorithmCliParameters {
         res.add(DBSCANClusteringBuilder.DBSCANParameters.DBSCAN_RADIUS);
         res.add(radius);
 
-        return res;
+        return res.toArray();
     }
 
     @Override
@@ -65,4 +65,12 @@ public class DBScanCliParameters extends AlgorithmCliParameters {
     public void setRadius(Double radius) {
         this.radius = radius;
     }
+
+    @Override
+    public String toString() {
+        return "DBScan{" +
+                "min_pts=" + min_pts +
+                ", radius=" + radius +
+                '}';
+    }
 }

src/main/java/de/monticore/lang/monticar/generator/middleware/cli/algorithms/MarkovCliParameters.java

@@ -30,7 +30,7 @@ public class MarkovCliParameters extends AlgorithmCliParameters {
     }
 
     @Override
-    public List<Object> asAlgorithmArgs() {
+    public Object[] asAlgorithmArgs() {
         List<Object> res = new ArrayList<>();
         if(max_residual != null){
             res.add(MarkovClusteringBuilder.MarkovParameters.MARKOV_MAX_RESIDUAL);
@@ -48,7 +48,7 @@ public class MarkovCliParameters extends AlgorithmCliParameters {
             res.add(MarkovClusteringBuilder.MarkovParameters.MARKOV_ZERO_MAX);
             res.add(zero_max);
         }
-        return res;
+        return res.toArray();
     }
 
     @Override
@@ -87,4 +87,14 @@ public class MarkovCliParameters extends AlgorithmCliParameters {
     public void setZeroMax(Double zero_max) {
         this.zero_max = zero_max;
     }
+
+    @Override
+    public String toString() {
+        return "Markov{" +
+                "max_residual=" + max_residual +
+                ", gamma_exp=" + gamma_exp +
+                ", loop_gain=" + loop_gain +
+                ", zero_max=" + zero_max +
+                '}';
+    }
 }

src/main/java/de/monticore/lang/monticar/generator/middleware/cli/algorithms/SpectralClusteringCliParameters.java

@@ -6,7 +6,6 @@ import de.monticore.lang.monticar.generator.middleware.clustering.algorithms.Spe
 import de.se_rwth.commons.logging.Log;
 
 import java.util.ArrayList;
-import java.util.List;
 import java.util.Optional;
 
 public class SpectralClusteringCliParameters extends AlgorithmCliParameters {
@@ -30,12 +29,12 @@ public class SpectralClusteringCliParameters extends AlgorithmCliParameters {
     }
 
     @Override
-    public List<Object> asAlgorithmArgs(){
+    public Object[] asAlgorithmArgs(){
         ArrayList<Object> res = new ArrayList<>();
 
         if(!isValid()){
             Log.error("SpectralClusteringCliParameters: The numberOfClusters parameter is mandatory but unset!");
-            return res;
+            return res.toArray();
         }
 
         res.add(SpectralClusteringBuilder.SpectralParameters.SPECTRAL_NUM_CLUSTERS);
@@ -51,7 +50,7 @@ public class SpectralClusteringCliParameters extends AlgorithmCliParameters {
             res.add(sigma);
         }
 
-        return res;
+        return res.toArray();
     }
 
     @Override
@@ -82,4 +81,13 @@ public class SpectralClusteringCliParameters extends AlgorithmCliParameters {
     public void setSigma(Double sigma) {
         this.sigma = sigma;
     }
+
+    @Override
+    public String toString() {
+        return "SpectralClustering{" +
+                "numberOfClusters=" + numberOfClusters +
+                ", l=" + l +
+                ", sigma=" + sigma +
+                '}';
+    }
 }

src/main/java/de/monticore/lang/monticar/generator/middleware/cli/algorithms/UnknownAlgorithmCliParameters.java

@@ -2,8 +2,6 @@ package de.monticore.lang.monticar.generator.middleware.cli.algorithms;
 
 import de.monticore.lang.monticar.generator.middleware.clustering.ClusteringAlgorithm;
 
-import java.util.List;
-
 public class UnknownAlgorithmCliParameters extends AlgorithmCliParameters {
 
     public UnknownAlgorithmCliParameters() {
@@ -20,7 +18,7 @@ public class UnknownAlgorithmCliParameters extends AlgorithmCliParameters {
     }
 
     @Override
-    public List<Object> asAlgorithmArgs() {
+    public Object[] asAlgorithmArgs() {
         return null;
     }
 

src/main/java/de/monticore/lang/monticar/generator/middleware/clustering/AutomaticClusteringHelper.java

 package de.monticore.lang.monticar.generator.middleware.clustering;
 
 import de.monticore.expressionsbasis._ast.ASTExpression;
-import de.monticore.lang.embeddedmontiarc.embeddedmontiarc._symboltable.*;
 import de.monticore.lang.embeddedmontiarc.embeddedmontiarc._symboltable.cncModel.EMAPortSymbol;
 import de.monticore.lang.embeddedmontiarc.embeddedmontiarc._symboltable.instanceStructure.EMAComponentInstanceSymbol;
 import de.monticore.lang.embeddedmontiarc.embeddedmontiarc._symboltable.instanceStructure.EMAConnectorInstanceSymbol;
 import de.monticore.lang.embeddedmontiarc.tagging.middleware.ros.RosConnectionSymbol;
 import de.monticore.lang.math._ast.ASTNumberExpression;
 import de.monticore.lang.monticar.common2._ast.ASTCommonMatrixType;
+import de.monticore.lang.monticar.generator.middleware.cli.algorithms.AlgorithmCliParameters;
 import de.monticore.lang.monticar.ts.MCTypeSymbol;
 import de.monticore.lang.monticar.ts.references.MCASTTypeSymbolReference;
 import de.monticore.lang.monticar.ts.references.MCTypeReference;
-import de.monticore.symboltable.CommonScope;
-import de.monticore.symboltable.MutableScope;
-import de.monticore.symboltable.Symbol;
-import de.monticore.symboltable.resolving.ResolvingFilter;
-import de.se_rwth.commons.logging.Log;
+import de.monticore.symboltable.CommonSymbol;
 
-import java.util.*;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
 import java.util.stream.Collectors;
 
 public class AutomaticClusteringHelper {
@@ -133,6 +132,41 @@ public class AutomaticClusteringHelper {
 
     }
 
+    public static double getTypeCostHeuristic(EMAComponentInstanceSymbol componentInstanceSymbol, List<Set<EMAComponentInstanceSymbol>> clustering){
+        List<Set<String>> clusteringAsNames = clustering.stream()
+                .map(s -> s.stream()
+                        .map(CommonSymbol::getFullName)
+                        .collect(Collectors.toSet()))
+                .collect(Collectors.toList());
+
+        List<EMAConnectorInstanceSymbol> interClusterConnectors = componentInstanceSymbol.getConnectorInstances().stream()
+                .filter(con -> {
+                    EMAComponentInstanceSymbol sourceComp = con.getSourcePort().getComponentInstance();
+                    EMAComponentInstanceSymbol targetComp = con.getTargetPort().getComponentInstance();
+
+                    int sourceClusterIndex = -1;
+                    int targetClusterIndex = -1;
+
+                    for (int i = 0; i < clusteringAsNames.size(); i++) {
+                        if (clusteringAsNames.get(i).contains(sourceComp.getFullName())) {
+                            sourceClusterIndex = i;
+                        }
+                        if (clusteringAsNames.get(i).contains(targetComp.getFullName())) {
+                            targetClusterIndex = i;
+                        }
+                    }
+
+                    return sourceClusterIndex != targetClusterIndex;
+                })
+                .collect(Collectors.toList());
+
+        return interClusterConnectors.stream()
+                .map(EMAConnectorInstanceSymbol::getTargetPort)
+                .map(AutomaticClusteringHelper::getTypeCostHeuristic)
+                .mapToDouble(d -> d)
+                .sum();
+    }
+
     public static double getTypeCostHeuristic(EMAPortSymbol port){
         return getTypeCostHeuristic(port.getTypeReference());
     }
@@ -174,5 +208,13 @@ public class AutomaticClusteringHelper {
     }
 
 
+    public static ClusteringResultList executeClusteringFromParams(EMAComponentInstanceSymbol emaComponentInstance, List<AlgorithmCliParameters> algoParams) {
+        ClusteringResultList res = new ClusteringResultList();
+        for (int i = 0; i < algoParams.size(); i++) {
+            System.out.println("Clustering with algorithm " + (i+1) + "/" + algoParams.size() + ": " +algoParams.get(i).toString());
+            res.add(ClusteringResult.fromParameters(emaComponentInstance, algoParams.get(i)));
+        }
+        return res;
+    }
 
 }

src/main/java/de/monticore/lang/monticar/generator/middleware/clustering/ClusteringAlgorithm.java

@@ -9,11 +9,14 @@ import java.util.Set;
 public interface ClusteringAlgorithm {
     List<Set<EMAComponentInstanceSymbol>> cluster(EMAComponentInstanceSymbol component, Object... args);
 
-    //TODO: add arguments as typed state of the algorithms
-    default List<Set<EMAComponentInstanceSymbol>> cluster(EMAComponentInstanceSymbol component){
-        return cluster(component ,getArgs());
+    //TODO: add arguments as typed state of the algorithms(instead of untyped)
+    default List<Set<EMAComponentInstanceSymbol>> clusterWithState(EMAComponentInstanceSymbol component){
+        Object[] args = getArgs();
+        return cluster(component, args);
     }
 
-    List<Object> getArgs();
+    default Object[] getArgs(){
+        return null;
+    }
 
 }

src/main/java/de/monticore/lang/monticar/generator/middleware/clustering/ClusteringResult.java

+package de.monticore.lang.monticar.generator.middleware.clustering;
+
+import de.monticore.lang.embeddedmontiarc.embeddedmontiarc._symboltable.instanceStructure.EMAComponentInstanceSymbol;
+import de.monticore.lang.monticar.generator.middleware.cli.algorithms.AlgorithmCliParameters;
+
+import java.util.List;
+import java.util.Set;
+
+public class ClusteringResult {
+    Double score = null;
+    private EMAComponentInstanceSymbol component;
+
+    private AlgorithmCliParameters parameters;
+    private List<Set<EMAComponentInstanceSymbol>> clustering;
+
+    private ClusteringResult(EMAComponentInstanceSymbol component, AlgorithmCliParameters parameters, List<Set<EMAComponentInstanceSymbol>> clustering) {
+        this.component = component;
+        this.parameters = parameters;
+        this.clustering = clustering;
+    }
+
+    public static ClusteringResult fromParameters(EMAComponentInstanceSymbol component, AlgorithmCliParameters parameters){
+        List<Set<EMAComponentInstanceSymbol>> res = parameters.asClusteringAlgorithm().clusterWithState(component);
+        return new ClusteringResult(component, parameters, res);
+    }
+
+    public double getScore(){
+        if(score == null){
+            score = AutomaticClusteringHelper.getTypeCostHeuristic(component, clustering);
+        }
+        return score;
+    }
+
+    public EMAComponentInstanceSymbol getComponent() {
+        return component;
+    }
+
+    public AlgorithmCliParameters getParameters() {
+        return parameters;
+    }
+
+    public List<Set<EMAComponentInstanceSymbol>> getClustering() {
+        return clustering;
+    }
+
+    public int getNumberOfClusters(){
+        return clustering.size();
+    }
+
+    public boolean hasNumberOfClusters(int n){
+        return getNumberOfClusters() == n;
+    }
+}

src/main/java/de/monticore/lang/monticar/generator/middleware/clustering/ClusteringResultList.java

+package de.monticore.lang.monticar.generator.middleware.clustering;
+
+import de.se_rwth.commons.logging.Log;
+
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.Optional;
+
+public class ClusteringResultList extends ArrayList<ClusteringResult> {
+
+    public ClusteringResultList() {
+        super();
+    }
+
+    public Optional<ClusteringResult> getBestResultWithFittingN(int n){
+        ClusteringResultList filteredList = new ClusteringResultList();
+        for(ClusteringResult c : this){
+            if(!c.hasNumberOfClusters(n)){
+                Log.warn("Not the right number of clusters! Ignoring the clustering from algorithm " + c.getParameters().toString());