Added ability to run multiple clustering algorithms on one component and...

Added ability to run multiple clustering algorithms on one component and compare the result, improved cli
parent 79d05e3f
Pipeline #102185 failed with stages
in 23 minutes and 55 seconds
......@@ -3,8 +3,12 @@ package de.monticore.lang.monticar.generator.middleware;
import de.monticore.lang.embeddedmontiarc.embeddedmontiarc._symboltable.instanceStructure.EMAComponentInstanceSymbol;
import de.monticore.lang.monticar.generator.FileContent;
import de.monticore.lang.monticar.generator.middleware.cli.ClusteringParameters;
import de.monticore.lang.monticar.generator.middleware.clustering.ClusterFromTagsHelper;
import de.monticore.lang.monticar.generator.middleware.helpers.*;
import de.monticore.lang.monticar.generator.middleware.cli.ResultChoosingStrategy;
import de.monticore.lang.monticar.generator.middleware.clustering.*;
import de.monticore.lang.monticar.generator.middleware.helpers.FileHelper;
import de.monticore.lang.monticar.generator.middleware.helpers.NameHelper;
import de.monticore.lang.monticar.generator.middleware.helpers.RosHelper;
import de.monticore.lang.monticar.generator.middleware.helpers.TemplateHelper;
import de.monticore.lang.monticar.generator.middleware.impls.GeneratorImpl;
import de.monticore.lang.monticar.generator.middleware.impls.MiddlewareTagGenImpl;
import de.monticore.lang.tagging._symboltable.TaggingResolver;
......@@ -32,16 +36,24 @@ public class DistributedTargetGenerator extends CMakeGenerator {
public DistributedTargetGenerator() {
}
public ClusteringParameters getClusteringParameters() {
return clusteringParameters;
}
public void setClusteringParameters(ClusteringParameters clusteringParameters) {
this.clusteringParameters = clusteringParameters;
}
@Override
public void setGenerationTargetPath(String path) {
super.setGenerationTargetPath(path);
}
@Override
public List<File> generate(EMAComponentInstanceSymbol componentInstanceSymbol, TaggingResolver taggingResolver) throws IOException {
public List<File> generate(EMAComponentInstanceSymbol genComp, TaggingResolver taggingResolver) throws IOException {
Map<EMAComponentInstanceSymbol, GeneratorImpl> generatorMap = new HashMap<>();
fixComponentInstance(componentInstanceSymbol);
EMAComponentInstanceSymbol componentInstanceSymbol = preprocessing(genComp);
List<EMAComponentInstanceSymbol> clusterSubcomponents = ClusterFromTagsHelper.getClusterSubcomponents(componentInstanceSymbol);
if (clusterSubcomponents.size() > 0) {
......@@ -78,6 +90,47 @@ public class DistributedTargetGenerator extends CMakeGenerator {
return files;
}
private EMAComponentInstanceSymbol preprocessing(EMAComponentInstanceSymbol genComp) {
EMAComponentInstanceSymbol componentInstanceSymbol = genComp;
if(clusteringParameters != null){
//Flatten
if(clusteringParameters.getFlatten()){
if(clusteringParameters.getFlattenLevel().isPresent()){
Integer level = clusteringParameters.getFlattenLevel().get();
componentInstanceSymbol = FlattenArchitecture.flattenArchitecture(genComp, new HashMap<>(), level);
}else {
componentInstanceSymbol = FlattenArchitecture.flattenArchitecture(genComp);
}
}
//Cluster
if(clusteringParameters.getAlgorithmParameters().size() > 0) {
ClusteringResultList clusteringResults = AutomaticClusteringHelper.executeClusteringFromParams(componentInstanceSymbol, clusteringParameters.getAlgorithmParameters());
Optional<Integer> nOpt = clusteringParameters.getNumberOfClusters();
for(ClusteringResult c : clusteringResults){
String prefix = nOpt.isPresent() && !c.hasNumberOfClusters(nOpt.get()) ? "[IGNORED]" : "";
System.out.println(prefix + "Score was " + c.getScore() + " for " + c.getParameters().toString());
}
Optional<ClusteringResult> clusteringOpt;
if(nOpt.isPresent() && clusteringParameters.getChooseBy().equals(ResultChoosingStrategy.bestWithFittingN)){
clusteringOpt = clusteringResults.getBestResultWithFittingN(nOpt.get());
}else{
clusteringOpt = clusteringResults.getBestResultOverall();
}
if(clusteringOpt.isPresent()){
ClusteringResult clusteringResult = clusteringOpt.get();
System.out.println("Best score was " + clusteringResult.getScore() + " for " + clusteringResult.getParameters().toString());
AutomaticClusteringHelper.annotateComponentWithRosTagsForClusters(componentInstanceSymbol, clusteringResult.getClustering());
}
}
}
fixComponentInstance(componentInstanceSymbol);
return componentInstanceSymbol;
}
private File generateRosMsgGen() throws IOException {
File file = new File(generationTargetPath + "rosMsg/CMakeLists.txt");
FileUtils.write(file, TemplateHelper.getStruct_msgsCmakeTemplate());
......@@ -103,7 +156,6 @@ public class DistributedTargetGenerator extends CMakeGenerator {
fileContent.setFileName("CMakeLists.txt");
StringBuilder content = new StringBuilder();
content.append("cmake_minimum_required(VERSION 3.5)\n");
//TODO setProjectName?
content.append("project (default)\n");
content.append("set (CMAKE_CXX_STANDARD 11)\n");
......
......@@ -3,6 +3,7 @@ package de.monticore.lang.monticar.generator.middleware.cli;
import com.google.gson.*;
import com.google.gson.stream.JsonReader;
import de.monticore.lang.monticar.generator.middleware.cli.algorithms.*;
import de.se_rwth.commons.logging.Log;
import java.io.FileNotFoundException;
import java.io.FileReader;
......@@ -36,7 +37,10 @@ public class CliParametersLoader {
case "dbscan": return DBScanCliParameters.class;
case "affinitypropagation": return AffinityPropagationCliParameters.class;
case "markov": return MarkovCliParameters.class;
default: return UnknownAlgorithmCliParameters.class;
default:{
Log.warn("Loaded config of unknown clustering algorithm: " + algoName);
return UnknownAlgorithmCliParameters.class;
}
}
}
}
......
package de.monticore.lang.monticar.generator.middleware.cli;
import de.monticore.lang.monticar.generator.middleware.cli.algorithms.AlgorithmCliParameters;
import de.monticore.lang.monticar.generator.middleware.cli.algorithms.SpectralClusteringCliParameters;
import java.util.ArrayList;
import java.util.List;
......@@ -8,6 +9,8 @@ import java.util.Optional;
public class ClusteringParameters {
private Integer numberOfClusters;
private Boolean flatten;
private Integer flattenLevel;
private ResultChoosingStrategy chooseBy = ResultChoosingStrategy.bestWithFittingN;
private List<AlgorithmCliParameters> algorithmParameters = new ArrayList<>();
......@@ -23,6 +26,22 @@ public class ClusteringParameters {
}
public List<AlgorithmCliParameters> getAlgorithmParameters() {
//Override numberOfClusters for all spectral clustering parameters
if(getNumberOfClusters().isPresent()){
Integer n = getNumberOfClusters().get();
algorithmParameters.stream()
.filter(a -> a.getName().equals(AlgorithmCliParameters.TYPE_SPECTRAL_CLUSTERING))
.forEach(a -> ((SpectralClusteringCliParameters)a).setNumberOfClusters(n));
}
return algorithmParameters;
}
public boolean getFlatten(){
return flatten == null ? false : flatten;
}
public Optional<Integer> getFlattenLevel() {
return Optional.ofNullable(flattenLevel);
}
}
......@@ -154,6 +154,15 @@ public final class DistributedTargetGeneratorCli {
generator.add(new ODVGenImpl(), "odv");
}
if (cliParameters.getClusteringParameters().isPresent()) {
ClusteringParameters clusteringParameters = cliParameters.getClusteringParameters().get();
generator.setClusteringParameters(clusteringParameters);
clusteringParameters.getAlgorithmParameters().stream()
.filter(alg -> !alg.isValid())
.forEach(alg -> Log.error("Parameters for the algorithm " + alg.getName() + " are invalid!"));
}
try {
generator.generate(componentInstanceSymbol, taggingResolver);
} catch (IOException e) {
......
......@@ -3,26 +3,33 @@ package de.monticore.lang.monticar.generator.middleware.cli.algorithms;
import de.monticore.lang.monticar.generator.middleware.clustering.ClusteringAlgorithm;
import de.monticore.lang.monticar.generator.middleware.clustering.algorithms.AffinityPropagationAlgorithm;
import java.util.ArrayList;
import java.util.List;
public class AffinityPropagationCliParameters extends AlgorithmCliParameters{
public AffinityPropagationCliParameters() {
}
@Override
public String getName() {
return TYPE_AFFINITY_PROPAGATION;
}
@Override
public ClusteringAlgorithm asClusteringAlgorithm() {
return new AffinityPropagationAlgorithm();
}
@Override
public List<Object> asAlgorithmArgs(){
return new ArrayList<>();
public Object[] asAlgorithmArgs(){
return new Object[]{};
}
@Override
public boolean isValid() {
return true;
}
@Override
public String toString() {
return "AffinityPropagation";
}
}
\ No newline at end of file
......@@ -2,13 +2,12 @@ package de.monticore.lang.monticar.generator.middleware.cli.algorithms;
import de.monticore.lang.monticar.generator.middleware.clustering.ClusteringAlgorithm;
import java.util.List;
public abstract class AlgorithmCliParameters {
public static final String TYPE_SPECTRAL_CLUSTERING = "SpectralClustering";
public static final String TYPE_UNKOWN = "Unkown";
public static final String TYPE_DBSCAN = "DBScan";
public static final String TYPE_MARKOV = "Markov";
public static final String TYPE_AFFINITY_PROPAGATION = "AffinityPropagation";
protected String name;
public AlgorithmCliParameters() {
......@@ -20,7 +19,7 @@ public abstract class AlgorithmCliParameters {
public abstract ClusteringAlgorithm asClusteringAlgorithm();
public abstract List<Object> asAlgorithmArgs();
public abstract Object[] asAlgorithmArgs();
public abstract boolean isValid();
}
......@@ -30,11 +30,11 @@ public class DBScanCliParameters extends AlgorithmCliParameters {
}
@Override
public List<Object> asAlgorithmArgs() {
public Object[] asAlgorithmArgs() {
List<Object> res = new ArrayList<>();
if(!isValid()){
Log.error("DBScanCliParameters: The min_pts or radius parameters are mandatory but at least one is unset!");
return res;
return res.toArray();
}
res.add(DBSCANClusteringBuilder.DBSCANParameters.DBSCAN_MIN_PTS);
......@@ -42,7 +42,7 @@ public class DBScanCliParameters extends AlgorithmCliParameters {
res.add(DBSCANClusteringBuilder.DBSCANParameters.DBSCAN_RADIUS);
res.add(radius);
return res;
return res.toArray();
}
@Override
......@@ -65,4 +65,12 @@ public class DBScanCliParameters extends AlgorithmCliParameters {
public void setRadius(Double radius) {
this.radius = radius;
}
@Override
public String toString() {
return "DBScan{" +
"min_pts=" + min_pts +
", radius=" + radius +
'}';
}
}
......@@ -30,7 +30,7 @@ public class MarkovCliParameters extends AlgorithmCliParameters {
}
@Override
public List<Object> asAlgorithmArgs() {
public Object[] asAlgorithmArgs() {
List<Object> res = new ArrayList<>();
if(max_residual != null){
res.add(MarkovClusteringBuilder.MarkovParameters.MARKOV_MAX_RESIDUAL);
......@@ -48,7 +48,7 @@ public class MarkovCliParameters extends AlgorithmCliParameters {
res.add(MarkovClusteringBuilder.MarkovParameters.MARKOV_ZERO_MAX);
res.add(zero_max);
}
return res;
return res.toArray();
}
@Override
......@@ -87,4 +87,14 @@ public class MarkovCliParameters extends AlgorithmCliParameters {
public void setZeroMax(Double zero_max) {
this.zero_max = zero_max;
}
@Override
public String toString() {
return "Markov{" +
"max_residual=" + max_residual +
", gamma_exp=" + gamma_exp +
", loop_gain=" + loop_gain +
", zero_max=" + zero_max +
'}';
}
}
......@@ -6,7 +6,6 @@ import de.monticore.lang.monticar.generator.middleware.clustering.algorithms.Spe
import de.se_rwth.commons.logging.Log;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
public class SpectralClusteringCliParameters extends AlgorithmCliParameters {
......@@ -30,12 +29,12 @@ public class SpectralClusteringCliParameters extends AlgorithmCliParameters {
}
@Override
public List<Object> asAlgorithmArgs(){
public Object[] asAlgorithmArgs(){
ArrayList<Object> res = new ArrayList<>();
if(!isValid()){
Log.error("SpectralClusteringCliParameters: The numberOfClusters parameter is mandatory but unset!");
return res;
return res.toArray();
}
res.add(SpectralClusteringBuilder.SpectralParameters.SPECTRAL_NUM_CLUSTERS);
......@@ -51,7 +50,7 @@ public class SpectralClusteringCliParameters extends AlgorithmCliParameters {
res.add(sigma);
}
return res;
return res.toArray();
}
@Override
......@@ -82,4 +81,13 @@ public class SpectralClusteringCliParameters extends AlgorithmCliParameters {
public void setSigma(Double sigma) {
this.sigma = sigma;
}
@Override
public String toString() {
return "SpectralClustering{" +
"numberOfClusters=" + numberOfClusters +
", l=" + l +
", sigma=" + sigma +
'}';
}
}
......@@ -2,8 +2,6 @@ package de.monticore.lang.monticar.generator.middleware.cli.algorithms;
import de.monticore.lang.monticar.generator.middleware.clustering.ClusteringAlgorithm;
import java.util.List;
public class UnknownAlgorithmCliParameters extends AlgorithmCliParameters {
public UnknownAlgorithmCliParameters() {
......@@ -20,7 +18,7 @@ public class UnknownAlgorithmCliParameters extends AlgorithmCliParameters {
}
@Override
public List<Object> asAlgorithmArgs() {
public Object[] asAlgorithmArgs() {
return null;
}
......
package de.monticore.lang.monticar.generator.middleware.clustering;
import de.monticore.expressionsbasis._ast.ASTExpression;
import de.monticore.lang.embeddedmontiarc.embeddedmontiarc._symboltable.*;
import de.monticore.lang.embeddedmontiarc.embeddedmontiarc._symboltable.cncModel.EMAPortSymbol;
import de.monticore.lang.embeddedmontiarc.embeddedmontiarc._symboltable.instanceStructure.EMAComponentInstanceSymbol;
import de.monticore.lang.embeddedmontiarc.embeddedmontiarc._symboltable.instanceStructure.EMAConnectorInstanceSymbol;
import de.monticore.lang.embeddedmontiarc.tagging.middleware.ros.RosConnectionSymbol;
import de.monticore.lang.math._ast.ASTNumberExpression;
import de.monticore.lang.monticar.common2._ast.ASTCommonMatrixType;
import de.monticore.lang.monticar.generator.middleware.cli.algorithms.AlgorithmCliParameters;
import de.monticore.lang.monticar.ts.MCTypeSymbol;
import de.monticore.lang.monticar.ts.references.MCASTTypeSymbolReference;
import de.monticore.lang.monticar.ts.references.MCTypeReference;
import de.monticore.symboltable.CommonScope;
import de.monticore.symboltable.MutableScope;
import de.monticore.symboltable.Symbol;
import de.monticore.symboltable.resolving.ResolvingFilter;
import de.se_rwth.commons.logging.Log;
import de.monticore.symboltable.CommonSymbol;
import java.util.*;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
public class AutomaticClusteringHelper {
......@@ -133,6 +132,41 @@ public class AutomaticClusteringHelper {
}
public static double getTypeCostHeuristic(EMAComponentInstanceSymbol componentInstanceSymbol, List<Set<EMAComponentInstanceSymbol>> clustering){
List<Set<String>> clusteringAsNames = clustering.stream()
.map(s -> s.stream()
.map(CommonSymbol::getFullName)
.collect(Collectors.toSet()))
.collect(Collectors.toList());
List<EMAConnectorInstanceSymbol> interClusterConnectors = componentInstanceSymbol.getConnectorInstances().stream()
.filter(con -> {
EMAComponentInstanceSymbol sourceComp = con.getSourcePort().getComponentInstance();
EMAComponentInstanceSymbol targetComp = con.getTargetPort().getComponentInstance();
int sourceClusterIndex = -1;
int targetClusterIndex = -1;
for (int i = 0; i < clusteringAsNames.size(); i++) {
if (clusteringAsNames.get(i).contains(sourceComp.getFullName())) {
sourceClusterIndex = i;
}
if (clusteringAsNames.get(i).contains(targetComp.getFullName())) {
targetClusterIndex = i;
}
}
return sourceClusterIndex != targetClusterIndex;
})
.collect(Collectors.toList());
return interClusterConnectors.stream()
.map(EMAConnectorInstanceSymbol::getTargetPort)
.map(AutomaticClusteringHelper::getTypeCostHeuristic)
.mapToDouble(d -> d)
.sum();
}
public static double getTypeCostHeuristic(EMAPortSymbol port){
return getTypeCostHeuristic(port.getTypeReference());
}
......@@ -174,5 +208,13 @@ public class AutomaticClusteringHelper {
}
public static ClusteringResultList executeClusteringFromParams(EMAComponentInstanceSymbol emaComponentInstance, List<AlgorithmCliParameters> algoParams) {
ClusteringResultList res = new ClusteringResultList();
for (int i = 0; i < algoParams.size(); i++) {
System.out.println("Clustering with algorithm " + (i+1) + "/" + algoParams.size() + ": " +algoParams.get(i).toString());
res.add(ClusteringResult.fromParameters(emaComponentInstance, algoParams.get(i)));
}
return res;
}
}
......@@ -9,11 +9,14 @@ import java.util.Set;
public interface ClusteringAlgorithm {
List<Set<EMAComponentInstanceSymbol>> cluster(EMAComponentInstanceSymbol component, Object... args);
//TODO: add arguments as typed state of the algorithms
default List<Set<EMAComponentInstanceSymbol>> cluster(EMAComponentInstanceSymbol component){
return cluster(component ,getArgs());
//TODO: add arguments as typed state of the algorithms(instead of untyped)
default List<Set<EMAComponentInstanceSymbol>> clusterWithState(EMAComponentInstanceSymbol component){
Object[] args = getArgs();
return cluster(component, args);
}
List<Object> getArgs();
default Object[] getArgs(){
return null;
}
}
package de.monticore.lang.monticar.generator.middleware.clustering;
import de.monticore.lang.embeddedmontiarc.embeddedmontiarc._symboltable.instanceStructure.EMAComponentInstanceSymbol;
import de.monticore.lang.monticar.generator.middleware.cli.algorithms.AlgorithmCliParameters;
import java.util.List;
import java.util.Set;
public class ClusteringResult {
Double score = null;
private EMAComponentInstanceSymbol component;
private AlgorithmCliParameters parameters;
private List<Set<EMAComponentInstanceSymbol>> clustering;
private ClusteringResult(EMAComponentInstanceSymbol component, AlgorithmCliParameters parameters, List<Set<EMAComponentInstanceSymbol>> clustering) {
this.component = component;
this.parameters = parameters;
this.clustering = clustering;
}
public static ClusteringResult fromParameters(EMAComponentInstanceSymbol component, AlgorithmCliParameters parameters){
List<Set<EMAComponentInstanceSymbol>> res = parameters.asClusteringAlgorithm().clusterWithState(component);
return new ClusteringResult(component, parameters, res);
}
public double getScore(){
if(score == null){
score = AutomaticClusteringHelper.getTypeCostHeuristic(component, clustering);
}
return score;
}
public EMAComponentInstanceSymbol getComponent() {
return component;
}
public AlgorithmCliParameters getParameters() {
return parameters;
}
public List<Set<EMAComponentInstanceSymbol>> getClustering() {
return clustering;
}
public int getNumberOfClusters(){
return clustering.size();
}
public boolean hasNumberOfClusters(int n){
return getNumberOfClusters() == n;
}
}
package de.monticore.lang.monticar.generator.middleware.clustering;
import de.se_rwth.commons.logging.Log;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.Optional;
public class ClusteringResultList extends ArrayList<ClusteringResult> {
public ClusteringResultList() {
super();
}
public Optional<ClusteringResult> getBestResultWithFittingN(int n){
ClusteringResultList filteredList = new ClusteringResultList();
for(ClusteringResult c : this){
if(!c.hasNumberOfClusters(n)){
Log.warn("Not the right number of clusters! Ignoring the clustering from algorithm " + c.getParameters().toString());
}else{
filteredList.add(c);
}
}
return filteredList.getBestResultOverall();
}
public Optional<ClusteringResult> getBestResultOverall(){
this.sort(Comparator.comparing(ClusteringResult::getScore));
return this.size() == 0 ? Optional.empty() : Optional.of(this.get(0));
}
}
......@@ -13,14 +13,14 @@ import java.util.*;
public class AffinityPropagationAlgorithm implements ClusteringAlgorithm {
private List<Object> args;
private Object[] args;
public void setArgs(List<Object> args) {
public void setArgs(Object[] args) {
this.args = args;
}
@Override
public List<Object> getArgs() {
public Object[] getArgs() {
return args;
}
......
......@@ -11,14 +11,14 @@ import java.util.*;
// DBSCAN clusterer product implementation
public class DBSCANClusteringAlgorithm implements ClusteringAlgorithm {
private List<Object> args;
private Object[] args;
public void setArgs(List<Object> args) {
public void setArgs(Object[] args) {
this.args = args;
}
@Override
public List<Object> getArgs() {
public Object[] getArgs() {
return args;
}
......
......@@ -15,14 +15,14 @@ import java.util.*;
// markov mcl clusterer product implementation
public class MarkovClusteringAlgorithm implements ClusteringAlgorithm {
private List<Object> args;
private Object[] args;
public void setArgs(List<Object> args) {