Commit 80b994bc authored by Michael Günther Beyer's avatar Michael Günther Beyer

Merge branch 'ML_clustering' of...

Merge branch 'ML_clustering' of https://git.rwth-aachen.de/monticore/EmbeddedMontiArc/generators/EMAM2Middleware into ML_clustering
parents 5f161a2f b41df725
Pipeline #104122 failed with stages
in 19 minutes and 40 seconds
......@@ -80,7 +80,7 @@ ROS Kinetic currently only supports Linux and the installation is described [her
## Usage
### CLI
Maven generates the jar `embedded-montiarc-math-middleware-generator-{Version}-jar-with-dependencies.jar`
and the cli is located in `de.monticore.lang.monticar.generator.middleware.DistributedTargetGeneratorCli`.
and the cli is located in `de.monticore.lang.monticar.generator.middleware.cli.DistributedTargetGeneratorCli`.
Parameters: `${file path to config json}` OR `-r ${raw json config string}`
```
......
......@@ -226,7 +226,7 @@
<configuration>
<archive>
<manifest>
<mainClass>de.monticore.lang.monticar.generator.middleware.DistributedTargetGeneratorCli</mainClass>
<mainClass>de.monticore.lang.monticar.generator.middleware.cli.DistributedTargetGeneratorCli</mainClass>
</manifest>
</archive>
<descriptorRefs>
......
......@@ -12,7 +12,7 @@ o Order all sub components by name (neccessary for the adjacency matrix).
o Create adjacency matrix to use with a clustering algorithm, with subcomponents as nodes and connectors between subcomponents as vertices. Sift out all connectors to the super component.
2) Feed adjacency matrix into the selected clustering algorithm
o We are using the machine learning library "smile ml" (see: https://github.com/haifengl/smile) which provides a broad range of different clustering and partitioning approaches. As a prime example we are using "spectral clustering" here. For a closer look at this approach, see the section below.
o The clustring algorithm yields multiple cluster labels with the clustered entries of the adjacency matrix assigned to them. We have to convert them back to a set of symbol tables of components representing the clusters.
o The clustering algorithm yields multiple cluster labels with the clustered entries of the adjacency matrix assigned to them. We have to convert them back to a set of symbol tables of components representing the clusters.
3) Generate middleware tags separating the clusters
o This will build the cluster-to-ROS connections.
o We won’t take account of ports of the super component and only consider connected top level components.
......
......@@ -2,8 +2,13 @@ package de.monticore.lang.monticar.generator.middleware;
import de.monticore.lang.embeddedmontiarc.embeddedmontiarc._symboltable.instanceStructure.EMAComponentInstanceSymbol;
import de.monticore.lang.monticar.generator.FileContent;
import de.monticore.lang.monticar.generator.middleware.clustering.ClusterFromTagsHelper;
import de.monticore.lang.monticar.generator.middleware.helpers.*;
import de.monticore.lang.monticar.generator.middleware.cli.ClusteringParameters;
import de.monticore.lang.monticar.generator.middleware.cli.ResultChoosingStrategy;
import de.monticore.lang.monticar.generator.middleware.clustering.*;
import de.monticore.lang.monticar.generator.middleware.helpers.FileHelper;
import de.monticore.lang.monticar.generator.middleware.helpers.NameHelper;
import de.monticore.lang.monticar.generator.middleware.helpers.RosHelper;
import de.monticore.lang.monticar.generator.middleware.helpers.TemplateHelper;
import de.monticore.lang.monticar.generator.middleware.impls.GeneratorImpl;
import de.monticore.lang.monticar.generator.middleware.impls.MiddlewareTagGenImpl;
import de.monticore.lang.tagging._symboltable.TaggingResolver;
......@@ -15,6 +20,7 @@ import java.util.*;
public class DistributedTargetGenerator extends CMakeGenerator {
private boolean generateMiddlewareTags = false;
private ClusteringResultList clusteringResults = new ClusteringResultList();
public boolean isGenerateMiddlewareTags() {
return generateMiddlewareTags;
......@@ -26,20 +32,29 @@ public class DistributedTargetGenerator extends CMakeGenerator {
private Set<String> subDirs = new HashSet<>();
private ClusteringParameters clusteringParameters;
public DistributedTargetGenerator() {
}
public ClusteringParameters getClusteringParameters() {
return clusteringParameters;
}
public void setClusteringParameters(ClusteringParameters clusteringParameters) {
this.clusteringParameters = clusteringParameters;
}
@Override
public void setGenerationTargetPath(String path) {
super.setGenerationTargetPath(path);
}
@Override
public List<File> generate(EMAComponentInstanceSymbol componentInstanceSymbol, TaggingResolver taggingResolver) throws IOException {
public List<File> generate(EMAComponentInstanceSymbol genComp, TaggingResolver taggingResolver) throws IOException {
Map<EMAComponentInstanceSymbol, GeneratorImpl> generatorMap = new HashMap<>();
fixComponentInstance(componentInstanceSymbol);
EMAComponentInstanceSymbol componentInstanceSymbol = preprocessing(genComp);
List<EMAComponentInstanceSymbol> clusterSubcomponents = ClusterFromTagsHelper.getClusterSubcomponents(componentInstanceSymbol);
if (clusterSubcomponents.size() > 0) {
......@@ -69,6 +84,7 @@ public class DistributedTargetGenerator extends CMakeGenerator {
if(generateMiddlewareTags){
MiddlewareTagGenImpl middlewareTagGen = new MiddlewareTagGenImpl();
middlewareTagGen.setGenerationTargetPath(generationTargetPath + "emam/");
middlewareTagGen.setClusteringResults(clusteringResults);
files.addAll(middlewareTagGen.generate(componentInstanceSymbol,taggingResolver));
}
......@@ -76,6 +92,47 @@ public class DistributedTargetGenerator extends CMakeGenerator {
return files;
}
private EMAComponentInstanceSymbol preprocessing(EMAComponentInstanceSymbol genComp) {
EMAComponentInstanceSymbol componentInstanceSymbol = genComp;
if(clusteringParameters != null){
//Flatten
if(clusteringParameters.getFlatten()){
if(clusteringParameters.getFlattenLevel().isPresent()){
Integer level = clusteringParameters.getFlattenLevel().get();
componentInstanceSymbol = FlattenArchitecture.flattenArchitecture(genComp, new HashMap<>(), level);
}else {
componentInstanceSymbol = FlattenArchitecture.flattenArchitecture(genComp);
}
}
//Cluster
if(clusteringParameters.getAlgorithmParameters().size() > 0) {
clusteringResults = AutomaticClusteringHelper.executeClusteringFromParams(componentInstanceSymbol, clusteringParameters.getAlgorithmParameters());
Optional<Integer> nOpt = clusteringParameters.getNumberOfClusters();
for(ClusteringResult c : clusteringResults){
String prefix = nOpt.isPresent() && !c.hasNumberOfClusters(nOpt.get()) ? "[IGNORED]" : "";
System.out.println(prefix + "Score was " + c.getScore() + " for " + c.getParameters().toString());
}
Optional<ClusteringResult> clusteringOpt;
if(nOpt.isPresent() && clusteringParameters.getChooseBy().equals(ResultChoosingStrategy.bestWithFittingN)){
clusteringOpt = clusteringResults.getBestResultWithFittingN(nOpt.get());
}else{
clusteringOpt = clusteringResults.getBestResultOverall();
}
if(clusteringOpt.isPresent()){
ClusteringResult clusteringResult = clusteringOpt.get();
System.out.println("Best score was " + clusteringResult.getScore() + " for " + clusteringResult.getParameters().toString());
AutomaticClusteringHelper.annotateComponentWithRosTagsForClusters(componentInstanceSymbol, clusteringResult.getClustering());
}
}
}
fixComponentInstance(componentInstanceSymbol);
return componentInstanceSymbol;
}
private File generateRosMsgGen() throws IOException {
File file = new File(generationTargetPath + "rosMsg/CMakeLists.txt");
FileUtils.write(file, TemplateHelper.getStruct_msgsCmakeTemplate());
......@@ -101,7 +158,6 @@ public class DistributedTargetGenerator extends CMakeGenerator {
fileContent.setFileName("CMakeLists.txt");
StringBuilder content = new StringBuilder();
content.append("cmake_minimum_required(VERSION 3.5)\n");
//TODO setProjectName?
content.append("project (default)\n");
content.append("set (CMAKE_CXX_STANDARD 11)\n");
......
package de.monticore.lang.monticar.generator.middleware.Simulation;
import de.monticore.lang.embeddedmontiarc.embeddedmontiarc._symboltable.instanceStructure.EMAComponentInstanceSymbol;
import de.monticore.lang.embeddedmontiarc.embeddedmontiarc._symboltable.instanceStructure.EMAConnectorInstanceSymbol;
import de.monticore.lang.monticar.generator.middleware.clustering.AutomaticClusteringHelper;
import de.monticore.lang.monticar.generator.middleware.clustering.FlattenArchitecture;
import de.monticore.lang.monticar.generator.middleware.clustering.algorithms.SpectralClusteringAlgorithm;
import de.monticore.lang.monticar.generator.middleware.clustering.algorithms.SpectralClusteringBuilder;
import java.util.Collection;
import java.util.List;
import java.util.Random;
import java.util.Set;
public class MonteCarloIntegration {
public double simulate(int iterations, EMAComponentInstanceSymbol componentInstanceSymbol){
EMAComponentInstanceSymbol flattenedComponent = FlattenArchitecture.flattenArchitecture(componentInstanceSymbol);
double sum = 0;
for(long i = 0; i<iterations; i++){
// Cluster with Spectral + save the cost: Parameter(Number of clusters, data)
int randNumClusters = randomNumberInRange(1, componentInstanceSymbol.getSubComponents().size());
SpectralClusteringAlgorithm spectralClusteringAlgorithm = new SpectralClusteringAlgorithm();
Object[] params = new Object[]{SpectralClusteringBuilder.SpectralParameters.SPECTRAL_NUM_CLUSTERS, randNumClusters};
List<Set<EMAComponentInstanceSymbol>> spectralClusters = spectralClusteringAlgorithm.cluster(flattenedComponent, params);
AutomaticClusteringHelper.annotateComponentWithRosTagsForClusters(flattenedComponent, spectralClusters);
//iterate through all clusters and add all cost of the ROS Tags between clusters
sum+=calculateCostOfClusters(componentInstanceSymbol, spectralClusters);
}
return sum/iterations;
}
public int randomNumberInRange(int min, int max) {
Random random = new Random();
return random.nextInt((max - min) + 1) + min;
}
public static double calculateCostOfClusters(EMAComponentInstanceSymbol componentInstanceSymbol, List<Set<EMAComponentInstanceSymbol>> clusters) {
Collection<EMAConnectorInstanceSymbol> connectors = componentInstanceSymbol.getConnectorInstances();
double sum = 0;
for(EMAConnectorInstanceSymbol con : connectors){
// -1 = super comp
int sourceClusterLabel = -1;
int targetClusterLabel = -1;
EMAComponentInstanceSymbol sourceComp = con.getSourcePort().getComponentInstance();
EMAComponentInstanceSymbol targetComp = con.getTargetPort().getComponentInstance();
for(int i = 0; i < clusters.size(); i++){
if(clusters.get(i).contains(sourceComp)){
sourceClusterLabel = i;
}
if(clusters.get(i).contains(targetComp)){
targetClusterLabel = i;
}
}
if(sourceClusterLabel != targetClusterLabel){
sum +=AutomaticClusteringHelper.getTypeCostHeuristic(con.getSourcePort());
}
}
return sum;
}
}
package de.monticore.lang.monticar.generator.middleware;
package de.monticore.lang.monticar.generator.middleware.cli;
import java.util.Optional;
import java.util.Set;
public class CliParameters {
......@@ -12,17 +13,19 @@ public class CliParameters {
private Set<String> generators;
private String emadlBackend;
private Boolean writeTagFile;
private ClusteringParameters clusteringParameters;
public CliParameters() {
}
public CliParameters(String modelsDir, String outputDir, String rootModel, Set<String> generators, String emadlBackend, Boolean writeTagFile) {
public CliParameters(String modelsDir, String outputDir, String rootModel, Set<String> generators, String emadlBackend, Boolean writeTagFile, ClusteringParameters clusteringParameters) {
this.modelsDir = modelsDir;
this.outputDir = outputDir;
this.rootModel = rootModel;
this.generators = generators;
this.emadlBackend = emadlBackend == null ? DEFAULT_EMADL_BACKEND : emadlBackend;
this.writeTagFile = writeTagFile == null ? DEFAULT_WRITE_TAG_FILE : writeTagFile;
this.emadlBackend = emadlBackend;
this.writeTagFile = writeTagFile;
this.clusteringParameters = clusteringParameters;
}
public String getModelsDir() {
......@@ -42,10 +45,15 @@ public class CliParameters {
}
public String getEmadlBackend() {
return emadlBackend;
return emadlBackend == null ? DEFAULT_EMADL_BACKEND : emadlBackend;
}
public boolean getWriteTagFile() {
return writeTagFile;
return writeTagFile == null ? DEFAULT_WRITE_TAG_FILE : writeTagFile;
}
public Optional<ClusteringParameters> getClusteringParameters() {
return Optional.ofNullable(clusteringParameters);
}
}
package de.monticore.lang.monticar.generator.middleware.cli;
import com.google.gson.*;
import com.google.gson.stream.JsonReader;
import de.monticore.lang.monticar.generator.middleware.cli.algorithms.*;
import de.se_rwth.commons.logging.Log;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.lang.reflect.Type;
public class CliParametersLoader {
private CliParametersLoader() {
}
public static CliParameters loadCliParameters(String filePath) throws FileNotFoundException {
JsonReader jsonReader = new JsonReader(new FileReader(filePath));
Gson gson = new GsonBuilder().registerTypeAdapter(AlgorithmCliParameters.class, new AlgorithmParametersInterfaceAdapter()).create();
return gson.fromJson(jsonReader, CliParameters.class);
}
static final class AlgorithmParametersInterfaceAdapter implements JsonSerializer<AlgorithmCliParameters>, JsonDeserializer<AlgorithmCliParameters> {
public JsonElement serialize(AlgorithmCliParameters object, Type interfaceType, JsonSerializationContext context) {
return context.serialize(object);
}
public AlgorithmCliParameters deserialize(JsonElement elem, Type interfaceType, JsonDeserializationContext context) throws JsonParseException {
final Type actualType = typeForName(((JsonObject)elem).get("name"));
return context.deserialize(elem, actualType);
}
private Type typeForName(final JsonElement typeElem){
String algoName = typeElem.getAsString().toLowerCase();
switch (algoName){
case "spectralclustering": return SpectralClusteringCliParameters.class;
case "dbscan": return DBScanCliParameters.class;
case "affinitypropagation": return AffinityPropagationCliParameters.class;
case "markov": return MarkovCliParameters.class;
default:{
Log.warn("Loaded config of unknown clustering algorithm: " + algoName);
return UnknownAlgorithmCliParameters.class;
}
}
}
}
}
package de.monticore.lang.monticar.generator.middleware.cli;
import de.monticore.lang.monticar.generator.middleware.cli.algorithms.AlgorithmCliParameters;
import de.monticore.lang.monticar.generator.middleware.cli.algorithms.SpectralClusteringCliParameters;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
public class ClusteringParameters {
private Integer numberOfClusters;
private Boolean flatten;
private Integer flattenLevel;
private ResultChoosingStrategy chooseBy = ResultChoosingStrategy.bestWithFittingN;
private List<AlgorithmCliParameters> algorithmParameters = new ArrayList<>();
public ClusteringParameters() {
}
public Optional<Integer> getNumberOfClusters() {
return Optional.ofNullable(numberOfClusters);
}
public ResultChoosingStrategy getChooseBy() {
return chooseBy;
}
public List<AlgorithmCliParameters> getAlgorithmParameters() {
//Override numberOfClusters for all spectral clustering parameters
if(getNumberOfClusters().isPresent()){
Integer n = getNumberOfClusters().get();
algorithmParameters.stream()
.filter(a -> a.getName().equals(AlgorithmCliParameters.TYPE_SPECTRAL_CLUSTERING))
.forEach(a -> ((SpectralClusteringCliParameters)a).setNumberOfClusters(n));
}
return algorithmParameters;
}
public boolean getFlatten(){
return flatten == null ? false : flatten;
}
public Optional<Integer> getFlattenLevel() {
return Optional.ofNullable(flattenLevel);
}
}
package de.monticore.lang.monticar.generator.middleware;
package de.monticore.lang.monticar.generator.middleware.cli;
import com.google.gson.Gson;
import com.google.gson.stream.JsonReader;
import de.monticore.lang.embeddedmontiarc.embeddedmontiarc._symboltable.instanceStructure.EMAComponentInstanceSymbol;
import de.monticore.lang.embeddedmontiarc.tagging.middleware.ros.RosToEmamTagSchema;
import de.monticore.lang.monticar.emadl.generator.EMADLAbstractSymtab;
import de.monticore.lang.monticar.generator.middleware.DistributedTargetGenerator;
import de.monticore.lang.monticar.generator.middleware.impls.CPPGenImpl;
import de.monticore.lang.monticar.generator.middleware.impls.EMADLGeneratorImpl;
import de.monticore.lang.monticar.generator.middleware.impls.ODVGenImpl;
......@@ -15,7 +15,6 @@ import de.monticore.lang.tagging._symboltable.TaggingResolver;
import de.se_rwth.commons.logging.Log;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Paths;
......@@ -58,8 +57,7 @@ public final class DistributedTargetGeneratorCli {
String filePath = expandHomeDir(args[0]);
try {
JsonReader jsonReader = new JsonReader(new FileReader(filePath));
parameters = gson.fromJson(jsonReader, CliParameters.class);
parameters = CliParametersLoader.loadCliParameters(filePath);
} catch (FileNotFoundException e) {
Log.error("0x49E6A: Can not find specified config file: " + args[0] + "!");
}
......@@ -156,6 +154,15 @@ public final class DistributedTargetGeneratorCli {
generator.add(new ODVGenImpl(), "odv");
}
if (cliParameters.getClusteringParameters().isPresent()) {
ClusteringParameters clusteringParameters = cliParameters.getClusteringParameters().get();
generator.setClusteringParameters(clusteringParameters);
clusteringParameters.getAlgorithmParameters().stream()
.filter(alg -> !alg.isValid())
.forEach(alg -> Log.error("Parameters for the algorithm " + alg.getName() + " are invalid!"));
}
try {
generator.generate(componentInstanceSymbol, taggingResolver);
} catch (IOException e) {
......
package de.monticore.lang.monticar.generator.middleware.cli;
public enum ResultChoosingStrategy {
bestWithFittingN,
bestOverall;
}
package de.monticore.lang.monticar.generator.middleware.cli.algorithms;
import de.monticore.lang.monticar.generator.middleware.clustering.ClusteringAlgorithm;
import de.monticore.lang.monticar.generator.middleware.clustering.algorithms.AffinityPropagationAlgorithm;
public class AffinityPropagationCliParameters extends AlgorithmCliParameters{
public AffinityPropagationCliParameters() {
}
@Override
public String getName() {
return TYPE_AFFINITY_PROPAGATION;
}
@Override
public ClusteringAlgorithm asClusteringAlgorithm() {
return new AffinityPropagationAlgorithm();
}
@Override
public Object[] asAlgorithmArgs(){
return new Object[]{};
}
@Override
public boolean isValid() {
return true;
}
@Override
public String toString() {
return "AffinityPropagation";
}
}
\ No newline at end of file
package de.monticore.lang.monticar.generator.middleware.cli.algorithms;
import de.monticore.lang.monticar.generator.middleware.clustering.ClusteringAlgorithm;
public abstract class AlgorithmCliParameters {
public static final String TYPE_SPECTRAL_CLUSTERING = "SpectralClustering";
public static final String TYPE_UNKOWN = "Unkown";
public static final String TYPE_DBSCAN = "DBScan";
public static final String TYPE_MARKOV = "Markov";
public static final String TYPE_AFFINITY_PROPAGATION = "AffinityPropagation";
protected String name;
public AlgorithmCliParameters() {
}
public String getName() {
return name;
}
public abstract ClusteringAlgorithm asClusteringAlgorithm();
public abstract Object[] asAlgorithmArgs();
public abstract boolean isValid();
}
package de.monticore.lang.monticar.generator.middleware.cli.algorithms;
import de.monticore.lang.monticar.generator.middleware.clustering.ClusteringAlgorithm;
import de.monticore.lang.monticar.generator.middleware.clustering.algorithms.DBSCANClusteringAlgorithm;
import de.monticore.lang.monticar.generator.middleware.clustering.algorithms.DBSCANClusteringBuilder;
import de.se_rwth.commons.logging.Log;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
public class DBScanCliParameters extends AlgorithmCliParameters {
private Integer min_pts;
private Double radius;
public DBScanCliParameters() {
}
@Override
public String getName() {
return TYPE_DBSCAN;
}
@Override
public ClusteringAlgorithm asClusteringAlgorithm() {
DBSCANClusteringAlgorithm dbscanClusteringAlgorithm = new DBSCANClusteringAlgorithm();
dbscanClusteringAlgorithm.setArgs(asAlgorithmArgs());
return dbscanClusteringAlgorithm;
}
@Override
public Object[] asAlgorithmArgs() {
List<Object> res = new ArrayList<>();
if(!isValid()){
Log.error("DBScanCliParameters: The min_pts or radius parameters are mandatory but at least one is unset!");
return res.toArray();
}
res.add(DBSCANClusteringBuilder.DBSCANParameters.DBSCAN_MIN_PTS);
res.add(min_pts);
res.add(DBSCANClusteringBuilder.DBSCANParameters.DBSCAN_RADIUS);
res.add(radius