Skip to content
Snippets Groups Projects
Commit cbeb24e3 authored by Muhammad Raufu Miah's avatar Muhammad Raufu Miah
Browse files

Add new file

parent feaa9aa3
No related branches found
No related tags found
No related merge requests found
Show whitespace changes
Inline Side-by-side
ExperimentsComparisonBlackBoxRandom.java 0 → 100644
+ 349
− 0
View file @ cbeb24e3
import de.learnlib.acex.AcexAnalyzer;
import de.learnlib.algorithm.kv.dfa.KearnsVaziraniDFA;
import de.learnlib.algorithm.kv.dfa.KearnsVaziraniDFABuilder;
import de.learnlib.algorithm.lstar.dfa.ClassicLStarDFABuilder;
import de.learnlib.algorithm.observationpack.dfa.OPLearnerDFA;
import de.learnlib.algorithm.observationpack.dfa.OPLearnerDFABuilder;
import de.learnlib.algorithm.rivestschapire.RivestSchapireDFA;
import de.learnlib.algorithm.ttt.base.AbstractTTTLearner;
import de.learnlib.algorithm.ttt.dfa.TTTLearnerDFA;
import de.learnlib.algorithm.ttt.dfa.TTTLearnerDFABuilder;
import de.learnlib.filter.statistic.oracle.DFACounterOracle;
import de.learnlib.oracle.EquivalenceOracle;
import de.learnlib.oracle.MembershipOracle;
import de.learnlib.oracle.equivalence.DFARandomWordsEQOracle;
import de.learnlib.oracle.equivalence.DFASimulatorEQOracle;
import de.learnlib.oracle.equivalence.DFAWMethodEQOracle;
import de.learnlib.oracle.equivalence.WMethodEQOracle;
import de.learnlib.oracle.membership.DFASimulatorOracle;
import de.learnlib.query.DefaultQuery;
import net.automatalib.alphabet.Alphabet;
import net.automatalib.alphabet.Alphabets;
import net.automatalib.automaton.fsa.DFA;
import de.learnlib.algorithm.lstar.dfa.ClassicLStarDFA;
import de.learnlib.algorithm.rivestschapire.RivestSchapireDFABuilder;
import net.automatalib.word.Word;
import org.knowm.xchart.QuickChart;
import org.knowm.xchart.SwingWrapper;
import org.knowm.xchart.XYChart;
import java.util.ArrayList;
import java.util.List;
import java.util.Random;
public class ExperimentsComparisonBlackBoxRandom {
public static void main(String[] args) {
// Define configurations
int[] yValues = { 5 };
int[] zValues = { 50 };
double[] xValues = {1000,3000,9000,27000};
// Declare arrays to store the average membership queries and average equivalence queries
double[] avgMemQueriesLstar = new double[xValues.length];
double[] avgMemQueriesRS = new double[xValues.length];
double[] avgMemQueriesKV = new double[xValues.length];
double[] avgMemQueriesOP = new double[xValues.length];
double[] avgMemQueriesTTT = new double[xValues.length];
double[] avgMemQueriesLstarLearn = new double[xValues.length];
double[] avgMemQueriesRSLearn = new double[xValues.length];
double[] avgMemQueriesKVLearn = new double[xValues.length];
double[] avgMemQueriesOPLearn = new double[xValues.length];
double[] avgMemQueriesTTTLearn = new double[xValues.length];
double[] avgEqQueriesLstar = new double[xValues.length];
double[] avgEqQueriesRS = new double[xValues.length];
double[] avgEqQueriesKV = new double[xValues.length];
double[] avgEqQueriesOP = new double[xValues.length];
double[] avgEqQueriesTTT = new double[xValues.length];
double[] accuracyLstar = new double[xValues.length];
double[] accuracyRS = new double[xValues.length];
double[] accuracyKV = new double[xValues.length];
double[] accuracyOP = new double[xValues.length];
double[] accuracyTTT = new double[xValues.length];
// Iterate over each configuration: y denotes the size of the alphabet of the target DFA, z denotes the size of the target DFA, x denotes the acceptance ratio of the target DFA
for(int y:yValues) {
for (int z : zValues) {
int k=0;
for (double x : xValues) {
// Create the alphabet
Alphabet<Integer> alphabet = Alphabets.integers(0, y - 1);
int memCounterLstar = 0;
int memCounterRS = 0;
int memCounterKV = 0;
int memCounterOP = 0;
int memCounterTTT = 0;
int memCounterLstarLearn = 0;
int memCounterRSLearn = 0;
int memCounterKVLearn = 0;
int memCounterOPLearn = 0;
int memCounterTTTLearn = 0;
int eqCounterLstar = 0;
int eqCounterRS = 0;
int eqCounterKV = 0;
int eqCounterOP = 0;
int eqCounterTTT = 0;
int controleLstar = 0;
int controleRS = 0;
int controleKV = 0;
int controleOP = 0;
int controleTTT = 0;
// Generate 100 random DFAs for each configuration
for (int i = 0; i < 20; i++) {
//System.out.println("Generating random DFA " + i + " for configuration (x,y,z):" + 0.01 + "," + y + "," + z + ":" + x);
Random rand = new Random();
ExampleRandomDFA example = new ExampleRandomDFA(rand, y, z, 0.01, alphabet);
DFA<?, Integer> dfa = example.getReferenceAutomaton();
// Initialize the Query Counters and the Oracles - we used the white-box oracles
EquivalenceOracle<DFA<?, Integer>, Integer, Boolean> controle = new DFASimulatorEQOracle<>(dfa); // for controle
MembershipOracle.DFAMembershipOracle<Integer> memOraclePlainLstar = new DFASimulatorOracle<>(dfa);
DFACounterOracle<Integer> memCounterOracleLstarLearn = new DFACounterOracle<>(memOraclePlainLstar);
MembershipOracle.DFAMembershipOracle<Integer> memOracleLstar = new DFASimulatorOracle<>(dfa);
DFACounterOracle<Integer> memCounterOracleLstar = new DFACounterOracle<>(memOracleLstar);
EquivalenceOracle<DFA<?, Integer>, Integer, Boolean> eqOracleLstar = new DFARandomWordsEQOracle<>(memCounterOracleLstar,10,200,(int)x);
EQCounterOracle<DFA<?, Integer>, Integer, Boolean> eqCounterOracleLstar = new EQCounterOracle<>(eqOracleLstar);
MembershipOracle.DFAMembershipOracle<Integer> memOraclePlainRS = new DFASimulatorOracle<>(dfa);
DFACounterOracle<Integer> memCounterOracleRSLearn = new DFACounterOracle<>(memOraclePlainRS);
MembershipOracle.DFAMembershipOracle<Integer> memOracleRS = new DFASimulatorOracle<>(dfa);
DFACounterOracle<Integer> memCounterOracleRS = new DFACounterOracle<>(memOracleRS);
EquivalenceOracle<DFA<?, Integer>, Integer, Boolean> eqOracleRS = new DFARandomWordsEQOracle<>(memCounterOracleRS,10,200,(int)x);
EQCounterOracle<DFA<?, Integer>, Integer, Boolean> eqCounterOracleRS = new EQCounterOracle<>(eqOracleRS);
MembershipOracle.DFAMembershipOracle<Integer> memOraclePlainKV = new DFASimulatorOracle<>(dfa);
DFACounterOracle<Integer> memCounterOracleKVLearn = new DFACounterOracle<>(memOraclePlainKV);
MembershipOracle.DFAMembershipOracle<Integer> memOracleKV = new DFASimulatorOracle<>(dfa);
DFACounterOracle<Integer> memCounterOracleKV = new DFACounterOracle<>(memOracleKV);
EquivalenceOracle<DFA<?, Integer>, Integer, Boolean> eqOracleKV = new DFARandomWordsEQOracle<>(memCounterOracleKV,10,200,(int)x);
EQCounterOracle<DFA<?, Integer>, Integer, Boolean> eqCounterOracleKV = new EQCounterOracle<>(eqOracleKV);
MembershipOracle.DFAMembershipOracle<Integer> memOraclePlainOP = new DFASimulatorOracle<>(dfa);
DFACounterOracle<Integer> memCounterOracleOPLearn = new DFACounterOracle<>(memOraclePlainOP);
MembershipOracle.DFAMembershipOracle<Integer> memOracleOP = new DFASimulatorOracle<>(dfa);
DFACounterOracle<Integer> memCounterOracleOP = new DFACounterOracle<>(memOracleOP);
EquivalenceOracle<DFA<?, Integer>, Integer, Boolean> eqOracleOP = new DFARandomWordsEQOracle<>(memCounterOracleOP,10,200,(int)x);
EQCounterOracle<DFA<?, Integer>, Integer, Boolean> eqCounterOracleOP = new EQCounterOracle<>(eqOracleOP);
MembershipOracle.DFAMembershipOracle<Integer> memOraclePlainTTT = new DFASimulatorOracle<>(dfa);
DFACounterOracle<Integer> memCounterOracleTTTLearn = new DFACounterOracle<>(memOraclePlainTTT);
MembershipOracle.DFAMembershipOracle<Integer> memOracleTTT = new DFASimulatorOracle<>(dfa);
DFACounterOracle<Integer> memCounterOracleTTT = new DFACounterOracle<>(memOracleTTT);
EquivalenceOracle<DFA<?, Integer>, Integer, Boolean> eqOracleTTT = new DFARandomWordsEQOracle<>(memCounterOracleTTT,10,200,(int)x);
EQCounterOracle<DFA<?, Integer>, Integer, Boolean> eqCounterOracleTTT = new EQCounterOracle<>(eqOracleTTT);
// Learn the DFA using each learning algorithm
if(learnWithClassicLStarDFA(dfa, alphabet, eqCounterOracleLstar, memCounterOracleLstarLearn, controle)==1){controleLstar++;};
if(learnWithRivestShapireDFA(dfa, alphabet, eqCounterOracleRS, memCounterOracleRSLearn, controle)==1){controleRS++;};
if((learnWithKearnsVauiraniDFA(dfa, alphabet, eqCounterOracleKV, memCounterOracleKVLearn, controle)==1)){controleKV++;};
if(learnWithOPLearnerDFA(dfa, alphabet, eqCounterOracleOP, memCounterOracleOPLearn, controle)==1){controleOP++;};
if(learnWithTTTLearnerDFA(dfa, alphabet, eqCounterOracleTTT, memCounterOracleTTTLearn, controle)==1){controleTTT++;};
// Update the counters
memCounterLstar += memCounterOracleLstar.getQueryCounter().getCount();
memCounterRS += memCounterOracleRS.getQueryCounter().getCount();
memCounterKV += memCounterOracleKV.getQueryCounter().getCount();
memCounterOP += memCounterOracleOP.getQueryCounter().getCount();
memCounterTTT += memCounterOracleTTT.getQueryCounter().getCount();
memCounterLstarLearn += memCounterOracleLstarLearn.getQueryCounter().getCount();
memCounterRSLearn += memCounterOracleRSLearn.getQueryCounter().getCount();
memCounterKVLearn += memCounterOracleKVLearn.getQueryCounter().getCount();
memCounterOPLearn += memCounterOracleOPLearn.getQueryCounter().getCount();
memCounterTTTLearn += memCounterOracleTTTLearn.getQueryCounter().getCount();
eqCounterLstar += eqCounterOracleLstar.getCounter();
eqCounterRS += eqCounterOracleRS.getCounter();
eqCounterKV += eqCounterOracleKV.getCounter();
eqCounterOP += eqCounterOracleOP.getCounter();
eqCounterTTT += eqCounterOracleTTT.getCounter();
}
//Calculate average for configuration (x,y,z) for the counters
System.out.println("For configuration (x,y,z):" + x + "," + y + "," + z + " :");
System.out.println("Average memCounterLstar: " + memCounterLstar/20);
System.out.println("Average memCounterRS: " + memCounterRS/20);
System.out.println("Average memCounterKV: " + memCounterKV/20);
System.out.println("Average memCounterOP: " + memCounterOP/20);
System.out.println("Average memCounterTTT: " + memCounterTTT/20);
System.out.println("Average eqCounterLstar: " + eqCounterLstar/20);
System.out.println("Average eqCounterRS: " + eqCounterRS/20);
System.out.println("Average eqCounterKV: " + eqCounterKV/20);
System.out.println("Average eqCounterOP: " + eqCounterOP/20);
System.out.println("Average eqCounterTTT: " + eqCounterTTT/20);
// Calculate the averages and store them in the arrays
avgMemQueriesLstar[k] = memCounterLstar / 20;
avgMemQueriesRS[k] = memCounterRS / 20;
avgMemQueriesKV[k] = memCounterKV / 20;
avgMemQueriesOP[k] = memCounterOP / 20;
avgMemQueriesTTT[k] = memCounterTTT / 20;
avgEqQueriesLstar[k] = eqCounterLstar / 20.0;
avgEqQueriesRS[k] = eqCounterRS / 20.0;
avgEqQueriesKV[k] = eqCounterKV / 20.0;
avgEqQueriesOP[k] = eqCounterOP / 20.0;
avgEqQueriesTTT[k] = eqCounterTTT / 20.0;
avgMemQueriesLstarLearn[k] = memCounterLstarLearn /20;
avgMemQueriesRSLearn[k] = memCounterRSLearn / 20;
avgMemQueriesKVLearn[k] = memCounterKVLearn / 20;
avgMemQueriesOPLearn[k] = memCounterOPLearn / 20;
avgMemQueriesTTTLearn[k] = memCounterTTTLearn / 20;
accuracyLstar[k] = (double)controleLstar/20;
accuracyRS[k] = (double)controleRS/20;
accuracyKV[k] = (double)controleKV/20;
accuracyOP[k] = (double)controleOP/20;
accuracyTTT[k] = (double)controleTTT/20;
k++;
}
}
}
String[] seriesNames = {"LStar","RS","KV", "OP", "TTT"};
// Create the first XY chart (membership queries)
XYChart chart1 = QuickChart.getChart("Average Additional Membership Queries,n=50,k=5", "TestSize", "Queries", seriesNames, xValues, new double[][]{avgMemQueriesLstar,avgMemQueriesRS, avgMemQueriesKV, avgMemQueriesOP, avgMemQueriesTTT});
new SwingWrapper(chart1).displayChart();
// Create the second XY chart (membership queries)
XYChart chart3 = QuickChart.getChart("Average Membership Queries,n=50,k=5", "TestSize", "Queries", seriesNames, xValues, new double[][]{avgMemQueriesLstarLearn,avgMemQueriesRSLearn, avgMemQueriesKVLearn, avgMemQueriesOPLearn, avgMemQueriesTTTLearn});
new SwingWrapper(chart3).displayChart();
double[] avgMemQueriesLstarRatio = new double[xValues.length];
double[] avgMemQueriesRSRatio = new double[xValues.length];
double[] avgMemQueriesKVRatio = new double[xValues.length];
double[] avgMemQueriesOPRatio = new double[xValues.length];
double[] avgMemQueriesTTTRatio = new double[xValues.length];
for (int i = 0; i < xValues.length; i++) {
avgMemQueriesLstarRatio[i] = avgMemQueriesLstar[i] / avgMemQueriesLstarLearn[i];
avgMemQueriesRSRatio[i] = avgMemQueriesRS[i] / avgMemQueriesRSLearn[i];
avgMemQueriesKVRatio[i] = avgMemQueriesKV[i] / avgMemQueriesKVLearn[i];
avgMemQueriesOPRatio[i] = avgMemQueriesOP[i] / avgMemQueriesOPLearn[i];
avgMemQueriesTTTRatio[i] = avgMemQueriesTTT[i] / avgMemQueriesTTTLearn[i];
}
// Create the third XY chart (membership queries)
XYChart chart4 = QuickChart.getChart("Average Membership Queries Ratio,n=50,k=5", "TestSize", "Factor (Additional/Base)", seriesNames, xValues, new double[][]{avgMemQueriesLstarRatio, avgMemQueriesRSRatio, avgMemQueriesKVRatio, avgMemQueriesOPRatio, avgMemQueriesTTTRatio});
new SwingWrapper(chart4).displayChart();
// Create the fourth XY chart (equivalence queries)
XYChart chart2 = QuickChart.getChart("Average Equivalence Queries,n=50,k=5", "TestSize", "Queries", seriesNames, xValues, new double[][]{avgEqQueriesLstar, avgEqQueriesRS,avgEqQueriesKV, avgEqQueriesOP, avgEqQueriesTTT});
new SwingWrapper(chart2).displayChart();
// Create the fifth XY chart (accuracy)
XYChart chart5 = QuickChart.getChart("n=50,k=5", "TestSize", "Accuracy", seriesNames, xValues, new double[][]{accuracyLstar, accuracyRS, accuracyKV, accuracyOP, accuracyTTT});
new SwingWrapper(chart5).displayChart();
}
// Define the learning methods
public static int learnWithClassicLStarDFA(DFA<?, Integer> dfa, Alphabet<Integer> alphabet, EquivalenceOracle<DFA<?, Integer>, Integer, Boolean> eqOracleLstar, MembershipOracle.DFAMembershipOracle<Integer> memOracleLstar, EquivalenceOracle<DFA<?, Integer>, Integer, Boolean> controle){
// Create an instance of the ClassicLStar algorithm
ClassicLStarDFA<Integer> learner =
new ClassicLStarDFABuilder<Integer>().withAlphabet(alphabet) // input alphabet
.withOracle(memOracleLstar) // membership oracle
.create();
// Start the learning process
learner.startLearning();
// Perform equivalence queries until no counterexample is found
DefaultQuery<Integer, Boolean> counterexample;
while ((counterexample = eqOracleLstar.findCounterExample(learner.getHypothesisModel(), alphabet)) != null) {
learner.refineHypothesis(counterexample);
}
if((counterexample = controle.findCounterExample(learner.getHypothesisModel(), alphabet)) == null){return 1;}
return 0;
}
public static int learnWithRivestShapireDFA(DFA<?, Integer> dfa, Alphabet<Integer> alphabet, EquivalenceOracle<DFA<?, Integer>, Integer, Boolean> eqOracleRS, MembershipOracle.DFAMembershipOracle<Integer> memOracleRS, EquivalenceOracle<DFA<?, Integer>, Integer, Boolean> controle) {
// Implement learning with RivestShapireDFA
RivestSchapireDFABuilder<Integer> rsBuilder = new RivestSchapireDFABuilder<>();
rsBuilder.withAlphabet(alphabet); // input alphabet
rsBuilder.withOracle(memOracleRS); // membership oracle
RivestSchapireDFA<Integer> learner = rsBuilder.create();
learner.startLearning();
DefaultQuery<Integer, Boolean> counterexample;
while ((counterexample = eqOracleRS.findCounterExample(learner.getHypothesisModel(), alphabet)) != null) {
learner.refineHypothesis(counterexample);
}
if((counterexample = controle.findCounterExample(learner.getHypothesisModel(), alphabet)) == null){return 1;}
return 0;
}
public static int learnWithKearnsVauiraniDFA(DFA<?, Integer> dfa, Alphabet<Integer> alphabet, EquivalenceOracle<DFA<?, Integer>, Integer, Boolean> eqOracleKV, MembershipOracle.DFAMembershipOracle<Integer> memOracleKV, EquivalenceOracle<DFA<?, Integer>, Integer, Boolean> controle) {
KearnsVaziraniDFABuilder<Integer> kol= new KearnsVaziraniDFABuilder<Integer>();
kol.withAlphabet(alphabet);
kol.withOracle(memOracleKV);
kol.withRepeatedCounterexampleEvaluation(true);
KearnsVaziraniDFA<Integer> learner = kol.create();
learner.startLearning();
DefaultQuery<Integer, Boolean> counterexample;
while ((counterexample = eqOracleKV.findCounterExample(learner.getHypothesisModel(), alphabet)) != null) {
learner.refineHypothesis(counterexample);
}
if((counterexample = controle.findCounterExample(learner.getHypothesisModel(), alphabet)) == null){return 1;}
return 0;
}
public static int learnWithOPLearnerDFA(DFA<?, Integer> dfa, Alphabet<Integer> alphabet, EquivalenceOracle<DFA<?, Integer>, Integer, Boolean> eqOracleOP, MembershipOracle.DFAMembershipOracle<Integer> memOracleOP, EquivalenceOracle<DFA<?, Integer>, Integer, Boolean> controle) {
OPLearnerDFABuilder<Integer> opBuilder = new OPLearnerDFABuilder<>();
opBuilder.withAlphabet(alphabet);
opBuilder.withOracle(memOracleOP);
opBuilder.setRepeatedCounterexampleEvaluation(true);
OPLearnerDFA<Integer> learner = opBuilder.create();
// Perform equivalence queries until no counterexample is found
DefaultQuery<Integer, Boolean> counterexample;
learner.startLearning();
while ((counterexample = eqOracleOP.findCounterExample(learner.getHypothesisModel(), alphabet)) != null) {
learner.refineHypothesis(counterexample);
}
if((counterexample = controle.findCounterExample(learner.getHypothesisModel(), alphabet)) == null){return 1;}
return 0;
}
public static int learnWithTTTLearnerDFA(DFA<?, Integer> dfa, Alphabet<Integer> alphabet, EquivalenceOracle<DFA<?, Integer>, Integer, Boolean> oracle, MembershipOracle.DFAMembershipOracle<Integer> mem, EquivalenceOracle<DFA<?, Integer>, Integer, Boolean> controle) {
TTTLearnerDFA<Integer> learner =
new TTTLearnerDFABuilder<Integer>().withAlphabet(alphabet)
.withOracle(mem)
.create();
learner.startLearning();
// Perform equivalence queries until no counterexample is found
DefaultQuery counterexample;
while ((counterexample = oracle.findCounterExample(learner.getHypothesisModel(), alphabet)) != null) {
learner.refineHypothesis(counterexample);
}
if((counterexample = controle.findCounterExample(learner.getHypothesisModel(), alphabet)) == null){return 1;}
return 0;
}
}
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment