diff --git a/ExperimentsRandomTestsVarSystemFixedAcceptanceRatioFixedTestSize.java b/ExperimentsRandomTestsVarSystemFixedAcceptanceRatioFixedTestSize.java new file mode 100644 index 0000000000000000000000000000000000000000..5a9d804e77c2a873a96ab77d13e51d5b6d3b3b98 --- /dev/null +++ b/ExperimentsRandomTestsVarSystemFixedAcceptanceRatioFixedTestSize.java @@ -0,0 +1,296 @@ +import de.learnlib.acex.AcexAnalyzer; +import de.learnlib.algorithm.kv.dfa.KearnsVaziraniDFA; +import de.learnlib.algorithm.kv.dfa.KearnsVaziraniDFABuilder; +import de.learnlib.algorithm.lstar.dfa.ClassicLStarDFABuilder; +import de.learnlib.algorithm.observationpack.dfa.OPLearnerDFA; +import de.learnlib.algorithm.observationpack.dfa.OPLearnerDFABuilder; +import de.learnlib.algorithm.rivestschapire.RivestSchapireDFA; +import de.learnlib.algorithm.ttt.base.AbstractTTTLearner; +import de.learnlib.algorithm.ttt.dfa.TTTLearnerDFA; +import de.learnlib.algorithm.ttt.dfa.TTTLearnerDFABuilder; +import de.learnlib.filter.statistic.oracle.DFACounterOracle; +import de.learnlib.oracle.EquivalenceOracle; +import de.learnlib.oracle.MembershipOracle; +import de.learnlib.oracle.equivalence.DFARandomWordsEQOracle; +import de.learnlib.oracle.equivalence.DFASimulatorEQOracle; +import de.learnlib.oracle.equivalence.WMethodEQOracle; +import de.learnlib.oracle.membership.DFASimulatorOracle; +import de.learnlib.query.DefaultQuery; +import net.automatalib.alphabet.Alphabet; +import net.automatalib.alphabet.Alphabets; +import net.automatalib.automaton.fsa.DFA; +import de.learnlib.algorithm.lstar.dfa.ClassicLStarDFA; +import de.learnlib.algorithm.rivestschapire.RivestSchapireDFABuilder; +import net.automatalib.word.Word; +import org.knowm.xchart.QuickChart; +import org.knowm.xchart.SwingWrapper; +import org.knowm.xchart.XYChart; + +import java.util.ArrayList; +import java.util.List; +import java.util.Random; + +public class ExperimentsRandomTestsVarSystemFixedAcceptanceRatioFixedTestSize { + + public static void main(String[] args) { + // Define configurations model easy to complex systems by increasing n and k in parallel + Pair[] yzValues = { new Pair(5,10), new Pair(10,60), new Pair(14,100), new Pair(20,150)}; + + + // Declare arrays to store the average membership queries and average equivalence queries + double[] avgMemQueriesLstar = new double[yzValues.length]; + double[] avgMemQueriesRS = new double[yzValues.length]; + double[] avgMemQueriesKV = new double[yzValues.length]; + double[] avgMemQueriesOP = new double[yzValues.length]; + double[] avgMemQueriesTTT = new double[yzValues.length]; + + double[] avgEqQueriesLstar = new double[yzValues.length]; + double[] avgEqQueriesRS = new double[yzValues.length]; + double[] avgEqQueriesKV = new double[yzValues.length]; + double[] avgEqQueriesOP = new double[yzValues.length]; + double[] avgEqQueriesTTT = new double[yzValues.length]; + + + double[] accuracyLstar = new double[yzValues.length]; + double[] accuracyRS = new double[yzValues.length]; + double[] accuracyKV = new double[yzValues.length]; + double[] accuracyOP = new double[yzValues.length]; + double[] accuracyTTT = new double[yzValues.length]; + + + + + // Iterate over each configuration: y denotes the size of the alphabet of the target DFA, z denotes the size of the target DFA, x denotes the acceptance ratio of the target DFA + int k=0; + for (Pair x : yzValues) { + + // Create the alphabet + Alphabet<Integer> alphabet = Alphabets.integers(0, x.first - 1); + + int memCounterLstar = 0; + int memCounterRS = 0; + int memCounterKV = 0; + int memCounterOP = 0; + int memCounterTTT = 0; + + int eqCounterLstar = 0; + int eqCounterRS = 0; + int eqCounterKV = 0; + int eqCounterOP = 0; + int eqCounterTTT = 0; + + int controleLstar = 0; + int controleRS = 0; + int controleKV = 0; + int controleOP = 0; + int controleTTT = 0; + + // Generate 100 random DFAs for each configuration + for (int i = 0; i < 100; i++) { + + //System.out.println("Generating random DFA " + i + " for configuration (x,y,z):" + 0.1 + "," + x.first + "," + x.second + "Test Size: " + "30"); + + Random rand = new Random(); + ExampleRandomDFA example = new ExampleRandomDFA(rand, x.first, x.second, 0.1, alphabet); + DFA<?, Integer> dfa = example.getReferenceAutomaton(); + + + EquivalenceOracle<DFA<?, Integer>, Integer, Boolean> eqOracleControle = new DFASimulatorEQOracle<>(dfa); + + // Initialize the Query Counters and the Oracles - we used the white-box oracles + + MembershipOracle.DFAMembershipOracle<Integer> memOracleLstar = new DFASimulatorOracle<>(dfa); + DFACounterOracle<Integer> memCounterOracleLstar = new DFACounterOracle<>(memOracleLstar); + MembershipOracle.DFAMembershipOracle<Integer> memUtilOracleLstar = new DFASimulatorOracle<>(dfa); + EquivalenceOracle<DFA<?, Integer>, Integer, Boolean> eqOracleLstar = new DFARandomWordsEQOracle<Integer>(memUtilOracleLstar,1,100, 15,rand); + EQCounterOracle<DFA<?, Integer>, Integer, Boolean> eqCounterOracleLstar = new EQCounterOracle<>(eqOracleLstar); + + MembershipOracle.DFAMembershipOracle<Integer> memOracleRS = new DFASimulatorOracle<>(dfa); + DFACounterOracle<Integer> memCounterOracleRS = new DFACounterOracle<>(memOracleRS); + MembershipOracle.DFAMembershipOracle<Integer> memUtilOracleRS = new DFASimulatorOracle<>(dfa); + EquivalenceOracle<DFA<?, Integer>, Integer, Boolean> eqOracleRS = new DFARandomWordsEQOracle<Integer>(memUtilOracleRS,1,100, 15,rand); + EQCounterOracle<DFA<?, Integer>, Integer, Boolean> eqCounterOracleRS = new EQCounterOracle<>(eqOracleRS); + + MembershipOracle.DFAMembershipOracle<Integer> memOracleKV = new DFASimulatorOracle<>(dfa); + DFACounterOracle<Integer> memCounterOracleKV = new DFACounterOracle<>(memOracleKV); + MembershipOracle.DFAMembershipOracle<Integer> memUtilOracleKV = new DFASimulatorOracle<>(dfa); + EquivalenceOracle<DFA<?, Integer>, Integer, Boolean> eqOracleKV = new DFARandomWordsEQOracle<Integer>(memUtilOracleKV,1,100, 15,rand); + EQCounterOracle<DFA<?, Integer>, Integer, Boolean> eqCounterOracleKV = new EQCounterOracle<>(eqOracleRS); + + MembershipOracle.DFAMembershipOracle<Integer> memOracleOP = new DFASimulatorOracle<>(dfa); + DFACounterOracle<Integer> memCounterOracleOP = new DFACounterOracle<>(memOracleOP); + MembershipOracle.DFAMembershipOracle<Integer> memUtilOracleOP = new DFASimulatorOracle<>(dfa); + EquivalenceOracle<DFA<?, Integer>, Integer, Boolean> eqOracleOP = new DFARandomWordsEQOracle<Integer>(memUtilOracleRS,1,100, 15,rand); + EQCounterOracle<DFA<?, Integer>, Integer, Boolean> eqCounterOracleOP = new EQCounterOracle<>(eqOracleRS); + + MembershipOracle.DFAMembershipOracle<Integer> memOracleTTT = new DFASimulatorOracle<>(dfa); + DFACounterOracle<Integer> memCounterOracleTTT = new DFACounterOracle<>(memOracleTTT); + MembershipOracle.DFAMembershipOracle<Integer> memUtilOracleTTT = new DFASimulatorOracle<>(dfa); + EquivalenceOracle<DFA<?, Integer>, Integer, Boolean> eqOracleTTT = new DFARandomWordsEQOracle<Integer>(memUtilOracleRS,1,100, 15,rand); + EQCounterOracle<DFA<?, Integer>, Integer, Boolean> eqCounterOracleTTT = new EQCounterOracle<>(eqOracleRS); + + + // Learn the DFA using each learning algorithm + if(learnWithClassicLStarDFA(dfa, alphabet, eqCounterOracleLstar, memCounterOracleLstar, eqOracleControle)==1){controleLstar++;}; + if(learnWithRivestShapireDFA(dfa, alphabet, eqCounterOracleRS, memCounterOracleRS, eqOracleControle)==1){controleRS++;}; + if((learnWithKearnsVauiraniDFA(dfa, alphabet, eqCounterOracleKV, memCounterOracleKV, eqOracleControle)==1)){controleKV++;}; + if(learnWithOPLearnerDFA(dfa, alphabet, eqCounterOracleOP, memCounterOracleOP, eqOracleControle)==1){controleOP++;}; + if(learnWithTTTLearnerDFA(dfa, alphabet, eqCounterOracleTTT, memCounterOracleTTT, eqOracleControle)==1){controleTTT++;}; + + + // Update the counters + memCounterLstar += memCounterOracleLstar.getQueryCounter().getCount(); + memCounterRS += memCounterOracleRS.getQueryCounter().getCount(); + memCounterKV += memCounterOracleKV.getQueryCounter().getCount(); + memCounterOP += memCounterOracleOP.getQueryCounter().getCount(); + memCounterTTT += memCounterOracleTTT.getQueryCounter().getCount(); + + eqCounterLstar += eqCounterOracleLstar.getCounter(); + eqCounterRS += eqCounterOracleRS.getCounter(); + eqCounterKV += eqCounterOracleKV.getCounter(); + eqCounterOP += eqCounterOracleOP.getCounter(); + eqCounterTTT += eqCounterOracleTTT.getCounter(); + } + //Calculate average for configuration (x,y,z) for the counters + { + System.out.println("For configuration (x,y,z):" + 0.1 + "," + x.first + "," + x.second + " :"); + System.out.println("Average memCounterLstar: " + memCounterLstar / 100); + System.out.println("Average memCounterRS: " + memCounterRS / 100); + System.out.println("Average memCounterKV: " + memCounterKV / 100); + System.out.println("Average memCounterOP: " + memCounterOP / 100); + System.out.println("Average memCounterTTT: " + memCounterTTT / 100); + + System.out.println("Average eqCounterLstar: " + eqCounterLstar / 100); + System.out.println("Average eqCounterRS: " + eqCounterRS / 100); + System.out.println("Average eqCounterKV: " + eqCounterKV / 100); + System.out.println("Average eqCounterOP: " + eqCounterOP / 100); + System.out.println("Average eqCounterTTT: " + eqCounterTTT / 100); + + // Calculate the averages and store them in the arrays + + avgMemQueriesLstar[k] = memCounterLstar / 100.0; + avgMemQueriesRS[k] = memCounterRS / 100.0; + avgMemQueriesKV[k] = memCounterKV / 100.0; + avgMemQueriesOP[k] = memCounterOP / 100.0; + avgMemQueriesTTT[k] = memCounterTTT / 100.0; + + avgEqQueriesLstar[k] = eqCounterLstar / 100.0; + avgEqQueriesRS[k] = eqCounterRS / 100.0; + avgEqQueriesKV[k] = eqCounterKV / 100.0; + avgEqQueriesOP[k] = eqCounterOP / 100.0; + avgEqQueriesTTT[k] = eqCounterTTT / 100.0; + + accuracyLstar[k] = (double) controleLstar / 100; + accuracyRS[k] = (double) controleRS / 100; + accuracyKV[k] = (double) controleKV / 100; + accuracyOP[k] = (double) controleOP / 100; + accuracyTTT[k] = (double) controleTTT / 100; + } + k++; + } + + String[] seriesNames = {"LStar","RS","KV", "OP", "TTT"}; + + + double xValues[] = {1.0,2.0,3.0,4.0}; + + + XYChart chart5 = QuickChart.getChart("AcceptanceRatio=0.1, Test Size = 15", "Complexity", "Accuracy", seriesNames, xValues, new double[][]{accuracyLstar, accuracyRS, accuracyKV, accuracyOP, accuracyTTT}); + new SwingWrapper(chart5).displayChart(); + + } + + + + // Define the learning methods + public static int learnWithClassicLStarDFA(DFA<?, Integer> dfa, Alphabet<Integer> alphabet, EquivalenceOracle<DFA<?, Integer>, Integer, Boolean> eqOracleLstar, MembershipOracle.DFAMembershipOracle<Integer> memOracleLstar, EquivalenceOracle<DFA<?, Integer>, Integer, Boolean> controle){ + // Create an instance of the ClassicLStar algorithm + ClassicLStarDFA<Integer> learner = + new ClassicLStarDFABuilder<Integer>().withAlphabet(alphabet) // input alphabet + .withOracle(memOracleLstar) // membership oracle + .create(); + + // Start the learning process + learner.startLearning(); + // Perform equivalence queries until no counterexample is found + DefaultQuery<Integer, Boolean> counterexample; + while ((counterexample = eqOracleLstar.findCounterExample(learner.getHypothesisModel(), alphabet)) != null) { + learner.refineHypothesis(counterexample); + } + if((counterexample = controle.findCounterExample(learner.getHypothesisModel(), alphabet)) == null){return 1;} + return 0; + } + + public static int learnWithRivestShapireDFA(DFA<?, Integer> dfa, Alphabet<Integer> alphabet, EquivalenceOracle<DFA<?, Integer>, Integer, Boolean> eqOracleRS, MembershipOracle.DFAMembershipOracle<Integer> memOracleRS, EquivalenceOracle<DFA<?, Integer>, Integer, Boolean> controle) { + // Implement learning with RivestShapireDFA + RivestSchapireDFABuilder<Integer> rsBuilder = new RivestSchapireDFABuilder<>(); + rsBuilder.withAlphabet(alphabet); // input alphabet + rsBuilder.withOracle(memOracleRS); // membership oracle + RivestSchapireDFA<Integer> learner = rsBuilder.create(); + learner.startLearning(); + DefaultQuery<Integer, Boolean> counterexample; + + while ((counterexample = eqOracleRS.findCounterExample(learner.getHypothesisModel(), alphabet)) != null) { + learner.refineHypothesis(counterexample); + } + if((counterexample = controle.findCounterExample(learner.getHypothesisModel(), alphabet)) == null){return 1;} + return 0; + } + + public static int learnWithKearnsVauiraniDFA(DFA<?, Integer> dfa, Alphabet<Integer> alphabet, EquivalenceOracle<DFA<?, Integer>, Integer, Boolean> eqOracleKV, MembershipOracle.DFAMembershipOracle<Integer> memOracleKV, EquivalenceOracle<DFA<?, Integer>, Integer, Boolean> controle) { + KearnsVaziraniDFABuilder<Integer> kol= new KearnsVaziraniDFABuilder<Integer>(); + kol.withAlphabet(alphabet); + kol.withOracle(memOracleKV); + kol.withRepeatedCounterexampleEvaluation(true); + KearnsVaziraniDFA<Integer> learner = kol.create(); + learner.startLearning(); + DefaultQuery<Integer, Boolean> counterexample; + + while ((counterexample = eqOracleKV.findCounterExample(learner.getHypothesisModel(), alphabet)) != null) { + learner.refineHypothesis(counterexample); + } + if((counterexample = controle.findCounterExample(learner.getHypothesisModel(), alphabet)) == null){return 1;} + return 0; + } + + public static int learnWithOPLearnerDFA(DFA<?, Integer> dfa, Alphabet<Integer> alphabet, EquivalenceOracle<DFA<?, Integer>, Integer, Boolean> eqOracleOP, MembershipOracle.DFAMembershipOracle<Integer> memOracleOP, EquivalenceOracle<DFA<?, Integer>, Integer, Boolean> controle) { + + OPLearnerDFABuilder<Integer> opBuilder = new OPLearnerDFABuilder<>(); + opBuilder.withAlphabet(alphabet); + opBuilder.withOracle(memOracleOP); + opBuilder.setRepeatedCounterexampleEvaluation(true); + OPLearnerDFA<Integer> learner = opBuilder.create(); + + + // Perform equivalence queries until no counterexample is found + DefaultQuery<Integer, Boolean> counterexample; + learner.startLearning(); + while ((counterexample = eqOracleOP.findCounterExample(learner.getHypothesisModel(), alphabet)) != null) { + learner.refineHypothesis(counterexample); + } + if((counterexample = controle.findCounterExample(learner.getHypothesisModel(), alphabet)) == null){return 1;} + return 0; + } + + public static int learnWithTTTLearnerDFA(DFA<?, Integer> dfa, Alphabet<Integer> alphabet, EquivalenceOracle<DFA<?, Integer>, Integer, Boolean> oracle, MembershipOracle.DFAMembershipOracle<Integer> mem, EquivalenceOracle<DFA<?, Integer>, Integer, Boolean> controle) { + TTTLearnerDFA<Integer> learner = + new TTTLearnerDFABuilder<Integer>().withAlphabet(alphabet) + .withOracle(mem) + .create(); + + learner.startLearning(); + + + // Perform equivalence queries until no counterexample is found + DefaultQuery counterexample; + + while ((counterexample = oracle.findCounterExample(learner.getHypothesisModel(), alphabet)) != null) { + learner.refineHypothesis(counterexample); + } + if((counterexample = controle.findCounterExample(learner.getHypothesisModel(), alphabet)) == null){return 1;} + return 0; + } + + + +}