Skip to content
Snippets Groups Projects
Select Git revision
  • 58b23bef1e5ba6d4f194e975f5f7daa20e7b2932
  • main default protected
2 results

ExperimentsRandomTestsFixedSystemVarTestSize.java

Blame
    • Code owners
    Assign users and groups as approvers for specific file changes. Learn more.
    ExperimentsRandomTestsFixedSystemVarTestSize.java 17.32 KiB 
    import de.learnlib.acex.AcexAnalyzer;
import de.learnlib.algorithm.kv.dfa.KearnsVaziraniDFA;
import de.learnlib.algorithm.kv.dfa.KearnsVaziraniDFABuilder;
import de.learnlib.algorithm.lstar.dfa.ClassicLStarDFABuilder;
import de.learnlib.algorithm.observationpack.dfa.OPLearnerDFA;
import de.learnlib.algorithm.observationpack.dfa.OPLearnerDFABuilder;
import de.learnlib.algorithm.rivestschapire.RivestSchapireDFA;
import de.learnlib.algorithm.ttt.base.AbstractTTTLearner;
import de.learnlib.algorithm.ttt.dfa.TTTLearnerDFA;
import de.learnlib.algorithm.ttt.dfa.TTTLearnerDFABuilder;
import de.learnlib.filter.statistic.oracle.DFACounterOracle;
import de.learnlib.oracle.EquivalenceOracle;
import de.learnlib.oracle.MembershipOracle;
import de.learnlib.oracle.equivalence.DFARandomWordsEQOracle;
import de.learnlib.oracle.equivalence.DFASimulatorEQOracle;
import de.learnlib.oracle.equivalence.WMethodEQOracle;
import de.learnlib.oracle.membership.DFASimulatorOracle;
import de.learnlib.query.DefaultQuery;
import net.automatalib.alphabet.Alphabet;
import net.automatalib.alphabet.Alphabets;
import net.automatalib.automaton.fsa.DFA;
import de.learnlib.algorithm.lstar.dfa.ClassicLStarDFA;
import de.learnlib.algorithm.rivestschapire.RivestSchapireDFABuilder;
import net.automatalib.word.Word;
import org.knowm.xchart.QuickChart;
import org.knowm.xchart.SwingWrapper;
import org.knowm.xchart.XYChart;

import java.util.ArrayList;
import java.util.List;
import java.util.Random;

public class ExperimentsRandomTestsFixedSystemVarTestSize {

    public static void main(String[] args) {
        // Define configurations
        int[] yValues = { 5 };
        int[] zValues = { 100 };
        double[] xValues = { 10,20,30,40,50,60,70,80,90,100,150,200,500};

        // Declare arrays to store the average membership queries and average equivalence queries
        double[] avgMemQueriesLstar = new double[xValues.length];
        double[] avgMemQueriesRS = new double[xValues.length];
        double[] avgMemQueriesKV = new double[xValues.length];
        double[] avgMemQueriesOP = new double[xValues.length];
        double[] avgMemQueriesTTT = new double[xValues.length];

        double[] avgEqQueriesLstar = new double[xValues.length];
        double[] avgEqQueriesRS = new double[xValues.length];
        double[] avgEqQueriesKV = new double[xValues.length];
        double[] avgEqQueriesOP = new double[xValues.length];
        double[] avgEqQueriesTTT = new double[xValues.length];


        double[] accuracyLstar = new double[xValues.length];
        double[] accuracyRS = new double[xValues.length];
        double[] accuracyKV = new double[xValues.length];
        double[] accuracyOP = new double[xValues.length];
        double[] accuracyTTT = new double[xValues.length];




        // Iterate over each configuration: y denotes the size of the alphabet of the target DFA, z denotes the size of the target DFA, x denotes the acceptance ratio of the target DFA
        for(int y:yValues) {
            for (int z : zValues) {
                int k=0;
                for (double x : xValues) {
                    // Create the alphabet
                    Alphabet<Integer> alphabet = Alphabets.integers(0, y - 1);
    
                    int memCounterLstar = 0;
                    int memCounterRS = 0;
                    int memCounterKV = 0;
                    int memCounterOP = 0;
                    int memCounterTTT = 0;

                    int eqCounterLstar = 0;
                    int eqCounterRS = 0;
                    int eqCounterKV = 0;
                    int eqCounterOP = 0;
                    int eqCounterTTT = 0;

                    int controleLstar = 0;
                    int controleRS = 0;
                    int controleKV = 0;
                    int controleOP = 0;
                    int controleTTT = 0;

                    // Generate 100 random DFAs for each configuration
                    for (int i = 0; i < 100; i++) {
                        System.out.println("Generating random DFA " + i + " for configuration (x,y,z):" + 0.1 + "," + y + "," + z + "Test Size: " + x);
                        Random rand = new Random();
                        ExampleRandomDFA example = new ExampleRandomDFA(rand, y, z, 0.1, alphabet);
                        DFA<?, Integer> dfa = example.getReferenceAutomaton();


                        EquivalenceOracle<DFA<?, Integer>, Integer, Boolean> eqOracleControle = new DFASimulatorEQOracle<>(dfa);

                        // Initialize the Query Counters and the Oracles - we used the white-box oracles

                        MembershipOracle.DFAMembershipOracle<Integer> memOracleLstar = new DFASimulatorOracle<>(dfa);
                        DFACounterOracle<Integer> memCounterOracleLstar = new DFACounterOracle<>(memOracleLstar);
                        MembershipOracle.DFAMembershipOracle<Integer> memUtilOracleLstar = new DFASimulatorOracle<>(dfa);
                        EquivalenceOracle<DFA<?, Integer>, Integer, Boolean> eqOracleLstar = new DFARandomWordsEQOracle<Integer>(memUtilOracleLstar,1,100, (int) x,rand);
                        EQCounterOracle<DFA<?, Integer>, Integer, Boolean> eqCounterOracleLstar = new EQCounterOracle<>(eqOracleLstar);

                        MembershipOracle.DFAMembershipOracle<Integer> memOracleRS = new DFASimulatorOracle<>(dfa);
                        DFACounterOracle<Integer> memCounterOracleRS = new DFACounterOracle<>(memOracleRS);
                        MembershipOracle.DFAMembershipOracle<Integer> memUtilOracleRS = new DFASimulatorOracle<>(dfa);
                        EquivalenceOracle<DFA<?, Integer>, Integer, Boolean> eqOracleRS = new DFARandomWordsEQOracle<Integer>(memUtilOracleRS,1,100, (int) x,rand);
                        EQCounterOracle<DFA<?, Integer>, Integer, Boolean> eqCounterOracleRS = new EQCounterOracle<>(eqOracleRS);

                        MembershipOracle.DFAMembershipOracle<Integer> memOracleKV = new DFASimulatorOracle<>(dfa);
                        DFACounterOracle<Integer> memCounterOracleKV = new DFACounterOracle<>(memOracleKV);
                        MembershipOracle.DFAMembershipOracle<Integer> memUtilOracleKV = new DFASimulatorOracle<>(dfa);
                        EquivalenceOracle<DFA<?, Integer>, Integer, Boolean> eqOracleKV = new DFARandomWordsEQOracle<Integer>(memUtilOracleKV,1,100, (int) x,rand);
                        EQCounterOracle<DFA<?, Integer>, Integer, Boolean> eqCounterOracleKV = new EQCounterOracle<>(eqOracleRS);

                        MembershipOracle.DFAMembershipOracle<Integer> memOracleOP = new DFASimulatorOracle<>(dfa);
                        DFACounterOracle<Integer> memCounterOracleOP = new DFACounterOracle<>(memOracleOP);
                        MembershipOracle.DFAMembershipOracle<Integer> memUtilOracleOP = new DFASimulatorOracle<>(dfa);
                        EquivalenceOracle<DFA<?, Integer>, Integer, Boolean> eqOracleOP = new DFARandomWordsEQOracle<Integer>(memUtilOracleRS,1,100, (int) x,rand);
                        EQCounterOracle<DFA<?, Integer>, Integer, Boolean> eqCounterOracleOP = new EQCounterOracle<>(eqOracleRS);

                        MembershipOracle.DFAMembershipOracle<Integer> memOracleTTT = new DFASimulatorOracle<>(dfa);
                        DFACounterOracle<Integer> memCounterOracleTTT = new DFACounterOracle<>(memOracleTTT);
                        MembershipOracle.DFAMembershipOracle<Integer> memUtilOracleTTT = new DFASimulatorOracle<>(dfa);
                        EquivalenceOracle<DFA<?, Integer>, Integer, Boolean> eqOracleTTT = new DFARandomWordsEQOracle<Integer>(memUtilOracleRS,1,100, (int) x,rand);
                        EQCounterOracle<DFA<?, Integer>, Integer, Boolean> eqCounterOracleTTT = new EQCounterOracle<>(eqOracleRS);


                        // Learn the DFA using each learning algorithm
                        if(learnWithClassicLStarDFA(dfa, alphabet, eqCounterOracleLstar, memCounterOracleLstar, eqOracleControle)==1){controleLstar++;};
                        if(learnWithRivestShapireDFA(dfa, alphabet, eqCounterOracleRS, memCounterOracleRS, eqOracleControle)==1){controleRS++;};
                        if((learnWithKearnsVauiraniDFA(dfa, alphabet, eqCounterOracleKV, memCounterOracleKV, eqOracleControle)==1)){controleKV++;};
                        if(learnWithOPLearnerDFA(dfa, alphabet, eqCounterOracleOP, memCounterOracleOP, eqOracleControle)==1){controleOP++;};
                        if(learnWithTTTLearnerDFA(dfa, alphabet, eqCounterOracleTTT, memCounterOracleTTT, eqOracleControle)==1){controleTTT++;};


                            // Update the counters
                        memCounterLstar += memCounterOracleLstar.getQueryCounter().getCount();
                        memCounterRS += memCounterOracleRS.getQueryCounter().getCount();
                        memCounterKV += memCounterOracleKV.getQueryCounter().getCount();
                        memCounterOP += memCounterOracleOP.getQueryCounter().getCount();
                        memCounterTTT += memCounterOracleTTT.getQueryCounter().getCount();

                        eqCounterLstar += eqCounterOracleLstar.getCounter();
                        eqCounterRS += eqCounterOracleRS.getCounter();
                        eqCounterKV += eqCounterOracleKV.getCounter();
                        eqCounterOP += eqCounterOracleOP.getCounter();
                        eqCounterTTT += eqCounterOracleTTT.getCounter();
                    }
                    //Calculate average for configuration (x,y,z) for the counters
                    System.out.println("For configuration (x,y,z):" + x + "," + y + "," + z + " :");
                    System.out.println("Average memCounterLstar: " + memCounterLstar/100);
                    System.out.println("Average memCounterRS: " + memCounterRS/100);
                    System.out.println("Average memCounterKV: " + memCounterKV/100);
                    System.out.println("Average memCounterOP: " + memCounterOP/100);
                    System.out.println("Average memCounterTTT: " + memCounterTTT/100);

                    System.out.println("Average eqCounterLstar: " + eqCounterLstar/100);
                    System.out.println("Average eqCounterRS: " + eqCounterRS/100);
                    System.out.println("Average eqCounterKV: " + eqCounterKV/100);
                    System.out.println("Average eqCounterOP: " + eqCounterOP/100);
                    System.out.println("Average eqCounterTTT: " + eqCounterTTT/100);

                    // Calculate the averages and store them in the arrays

                    avgMemQueriesLstar[k] = memCounterLstar / 100.0;
                    avgMemQueriesRS[k] = memCounterRS / 100.0;
                    avgMemQueriesKV[k] = memCounterKV / 100.0;
                    avgMemQueriesOP[k] = memCounterOP / 100.0;
                    avgMemQueriesTTT[k] = memCounterTTT / 100.0;

                    avgEqQueriesLstar[k] = eqCounterLstar / 100.0;
                    avgEqQueriesRS[k] = eqCounterRS / 100.0;
                    avgEqQueriesKV[k] = eqCounterKV / 100.0;
                    avgEqQueriesOP[k] = eqCounterOP / 100.0;
                    avgEqQueriesTTT[k] = eqCounterTTT / 100.0;

                    accuracyLstar[k] = (double)controleLstar/100;
                    accuracyRS[k] = (double)controleRS/100;
                    accuracyKV[k] = (double)controleKV/100;
                    accuracyOP[k] = (double)controleOP/100;
                    accuracyTTT[k] = (double)controleTTT/100;

                    k++;
                }
            }
        }
        String[] seriesNames = {"LStar","RS","KV", "OP", "TTT"};

// Create the first XY chart (membership queries)
        XYChart chart1 = QuickChart.getChart("Average Membership Queries", "Test Size", "Queries", seriesNames, xValues, new double[][]{avgMemQueriesLstar,avgMemQueriesRS, avgMemQueriesKV, avgMemQueriesOP, avgMemQueriesTTT});
        new SwingWrapper(chart1).displayChart();

// Create the second XY chart (equivalence queries)
        XYChart chart2 = QuickChart.getChart("Average Equivalence Queries", "Test Size", "Queries", seriesNames, xValues, new double[][]{avgEqQueriesLstar, avgEqQueriesRS,avgEqQueriesKV, avgEqQueriesOP, avgEqQueriesTTT});
        new SwingWrapper(chart2).displayChart();

        XYChart chart5 = QuickChart.getChart("Accuracy", "Test Size", "Queries", seriesNames, xValues, new double[][]{accuracyLstar, accuracyRS, accuracyKV, accuracyOP, accuracyTTT});
        new SwingWrapper(chart5).displayChart();

    }



    // Define the learning methods
    public static int learnWithClassicLStarDFA(DFA<?, Integer> dfa, Alphabet<Integer> alphabet, EquivalenceOracle<DFA<?, Integer>, Integer, Boolean> eqOracleLstar, MembershipOracle.DFAMembershipOracle<Integer> memOracleLstar, EquivalenceOracle<DFA<?, Integer>, Integer, Boolean> controle){
            // Create an instance of the ClassicLStar algorithm
        ClassicLStarDFA<Integer> learner =
                new ClassicLStarDFABuilder<Integer>().withAlphabet(alphabet) // input alphabet
                        .withOracle(memOracleLstar) // membership oracle
                        .create();

        // Start the learning process
        learner.startLearning();
        // Perform equivalence queries until no counterexample is found
        DefaultQuery<Integer, Boolean> counterexample;
        while ((counterexample = eqOracleLstar.findCounterExample(learner.getHypothesisModel(), alphabet)) != null) {
            learner.refineHypothesis(counterexample);
        }
        if((counterexample = controle.findCounterExample(learner.getHypothesisModel(), alphabet)) == null){return 1;}
        return 0;
    }

    public static int learnWithRivestShapireDFA(DFA<?, Integer> dfa, Alphabet<Integer> alphabet, EquivalenceOracle<DFA<?, Integer>, Integer, Boolean> eqOracleRS, MembershipOracle.DFAMembershipOracle<Integer> memOracleRS, EquivalenceOracle<DFA<?, Integer>, Integer, Boolean> controle) {
        // Implement learning with RivestShapireDFA
        RivestSchapireDFABuilder<Integer> rsBuilder = new RivestSchapireDFABuilder<>();
        rsBuilder.withAlphabet(alphabet); // input alphabet
        rsBuilder.withOracle(memOracleRS); // membership oracle
        RivestSchapireDFA<Integer> learner = rsBuilder.create();
        learner.startLearning();
        DefaultQuery<Integer, Boolean> counterexample;

        while ((counterexample = eqOracleRS.findCounterExample(learner.getHypothesisModel(), alphabet)) != null) {
            learner.refineHypothesis(counterexample);
        }
        if((counterexample = controle.findCounterExample(learner.getHypothesisModel(), alphabet)) == null){return 1;}
        return 0;
    }

    public static int learnWithKearnsVauiraniDFA(DFA<?, Integer> dfa, Alphabet<Integer> alphabet, EquivalenceOracle<DFA<?, Integer>, Integer, Boolean> eqOracleKV, MembershipOracle.DFAMembershipOracle<Integer> memOracleKV, EquivalenceOracle<DFA<?, Integer>, Integer, Boolean> controle) {
        KearnsVaziraniDFABuilder<Integer> kol= new KearnsVaziraniDFABuilder<Integer>();
        kol.withAlphabet(alphabet);
        kol.withOracle(memOracleKV);
        kol.withRepeatedCounterexampleEvaluation(true);
        KearnsVaziraniDFA<Integer> learner = kol.create();
        learner.startLearning();
        DefaultQuery<Integer, Boolean> counterexample;

        while ((counterexample = eqOracleKV.findCounterExample(learner.getHypothesisModel(), alphabet)) != null) {
            learner.refineHypothesis(counterexample);
        }
        if((counterexample = controle.findCounterExample(learner.getHypothesisModel(), alphabet)) == null){return 1;}
        return 0;
    }

    public static int learnWithOPLearnerDFA(DFA<?, Integer> dfa, Alphabet<Integer> alphabet, EquivalenceOracle<DFA<?, Integer>, Integer, Boolean> eqOracleOP, MembershipOracle.DFAMembershipOracle<Integer> memOracleOP, EquivalenceOracle<DFA<?, Integer>, Integer, Boolean> controle) {

        OPLearnerDFABuilder<Integer> opBuilder = new OPLearnerDFABuilder<>();
        opBuilder.withAlphabet(alphabet);
        opBuilder.withOracle(memOracleOP);
        opBuilder.setRepeatedCounterexampleEvaluation(true);
        OPLearnerDFA<Integer> learner = opBuilder.create();


        // Perform equivalence queries until no counterexample is found
        DefaultQuery<Integer, Boolean> counterexample;
        learner.startLearning();
        while ((counterexample = eqOracleOP.findCounterExample(learner.getHypothesisModel(), alphabet)) != null) {
            learner.refineHypothesis(counterexample);
        }
        if((counterexample = controle.findCounterExample(learner.getHypothesisModel(), alphabet)) == null){return 1;}
        return 0;
    }

    public static int learnWithTTTLearnerDFA(DFA<?, Integer> dfa, Alphabet<Integer> alphabet, EquivalenceOracle<DFA<?, Integer>, Integer, Boolean> oracle, MembershipOracle.DFAMembershipOracle<Integer> mem, EquivalenceOracle<DFA<?, Integer>, Integer, Boolean> controle) {
        TTTLearnerDFA<Integer> learner =
                    new TTTLearnerDFABuilder<Integer>().withAlphabet(alphabet)
                        .withOracle(mem)
                        .create();

        learner.startLearning();


        // Perform equivalence queries until no counterexample is found
        DefaultQuery counterexample;

        while ((counterexample = oracle.findCounterExample(learner.getHypothesisModel(), alphabet)) != null) {
            learner.refineHypothesis(counterexample);
        }
        if((counterexample = controle.findCounterExample(learner.getHypothesisModel(), alphabet)) == null){return 1;}
        return 0;
    }


}