<commit_before>package generator;

import util.State;

public class BigFstGenerator {

    private StringBuffer strBuff;

    public BigFstGenerator() {
    }

    public StringBuffer compute(State initState, String className) {
        strBuff = new StringBuffer();
        System.out.println("[FstGenerator] Transforming Fst to a java class");
        append("package generated;");

        append("public class " + className + " {");

        appendWithTab("public static float compute(int[] token) {", 1);

        appendWithTab("int pos=0;", 2);
        appendWithTab("float result=0f;", 2);
        appendWithTab("return state_0(token, pos, result);", 2);

        appendWithTab("}", 1);

        generateCases(initState);

        append("}");
        System.out.println("[FstGenerator] Successfully transformed fst to " + className + ".java");
        return strBuff;
    }


    private void generateCases(State currentState) {
        append("\n\tprivate static float state_" + currentState.getId() +
                "(int[] token, int pos, float result) {");
       
        if( currentState.getNumArcs() > 0) { 
            generateTokenLengthTest(2);
            appendWithTab("switch(token[pos++]) {", 2);
            for (int i = 0; i < currentState.getNumArcs(); i++) {
                appendWithTab("case " + currentState.getArc(i).getIlabel()  + ":", 3); 
                if (currentState.getArc(i).getWeight() != 0f) {
                    appendWithTab("result+=" + currentState.getArc(i).getWeight() + "f;", 4);
                }
                generateCases(currentState.getArc(i).getNextState(), 4);
            }
            appendWithTab("default:", 3);
            appendWithTab("return -1;", 4);
            appendWithTab("}", 2);
        } else {
            appendWithTab("return (pos!=token.length) ? -1 : result;", 2);
        }
        appendWithTab("}", 1);
    }

    private void generateTokenLengthTest(int tab) {
        appendWithTab("if(pos>=token.length) {return -1;}", tab);
    }

    private void append(String strToAppend) {
        strBuff.append(strToAppend);
        strBuff.append("\n");
    }

    private void appendWithTab(String strToAppend, int numberOfTab) {
        for (int i = 0; i < numberOfTab; i++) {
            strBuff.append("\t");
        }
        strBuff.append(strToAppend);
        strBuff.append("\n");
    }

}<commit_msg>FEATURE/MAJOR: Create a list of next node to be written in the end<commit_after>package generator;

import util.State;
import java.util.ArrayList;

public class BigFstGenerator {

    private StringBuffer strBuff;

    public BigFstGenerator() {
    }

    public StringBuffer compute(State initState, String className) {
        strBuff = new StringBuffer();
        System.out.println("[FstGenerator] Transforming Fst to a java class");
        append("package generated;");

        append("public class " + className + " {");

        appendWithTab("public static float compute(int[] token) {", 1);

        appendWithTab("int pos=0;", 2);
        appendWithTab("float result=0f;", 2);
        appendWithTab("return state_0(token, pos, result);", 2);

        appendWithTab("}", 1);

        generateCases(initState);

        append("}");
        System.out.println("[FstGenerator] Successfully transformed fst to " + className + ".java");
        return strBuff;
    }


    private void generateCases(State currentState) {
        List<State> nextStates = new ArrayList<>();

        append("\n\tprivate static float state_" + currentState.getId() +
                "(int[] token, int pos, float result) {");
       
        if( currentState.getNumArcs() > 0) { 
            generateTokenLengthTest(2);
            appendWithTab("switch(token[pos++]) {", 2);
            for (int i = 0; i < currentState.getNumArcs(); i++) {
                appendWithTab("case " + currentState.getArc(i).getIlabel()  + ":", 3); 
                if (currentState.getArc(i).getWeight() != 0f) {
                    appendWithTab("result+=" + currentState.getArc(i).getWeight() + "f;", 4);
                }
                appendWithTab("return state_" + currentState.getArc(i).getNextState().getId() +
                        "(token, pos, result);", 4);
                nextStates.add(currentState.getArc(i).getNextState());
            }
            appendWithTab("default:", 3);
            appendWithTab("return -1;", 4);
            appendWithTab("}", 2);
        } else {
            appendWithTab("return (pos!=token.length) ? -1 : result;", 2);
        }
        appendWithTab("}", 1);

        for (State next: nextStates) {
            generateCases(next);
        }
    }

    private void generateTokenLengthTest(int tab) {
        appendWithTab("if(pos>=token.length) {return -1;}", tab);
    }

    private void append(String strToAppend) {
        strBuff.append(strToAppend);
        strBuff.append("\n");
    }

    private void appendWithTab(String strToAppend, int numberOfTab) {
        for (int i = 0; i < numberOfTab; i++) {
            strBuff.append("\t");
        }
        strBuff.append(strToAppend);
        strBuff.append("\n");
    }

}
{'fraction_non_alphanumeric': 0.14432789432789433, 'fraction_numerical': 0.009712509712509712, 'mean_word_length': 2.3048780487804876, 'pattern_counts': {'":': 2, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4362254', 'n_tokens_mistral': 1535, 'n_tokens_neox': 1460, 'n_words': 325}