<commit_before>package beast.util;

import beast.core.util.Log;
import beast.evolution.alignment.*;
import beast.evolution.datatype.DataType;
import beast.evolution.datatype.StandardData;
import beast.evolution.datatype.UserDataType;
import beast.evolution.tree.TraitSet;
import beast.evolution.tree.Tree;

import java.io.*;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;


/**
 * parses nexus file and grabs alignment and calibration from the file *
 */
public class NexusParser {
    /**
     * keep track of nexus file line number, to report when the file does not parse *
     */
    int lineNr;

    /**
     * Beast II objects reconstructed from the file*
     */
    public Alignment m_alignment;
    public List<Alignment> filteredAlignments = new ArrayList<Alignment>();
    public TraitSet traitSet;

    public List<String> taxa;
    public List<Tree> trees;

    static Set<String> g_sequenceIDs;

    public Map<String, String> translationMap = null;

    static {
        g_sequenceIDs = new HashSet<String>();
    }

    public List<TaxonSet> taxonsets = new ArrayList<TaxonSet>();

//    private List<NexusParserListener> listeners = new ArrayList<NexusParserListener>();

//    public void addListener(final NexusParserListener listener) {
//        listeners.add(listener);
//    }

    /**
     * Try to parse BEAST 2 objects from the given file
     *
     * @param file the file to parse.
     */
    public void parseFile(final File file) throws Exception {
        final String fileName = file.getName().replaceAll(".*[\\/\\\\]", "").replaceAll("\\..*", "");

        parseFile(fileName, new FileReader(file));
    }

    /**
     * try to reconstruct Beast II objects from the given reader
     *
     * @param id     a name to give to the parsed results
     * @param reader a reader to parse from
     */
    public void parseFile(final String id, final Reader reader) throws Exception {
        lineNr = 0;
        final BufferedReader fin;
        if (reader instanceof BufferedReader) {
            fin = (BufferedReader) reader;
        } else {
            fin = new BufferedReader(reader);
        }
        try {
            while (fin.ready()) {
                final String sStr = nextLine(fin);
                if (sStr == null) {
                    return;
                }
                final String sLower = sStr.toLowerCase();
                if (sLower.matches("^\\s*begin\\s+data;\\s*$") || sLower.matches("^\\s*begin\\s+characters;\\s*$")) {
                    m_alignment = parseDataBlock(fin);
                    m_alignment.setID(id);
                } else if (sLower.matches("^\\s*begin\\s+calibration;\\s*$")) {
                    traitSet = parseCalibrationsBlock(fin);
                } else if (sLower.matches("^\\s*begin\\s+assumptions;\\s*$") ||
                        sLower.matches("^\\s*begin\\s+sets;\\s*$") ||
                        sLower.matches("^\\s*begin\\s+mrbayes;\\s*$")) {
                    parseAssumptionsBlock(fin);
                } else if (sLower.matches("^\\s*begin\\s+taxa;\\s*$")) {
                    parseTaxaBlock(fin);
                } else if (sLower.matches("^\\s*begin\\s+trees;\\s*$")) {
                    parseTreesBlock(fin);
                }
            }
        } catch (Exception e) {
            e.printStackTrace();
            throw new Exception("Around line " + lineNr + "\n" + e.getMessage());
        }
    } // parseFile

    private void parseTreesBlock(final BufferedReader fin) throws Exception {
        trees = new ArrayList<Tree>();
        // read to first non-empty line within trees block
        String sStr = fin.readLine().trim();
        while (sStr.equals("")) {
            sStr = fin.readLine().trim();
        }

        int origin = -1;

        // if first non-empty line is "translate" then parse translate block
        if (sStr.toLowerCase().contains("translate")) {
            translationMap = parseTranslateBlock(fin);
            origin = getIndexedTranslationMapOrigin(translationMap);
            if (origin != -1) {
                taxa = getIndexedTranslationMap(translationMap, origin);
            }
        }

        // read trees
        while (sStr != null) {
            if (sStr.toLowerCase().startsWith("tree ")) {
                final int i = sStr.indexOf('(');
                if (i > 0) {
                    sStr = sStr.substring(i);
                }
                TreeParser treeParser;

                if (origin != -1) {
                    treeParser = new TreeParser(taxa, sStr, origin, false);
                } else {
                    try {
                        treeParser = new TreeParser(taxa, sStr, 0, false);
                    } catch (ArrayIndexOutOfBoundsException e) {
                        treeParser = new TreeParser(taxa, sStr, 1, false);
                    }
                }
//                catch (NullPointerException e) {
//                    treeParser = new TreeParser(m_taxa, sStr, 1);
//                }

//                for (final NexusParserListener listener : listeners) {
//                    listener.treeParsed(trees.size(), treeParser);
//                }

                if (translationMap != null) treeParser.translateLeafIds(translationMap);

                trees.add(treeParser);

//				Node tree = treeParser.getRoot();
//				tree.sort();
//				tree.labelInternalNodes(nNrOfLabels);
            }
            sStr = fin.readLine();
            if (sStr != null) sStr = sStr.trim();
        }
    }

    private List<String> getIndexedTranslationMap(final Map<String, String> translationMap, final int origin) {

        System.out.println("translation map size = " + translationMap.size());

        final String[] taxa = new String[translationMap.size()];

        for (final String key : translationMap.keySet()) {
            taxa[Integer.parseInt(key) - origin] = translationMap.get(key);
        }
        return Arrays.asList(taxa);
    }

    /**
     * @param translationMap
     * @return minimum key value if keys are a contiguous set of integers starting from zero or one, -1 otherwise<commit_msg>TreeStat2 was broken because somebody removed NexusParserListener support from the NexusParser class. Reinstated.<commit_after>package beast.util;

import beast.core.util.Log;
import beast.evolution.alignment.*;
import beast.evolution.datatype.DataType;
import beast.evolution.datatype.StandardData;
import beast.evolution.datatype.UserDataType;
import beast.evolution.tree.TraitSet;
import beast.evolution.tree.Tree;

import java.io.*;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;


/**
 * parses nexus file and grabs alignment and calibration from the file *
 */
public class NexusParser {
    /**
     * keep track of nexus file line number, to report when the file does not parse *
     */
    int lineNr;

    /**
     * Beast II objects reconstructed from the file*
     */
    public Alignment m_alignment;
    public List<Alignment> filteredAlignments = new ArrayList<Alignment>();
    public TraitSet traitSet;

    public List<String> taxa;
    public List<Tree> trees;

    static Set<String> g_sequenceIDs;

    public Map<String, String> translationMap = null;

    static {
        g_sequenceIDs = new HashSet<>();
    }

    public List<TaxonSet> taxonsets = new ArrayList<>();

    private List<NexusParserListener> listeners = new ArrayList<>();

    /**
     * Adds a listener for client classes that want to monitor progress of the parsing.
     * @param listener
     */
    public void addListener(final NexusParserListener listener) {
        listeners.add(listener);
    }

    /**
     * Try to parse BEAST 2 objects from the given file
     *
     * @param file the file to parse.
     */
    public void parseFile(final File file) throws Exception {
        final String fileName = file.getName().replaceAll(".*[\\/\\\\]", "").replaceAll("\\..*", "");

        parseFile(fileName, new FileReader(file));
    }

    /**
     * try to reconstruct Beast II objects from the given reader
     *
     * @param id     a name to give to the parsed results
     * @param reader a reader to parse from
     */
    public void parseFile(final String id, final Reader reader) throws Exception {
        lineNr = 0;
        final BufferedReader fin;
        if (reader instanceof BufferedReader) {
            fin = (BufferedReader) reader;
        } else {
            fin = new BufferedReader(reader);
        }
        try {
            while (fin.ready()) {
                final String sStr = nextLine(fin);
                if (sStr == null) {
                    return;
                }
                final String sLower = sStr.toLowerCase();
                if (sLower.matches("^\\s*begin\\s+data;\\s*$") || sLower.matches("^\\s*begin\\s+characters;\\s*$")) {
                    m_alignment = parseDataBlock(fin);
                    m_alignment.setID(id);
                } else if (sLower.matches("^\\s*begin\\s+calibration;\\s*$")) {
                    traitSet = parseCalibrationsBlock(fin);
                } else if (sLower.matches("^\\s*begin\\s+assumptions;\\s*$") ||
                        sLower.matches("^\\s*begin\\s+sets;\\s*$") ||
                        sLower.matches("^\\s*begin\\s+mrbayes;\\s*$")) {
                    parseAssumptionsBlock(fin);
                } else if (sLower.matches("^\\s*begin\\s+taxa;\\s*$")) {
                    parseTaxaBlock(fin);
                } else if (sLower.matches("^\\s*begin\\s+trees;\\s*$")) {
                    parseTreesBlock(fin);
                }
            }
        } catch (Exception e) {
            e.printStackTrace();
            throw new Exception("Around line " + lineNr + "\n" + e.getMessage());
        }
    } // parseFile

    private void parseTreesBlock(final BufferedReader fin) throws Exception {
        trees = new ArrayList<Tree>();
        // read to first non-empty line within trees block
        String sStr = fin.readLine().trim();
        while (sStr.equals("")) {
            sStr = fin.readLine().trim();
        }

        int origin = -1;

        // if first non-empty line is "translate" then parse translate block
        if (sStr.toLowerCase().contains("translate")) {
            translationMap = parseTranslateBlock(fin);
            origin = getIndexedTranslationMapOrigin(translationMap);
            if (origin != -1) {
                taxa = getIndexedTranslationMap(translationMap, origin);
            }
        }

        // read trees
        while (sStr != null) {
            if (sStr.toLowerCase().startsWith("tree ")) {
                final int i = sStr.indexOf('(');
                if (i > 0) {
                    sStr = sStr.substring(i);
                }
                TreeParser treeParser;

                if (origin != -1) {
                    treeParser = new TreeParser(taxa, sStr, origin, false);
                } else {
                    try {
                        treeParser = new TreeParser(taxa, sStr, 0, false);
                    } catch (ArrayIndexOutOfBoundsException e) {
                        treeParser = new TreeParser(taxa, sStr, 1, false);
                    }
                }
//                catch (NullPointerException e) {
//                    treeParser = new TreeParser(m_taxa, sStr, 1);
//                }

                for (final NexusParserListener listener : listeners) {
                    listener.treeParsed(trees.size(), treeParser);
                }

                if (translationMap != null) treeParser.translateLeafIds(translationMap);

                trees.add(treeParser);

//				Node tree = treeParser.getRoot();
//				tree.sort();
//				tree.labelInternalNodes(nNrOfLabels);
            }
            sStr = fin.readLine();
            if (sStr != null) sStr = sStr.trim();
        }
    }

    private List<String> getIndexedTranslationMap(final Map<String, String> translationMap, final int origin) {

        System.out.println("translation map size = " + translationMap.size());

        final String[] taxa = new String[translationMap.size()];

        for (final String key : translationMap.keySet()) {
            taxa[Integer.parseInt(key) - origin] = translationMap.get(key);
        }
        return Arrays.asList(taxa);
    }

    /**
     * @param translationMap
     * @return minimum key value if keys are a contiguous set of integers starting from zero or one, -1 otherwise
{'fraction_non_alphanumeric': 0.12058494486175483, 'fraction_numerical': 0.0016781205050343615, 'mean_word_length': 1.861879716441802, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2623852', 'n_tokens_mistral': 3536, 'n_tokens_neox': 3339, 'n_words': 863}