<commit_before>/*
 * LogCombiner.java
 *
 * Copyright (c) 2002-2015 <NAME>, <NAME> and <NAME>
 *
 * This file is part of BEAST.
 * See the NOTICE file distributed with this work for additional
 * information regarding copyright ownership and licensing.
 *
 * BEAST is free software; you can redistribute it and/or modify
 * it under the terms of the GNU Lesser General Public License as
 * published by the Free Software Foundation; either version 2
 * of the License, or (at your option) any later version.
 *
 *  BEAST is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with BEAST; if not, write to the
 * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
 * Boston, MA  02110-1301  USA
 */

package dr.app.tools;

import dr.app.beast.BeastVersion;
import dr.app.util.Arguments;
import dr.evolution.io.Importer;
import dr.evolution.io.NexusImporter;
import dr.evolution.io.TreeImporter;
import dr.evolution.tree.MutableTree;
import dr.evolution.tree.NodeRef;
import dr.evolution.tree.Tree;
import dr.evolution.util.Taxon;
import dr.util.Version;
import jam.console.ConsoleApplication;

import javax.swing.*;
import java.io.*;
import java.text.DecimalFormat;
import java.text.DecimalFormatSymbols;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Locale;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * @author <NAME>
 * @author <NAME>
 * @version $Id:$
 */
public class LogCombiner {

    private final static Version version = new BeastVersion();

    public LogCombiner(long[] burnins, long resample, String[] inputFileNames, String outputFileName, boolean treeFiles,
                       boolean convertToDecimal, boolean stripAnnotations,
                       boolean renumberOutput, boolean useScale, double scale) throws IOException {

        System.out.println("Creating combined " + (treeFiles ? "tree" : "log") + " file: '" + outputFileName + "'");

        if (convertToDecimal) {
            System.out.println("Converting to decimal.");
        }

        if (treeFiles && stripAnnotations) {
            System.out.println("Stripping annotations from trees.");
        }

        if (renumberOutput) {
            System.out.println("Renumbering output.");
        }

        if (useScale) {
            System.out.println("Rescaling using scale factor: " + scale);
        }

        System.out.println();

        PrintWriter writer = new PrintWriter(new FileOutputStream(outputFileName));

        boolean firstFile = true;
        boolean firstTree = true;
        long stateCount = (renumberOutput ? -1 : 0);
        long stateStep = -1;
        int columnCount = 0;

        String[] titles = null;

        System.out.println();
        for (int i = 0; i < inputFileNames.length; i++) {
            File inputFile = new File(inputFileNames[i]);

            if (!inputFile.exists()) {
                System.err.println(inputFileNames[i] + " does not exist!");
                return;
            } else if (inputFile.isDirectory()) {
                System.err.println(inputFileNames[i] + " is a directory.");
                return;
            }

            long burnin = burnins[0];
            if (burnins.length > i) {
                burnin = burnins[i];
            }

            if (burnin > 0) {
                System.out.print("Combining file: '" + inputFileNames[i] + "' removing burnin: " + burnin);
            } else {
                System.out.print("Combining file: '" + inputFileNames[i] + "' without removing burnin");
            }

            if (resample > 0) {
                System.out.print(", resampling with frequency: " + resample);
            }

            if (useScale) {
                System.out.println(", rescaling by: " + scale);
            } else {
                System.out.println();
            }

            boolean processTrees = stripAnnotations || convertToDecimal;

            if (treeFiles) {

                if (processTrees) { // then we need to read each tree and write it.
                    TreeImporter importer = new NexusImporter(new FileReader(inputFile), stripAnnotations);
                    try {
                        while (importer.hasTree()) {
                            Tree tree = importer.importNextTree();
                            if (firstTree) {
                                startLog(tree, writer);
                                firstTree = false;
                            }

                            String name = tree.getId();
                            if (name == null) {
                                System.err.println("ERROR: Trees do not give state numbers as tree attributes.");
                                return;
                            }

                            // split on underscore in STATE_xxxx
                            String[] bits = name.split("_");
                            long state = Long.parseLong(bits[1]);

                            if (stateStep < 0 && state > 0) {
                                stateStep = state;
                            }

                            if (state >= burnin) {
                                if (stateStep > 0) {
                                    if (!renumberOutput) {
                                        stateCount += stateStep;
                                    } else {
                                        stateCount += 1;
                                    }
                                }

                                if (resample >= 0) {
                                    if (resample % stateStep != 0) {
                                        System.err.println("ERROR: Resampling frequency is not a multiple of existing sampling frequency");
                                        return;
                                    }
                                }

                                boolean logThis;
                                if (resample < 0) {
                                    // not resampling, log every state
                                    logThis = true;
                                } else if (!renumberOutput) {
                                    // resampling but not renumbering
                                    logThis = (stateCount % resample == 0);
                                } else {
                                    logThis = (stateCount * stateStep % resample == 0);
                                }

                                long stateLineEntry;
                                if (!renumberOutput) {
                                    stateLineEntry = stateCount;
                                } else {
                                    stateLineEntry = stateCount / (resample / stateStep);
                                }

                                if (logThis) {
                                    writeTree(stateLineEntry, tree, convertToDecimal, writer);
                                }

                            }
                        }
                    } catch (Importer.ImportException e) {
                        System.err.println("Error Parsing Input Tree: " + e.getMessage());
                        return;
                    }
                } else {
                    BufferedReader reader = new BufferedReader(new FileReader(inputFile));

                    String line = reader.readLine();

                    // skip (or write) the headers
                    while (line != null && !line.trim().startsWith("tree ")) {
                        if (firstFile) {
                            writer.println(line);
                        }
                        line = reader.readLine();
                    }

                    Pattern pattern = Pattern.compile("tree STATE_(\\d+)\\s(.*)");

                    while (line != null) {<commit_msg>Update LogCombiner.java<commit_after>/*
 * LogCombiner.java
 *
 * Copyright (c) 2002-2015 <NAME>, <NAME> and <NAME>
 *
 * This file is part of BEAST.
 * See the NOTICE file distributed with this work for additional
 * information regarding copyright ownership and licensing.
 *
 * BEAST is free software; you can redistribute it and/or modify
 * it under the terms of the GNU Lesser General Public License as
 * published by the Free Software Foundation; either version 2
 * of the License, or (at your option) any later version.
 *
 *  BEAST is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with BEAST; if not, write to the
 * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
 * Boston, MA  02110-1301  USA
 */

package dr.app.tools;

import dr.app.beast.BeastVersion;
import dr.app.util.Arguments;
import dr.evolution.io.Importer;
import dr.evolution.io.NexusImporter;
import dr.evolution.io.TreeImporter;
import dr.evolution.tree.MutableTree;
import dr.evolution.tree.NodeRef;
import dr.evolution.tree.Tree;
import dr.evolution.util.Taxon;
import dr.util.Version;
import jam.console.ConsoleApplication;

import javax.swing.*;
import java.io.*;
import java.text.DecimalFormat;
import java.text.DecimalFormatSymbols;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Locale;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * @author <NAME>
 * @author <NAME>
 * @version $Id:$
 */
public class LogCombiner {

    private final static Version version = new BeastVersion();

    public LogCombiner(long[] burnins, long resample, String[] inputFileNames, String outputFileName, boolean treeFiles,
                       boolean convertToDecimal, boolean stripAnnotations,
                       boolean renumberOutput, boolean useScale, double scale) throws IOException {

        System.out.println("Creating combined " + (treeFiles ? "tree" : "log") + " file: '" + outputFileName + "'");

        if (convertToDecimal) {
            System.out.println("Converting to decimal.");
        }

        if (treeFiles && stripAnnotations) {
            System.out.println("Stripping annotations from trees.");
        }

        if (renumberOutput) {
            System.out.println("Renumbering output.");
        }

        if (useScale) {
            System.out.println("Rescaling using scale factor: " + scale);
        }

        System.out.println();

        PrintWriter writer = new PrintWriter(new FileOutputStream(outputFileName));

        boolean firstFile = true;
        boolean firstTree = true;
        long stateCount = (renumberOutput ? -1 : 0);
        long stateStep = -1;
        int columnCount = 0;

        String[] titles = null;

        System.out.println();
        for (int i = 0; i < inputFileNames.length; i++) {
            File inputFile = new File(inputFileNames[i]);

            if (!inputFile.exists()) {
                System.err.println(inputFileNames[i] + " does not exist!");
                return;
            } else if (inputFile.isDirectory()) {
                System.err.println(inputFileNames[i] + " is a directory.");
                return;
            }

            long burnin = burnins[0];
            if (burnins.length > i) {
                burnin = burnins[i];
            }

            if (burnin > 0) {
                System.out.print("Combining file: '" + inputFileNames[i] + "' removing burnin: " + burnin);
            } else {
                System.out.print("Combining file: '" + inputFileNames[i] + "' without removing burnin");
            }

            if (resample > 0) {
                System.out.print(", resampling with frequency: " + resample);
            }

            if (useScale) {
                System.out.println(", rescaling by: " + scale);
            } else {
                System.out.println();
            }

            boolean processTrees = stripAnnotations || convertToDecimal;

            if (treeFiles) {

                if (processTrees) { // then we need to read each tree and write it.
                    TreeImporter importer = new NexusImporter(new FileReader(inputFile), stripAnnotations);
                    try {
                        while (importer.hasTree()) {
                            Tree tree = importer.importNextTree();
                            if (firstTree) {
                                startLog(tree, writer);
                                firstTree = false;
                            }

                            String name = tree.getId();
                            if (name == null) {
                                System.err.println("ERROR: Trees do not give state numbers as tree attributes.");
                                return;
                            }

                            // split on underscore in STATE_xxxx
                            String[] bits = name.split("_");
                            long state = Long.parseLong(bits[1]);

                            if (stateStep < 0 && state > 0) {
                                stateStep = state;
                            }

                            if (state >= burnin) {
                                if (stateStep > 0) {
                                    if (!renumberOutput) {
                                        stateCount += stateStep;
                                    } else {
                                        stateCount += 1;
                                    }
                                }

                                if (resample >= 0) {
                                    if (resample % stateStep != 0) {
                                        System.err.println("ERROR: Resampling frequency is not a multiple of existing sampling frequency");
                                        return;
                                    }
                                }

                                boolean logThis;
                                if (resample < 0) {
                                    // not resampling, log every state
                                    logThis = true;
                                } else if (!renumberOutput) {
                                    // resampling but not renumbering
                                    logThis = (stateCount % resample == 0);
                                } else {
                                    logThis = (stateCount * stateStep % resample == 0);
                                }

                                long stateLineEntry;
                                if (!renumberOutput) {
                                    stateLineEntry = stateCount;
                                } else {
                                    stateLineEntry = stateCount / (resample / stateStep);
                                }

                                if (logThis) {
                                    writeTree(stateLineEntry, tree, convertToDecimal, writer);
                                }

                            }
                        }
                    } catch (Importer.ImportException e) {
                        System.err.println("Error Parsing Input Tree: " + e.getMessage());
                        return;
                    }
                } else {
                    BufferedReader reader = new BufferedReader(new FileReader(inputFile));

                    String line = reader.readLine();

                    // skip (or write) the headers
                    while (line != null && !line.trim().startsWith("tree ")) {
                        if (firstFile) {
                            writer.println(line);
                        }
                        line = reader.readLine();
                    }

                    Pattern pattern = Pattern.compile("tree STATE_(\\d+)(\\s.*)");

                    while (line != null) {
{'fraction_non_alphanumeric': 0.08901260247155267, 'fraction_numerical': 0.0046494555242872875, 'mean_word_length': 1.2396218660090423, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 50, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3847868', 'n_tokens_mistral': 3945, 'n_tokens_neox': 3665, 'n_words': 1077}