<commit_before>/**
 * @file POSWriter.java
 * Contains some simple preprocessing code in Java using the Stanford Parser.
 */

import java.io.BufferedReader;
import java.io.FileReader;
import java.util.ArrayList;
import java.util.List;
import java.io.FileWriter;
import java.io.IOException;
import edu.stanford.nlp.ling.Sentence;
import edu.stanford.nlp.ling.TaggedWord;
import edu.stanford.nlp.ling.HasWord;
import edu.stanford.nlp.tagger.maxent.MaxentTagger;

class POSWriter
{
    /**
     * Reads a file and returns all its lines.
     * @param filename - the file to read from
     * @return lines an ArrayList of lines from the file
     */
    private static ArrayList<String> readLines(String filename)
    {
        ArrayList<String> lines = new ArrayList<String>();
        try
        {
            String line = null;
            BufferedReader bufferedReader = new BufferedReader(new FileReader(filename));
            while((line = bufferedReader.readLine()) != null)
                lines.add(line);
            bufferedReader.close();
        }
        catch(Exception e)
        {
            System.out.println("Error reading " + filename);
        }
        return lines;
    }

    public static void main(String[] args) throws Exception
    {
        ArrayList<String> lines = readLines(args[0]);
        System.out.println("Processing " + lines.size() + " files.");
        String modelPath = "../data/english-left3words-distsim.tagger";
        MaxentTagger tagger = new MaxentTagger(modelPath);
        int filesDone = 0;
        for(String filename: lines)
        {
            FileWriter writer = new FileWriter(filename + ".pos");
            System.out.println("Parsing " + filename + " ... ");
            String numDone = "[" + (filesDone + 1) + "/" + lines.size() + "]: ";
            List<List<HasWord>> sentences = MaxentTagger.tokenizeText(new BufferedReader(new FileReader(filename)));
            double sentencesProcessed = 0;
            for(List<HasWord> sentence: sentences)
            {
                ArrayList<TaggedWord> tSentence = tagger.tagSentence(sentence);
                String tags = "<s> ";
                for(TaggedWord tw: tSentence)
                    tags += tw.tag() + " ";
                tags += "</s>";
                System.out.print("  " + numDone);
                System.out.printf("%2.2f%% ", sentencesProcessed / sentences.size() * 100.0);
                System.out.println(tags.substring(0, Math.min(100, tags.length())));
                ++sentencesProcessed;
                writer.write(tags + "\n");
            }
            System.out.println();
            writer.close();
            ++filesDone;
        }
    }
}
<commit_msg>fix path in preprocessor<commit_after>/**
 * @file POSWriter.java
 * Contains some simple preprocessing code in Java using the Stanford Parser.
 */

import java.io.BufferedReader;
import java.io.FileReader;
import java.util.ArrayList;
import java.util.List;
import java.io.FileWriter;
import java.io.IOException;
import edu.stanford.nlp.ling.Sentence;
import edu.stanford.nlp.ling.TaggedWord;
import edu.stanford.nlp.ling.HasWord;
import edu.stanford.nlp.tagger.maxent.MaxentTagger;

class POSWriter
{
    /**
     * Reads a file and returns all its lines.
     * @param filename - the file to read from
     * @return lines an ArrayList of lines from the file
     */
    private static ArrayList<String> readLines(String filename)
    {
        ArrayList<String> lines = new ArrayList<String>();
        try
        {
            String line = null;
            BufferedReader bufferedReader = new BufferedReader(new FileReader(filename));
            while((line = bufferedReader.readLine()) != null)
                lines.add(line);
            bufferedReader.close();
        }
        catch(Exception e)
        {
            System.out.println("Error reading " + filename);
        }
        return lines;
    }

    public static void main(String[] args) throws Exception
    {
        ArrayList<String> lines = readLines(args[0]);
        System.out.println("Processing " + lines.size() + " files.");
        String modelPath = "../../data/english-left3words-distsim.tagger";
        MaxentTagger tagger = new MaxentTagger(modelPath);
        int filesDone = 0;
        for(String filename: lines)
        {
            FileWriter writer = new FileWriter(filename + ".pos");
            System.out.println("Parsing " + filename + " ... ");
            String numDone = "[" + (filesDone + 1) + "/" + lines.size() + "]: ";
            List<List<HasWord>> sentences = MaxentTagger.tokenizeText(new BufferedReader(new FileReader(filename)));
            double sentencesProcessed = 0;
            for(List<HasWord> sentence: sentences)
            {
                ArrayList<TaggedWord> tSentence = tagger.tagSentence(sentence);
                String tags = "<s> ";
                for(TaggedWord tw: tSentence)
                    tags += tw.tag() + " ";
                tags += "</s>";
                System.out.print("  " + numDone);
                System.out.printf("%2.2f%% ", sentencesProcessed / sentences.size() * 100.0);
                System.out.println(tags.substring(0, Math.min(100, tags.length())));
                ++sentencesProcessed;
                writer.write(tags + "\n");
            }
            System.out.println();
            writer.close();
            ++filesDone;
        }
    }
}

{'fraction_non_alphanumeric': 0.12421788737578211, 'fraction_numerical': 0.005520794994479205, 'mean_word_length': 2.086314593980693, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6334756', 'n_tokens_mistral': 1547, 'n_tokens_neox': 1495, 'n_words': 337}