<commit_before>/*
 * @author <NAME>, <NAME>
 *
 */
import java.lang.IllegalStateException;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.FileInputStream;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileOutputStream;
import java.io.OutputStreamWriter;
import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.util.Calendar;
import java.util.Vector;
import java.util.Properties;

import twitter4j.Twitter;
import twitter4j.TwitterFactory;
import twitter4j.FilterQuery;
import twitter4j.Status;
import twitter4j.GeoLocation;
import twitter4j.StatusDeletionNotice;
import twitter4j.StatusListener;
import twitter4j.TwitterException;
import twitter4j.TwitterStream;
import twitter4j.TwitterStreamFactory;
import twitter4j.StallWarning;
import twitter4j.auth.BasicAuthorization;
import twitter4j.auth.AccessToken;
import twitter4j.auth.RequestToken;
import twitter4j.conf.Configuration;
import twitter4j.conf.ConfigurationBuilder;
import twitter4j.json.DataObjectFactory;

class StringsToProcess { 

    private Vector<String> list;

    public StringsToProcess()
    {
        list = new Vector<String>();
    }
    
    public synchronized void add(String st)
    {
        list.add(st);
        notifyAll();
    }

    public synchronized String pop()
    {   
        if (list.size() == 0)
        {
            try {
                // wait for Consumer to get value
                wait();
            } catch (InterruptedException e) {
            }
        }

        String lastElement = list.lastElement();
        list.remove(list.size()-1);
        
        // notify Consumer that value has been set
        // notifyAll();
        return lastElement;
    }
}

class Storage extends Thread {
    
    private BufferedWriter writer;
    private String   path;
    private String   radical;
    private Calendar current_calendar;
    private String   current_filename;
    
    StringsToProcess zip_stack;
    
    public Storage(String path, String radical)
    {
        this.path             = path;
        this.radical          = radical;
        this.writer           = null;
        this.current_calendar = null;
        this.current_filename = null;
        this.zip_stack        = new StringsToProcess();
    }

    private Calendar getCurrentCalendar()
    {
        Calendar cal = Calendar.getInstance();
        cal.set(Calendar.MINUTE, 0);
        cal.set(Calendar.SECOND, 0);
        cal.set(Calendar.MILLISECOND, 0);
        return cal;
    }
    
    private void updateWriter() throws IOException
    {
        if (writer != null)
        {
            writer.close();
        }
        
        if (current_filename != null)
        {
            zip_stack.add(current_filename);
        }
    
        current_calendar = getCurrentCalendar();

        String directory = String.format(
            "%s/data/%04d-%02d-%02d",
            path,
            current_calendar.get(Calendar.YEAR),
            current_calendar.get(Calendar.MONTH)+1,
            current_calendar.get(Calendar.DAY_OF_MONTH)); 

        (new File(directory)).mkdirs(); // make sure directories exist
         
        current_filename = String.format(
            "%s/%s_%04d-%02d-%02d_%02dh.json",
            directory,
            this.radical,
            current_calendar.get(Calendar.YEAR),
            current_calendar.get(Calendar.MONTH)+1,
            current_calendar.get(Calendar.DAY_OF_MONTH),
            current_calendar.get(Calendar.HOUR_OF_DAY)
            );

        writer = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(current_filename), "UTF-8"));
    }
    
    public synchronized void store(String data) throws IOException
    {
        Calendar c = this.getCurrentCalendar();
        if (writer == null || c.compareTo(this.current_calendar) > 0)
        {
            updateWriter();
        }
        writer.write(data);
        writer.write("\n");
        writer.flush();
    }
    
    public void run()
    {
        System.out.println("Starting Zip Monitor");
        
        // zip stuff
        while (true)
        {
            String filename = zip_stack.pop();
            try {
                {
                    String arr = String.format("tar cvfz %s.tar.gz %s", filename, filename);  
                    Process proc = Runtime.getRuntime().exec(arr);  
                    proc.waitFor();  
                }
                {
                    String arr = String.format("rm  %s", filename);  
                    Process proc = Runtime.getRuntime().exec(arr);  <commit_msg>remove extra space<commit_after>/*
 * @author <NAME>, <NAME>
 *
 */
import java.lang.IllegalStateException;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.FileInputStream;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileOutputStream;
import java.io.OutputStreamWriter;
import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.util.Calendar;
import java.util.Vector;
import java.util.Properties;

import twitter4j.Twitter;
import twitter4j.TwitterFactory;
import twitter4j.FilterQuery;
import twitter4j.Status;
import twitter4j.GeoLocation;
import twitter4j.StatusDeletionNotice;
import twitter4j.StatusListener;
import twitter4j.TwitterException;
import twitter4j.TwitterStream;
import twitter4j.TwitterStreamFactory;
import twitter4j.StallWarning;
import twitter4j.auth.BasicAuthorization;
import twitter4j.auth.AccessToken;
import twitter4j.auth.RequestToken;
import twitter4j.conf.Configuration;
import twitter4j.conf.ConfigurationBuilder;
import twitter4j.json.DataObjectFactory;

class StringsToProcess { 

    private Vector<String> list;

    public StringsToProcess()
    {
        list = new Vector<String>();
    }
    
    public synchronized void add(String st)
    {
        list.add(st);
        notifyAll();
    }

    public synchronized String pop()
    {   
        if (list.size() == 0)
        {
            try {
                // wait for Consumer to get value
                wait();
            } catch (InterruptedException e) {
            }
        }

        String lastElement = list.lastElement();
        list.remove(list.size()-1);
        
        // notify Consumer that value has been set
        // notifyAll();
        return lastElement;
    }
}

class Storage extends Thread {
    
    private BufferedWriter writer;
    private String   path;
    private String   radical;
    private Calendar current_calendar;
    private String   current_filename;
    
    StringsToProcess zip_stack;
    
    public Storage(String path, String radical)
    {
        this.path             = path;
        this.radical          = radical;
        this.writer           = null;
        this.current_calendar = null;
        this.current_filename = null;
        this.zip_stack        = new StringsToProcess();
    }

    private Calendar getCurrentCalendar()
    {
        Calendar cal = Calendar.getInstance();
        cal.set(Calendar.MINUTE, 0);
        cal.set(Calendar.SECOND, 0);
        cal.set(Calendar.MILLISECOND, 0);
        return cal;
    }
    
    private void updateWriter() throws IOException
    {
        if (writer != null)
        {
            writer.close();
        }
        
        if (current_filename != null)
        {
            zip_stack.add(current_filename);
        }
    
        current_calendar = getCurrentCalendar();

        String directory = String.format(
            "%s/data/%04d-%02d-%02d",
            path,
            current_calendar.get(Calendar.YEAR),
            current_calendar.get(Calendar.MONTH)+1,
            current_calendar.get(Calendar.DAY_OF_MONTH)); 

        (new File(directory)).mkdirs(); // make sure directories exist
         
        current_filename = String.format(
            "%s/%s_%04d-%02d-%02d_%02dh.json",
            directory,
            this.radical,
            current_calendar.get(Calendar.YEAR),
            current_calendar.get(Calendar.MONTH)+1,
            current_calendar.get(Calendar.DAY_OF_MONTH),
            current_calendar.get(Calendar.HOUR_OF_DAY)
            );

        writer = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(current_filename), "UTF-8"));
    }
    
    public synchronized void store(String data) throws IOException
    {
        Calendar c = this.getCurrentCalendar();
        if (writer == null || c.compareTo(this.current_calendar) > 0)
        {
            updateWriter();
        }
        writer.write(data);
        writer.write("\n");
        writer.flush();
    }
    
    public void run()
    {
        System.out.println("Starting Zip Monitor");
        
        // zip stuff
        while (true)
        {
            String filename = zip_stack.pop();
            try {
                {
                    String arr = String.format("tar cvfz %s.tar.gz %s", filename, filename);  
                    Process proc = Runtime.getRuntime().exec(arr);  
                    proc.waitFor();  
                }
                {
                    String arr = String.format("rm %s", filename);  
                    Process proc = Runtime.getRuntime().exec(arr);  
{'fraction_non_alphanumeric': 0.09831095712429623, 'fraction_numerical': 0.0086617583369424, 'mean_word_length': 2.0485148514851486, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5766590', 'n_tokens_mistral': 2618, 'n_tokens_neox': 2495, 'n_words': 528}