<commit_before>package org.rapidandroid.data.controller;

import java.io.BufferedInputStream;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.util.Calendar;
import java.util.Date;
import java.util.zip.GZIPOutputStream;

import org.rapidandroid.content.translation.MessageTranslator;
import org.rapidandroid.data.RapidSmsDBConstants;
import org.rapidandroid.data.SmsDbHelper;
import org.rapidsms.java.core.model.Form;
import org.rapidsms.java.core.model.Message;

import android.content.Context;
import android.database.Cursor;
import android.net.Uri;
import android.os.Environment;
import android.util.Log;

/**
 * 
 * @author <NAME> <EMAIL>
 * @created Jan 30, 2009
 * 
 */
public class ParsedDataReporter {

	SmsDbHelper mHelper;
	Context mContext;
	
	private String[] messageColumns = new String[] {"message_time", "monitor_id", "monitor_phone", "message_text"};
	
	public ParsedDataReporter(Context context) {
		mHelper = new SmsDbHelper(context);
		mContext = context;
	}
	
	public void exportFormDataToCSV(Form f, Calendar startDate, Calendar endDate) {

		//build the query
		StringBuilder query = new StringBuilder();
		query.append("select " + RapidSmsDBConstants.FormData.TABLE_PREFIX);
		query.append(f.getPrefix() + ".*");
		query.append(", rapidandroid_message.message,rapidandroid_message.time, rapidandroid_monitor._id as monitor_id, rapidandroid_monitor.phone as monitor_phone ");
		query.append(" from " + RapidSmsDBConstants.FormData.TABLE_PREFIX + f.getPrefix());
		query.append(" join rapidandroid_message on (");
		query.append(RapidSmsDBConstants.FormData.TABLE_PREFIX + f.getPrefix());
		query.append(".message_id = rapidandroid_message._id");
		query.append(") ");
		
		query.append(" join rapidandroid_monitor on (");
		query.append("rapidandroid_monitor._id = rapidandroid_message.monitor_id");
		query.append(") ");
		
		query.append("WHERE rapidandroid_message.time > '" + startDate.get(Calendar.YEAR) + "-" + (startDate.get(Calendar.MONTH)+1) + "-" + startDate.get(Calendar.DATE) + "' AND ");
		query.append(" rapidandroid_message.time < '" + endDate.get(Calendar.YEAR) + "-" + (1+endDate.get(Calendar.MONTH)) + "-" + endDate.get(Calendar.DATE) + "';");
		
		Cursor cr = mHelper.getReadableDatabase().rawQuery(query.toString(), null);

		File sdcard = Environment.getExternalStorageDirectory();
		File destinationdir = new File(sdcard,"rapidandroid/exports");
		destinationdir.mkdir();
		Date now = new Date();
		File destinationfile = new File(destinationdir,"formdata_" + f.getPrefix() + now.getYear() + now.getMonth() + now.getDate() + "-" + now.getHours() + now.getMinutes() + ".csv");
		FileOutputStream fOut = null;
		try {
			destinationfile.createNewFile();
			fOut =  new FileOutputStream(destinationfile); 
			String[] cols = cr.getColumnNames();
			int colcount = cols.length;
			StringBuilder sbrow = new StringBuilder();
			for(int i = 0; i < colcount; i++) {
				//sbrow.append(cols[i] + ",");
				sbrow.append(cols[i]);
				if (i < colcount - 1) {
					sbrow.append(",");
				} else {
					sbrow.append("\n");
				}
			}			
			fOut.write(sbrow.toString().getBytes());
			cr.moveToFirst();
			do{
				sbrow = new StringBuilder();
				for(int i = 0; i < colcount; i++) {
					//sbrow.append(cr.getString(i) + ",");
					sbrow.append(cr.getString(i));
					if (i < colcount - 1) {
						sbrow.append(",");
					} else {
						sbrow.append("\n");
					}
				}
				fOut.write(sbrow.toString().getBytes());
			} while (cr.moveToNext());			
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		finally {
			cr.close();
			if(fOut != null) {
				try {
					fOut.close();
				} catch (IOException e) {
					// TODO Auto-generated catch block
					e.printStackTrace();
				}
			}			
			//compressFile(destinationfile);
		}
	
	}
	
	void compressFile(File rawFile) {
		try {		
			FileInputStream fin = new FileInputStream(rawFile);
			FileOutputStream fout = new FileOutputStream(rawFile.getAbsoluteFile() + ".gz");
		 	GZIPOutputStream gz = new GZIPOutputStream(fout);
		 	byte[] buf = new byte[4096];
		 	int readCount;		 	
		 	while ((readCount = fin.read(buf)) != -1) {
                gz.write(buf, 0, readCount);
		 	}
		 	fin.close();
			gz.close();

		} catch (Exception ex) {
		}
	}
}<commit_msg>tidy up the compression method to close the streams more gracefully<commit_after>package org.rapidandroid.data.controller;

import java.io.BufferedInputStream;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.util.Calendar;
import java.util.Date;
import java.util.zip.GZIPOutputStream;

import org.rapidandroid.content.translation.MessageTranslator;
import org.rapidandroid.data.RapidSmsDBConstants;
import org.rapidandroid.data.SmsDbHelper;
import org.rapidsms.java.core.model.Form;
import org.rapidsms.java.core.model.Message;

import android.content.Context;
import android.database.Cursor;
import android.net.Uri;
import android.os.Environment;
import android.util.Log;

/**
 * 
 * @author <NAME> <EMAIL>
 * @created Jan 30, 2009
 * 
 */
public class ParsedDataReporter {

	SmsDbHelper mHelper;
	Context mContext;
	
	private String[] messageColumns = new String[] {"message_time", "monitor_id", "monitor_phone", "message_text"};
	
	public ParsedDataReporter(Context context) {
		mHelper = new SmsDbHelper(context);
		mContext = context;
	}
	
	public void exportFormDataToCSV(Form f, Calendar startDate, Calendar endDate) {

		//build the query
		StringBuilder query = new StringBuilder();
		query.append("select " + RapidSmsDBConstants.FormData.TABLE_PREFIX);
		query.append(f.getPrefix() + ".*");
		query.append(", rapidandroid_message.message,rapidandroid_message.time, rapidandroid_monitor._id as monitor_id, rapidandroid_monitor.phone as monitor_phone ");
		query.append(" from " + RapidSmsDBConstants.FormData.TABLE_PREFIX + f.getPrefix());
		query.append(" join rapidandroid_message on (");
		query.append(RapidSmsDBConstants.FormData.TABLE_PREFIX + f.getPrefix());
		query.append(".message_id = rapidandroid_message._id");
		query.append(") ");
		
		query.append(" join rapidandroid_monitor on (");
		query.append("rapidandroid_monitor._id = rapidandroid_message.monitor_id");
		query.append(") ");
		
		query.append("WHERE rapidandroid_message.time > '" + startDate.get(Calendar.YEAR) + "-" + (startDate.get(Calendar.MONTH)+1) + "-" + startDate.get(Calendar.DATE) + "' AND ");
		query.append(" rapidandroid_message.time < '" + endDate.get(Calendar.YEAR) + "-" + (1+endDate.get(Calendar.MONTH)) + "-" + endDate.get(Calendar.DATE) + "';");
		
		Cursor cr = mHelper.getReadableDatabase().rawQuery(query.toString(), null);

		File sdcard = Environment.getExternalStorageDirectory();
		File destinationdir = new File(sdcard,"rapidandroid/exports");
		destinationdir.mkdir();
		Date now = new Date();
		File destinationfile = new File(destinationdir,"formdata_" + f.getPrefix() + now.getYear() + now.getMonth() + now.getDate() + "-" + now.getHours() + now.getMinutes() + ".csv");
		FileOutputStream fOut = null;
		try {
			destinationfile.createNewFile();
			fOut =  new FileOutputStream(destinationfile); 
			String[] cols = cr.getColumnNames();
			int colcount = cols.length;
			StringBuilder sbrow = new StringBuilder();
			for(int i = 0; i < colcount; i++) {
				//sbrow.append(cols[i] + ",");
				sbrow.append(cols[i]);
				if (i < colcount - 1) {
					sbrow.append(",");
				} else {
					sbrow.append("\n");
				}
			}			
			fOut.write(sbrow.toString().getBytes());
			cr.moveToFirst();
			do{
				sbrow = new StringBuilder();
				for(int i = 0; i < colcount; i++) {
					//sbrow.append(cr.getString(i) + ",");
					sbrow.append(cr.getString(i));
					if (i < colcount - 1) {
						sbrow.append(",");
					} else {
						sbrow.append("\n");
					}
				}
				fOut.write(sbrow.toString().getBytes());
			} while (cr.moveToNext());			
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		finally {
			cr.close();
			if(fOut != null) {
				try {
					fOut.close();
				} catch (IOException e) {
					// TODO Auto-generated catch block
					e.printStackTrace();
				}
			}			
			//compressFile(destinationfile);
		}
	
	}

	void compressFile(File rawFile) {
		FileInputStream fin = null;
		GZIPOutputStream gz = null;
		try {
			fin = new FileInputStream(rawFile);
			FileOutputStream fout = new FileOutputStream(rawFile
					.getAbsoluteFile()
					+ ".gz");
			gz = new GZIPOutputStream(fout);
			byte[] buf = new byte[4096];
			int readCount;
			while ((readCount = fin.read(buf)) != -1) {
				gz.write(buf, 0, readCount);
			}

		} catch (Exception ex) {
		} finally {
			// Close the BufferedInputStream
			try {
				if (fin != null)
					fin.close();
				if (gz != null)
					gz.close();
			} catch (IOException ex) {
				ex.printStackTrace();
			}

		}
	}
}
{'fraction_non_alphanumeric': 0.14974087550997905, 'fraction_numerical': 0.0039695666556400925, 'mean_word_length': 3.847675040085516, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3638358', 'n_tokens_mistral': 3399, 'n_tokens_neox': 2909, 'n_words': 574}