<commit_before>package graphene.walker.ingest;

//import graphene.util.FastNumberUtils;

import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Date;

public class Email {

	String senderName;
	String senderAddress;
	String receiverName;
	String receiverAddress;
	
	String subject;
	String payload;
	
	Date dt;
	//FIXME: This is not the length, it's the payload id (the whole email, stored in another table)
	int length;
	//static SimpleDateFormat df = new SimpleDateFormat("MM/dd/yyyy");
	static SimpleDateFormat df = new SimpleDateFormat("YYYY-MM-dd'T'HH:mm:ss");
	
	boolean parseFromLine(String line)
	{
		String[]cols;
		//0   1   2        3        4          5    6  7  8  9   10          11        12        13         14      15
		//num dir category datetime importance from ip to cc bcc attachments messageid inreplyto references subject body
		
		cols = line.split("\t");
		
		for (int i = 0; i < cols.length; ++i)
			cols[i]=cols[i].trim();
		
		//senderAddress = cols[0];
		//receiverAddress = cols[1];
		senderAddress = cols[5];
		receiverAddress = cols[7];
		subject = cols[14];
		payload = cols[15];
		
		if (senderAddress.length() == 0) {
			System.out.println("No sender in " + line);
			return false;
		}
		
		if (receiverAddress.length() == 0) {<commit_msg>Fixed a date ingest issue that prevented correct dates from appearing in the data.  Note: to see this fix, you will have to re-run the ingest process and produce a new data set<commit_after>package graphene.walker.ingest;

//import graphene.util.FastNumberUtils;

import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Date;

public class Email {

	String senderName;
	String senderAddress;
	String receiverName;
	String receiverAddress;
	
	String subject;
	String payload;
	
	Date dt;
	//FIXME: This is not the length, it's the payload id (the whole email, stored in another table)
	int length;
	//static SimpleDateFormat df = new SimpleDateFormat("MM/dd/yyyy");
	static SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss");
	
	boolean parseFromLine(String line)
	{
		String[]cols;
		//0   1   2        3        4          5    6  7  8  9   10          11        12        13         14      15
		//num dir category datetime importance from ip to cc bcc attachments messageid inreplyto references subject body
		
		cols = line.split("\t");
		
		for (int i = 0; i < cols.length; ++i)
			cols[i]=cols[i].trim();
		
		//senderAddress = cols[0];
		//receiverAddress = cols[1];
		senderAddress = cols[5];
		receiverAddress = cols[7];
		subject = cols[14];
		payload = cols[15];
		
		if (senderAddress.length() == 0) {
			System.out.println("No sender in " + line);
			return false;
		}
		
		if (receiverAddress.length() == 0) {
{'fraction_non_alphanumeric': 0.10890732059141724, 'fraction_numerical': 0.023800937612693834, 'mean_word_length': 3.020289855072464, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5940131', 'n_tokens_mistral': 983, 'n_tokens_neox': 859, 'n_words': 284}