<commit_before>/**
 * 
 */
package com.spacehopperstudios.epf;

import java.io.IOException;
import java.io.RandomAccessFile;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Pattern;

import com.google.common.base.Charsets;

/**
 * Parses an EPF file.
 * 
 * During initialization, all the file db metadata is stored, and the file seek position is set to the beginning of the first data record. The Parser object can
 * then be used directly by an Ingester to create and populate the table.
 * 
 * typeMap is a dictionary mapping datatype strings in the file to corresponding types for the database being used. The default map is for MySQL.
 */
public class Parser {

	private static final String PRIMARY_KEY_TAG = "primaryKey:";
	private static final String DATA_TYPES_TAG = "dbTypes:";
	private static final String EXPORT_MODE_TAG = "exportMode:";
	private static final String RECORD_COUNT_TAG = "recordsWritten:";

	private List<String> numberTypes;
	private List<String> dateTypes;
	private List<Integer> dateColumns;
	private Map<String, String> dataTypeMap;
	private List<Integer> numberColumns;
	// private Map<String, String> typeMap;

	private String recordDelim = "\u0002\n";
	private String fieldDelim = "\u0001";
	private String commentChar = "#";

	private long recordsExpected;
	private long latestRecordNum;
	private String exportMode;
	private List<String> columnNames;
	private List<String> primaryKey;
	private List<String> dataTypes;

	private RandomAccessFile eFile;

	public long getRecordsExpected() {
		return recordsExpected;
	}

	public long getLatestRecordNum() {
		return latestRecordNum;
	}

	public String getExportMode() {
		return exportMode;
	}

	public List<String> getColumnNames() {
		return columnNames;
	}

	public void setColumnNames(List<String> value) {
		columnNames = value;
	}

	public List<String> getPrimaryKey() {
		return primaryKey;
	}

	public List<String> getDataTypes() {
		return dataTypes;
	}

	public static Map<String, String> DEFAULT_TYPE_MAP;

	static {
		DEFAULT_TYPE_MAP = new HashMap<String, String>();
		DEFAULT_TYPE_MAP.put("CLOB", "LONGTEXT");
	}

	public Parser(String filePath, Map<String, String> typeMap/* ={"CLOB":"LONGTEXT"} */, String recordDelim/* ='\x02\n' */, String fieldDelim/* ='\x01' */)
			throws IOException, SubstringNotFoundException {
		dataTypeMap = typeMap;
		numberTypes = Arrays.asList(new String[] { "INTEGER", "INT", "BIGINT", "TINYINT" });
		dateTypes = Arrays.asList(new String[] { "DATE", "DATETIME", "TIME", "TIMESTAMP" });
		columnNames = new ArrayList<String>();
		this.primaryKey = new ArrayList<String>();
		this.dataTypes = new ArrayList<String>();
		this.exportMode = null;
		this.dateColumns = new ArrayList<Integer>(); // fields containing dates need special treatment; we'll cache the indexes here
		this.numberColumns = new ArrayList<Integer>(); // numeric fields don't accept NULL; we'll cache the indexes here to use later
		// this.typeMap = null;
		this.recordsExpected = 0;
		this.latestRecordNum = 0;
		this.recordDelim = recordDelim;
		this.fieldDelim = fieldDelim;

		this.eFile = new RandomAccessFile(filePath, "r");

		// Seek to the end and parse the recordsWritten line
		this.eFile.seek(eFile.length() - 40);
		String str = this.eFile.readLine();
		String[] lst = str.split(this.commentChar + Parser.RECORD_COUNT_TAG);
		String numStr = lst[0].split(this.recordDelim)[1];
		this.recordsExpected = Integer.parseInt(numStr);
		this.eFile.seek(0); // seek back to the beginning
		// Extract the column names
		String line1 = this.nextRowString(false);
		this.columnNames = this.splitRow(line1, this.commentChar);

		// We'll now grab the rest of the header data, without assuming a particular order
		String primStart = this.commentChar + Parser.PRIMARY_KEY_TAG;
		String dtStart = this.commentChar + Parser.DATA_TYPES_TAG;
		String exStart = this.commentChar + Parser.EXPORT_MODE_TAG;

		// Grab the next 6 lines, which should include all the header comments
		List<String> firstRows = new ArrayList<String>();
		for (int j = 0; j < 6; j++) {
			firstRows.add(this.nextRowString(false));
			List<String> firstRowsTmp = new ArrayList<String>();
			for (String aRow : firstRows) {
				if (aRow != null && aRow.length() > 0) {
					firstRowsTmp.add(aRow); // strip null rows (possible if the file is < 6 rows)
				}
			}
			firstRows = firstRowsTmp;
		}
<commit_msg>readline replaced with read bytes (may need more of those)<commit_after>/**
 * 
 */
package com.spacehopperstudios.epf;

import java.io.IOException;
import java.io.RandomAccessFile;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Pattern;

import com.google.common.base.Charsets;

/**
 * Parses an EPF file.
 * 
 * During initialization, all the file db metadata is stored, and the file seek position is set to the beginning of the first data record. The Parser object can
 * then be used directly by an Ingester to create and populate the table.
 * 
 * typeMap is a dictionary mapping datatype strings in the file to corresponding types for the database being used. The default map is for MySQL.
 */
public class Parser {

	private static final String PRIMARY_KEY_TAG = "primaryKey:";
	private static final String DATA_TYPES_TAG = "dbTypes:";
	private static final String EXPORT_MODE_TAG = "exportMode:";
	private static final String RECORD_COUNT_TAG = "recordsWritten:";

	private List<String> numberTypes;
	private List<String> dateTypes;
	private List<Integer> dateColumns;
	private Map<String, String> dataTypeMap;
	private List<Integer> numberColumns;
	// private Map<String, String> typeMap;

	private String recordDelim = "\u0002\n";
	private String fieldDelim = "\u0001";
	private String commentChar = "#";

	private long recordsExpected;
	private long latestRecordNum;
	private String exportMode;
	private List<String> columnNames;
	private List<String> primaryKey;
	private List<String> dataTypes;

	private RandomAccessFile eFile;

	public long getRecordsExpected() {
		return recordsExpected;
	}

	public long getLatestRecordNum() {
		return latestRecordNum;
	}

	public String getExportMode() {
		return exportMode;
	}

	public List<String> getColumnNames() {
		return columnNames;
	}

	public void setColumnNames(List<String> value) {
		columnNames = value;
	}

	public List<String> getPrimaryKey() {
		return primaryKey;
	}

	public List<String> getDataTypes() {
		return dataTypes;
	}

	public static Map<String, String> DEFAULT_TYPE_MAP;

	static {
		DEFAULT_TYPE_MAP = new HashMap<String, String>();
		DEFAULT_TYPE_MAP.put("CLOB", "LONGTEXT");
	}

	public Parser(String filePath, Map<String, String> typeMap/* ={"CLOB":"LONGTEXT"} */, String recordDelim/* ='\x02\n' */, String fieldDelim/* ='\x01' */)
			throws IOException, SubstringNotFoundException {
		dataTypeMap = typeMap;
		numberTypes = Arrays.asList(new String[] { "INTEGER", "INT", "BIGINT", "TINYINT" });
		dateTypes = Arrays.asList(new String[] { "DATE", "DATETIME", "TIME", "TIMESTAMP" });
		columnNames = new ArrayList<String>();
		this.primaryKey = new ArrayList<String>();
		this.dataTypes = new ArrayList<String>();
		this.exportMode = null;
		this.dateColumns = new ArrayList<Integer>(); // fields containing dates need special treatment; we'll cache the indexes here
		this.numberColumns = new ArrayList<Integer>(); // numeric fields don't accept NULL; we'll cache the indexes here to use later
		// this.typeMap = null;
		this.recordsExpected = 0;
		this.latestRecordNum = 0;
		this.recordDelim = recordDelim;
		this.fieldDelim = fieldDelim;

		this.eFile = new RandomAccessFile(filePath, "r");

		// Seek to the end and parse the recordsWritten line
		this.eFile.seek(eFile.length() - 40);
		byte [] b = new byte[40];
		this.eFile.read(b);
		String str = new String(b, Charsets.UTF_8);
		String[] lst = str.split(this.commentChar + Parser.RECORD_COUNT_TAG);
		String numStr = lst[0].split(this.recordDelim)[1];
		this.recordsExpected = Integer.parseInt(numStr);
		this.eFile.seek(0); // seek back to the beginning
		// Extract the column names
		String line1 = this.nextRowString(false);
		this.columnNames = this.splitRow(line1, this.commentChar);

		// We'll now grab the rest of the header data, without assuming a particular order
		String primStart = this.commentChar + Parser.PRIMARY_KEY_TAG;
		String dtStart = this.commentChar + Parser.DATA_TYPES_TAG;
		String exStart = this.commentChar + Parser.EXPORT_MODE_TAG;

		// Grab the next 6 lines, which should include all the header comments
		List<String> firstRows = new ArrayList<String>();
		for (int j = 0; j < 6; j++) {
			firstRows.add(this.nextRowString(false));
			List<String> firstRowsTmp = new ArrayList<String>();
			for (String aRow : firstRows) {
				if (aRow != null && aRow.length() > 0) {
					firstRowsTmp.add(aRow); // strip null rows (possible if the file is < 6 rows)
				}
			}
			firstRows = firstRowsTmp;
		}

{'fraction_non_alphanumeric': 0.11147939454203955, 'fraction_numerical': 0.006076676610319302, 'mean_word_length': 4.486060606060606, 'pattern_counts': {'":': 2, '<': 57, '<?xml version=': 0, '>': 55, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4543718', 'n_tokens_mistral': 2984, 'n_tokens_neox': 2760, 'n_words': 854}