<commit_before><?php
/**
 * $Id$
 *
 * Simple CSV parser
 *
 * @author <NAME>, 2008-2009 <<EMAIL>>
 */


/**
 * CSV data parser
 *
 * @param $filename string filename
 * @param $callback string callback function
 * @param $start_line int starting line number (counting from 0)
 * @param $delimiter character separating CSV cells (usually , or ;)
 */
function csvParse($filename, $callback, $start_line = 0, $delimiter = ',')
{
	$fp = fopen($filename, 'r');
	if (!$fp || !function_exists($callback)) {
		echo "FATAL: csvParse() callback not defined\n";
		return false;
	}

	$cols = 0;
	$i = 0;
	while (!feof($fp)) {
		$buffer = fgets($fp, 4096);
		if ($i >= $start_line) {
			if (!$buffer) break;
			$row = csvParseRow($buffer, $delimiter);
			if (!$cols) $cols = count($row);
			if ($cols != count($row)) {
				echo "FATAL: CSV format error in $filename at line ".($i+1).": ".count($row)." columns found, $cols expected\n";
				return false;
			}
			if ($row) call_user_func($callback, $row);
		}
		$i++;
	}

	fclose($fp);
}

/**
 * Parses a row of CSV data into a array
 *
 * @param $row line of raw CSV data to parse
 * @param $delimiter character separating CSV cells (usually , or ;)
 */
function csvParseRow($row, $delimiter = ',')
{
	if (strpos($row, $delimiter) === false) {
		echo "FATAL: csvParseRow() got bad input\n";
		return false;
	}

	$el = 0;
	$res = array();
	$in_esc = false;

	for ($i=0; $i<strlen($row); $i++) {
		if (!isset($res[$el])) $res[$el] = '';
		$c = substr($row, $i, 1);
		switch ($c) {
			case $delimiter:
				if (!$in_esc) $el++;
				else $res[$el] .= $c;
				break;

			case '"':
				$in_esc = !$in_esc;
				$res[$el] .= $c;
				break;

			default:
				$res[$el] .= $c;
		}
	}

	//Clean up escaped fields
	for ($i=0; $i<count($res); $i++) {
		$res[$i] = csvUnescape($res[$i]);
	}

	return $res;
}

/**
 * Unescapes CSV data
 */
function csvUnescape($str)<commit_msg>csv parser: fixed handling of linefeeds<commit_after><?php
/**
 * $Id$
 *
 * Simple CSV parser
 *
 * @author <NAME>, 2008-2009 <<EMAIL>>
 */


/**
 * CSV data parser
 *
 * @param $filename string filename
 * @param $callback string callback function
 * @param $start_line int starting line number (counting from 0)
 * @param $delimiter character separating CSV cells (usually , or ;)
 */
function csvParse($filename, $callback, $start_line = 0, $delimiter = ',')
{
	$fp = fopen($filename, 'r');
	if (!$fp || !function_exists($callback)) {
		echo "FATAL: csvParse() callback not defined\n";
		return false;
	}

	$cols = 0;
	$i = 0;
	while (!feof($fp)) {
		$buffer = fgets($fp, 4096);
		if ($i >= $start_line) {
			if (!$buffer) break;
			$row = csvParseRow($buffer, $delimiter);
			if (!$cols) $cols = count($row);
			if ($cols != count($row)) {
				echo "FATAL: CSV format error in $filename at line ".($i+1).": ".count($row)." columns found, $cols expected\n";
				return false;
			}
			if ($row) call_user_func($callback, $row);
		}
		$i++;
	}

	fclose($fp);
}

/**
 * Parses a row of CSV data into a array
 *
 * @param $row line of raw CSV data to parse
 * @param $delimiter character separating CSV cells (usually , or ;)
 * @return array of parsed values
 */
function csvParseRow($row, $delimiter = ',')
{
	if (strpos($row, $delimiter) === false) {
		echo "FATAL: csvParseRow() got bad input\n";
		return false;
	}

	$el = 0;
	$res = array();
	$in_esc = false;

	for ($i=0; $i<strlen($row); $i++) {
		if (!isset($res[$el])) $res[$el] = '';
		$c = substr($row, $i, 1);
		switch ($c) {
			case $delimiter:
				if (!$in_esc) $el++;
				else $res[$el] .= $c;
				break;

			case '"':
				$in_esc = !$in_esc;
				$res[$el] .= $c;
				break;

			default:
				$res[$el] .= $c;
		}
	}

	//Clean up escaped fields
	for ($i=0; $i<count($res); $i++) {
		$res[$i] = csvUnescape($res[$i]);
		if ($i == count($res)-1) $res[$i] = rtrim($res[$i]); //strip lf
	}

	return $res;
}

/**
 * Unescapes CSV data
 * @param $str string to unescape
 * @return unescaped string
 */
function csvUnescape($str)
{'fraction_non_alphanumeric': 0.20496738585047666, 'fraction_numerical': 0.010787757150025088, 'mean_word_length': 3.376509330406147, 'pattern_counts': {'":': 2, '<': 15, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '745139', 'n_tokens_mistral': 1692, 'n_tokens_neox': 1488, 'n_words': 426}