<commit_before><?php
include_once 'classes/UploadedFile.class.php';
include_once 'includes/settings.inc.php';
include_once 'includes/database.inc.php';

/**
 * Generates a name for the file, retrying until we get an unused one 
 *
 * @param UploadedFile $file
 * @returns string
 */
function generate_name ($file) {
	// We start at N retries, and --N until we give up
	$tries = POMF_FILES_RETRIES;
	// We rip out the extension using pathinfo
	// TODO: figure out a solution for .tar.gz and similar files?
	$ext = pathinfo($file->name, PATHINFO_EXTENSION);
	// Take the first 3 chars of the CRC32 checksum
	$hashchunk = substr($file->get_crc32(), 0, 3);
	do {
		// If we run out of tries, throw an exception.  Should be caught and JSONified.
		if ($tries-- == 0) throw new Exception('Gave up trying to find an unused name');

		// TODO: come up with a better name generating algorithm
		$newname  = '';                                  // Filename Generator:
		$newname .= chr(mt_rand(ord("a"), ord("z")));    // + random lowercase letter
		$newname .= $hashchunk;                          // + first 3 of crc32b checksum
		$newname .= chr(mt_rand(ord("a"), ord("z")));    // + random lowercase letter
		$newname .= '.' . $ext;                          // + '.' + extension

	} while (file_exists(POMF_FILES_ROOT . $newname)); // TODO: check the database instead?

	return $newname;
}

/**
 * Handles the uploading and db entry for a file
 *
 * @param UploadedFile $file
 * @returns array
 */
function upload_file ($file) {
	global $db;

	// If the file has an error attached, we just throw it as an exception.
	if ($file->error) throw new Exception($file->error);

	// Check if we have a file with that hash in the db
	$q = $db->prepare("SELECT hash, filename, size FROM files WHERE hash = (:hash)");
	$q->bindValue('hash', $file->get_sha1());
	$q->execute();
	$result = $q->fetch();

	// If we found a file with the same checksums, then we can assume it's a dupe
	// so we don't bother with it, and just unlink (delete) the tmpfile and return
	// the previous data.
	if ($result['hash'] === $file->get_sha1()) {
		unlink($file->tempfile);
		return $result;
	} else {
		// Generate a name for the file
		$newname = generate_name($file);

		// Attempt to move it to the static directory
		if (move_uploaded_file($file->tempfile, POMF_FILES_ROOT . $newname)) {
			// Add it to the database
			$q = $db->prepare('INSERT INTO files (hash, orginalname, filename, size, date, expire, delid)' .
			                  'VALUES (:hash, :orig, :name, :size, :date, :expires, :delid)');
			$q->bindValue(':hash', $file->get_sha1());
			$q->bindValue(':orig', $file->name);
			$q->bindValue(':name', $newname);
			$q->bindValue(':size', $file->size);
			$q->bindValue(':date', date('Y-m-d'));
			$q->bindValue(':expires', null);
			$q->bindValue(':delid', sha1($file->tempfile));
			$q->execute();

			return array(
				'hash' => $file->get_sha1(),
				'name' => $file->name,
				'url' => $newname,
				'size' => $file->size
			);
		} else {
			throw new Exception('Failed to move file to destination');
		}<commit_msg>Fix deduplication<commit_after><?php
include_once 'classes/UploadedFile.class.php';
include_once 'includes/settings.inc.php';
include_once 'includes/database.inc.php';

/**
 * Generates a name for the file, retrying until we get an unused one 
 *
 * @param UploadedFile $file
 * @returns string
 */
function generate_name ($file) {
	// We start at N retries, and --N until we give up
	$tries = POMF_FILES_RETRIES;
	// We rip out the extension using pathinfo
	// TODO: figure out a solution for .tar.gz and similar files?
	$ext = pathinfo($file->name, PATHINFO_EXTENSION);
	// Take the first 3 chars of the CRC32 checksum
	$hashchunk = substr($file->get_crc32(), 0, 3);
	do {
		// If we run out of tries, throw an exception.  Should be caught and JSONified.
		if ($tries-- == 0) throw new Exception('Gave up trying to find an unused name');

		// TODO: come up with a better name generating algorithm
		$newname  = '';                                  // Filename Generator:
		$newname .= chr(mt_rand(ord("a"), ord("z")));    // + random lowercase letter
		$newname .= $hashchunk;                          // + first 3 of crc32b checksum
		$newname .= chr(mt_rand(ord("a"), ord("z")));    // + random lowercase letter
		$newname .= '.' . $ext;                          // + '.' + extension

	} while (file_exists(POMF_FILES_ROOT . $newname)); // TODO: check the database instead?

	return $newname;
}

/**
 * Handles the uploading and db entry for a file
 *
 * @param UploadedFile $file
 * @returns array
 */
function upload_file ($file) {
	global $db;

	// If the file has an error attached, we just throw it as an exception.
	if ($file->error) throw new Exception($file->error);

	// Check if we have a file with that hash in the db
	$q = $db->prepare("SELECT hash, filename, size FROM files WHERE hash = (:hash)");
	$q->bindValue('hash', $file->get_sha1());
	$q->execute();
	$result = $q->fetch();

	// If we found a file with the same checksums, then we can assume it's a dupe
	// so we don't bother with it, and just unlink (delete) the tmpfile and return
	// the previous data.
	if ($result['hash'] === $file->get_sha1()) {
		unlink($file->tempfile);
		return array(
			'hash' => $result['hash'],
			'name' => $file->name,
			'url' => $result['filename'],
			'size' => $result['size']
		);
	} else {
		// Generate a name for the file
		$newname = generate_name($file);

		// Attempt to move it to the static directory
		if (move_uploaded_file($file->tempfile, POMF_FILES_ROOT . $newname)) {
			// Add it to the database
			$q = $db->prepare('INSERT INTO files (hash, orginalname, filename, size, date, expire, delid)' .
			                  'VALUES (:hash, :orig, :name, :size, :date, :expires, :delid)');
			$q->bindValue(':hash', $file->get_sha1());
			$q->bindValue(':orig', $file->name);
			$q->bindValue(':name', $newname);
			$q->bindValue(':size', $file->size);
			$q->bindValue(':date', date('Y-m-d'));
			$q->bindValue(':expires', null);
			$q->bindValue(':delid', sha1($file->tempfile));
			$q->execute();

			return array(
				'hash' => $file->get_sha1(),
				'name' => $file->name,
				'url' => $newname,
				'size' => $file->size
			);
		} else {
			throw new Exception('Failed to move file to destination');
		}
{'fraction_non_alphanumeric': 0.1654301499605367, 'fraction_numerical': 0.005051302288871349, 'mean_word_length': 3.1411764705882352, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 72, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6228427', 'n_tokens_mistral': 2253, 'n_tokens_neox': 2093, 'n_words': 667}