<commit_before><?php

require_once('FFDB.php');
require_once('BookDB.php');
require_once('credentials.php');
require_once('config.php');

$ffdb = new FFDB();
$bookdb = new BookDB();

$resume_data = $ffdb->get_worker_data('file_downloader', 'last_id');

$files = $ffdb->get_files($resume_data);

$types = $bookdb->types;
$finfo = finfo_open(FILEINFO_MIME_TYPE); // return mime type aka mimetype extension

$out_dir = Config::book_download_dir;

$ffdb->insert_worker_data('file_downloader', 'start_id', $resume_data + 1);

$bookdb->begin();

for($i=0; $i < sizeof($files); $i++)
{
	$file = $files[$i];
	$u = $file['url'];

	echo("Downloading... ");
	$c = file_get_contents($u);

	if(!$c)
	{
		echo("Cannot open '$u' . Aborting!\n");
		break;
	}

	$file_name = $out_dir . $file['name'];

	while(file_exists($file_name))
	{
		$p = pathinfo($file_name);
		$file_name =  $p['dirname'] . '/' .  $p['filename'] . "_copy." .  $p['extension'];
	}

	if(!file_put_contents($file_name, $c))
	{
		echo("Cannot write the file '$file_name' . Aborting!\n");
		break;
	}
	
	echo("Saved to $file_name \n");

	// Insert into booksDb
	$n = pathinfo($file_name, PATHINFO_BASENAME);
	$type = type_of($file_name);
	$p = pathinfo($n);
	$basename =  $p['filename'];
	$a = $t = '';
	if($d = strrpos($basename, "-")){
		$a = trim(substr($basename, 0, $d));
		$t = trim(substr($basename, $d + 1));
	}

	$bookdb->insert_a_book($a, $t, $n, $type['id'], $file['url'], 1);
}

$num_of_entries = $bookdb->end();
//echo("$num_of_entries books are added to database.");

$last_id = $ffdb->get_last_id_of_files();
$ffdb->insert_worker_data('file_downloader', 'last_id', $last_id);

finfo_close($finfo);

function type_of($f)
{<commit_msg>file_downloader now act wiser on failure.<commit_after><?php

require_once('FFDB.php');
require_once('BookDB.php');
require_once('credentials.php');
require_once('config.php');

$ffdb = new FFDB();
$bookdb = new BookDB();

$resume_data = $ffdb->get_worker_data('file_downloader', 'last_id');

$files = $ffdb->get_files($resume_data);

$types = $bookdb->types;
$finfo = finfo_open(FILEINFO_MIME_TYPE); // return mime type aka mimetype extension

$out_dir = Config::book_download_dir;

$ffdb->insert_worker_data('file_downloader', 'start_id', $resume_data + 1);

$bookdb->begin();

$last_successful_id = $resume_data;
for($i=0; $i < sizeof($files); $i++)
{
	$file = $files[$i];
	$u = $file['url'];

	echo("Downloading... ");
	$c = file_get_contents($u);

	if(!$c)
	{
		echo("Cannot open '$u' . Aborting!\n");
		break;
	}

	$file_name = $out_dir . $file['name'];

	while(file_exists($file_name))
	{
		$p = pathinfo($file_name);
		$file_name =  $p['dirname'] . '/' .  $p['filename'] . "_copy." .  $p['extension'];
	}

	if(!file_put_contents($file_name, $c))
	{
		echo("Cannot write the file '$file_name' . Aborting!\n");
		break;
	}
	
	echo("Saved to $file_name \n");

	// Insert into booksDb
	$n = pathinfo($file_name, PATHINFO_BASENAME);
	$type = type_of($file_name);
	$p = pathinfo($n);
	$basename =  $p['filename'];
	$a = $t = '';
	if($d = strrpos($basename, "-")){
		$a = trim(substr($basename, 0, $d));
		$t = trim(substr($basename, $d + 1));
	}

	$bookdb->insert_a_book($a, $t, $n, $type['id'], $file['url'], 1);

	$last_successful_id = $file['id'];
}

$num_of_entries = $bookdb->end();
//echo("$num_of_entries books are added to database.");

$last_id = $last_successful_id;
$ffdb->insert_worker_data('file_downloader', 'last_id', $last_id);

finfo_close($finfo);

function type_of($f)
{
{'fraction_non_alphanumeric': 0.22199942709825266, 'fraction_numerical': 0.002864508736751647, 'mean_word_length': 5.280575539568345, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 11, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6112179', 'n_tokens_mistral': 1471, 'n_tokens_neox': 1423, 'n_words': 249}