<commit_before><?php
require_once("libs/simple_html_dom.php");
require_once('common.inc.php');
require_once('config.inc.php');
require_once("vendor/autoload.php");

use LanguageDetection\Language;

//Check if the provided text is Catalan or Spanish (Catalan=true, Spanish=false)
function is_catalan($text, $only_beginning=TRUE){
	$ld = new Language(['ca', 'es']);
	$text = preg_replace('!\s+!', ' ', $text);
	if ($only_beginning){
		$text = implode(' ', array_slice(explode(' ', $text), 0, 30));
	}
	$result = $ld->detect($text)->close();
	return $result['ca']>=$result['es'];
}

//Get last occurrence in a string
function strrstr($h, $n, $before = false) {
    $rpos = strrpos($h, $n);
    if($rpos === false) return false;
    if($before == false) return substr($h, $rpos);
    else return substr($h, 0, $rpos);
}

//Removes the unwanted tags, double enters, etc. from descriptions
function parse_description($description){
	$description = strip_tags($description, '<br><b><strong><em><i><ul><li><ol><hr><sub><sup><u><tt><p>');
	$description = str_replace('&nbsp;',' ', $description);
	$description = str_replace(' & ','&amp;', $description);
	$description = str_replace('<br>','<br />', $description);
	$description = preg_replace('/(<br\s*\/?>\s*){3,}/', '<br /><br />', $description);
	return preg_replace('/(?:<br\s*\/?>\s*)+$/', '', preg_replace('/^(?:<br\s*\/?>\s*)+/', '', trim($description)));
}

function get_prepositioned_text($text, $twitter=FALSE){
	$first = substr($text, $twitter ? 1 : 0, 1);
	if ($first == 'A' || $first == 'E' || $first == 'I' || $first == 'O' || $first == 'U' || $first == 'a' || $first == 'e' || $first == 'i' || $first == 'o' || $first == 'u'){
		return "d'$text";
	}
	return "de $text";
}

//Gets the first image in the news content that is not a SVG, if available.
//Then copies it to our website directory
function fetch_and_parse_image($fansub_slug, $url, $description){
	global $website_directory;
	preg_match_all('/<img [^>]*src=["|\']([^"|\']+)/i', $description, $matches);

	$first_image_url=NULL;
	if (isset($matches) && isset($matches[1])){
		for ($i=0;$i<count($matches[1]);$i++){
			if (strpos($matches[1][$i], '.svg')===FALSE){
				$first_image_url = $matches[1][$i];
				break;
			}
		}
	}

	if ($first_image_url!=NULL){
		if (strpos($first_image_url,"://")===FALSE){
			$first_image_url=$url.$first_image_url;
		}<commit_msg>Fix image fetching from Blogspot when no protocol is specified<commit_after><?php
require_once("libs/simple_html_dom.php");
require_once('common.inc.php');
require_once('config.inc.php');
require_once("vendor/autoload.php");

use LanguageDetection\Language;

//Check if the provided text is Catalan or Spanish (Catalan=true, Spanish=false)
function is_catalan($text, $only_beginning=TRUE){
	$ld = new Language(['ca', 'es']);
	$text = preg_replace('!\s+!', ' ', $text);
	if ($only_beginning){
		$text = implode(' ', array_slice(explode(' ', $text), 0, 30));
	}
	$result = $ld->detect($text)->close();
	return $result['ca']>=$result['es'];
}

//Get last occurrence in a string
function strrstr($h, $n, $before = false) {
    $rpos = strrpos($h, $n);
    if($rpos === false) return false;
    if($before == false) return substr($h, $rpos);
    else return substr($h, 0, $rpos);
}

//Removes the unwanted tags, double enters, etc. from descriptions
function parse_description($description){
	$description = strip_tags($description, '<br><b><strong><em><i><ul><li><ol><hr><sub><sup><u><tt><p>');
	$description = str_replace('&nbsp;',' ', $description);
	$description = str_replace(' & ','&amp;', $description);
	$description = str_replace('<br>','<br />', $description);
	$description = preg_replace('/(<br\s*\/?>\s*){3,}/', '<br /><br />', $description);
	return preg_replace('/(?:<br\s*\/?>\s*)+$/', '', preg_replace('/^(?:<br\s*\/?>\s*)+/', '', trim($description)));
}

function get_prepositioned_text($text, $twitter=FALSE){
	$first = substr($text, $twitter ? 1 : 0, 1);
	if ($first == 'A' || $first == 'E' || $first == 'I' || $first == 'O' || $first == 'U' || $first == 'a' || $first == 'e' || $first == 'i' || $first == 'o' || $first == 'u'){
		return "d'$text";
	}
	return "de $text";
}

//Gets the first image in the news content that is not a SVG, if available.
//Then copies it to our website directory
function fetch_and_parse_image($fansub_slug, $url, $description){
	global $website_directory;
	preg_match_all('/<img [^>]*src=["|\']([^"|\']+)/i', $description, $matches);

	$first_image_url=NULL;
	if (isset($matches) && isset($matches[1])){
		for ($i=0;$i<count($matches[1]);$i++){
			if (strpos($matches[1][$i], '.svg')===FALSE){
				$first_image_url = $matches[1][$i];
				break;
			}
		}
	}

	if ($first_image_url!=NULL){
		if (substr($first_image_url,0,2)=="//"){
			$first_image_url="https:".$first_image_url;
		}
		if (strpos($first_image_url,"://")===FALSE){
			$first_image_url=$url.$first_image_url;
		}
{'fraction_non_alphanumeric': 0.24760155133700756, 'fraction_numerical': 0.005715452133088385, 'mean_word_length': 5.4136125654450264, 'pattern_counts': {'":': 2, '<': 51, '<?xml version=': 0, '>': 53, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5877063', 'n_tokens_mistral': 1863, 'n_tokens_neox': 1790, 'n_words': 382}