<commit_before><?php

require_once(mnminclude.'Zend/Search/Lucene.php');

// Configurable parameters
global $lucene_stopwords;
$lucene_stopwords = array('a', 'al', 'an', 'at', 'the', 'and', 'or', 'is', 'am', 'my', 'ho', 'la', 'el', 'las', 'los', 'de', 'me', 'mi', 'no', 'da',
		'ni', 'en', 'con', 'ha', 'he', 'es', 'un', 'una', 'tu', 'te', 'con', 'por', 'su', 'se', 'si', 'que', 'ya', 'yo', 'va', 'para', 'sin',
		'http', 'https', 'www', 'fue', 'le', 'del', 'lo', 'sus', 'ip', 'com');

function lucene_open() {
	global $globals, $lucene_stopwords;

	// Change the token analyzer
	$analyzer = new Mnm_Lucene_Analysis_Analyzer_Common_Utf8Num($lucene_stopwords);
	Zend_Search_Lucene_Analysis_Analyzer::setDefault($analyzer);

	if (file_exists($globals['lucene_dir'] )) {
		$index = Zend_Search_Lucene::open($globals['lucene_dir'] );
  	} else {
		print "Creando dir\n";
		$index = Zend_Search_Lucene::create($globals['lucene_dir']);
		@chmod($globals['lucene_dir'], 0777);
	}
	return $index;
}



function lucene_get_search_link_ids($by_date = false, $start = 0, $count = 50) {
	global $globals;

	$ids = array();

	if(!empty($_REQUEST['q'])) {
		if ($_REQUEST['p']) {
			//Allows variable "p" as prefix too
			$_REQUEST['q'] = $_REQUEST['p'].':'.$_REQUEST['q'];
		}

		$words = $_REQUEST['q'] = trim(substr(strip_tags($_REQUEST['q']), 0, 250)); 

		// Basic filtering to avoid Lucene errors
		$words = preg_replace('/\^([^1-9])/','$1',$words); // Only allow ^ followed by numbers
		//$words = preg_replace('/(.*) +(\w{1,2} *$)/', '$2 $1', $words); // Lucene dies if the last word has 2 chars, move it to the begining
		//$words = preg_replace('/[\~\*\(\)\[\]\|\{\}]/',' ',$words);
		//$words = preg_replace('/^ *(and|not|no|or|\&) *$/','',$words);

		if(preg_match('/^ *(\w+): *(.*)/', mb_strtolower($words), $matches)) {
			$prefix = $matches[1];
			$words = $matches[2];
		}
		if (preg_match('/^http[s]*/', $prefix)) { // It's an url search
			$words = "$prefix:$words";
			$prefix = false;
			$field = 'url';
		}
		$words_count = count(explode(" ", $words));
		if ($words_count == 1 || $prefix == 'date') {
			$by_date = true;
		}
		if ($prefix) {
			switch ($prefix) {
				case 'url';
					$field = 'url';
					break;
				case 'title';
					$field = 'title';
					break;
				case 'tag':
				case 'tags':
					$field = 'tags';
					break;
			}
		}
		// if there is only a word and is a number, do not search in urls
		if ($words_count == 1 && !$field && preg_match('/^\+*[0-9]{1,4}$/', $words)) {
			$words = preg_replace('/\+/', '', $words);
			$words = "title:$words tags:$words content:$words";
		}
		if ($field) {
			$query = "$field:($words)";
		} else {
			$query = $words;
		}
		if (empty($query)) return false;

		if ($globals['bot']) {
			Zend_Search_Lucene::setResultSetLimit(40);
		} else {
			Zend_Search_Lucene::setResultSetLimit(2000);
		}
		$index = lucene_open();
		$hits = lucene_search($index, $query, $by_date);
		if (!$by_date && count($hits) > 200 && $words_count > 1 && !preg_match('/[\+\-\"]|(^|[ :])(AND|OR|NOT|TO) /i', $words)) {
			Zend_Search_Lucene::setResultSetLimit(200);
			$query = preg_replace('/(^|[: ]+)(\w)/', '$1+$2', $query);
			echo "\n<!--  Trying to refine with a new query: $query -->\n";
			$hits2 = lucene_search($index, $query, $by_date);
			if (count($hits2) > 0) {
				$base_hits = 0;
				$nhits2 = min(count($hits2), 10);
				// Add the second hits if they are not already among the fist 10
				for ($i=$nhits2-1; $i>=0;$i--) {
					$found = false;
					for ($j=$base_hits; $j<$base_hits+10 && ! $found; $j++) {
						if ($hits2[$i]->link_id == $hits[$j]->link_id) {
							//$hits[$j] = false;
							unset($hits[$j]);
							$found = true;
						}
					}
					array_unshift($hits, $hits2[$i]);
					$base_hits++;
				}
				//$hits = array_merge($hits2, $hits);
			}
		}
		$globals['rows'] = count($hits); // Save the number of hits
		$elements = min($globals['rows'], $start+$count);
		if ($elements == 0 || $elements < $start) return false;<commit_msg>Cambia el número de resultados dependiendo del número de palabras<commit_after><?php

require_once(mnminclude.'Zend/Search/Lucene.php');

// Configurable parameters
global $lucene_stopwords;
$lucene_stopwords = array('a', 'al', 'an', 'at', 'the', 'and', 'or', 'is', 'am', 'my', 'ho', 'la', 'el', 'las', 'los', 'de', 'me', 'mi', 'no', 'da',
		'ni', 'en', 'con', 'ha', 'he', 'es', 'un', 'una', 'tu', 'te', 'con', 'por', 'su', 'se', 'si', 'que', 'ya', 'yo', 'va', 'para', 'sin',
		'http', 'https', 'www', 'fue', 'le', 'del', 'lo', 'sus', 'ip', 'com');

function lucene_open() {
	global $globals, $lucene_stopwords;

	// Change the token analyzer
	$analyzer = new Mnm_Lucene_Analysis_Analyzer_Common_Utf8Num($lucene_stopwords);
	Zend_Search_Lucene_Analysis_Analyzer::setDefault($analyzer);

	if (file_exists($globals['lucene_dir'] )) {
		$index = Zend_Search_Lucene::open($globals['lucene_dir'] );
  	} else {
		print "Creando dir\n";
		$index = Zend_Search_Lucene::create($globals['lucene_dir']);
		@chmod($globals['lucene_dir'], 0777);
	}
	return $index;
}



function lucene_get_search_link_ids($by_date = false, $start = 0, $count = 50) {
	global $globals;

	$ids = array();

	if(!empty($_REQUEST['q'])) {
		if ($_REQUEST['p']) {
			//Allows variable "p" as prefix too
			$_REQUEST['q'] = $_REQUEST['p'].':'.$_REQUEST['q'];
		}

		$words = $_REQUEST['q'] = trim(substr(strip_tags($_REQUEST['q']), 0, 250)); 

		// Basic filtering to avoid Lucene errors
		$words = preg_replace('/\^([^1-9])/','$1',$words); // Only allow ^ followed by numbers
		//$words = preg_replace('/(.*) +(\w{1,2} *$)/', '$2 $1', $words); // Lucene dies if the last word has 2 chars, move it to the begining
		//$words = preg_replace('/[\~\*\(\)\[\]\|\{\}]/',' ',$words);
		//$words = preg_replace('/^ *(and|not|no|or|\&) *$/','',$words);

		if(preg_match('/^ *(\w+): *(.*)/', mb_strtolower($words), $matches)) {
			$prefix = $matches[1];
			$words = $matches[2];
		}
		if (preg_match('/^http[s]*/', $prefix)) { // It's an url search
			$words = "$prefix:$words";
			$prefix = false;
			$field = 'url';
		}
		$words_count = count(explode(" ", $words));
		if ($words_count == 1 || $prefix == 'date') {
			$words = preg_replace('/(^| )(\w)/', '+$2', $words);
			$by_date = true;
		}
		if ($prefix) {
			switch ($prefix) {
				case 'url';
					$field = 'url';
					break;
				case 'title';
					$field = 'title';
					break;
				case 'tag':
				case 'tags':
					$field = 'tags';
					break;
			}
		}
		// if there is only a word and is a number, do not search in urls
		if ($words_count == 1 && !$field && preg_match('/^\+*[0-9]{1,4}$/', $words)) {
			//$words = preg_replace('/\+/', '', $words);
			$words = "title:$words tags:$words content:$words";
		}
		if ($field) {
			$query = "$field:($words)";
		} else {
			$query = $words;
		}
		if (empty($query)) return false;

		if ($globals['bot']) {
			Zend_Search_Lucene::setResultSetLimit(40);
		} elseif ($words_count == 1) {
			Zend_Search_Lucene::setResultSetLimit(6000);
		} else {
			Zend_Search_Lucene::setResultSetLimit(3000);
		}
		$index = lucene_open();
		echo "<!--Query: $query -->\n";
		$hits = lucene_search($index, $query, $by_date);
		if (!$by_date && count($hits) > 200 && $words_count > 1 && !preg_match('/[\+\-\"]|(^|[ :])(AND|OR|NOT|TO) /i', $words)) {
			Zend_Search_Lucene::setResultSetLimit(200);
			$query = preg_replace('/(^|[: ]+)(\w)/', '$1+$2', $query);
			echo "\n<!--  Trying to refine with a new query: $query -->\n";
			$hits2 = lucene_search($index, $query, $by_date);
			if (count($hits2) > 0) {
				$base_hits = 0;
				$nhits2 = min(count($hits2), 10);
				// Add the second hits if they are not already among the fist 10
				for ($i=$nhits2-1; $i>=0;$i--) {
					$found = false;
					for ($j=$base_hits; $j<$base_hits+10 && ! $found; $j++) {
						if ($hits2[$i]->link_id == $hits[$j]->link_id) {
							//$hits[$j] = false;
							unset($hits[$j]);
							$found = true;
						}
					}
					array_unshift($hits, $hits2[$i]);
					$base_hits++;
				}
				//$hits = array_merge($hits2, $hits);
			}
		}
		$globals['rows'] = count($hits); // Save the number of hits
		$elements = min($globals['rows'], $start+$count);
		if ($elements == 0 || $elements < $start) return false;
{'fraction_non_alphanumeric': 0.24705453662091584, 'fraction_numerical': 0.015790112960038866, 'mean_word_length': 3.681068789084707, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '655171', 'n_tokens_mistral': 3718, 'n_tokens_neox': 3202, 'n_words': 715}