<commit_before><?php
/**
 * @version		$Id: filter.php 1309 2011-05-17 16:47:27Z johanjanssens $
 * @category	Nooku
 * @package     Nooku_Server
 * @subpackage  Search
 * @copyright	Copyright (C) 2011 Timble CVBA and Contributors. (http://www.timble.net)
 * @license		GNU GPLv3 <http://www.gnu.org/licenses/gpl.html>
 * @link		http://www.nooku.org
 */

/**
 * Search Term Filter Class
 *
 * @author    	<NAME> <http://nooku.assembla.com/profile/amazeika>
 * @category 	Nooku
 * @package     Nooku_Server
 * @subpackage  Search
 */
class ComSearchFilterTerm extends KFilterString
{
    /**
     * Sanitize a value
     *
     * @param   mixed   Value to be sanitized
     * @return  string
     */
    protected function _sanitize($value)
    {
        $value = parent::_sanitize($value);
        
        // Apply length limitations
		$value = $this->_limitTerm($value);
			
		//Rmove ignored words
		$value = $this->_sanitizeTerm($value);
		
        return $value;
    }
	
    public function _sanitizeTerm($term, $searchphrase = 'exact')
	{
		$lang = KFactory::get('lib.joomla.language');

		$search_ignore	= array();
		$tag			= $lang->getTag();
		$ignoreFile		= $lang->getLanguagePath().DS.$tag.DS.$tag.'.ignore.php';
		
		if (file_exists($ignoreFile)) {
			include $ignoreFile;
		}

	 	//Check for words to ignore
		$aterms = explode(' ', JString::strtolower( $term ) );

		//First case is single ignored word
		if(count( $aterms ) == 1 && in_array( JString::strtolower( $term ), $search_ignore ) ) {
			$ignored = true;
		}

		//Filter out search terms that are too small
		foreach($aterms as $aterm ) 
		{
			if(JString::strlen( $aterm ) < 3) {
				$search_ignore[] = $aterm;
			}
		}

		//Remove ignored words from type 'all' or 'any' (not exact) searches with multiple words
		if(count( $aterms ) > 1 && $searchphrase != 'exact' ) 
		{
			$pruned = array_diff( $aterms, $search_ignore );
			$term   = implode( ' ', $pruned );
		}

		return $term;
	}
	
    public function _limitTerm($term)
	{
		// Limit term to 20 characters
		if( JString::strlen( $term ) > 20 ) {
			$term 	= JString::substr( $term, 0, 19 );
		}

		// Term must contain a minimum of 3 characters
		if( $term && JString::strlen( $term ) < 3 ) {
			$term = '';
		}

		return $term;
	}
}<commit_msg>re #106 : Removed ignored words fucntionality.<commit_after><?php
/**
 * @version		$Id: filter.php 1309 2011-05-17 16:47:27Z johanjanssens $
 * @category	Nooku
 * @package     Nooku_Server
 * @subpackage  Search
 * @copyright	Copyright (C) 2011 Timble CVBA and Contributors. (http://www.timble.net)
 * @license		GNU GPLv3 <http://www.gnu.org/licenses/gpl.html>
 * @link		http://www.nooku.org
 */

/**
 * Search Term Filter Class
 *
 * @author    	<NAME> <http://nooku.assembla.com/profile/amazeika>
 * @category 	Nooku
 * @package     Nooku_Server
 * @subpackage  Search
 */
class ComSearchFilterTerm extends KFilterString
{
    /**
     * Sanitize a value
     *
     * @param   mixed   Value to be sanitized
     * @return  string
     */
    protected function _sanitize($value)
    {
        $value = parent::_sanitize($value);
        
        // Limit term to 20 characters
		if( JString::strlen( $term ) > 20 ) {
			$term 	= JString::substr( $term, 0, 19 );
		}

		// Term must contain a minimum of 3 characters
		if( $term && JString::strlen( $term ) < 3 ) {
			$term = '';
		}
		
        //Filter out search terms that are too small
		$words = explode(' ', JString::strtolower( $term ) );

		foreach($words as $word ) 
		{
			if(JString::strlen( $word ) < 3) {
				$search_ignore[] = $word;
			}
		}

		if(count( $words ) > 1 ) 
		{
			$pruned = array_diff( $words, $search_ignore );
			$term   = implode( ' ', $pruned );
		}
		
        return $value;
    }
}
{'fraction_non_alphanumeric': 0.1518716577540107, 'fraction_numerical': 0.019251336898395723, 'mean_word_length': 2.7223880597014927, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 6, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3389623', 'n_tokens_mistral': 1489, 'n_tokens_neox': 1326, 'n_words': 319}