<commit_before><?php
/**
 * EmeraldView
 *
 * LICENSE
 *
 * This source file is subject to the new BSD license that is bundled
 * with this package in the file LICENSE.txt.
 * It is also available through the world-wide-web at this URL:
 * http://yitznewton.org/emeraldview/index.php?title=License
 * If you did not receive a copy of the license and are unable to
 * obtain it through the world-wide-web, please send an email
 * to <EMAIL> so we can send you a copy immediately.
 *
 * @version 0.2.0-b4
 * @package libraries
 */
/**
 * SearchHandler is a wrapper class which creates objects and delegates
 * responsibility for search functionality
 *
 * @package libraries
 * @copyright  Copyright (c) 2010 <NAME> (http://yitznewton.org/)
 * @license    http://yitznewton.org/emeraldview/index.php?title=License     New BSD License
 */
class SearchHandler
{
  /**
   * ASCII matching pattern for search term boundary detection -
   * must match the search term in the first backreference
   */
  const BOUNDARIES_PATTERN_ASCII   = '\\b(%s)\\b';
  /**
   * Unicode matching pattern for search term boundary detection -
   * must match the search term in the first backreference
   */
  const BOUNDARIES_PATTERN_UNICODE = '(?:[^_\pL\pN]|^)(%s)(?:[^_\pL\pN]|$)';

  /**
   * An array of the query parameters for this search
   *
   * @var array
   */
  protected $params;
  /**
   * The Collection being searched
   *
   * @var Collection
   */
  protected $collection;
  /**
   * The full path of the directory containing raw node text
   *
   * @var string
   */
  protected $rawTextDir;
  /**
   * A QueryBuilder representing this search
   *
   * @var QueryBuilder
   */
  protected $queryBuilder;
  /**
   * The level of node being searched (document or section)
   *
   * @var string
   */
  protected $indexLevel;
  /**
   * The Zend_Search_Lucene interface to the appropriate index
   *
   * @var Zend_Search_Lucene
   */
  protected $luceneObject;
  /**
   * The word boundary matching pattern to use with this search
   *
   * @var string
   */
  protected $boundariesPattern;

  /**
   * @param array $params An array of the query parameters
   * @param Collection $collection The Collection to search
   */
  public function __construct( array $params, Collection $collection )
  {
    $this->params = $this->filterParams( $params );
    $this->collection = $collection;
    $this->queryBuilder = QueryBuilder::factory( $params, $collection );
    $this->boundariesPattern = $collection->getConfig( 'boundaries_pattern', SearchHandler::BOUNDARIES_PATTERN_ASCII );

    $level_prefix = substr( $this->getIndexLevel(), 0, 1 );
    $index_dir = $collection->getGreenstoneDirectory()
               . "/index/$level_prefix" . 'idx';

    $this->rawTextDir = $collection->getGreenstoneDirectory()
                      . "/index/raw-text/$level_prefix" . 'idx';

    if (!is_dir( $index_dir ) || !is_readable( $index_dir )) {
      throw new Exception("Could not read index directory $index_dir");
    }

    $b_and = Zend_Search_Lucene_Search_QueryParser::B_AND;
    Zend_Search_Lucene_Search_QueryParser::setDefaultOperator( $b_and );
    Zend_Search_Lucene_Analysis_Analyzer::setDefault(
      new Zend_Search_Lucene_Analysis_Analyzer_Common_Utf8());

    $this->luceneObject = Zend_Search_Lucene::open( $index_dir );
  }

  /**
   * Performs query and returns an array of Zend_Search_Lucene_Search_QueryHit
   * objects
   *
   * @return array
   */
  public function execute()
  {
    $query = $this->queryBuilder->getQuery();
    
    try {
      $lucene_hits = @$this->luceneObject->find( $query );
    }
    catch (Zend_Search_Lucene_Exception $e) {
      // malformed search
      Kohana::log( 'error', $e->getMessage() );
      return array();
    }

    $hits = array();

    foreach ($lucene_hits as $lucene_hit) {
      $hits[] = new Hit( $lucene_hit, $this );
    }

    return $hits;
  }

  /**
   * Returns an array of the query parameters
   *
   * @return array
   */
  public function getParams()
  {
    return $this->params;
  }

  /**
   * Returns an array of query parameters with irrelevant ones filtered out -
   * prepares them for search history processing
   *
   * @param array $params An array of raw parameters
   * @param array $valid_params An array of parameters to allow
   * @return array
   */
  protected function filterParams( array $params, array $valid_params = null )
  {
    if ( ! $valid_params ) {
      $valid_params = array(
        'l', 'i', 'i1', 'i2', 'i3', 'q', 'q1', 'q2', 'q3',
      );
    }

    foreach( $params as $key => $value ) {
      if ( ! in_array( $key, $valid_params ) ) {
        unset( $params[ $key ] );
      }
    }

    return $params;
  }

  /**
   * Returns the Collection to search
   *
   * @return Collection
   */
  public function getCollection()
  {<commit_msg>rethink filterParams() - overload in child class if needed<commit_after><?php
/**
 * EmeraldView
 *
 * LICENSE
 *
 * This source file is subject to the new BSD license that is bundled
 * with this package in the file LICENSE.txt.
 * It is also available through the world-wide-web at this URL:
 * http://yitznewton.org/emeraldview/index.php?title=License
 * If you did not receive a copy of the license and are unable to
 * obtain it through the world-wide-web, please send an email
 * to <EMAIL> so we can send you a copy immediately.
 *
 * @version 0.2.0-b4
 * @package libraries
 */
/**
 * SearchHandler is a wrapper class which creates objects and delegates
 * responsibility for search functionality
 *
 * @package libraries
 * @copyright  Copyright (c) 2010 <NAME> (http://yitznewton.org/)
 * @license    http://yitznewton.org/emeraldview/index.php?title=License     New BSD License
 */
class SearchHandler
{
  /**
   * ASCII matching pattern for search term boundary detection -
   * must match the search term in the first backreference
   */
  const BOUNDARIES_PATTERN_ASCII   = '\\b(%s)\\b';
  /**
   * Unicode matching pattern for search term boundary detection -
   * must match the search term in the first backreference
   */
  const BOUNDARIES_PATTERN_UNICODE = '(?:[^_\pL\pN]|^)(%s)(?:[^_\pL\pN]|$)';

  /**
   * An array of the query parameters for this search
   *
   * @var array
   */
  protected $params;
  /**
   * The Collection being searched
   *
   * @var Collection
   */
  protected $collection;
  /**
   * The full path of the directory containing raw node text
   *
   * @var string
   */
  protected $rawTextDir;
  /**
   * A QueryBuilder representing this search
   *
   * @var QueryBuilder
   */
  protected $queryBuilder;
  /**
   * The level of node being searched (document or section)
   *
   * @var string
   */
  protected $indexLevel;
  /**
   * The Zend_Search_Lucene interface to the appropriate index
   *
   * @var Zend_Search_Lucene
   */
  protected $luceneObject;
  /**
   * The word boundary matching pattern to use with this search
   *
   * @var string
   */
  protected $boundariesPattern;

  /**
   * @param array $params An array of the query parameters
   * @param Collection $collection The Collection to search
   */
  public function __construct( array $params, Collection $collection )
  {
    $this->params = $this->filterParams( $params );
    $this->collection = $collection;
    $this->queryBuilder = QueryBuilder::factory( $params, $collection );
    $this->boundariesPattern = $collection->getConfig( 'boundaries_pattern', SearchHandler::BOUNDARIES_PATTERN_ASCII );

    $level_prefix = substr( $this->getIndexLevel(), 0, 1 );
    $index_dir = $collection->getGreenstoneDirectory()
               . "/index/$level_prefix" . 'idx';

    $this->rawTextDir = $collection->getGreenstoneDirectory()
                      . "/index/raw-text/$level_prefix" . 'idx';

    if (!is_dir( $index_dir ) || !is_readable( $index_dir )) {
      throw new Exception("Could not read index directory $index_dir");
    }

    $b_and = Zend_Search_Lucene_Search_QueryParser::B_AND;
    Zend_Search_Lucene_Search_QueryParser::setDefaultOperator( $b_and );
    Zend_Search_Lucene_Analysis_Analyzer::setDefault(
      new Zend_Search_Lucene_Analysis_Analyzer_Common_Utf8());

    $this->luceneObject = Zend_Search_Lucene::open( $index_dir );
  }

  /**
   * Performs query and returns an array of Zend_Search_Lucene_Search_QueryHit
   * objects
   *
   * @return array
   */
  public function execute()
  {
    $query = $this->queryBuilder->getQuery();
    
    try {
      $lucene_hits = @$this->luceneObject->find( $query );
    }
    catch (Zend_Search_Lucene_Exception $e) {
      // malformed search
      Kohana::log( 'error', $e->getMessage() );
      return array();
    }

    $hits = array();

    foreach ($lucene_hits as $lucene_hit) {
      $hits[] = new Hit( $lucene_hit, $this );
    }

    return $hits;
  }

  /**
   * Returns an array of the query parameters
   *
   * @return array
   */
  public function getParams()
  {
    return $this->params;
  }

  /**
   * Returns an array of query parameters with irrelevant ones filtered out -
   * prepares them for search history processing
   *
   * @param array $params An array of raw parameters
   * @return array
   */
  protected function filterParams( array $params )
  {
    $valid_params = array(
      'l', 'i', 'i1', 'i2', 'i3', 'q', 'q1', 'q2', 'q3',
    );

    foreach( $params as $key => $value ) {
      if ( ! in_array( $key, $valid_params ) ) {
        unset( $params[ $key ] );
      }
    }

    return $params;
  }

  /**
   * Returns the Collection to search
   *
   * @return Collection
   */
  public function getCollection()
  {
{'fraction_non_alphanumeric': 0.12049573005623829, 'fraction_numerical': 0.003540928973130598, 'mean_word_length': 2.9551070840197693, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 43, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3099485', 'n_tokens_mistral': 3033, 'n_tokens_neox': 2931, 'n_words': 857}