<commit_before><?php
/**
 * EmeraldView
 *
 * LICENSE
 *
 * This source file is subject to the new BSD license that is bundled
 * with this package in the file LICENSE.txt.
 * It is also available through the world-wide-web at this URL:
 * http://yitznewton.org/emeraldview/index.php?title=License
 * If you did not receive a copy of the license and are unable to
 * obtain it through the world-wide-web, please send an email
 * to <EMAIL> so we can send you a copy immediately.
 *
 * @version 0.2.0-b4
 * @package libraries
 */
/**
 * SearchHandler for Solr instances
 *
 * @package libraries
 * @copyright  Copyright (c) 2010 <NAME> (http://yitznewton.org/)
 * @license    http://yitznewton.org/emeraldview/index.php?title=License     New BSD License
 */
class SearchHandler_Solr extends SearchHandler
{
  public function execute()
  {
    $querystring = $this->query->getQuerystring();

    $solr_params = array(
      'hl'      => 'on',
      'hl.fl'   => '*',
      'qf'      => 'text EX^2.5',
      'wt'      => 'xslt',
      'tr'      => 'emeraldview.xsl',
      'start'   => $this->startAt-1,
      'rows'    => $this->hitsPerPage,
      'q'       => $querystring,
    );

    if ( $this->query instanceof Query_Simple ) {
      //$solr_params['defType'] = 'dismax';  // breaks apostrophes in Hebrew
    }

    $host = $this->query->getCollection()->getConfig( 'solr_host' );

    if ( ! $host ) {
      $msg = 'No Solr host specified in config for collection '
             . $this->query->getCollection()->getGreenstoneName();

      throw new Exception( $msg );
    }

    $query_url = 'http://' . $host . '/select/?'
                 . http_build_query( $solr_params );

    $query_url = substr( $query_url, 0, 2048 );

    $last_quote_pos = strrpos( $query_url, '+OR+%22' );

    if ( $last_quote_pos !== false && substr( $query_url, -3 ) != '%22' ) {
      // truncated in the middle of a quoted portion
      // FIXME: this hack doesn't always seem to work
      $query_url = substr( $query_url, 0, -3 ) . '%22';
    }
    
    $ctx = stream_context_create( array( 'http' => array(
      'timeout' => 6,
    )));
    
    $xml = @file_get_contents( $query_url, 0, $ctx );

    if ( ! $xml ) {
      throw new Exception( 'Unexpected or no response from Solr' );
    }
    
    $data = new SimpleXMLElement( $xml );
    
    $attributes          = $data->attributes();
    $this->totalHitCount = (int) $attributes['numFound'];

    $hits = array();

    foreach ( $data->children() as $child ) {
      $hits[] = new Hit_Solr( $this, $child );
    }

    return $hits;
  }

  /**
   * Returns an array of query parameters with irrelevant ones filtered out -
   * prepares them for search history processing
   *
   * @param array $params An array of raw parameters
   * @return array
   */
  protected function filterParams( array $params )
  {
    $valid_params = array(
      'l', 'i', 'i1', 'i2', 'i3', 'q', 'q1', 'q2', 'q3', 'b1', 'b2', 'b3',
    );

    foreach( $params as $key => $value ) {
      if ( ! in_array( $key, $valid_params ) ) {
        unset( $params[ $key ] );
      }
    }

    return $params;
  }
}<commit_msg>rework to use curl to POST Solr requests: adds support for very long query strings<commit_after><?php
/**
 * EmeraldView
 *
 * LICENSE
 *
 * This source file is subject to the new BSD license that is bundled
 * with this package in the file LICENSE.txt.
 * It is also available through the world-wide-web at this URL:
 * http://yitznewton.org/emeraldview/index.php?title=License
 * If you did not receive a copy of the license and are unable to
 * obtain it through the world-wide-web, please send an email
 * to <EMAIL> so we can send you a copy immediately.
 *
 * @version 0.2.0-b4
 * @package libraries
 */
/**
 * SearchHandler for Solr instances
 *
 * @package libraries
 * @copyright  Copyright (c) 2010 <NAME> (http://yitznewton.org/)
 * @license    http://yitznewton.org/emeraldview/index.php?title=License     New BSD License
 */
class SearchHandler_Solr extends SearchHandler
{
  public function execute()
  {
    $host = $this->query->getCollection()->getConfig( 'solr_host' );
    $path = $this->query->getCollection()->getConfig( 'solr_path', '/solr' );
    $port = $this->query->getCollection()->getConfig( 'solr_port' );

    if ( ! $host ) {
      $msg = 'No Solr host specified in config for collection '
             . $this->query->getCollection()->getGreenstoneName();

      throw new Exception( $msg );
    }

    if ( ! $port ) {
      $msg = 'No Solr port specified in config for collection '
             . $this->query->getCollection()->getGreenstoneName();

      throw new Exception( $msg );
    }

    $querystring = $this->query->getQuerystring();

    $solr_params = array(
      'hl'      => 'on',
      'hl.fl'   => '*',
      'qf'      => 'text EX^2.5',
      'wt'      => 'xslt',
      'tr'      => 'emeraldview.xsl',
      'start'   => $this->startAt-1,
      'rows'    => $this->hitsPerPage,
      'q'       => $querystring,
    );

    $path .= '/select';

    $xml = $this->post( $host, $path, $port, http_build_query( $solr_params ) );

    if ( ! $xml || substr( $xml, 0, 5 ) !== '<?xml' ) {
      throw new Exception( 'Unexpected or no response from Solr' );
    }
    
    return $this->parseHits( $xml );
  }

  /**
   * Returns an array of query parameters with irrelevant ones filtered out -
   * prepares them for search history processing
   *
   * @param array $params An array of raw parameters
   * @return array
   */
  protected function filterParams( array $params )
  {
    $valid_params = array(
      'l', 'i', 'i1', 'i2', 'i3', 'q', 'q1', 'q2', 'q3', 'b1', 'b2', 'b3',
    );

    foreach( $params as $key => $value ) {
      if ( ! in_array( $key, $valid_params ) ) {
        unset( $params[ $key ] );
      }
    }

    return $params;
  }

  protected function parseHits( $xml )
  {
    $data = new SimpleXMLElement( $xml );

    $attributes          = $data->attributes();
    $this->totalHitCount = (int) $attributes['numFound'];

    $hits = array();

    foreach ( $data->children() as $child ) {
      $hits[] = new Hit_Solr( $this, $child );
    }

    return $hits;
  }

  /**
   *
   * @param string $host The host on which Solr is running
   * @param string $path The path to Solr
   * @param int $port The port Solr is listening on
   * @param array|string $data The query data
   * @return string
   */
  protected function post( $host, $path, $port, $data )
  {
    $curl = curl_init();

    curl_setopt( $curl, CURLOPT_TIMEOUT, 5 );
    curl_setopt( $curl, CURLOPT_PORT, $port );
    curl_setopt( $curl, CURLOPT_URL, 'http://' . $host . $path );
    curl_setopt( $curl, CURLOPT_POST, true );
    curl_setopt( $curl, CURLOPT_RETURNTRANSFER, true );
    curl_setopt( $curl, CURLOPT_POSTFIELDS, $data );

    $response = curl_exec( $curl );

    curl_close( $curl );

    return $response;
  }
}
{'fraction_non_alphanumeric': 0.1532561505065123, 'fraction_numerical': 0.0085383502170767, 'mean_word_length': 2.586403736377789, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 65, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 10}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5406058', 'n_tokens_mistral': 2370, 'n_tokens_neox': 2261, 'n_words': 654}