<commit_before><?php

/**
 * Extension to provide a search interface when applied to ContentController
 * 
 * @package lucene-silverstripe-plugin
 * @author <NAME> <<EMAIL>>
 */
 
class ZendSearchLuceneContentController extends Extension { 

	static $allowed_actions = array(
		'ZendSearchLuceneForm',
		'ZendSearchLuceneResults',
	);
	
	/**
	 * Returns the Lucene-powered search Form object.
	 *
	 * @return  Form    A Form object representing the search form.
	 */
	function ZendSearchLuceneForm() {
		$searchText = isset($_REQUEST['Search']) ? $_REQUEST['Search'] : '';
		$fields = new FieldSet(
			new TextField(
			    'Search', 
			    _t('ZendSearchLuceneForm.SearchButtonText', 'Search'), 
			    $searchText
			)
		);
		$actions = new FieldSet(
			new FormAction('ZendSearchLuceneResults', 'Go')
		);
		$form = new Form($this->owner, 'ZendSearchLuceneForm', $fields, $actions);
        $form->disableSecurityToken();
        $form->setFormMethod('get');
		return $form;
	}

	/**
	 * Process and render search results.
	 * 
	 * @param   array           $data       The raw request data submitted by user
	 * @param   Form            $form       The form instance that was submitted
	 * @param   SS_HTTPRequest  $request    Request generated for this action
	 * @return  String                      The rendered form, for inclusion into the page template.
	 */
	function ZendSearchLuceneResults($data, $form, $request) {
		$query = $form->dataFieldByName('Search')->dataValue();
		$hits = ZendSearchLuceneWrapper::find($query);
        $data = $this->getDataArrayFromHits($hits, $request);
		return $this->owner->customise($data)->renderWith(array('Lucene_results', 'Page'));
	}

    /**
     * Returns a data array suitable for customising a Page with, containing
     * search result and pagination information.
     * 
     * @param   Array           $hits       An array of Zend_Search_Lucene_Search_QueryHit objects
     * @param   SS_HTTPRequest  $request    The request that generated the search
     * @return  Array                       A custom array containing pagination data.
     */
    protected function getDataArrayFromHits($hits, $request) {
		$data = array(
			'Results' => null,
			'Query' => null,
			'Title' => 'Search Results',
			'TotalResults' => null,
			'TotalPages' => null,
			'ThisPage' => null,
			'StartResult' => null,
			'EndResult' => null,
			'PrevUrl' => DBField::create('Text', 'false'),
			'NextUrl' => DBField::create('Text', 'false'),
			'SearchPages' => new DataObjectSet()
		);
		
        $pageLength = ZendSearchLuceneSearchable::$pageLength;              // number of results per page
        $alwaysShowPages = ZendSearchLuceneSearchable::$alwaysShowPages;    // always show this many pages in pagination
        $maxShowPages = ZendSearchLuceneSearchable::$maxShowPages;          // maximum number of pages shown in pagination

		$start = $request->requestVar('start') ? (int)$request->requestVar('start') : 0;
		$currentPage = floor( $start / $pageLength ) + 1;
		$totalPages = ceil( count($hits) / $pageLength );
		if ( $totalPages == 0 ) $totalPages = 1;
		if ( $currentPage > $totalPages ) $currentPage = $totalPages;

        // Assemble final results after page number mangling
        $results = new DataObjectSet();
		foreach($hits as $k => $hit) {
		    if ( $k < ($currentPage-1)*$pageLength || $k >= ($currentPage*$pageLength) ) continue;
			$className = $hit->ClassName;
			$vars = new $className();
			$vars = $vars->getSearchedVars();
			$obj = new DataObject();
			foreach ( $vars as $var ) {
			    try {
			        $obj->$var = $hit->$var;
                } catch (Exception $e) { }
			}
			$obj->score = $hit->score;
			$obj->Number = $k + 1; // which number result it is...
			$results->push($obj);
		}
		// filter by permission
/*
		if($results) foreach($results as $result) {
			if(!$result->canView()) $results->remove($result);
		}
*/
		
	    $data['Results'] = $results;
	    $data['Query']   = DBField::create('Text', $request->requestVar('Search'));
	    $data['TotalResults'] = DBField::create('Text', count($hits));
        $data['TotalPages'] = DBField::create('Text', $totalPages);
        $data['ThisPage'] = DBField::create('Text', $currentPage);
        $data['StartResult'] = $start + 1;
        $data['EndResult'] = $start + count($results);
<commit_msg>Carry through Content field for Files that have been scanned for content<commit_after><?php

/**
 * Extension to provide a search interface when applied to ContentController
 * 
 * @package lucene-silverstripe-plugin
 * @author <NAME> <<EMAIL>>
 */
 
class ZendSearchLuceneContentController extends Extension { 

	static $allowed_actions = array(
		'ZendSearchLuceneForm',
		'ZendSearchLuceneResults',
	);
	
	/**
	 * Returns the Lucene-powered search Form object.
	 *
	 * @return  Form    A Form object representing the search form.
	 */
	function ZendSearchLuceneForm() {
		$searchText = isset($_REQUEST['Search']) ? $_REQUEST['Search'] : '';
		$fields = new FieldSet(
			new TextField(
			    'Search', 
			    _t('ZendSearchLuceneForm.SearchButtonText', 'Search'), 
			    $searchText
			)
		);
		$actions = new FieldSet(
			new FormAction('ZendSearchLuceneResults', 'Go')
		);
		$form = new Form($this->owner, 'ZendSearchLuceneForm', $fields, $actions);
        $form->disableSecurityToken();
        $form->setFormMethod('get');
		return $form;
	}

	/**
	 * Process and render search results.
	 * 
	 * @param   array           $data       The raw request data submitted by user
	 * @param   Form            $form       The form instance that was submitted
	 * @param   SS_HTTPRequest  $request    Request generated for this action
	 * @return  String                      The rendered form, for inclusion into the page template.
	 */
	function ZendSearchLuceneResults($data, $form, $request) {
		$query = $form->dataFieldByName('Search')->dataValue();
		$hits = ZendSearchLuceneWrapper::find($query);
        $data = $this->getDataArrayFromHits($hits, $request);
		return $this->owner->customise($data)->renderWith(array('Lucene_results', 'Page'));
	}

    /**
     * Returns a data array suitable for customising a Page with, containing
     * search result and pagination information.
     * 
     * @param   Array           $hits       An array of Zend_Search_Lucene_Search_QueryHit objects
     * @param   SS_HTTPRequest  $request    The request that generated the search
     * @return  Array                       A custom array containing pagination data.
     */
    protected function getDataArrayFromHits($hits, $request) {
		$data = array(
			'Results' => null,
			'Query' => null,
			'Title' => 'Search Results',
			'TotalResults' => null,
			'TotalPages' => null,
			'ThisPage' => null,
			'StartResult' => null,
			'EndResult' => null,
			'PrevUrl' => DBField::create('Text', 'false'),
			'NextUrl' => DBField::create('Text', 'false'),
			'SearchPages' => new DataObjectSet()
		);
		
        $pageLength = ZendSearchLuceneSearchable::$pageLength;              // number of results per page
        $alwaysShowPages = ZendSearchLuceneSearchable::$alwaysShowPages;    // always show this many pages in pagination
        $maxShowPages = ZendSearchLuceneSearchable::$maxShowPages;          // maximum number of pages shown in pagination

		$start = $request->requestVar('start') ? (int)$request->requestVar('start') : 0;
		$currentPage = floor( $start / $pageLength ) + 1;
		$totalPages = ceil( count($hits) / $pageLength );
		if ( $totalPages == 0 ) $totalPages = 1;
		if ( $currentPage > $totalPages ) $currentPage = $totalPages;

        // Assemble final results after page number mangling
        $results = new DataObjectSet();
		foreach($hits as $k => $hit) {
		    if ( $k < ($currentPage-1)*$pageLength || $k >= ($currentPage*$pageLength) ) continue;
			$className = $hit->ClassName;
			$vars = new $className();
			$vars = $vars->getSearchedVars();
			$obj = new DataObject();
			foreach ( $vars as $var ) {
			    try {
                    // Copy scanned text into Content if Content is empty
                    // Zend uses 'body' for scanned text, & we can't change it 
                    // without altering the Zend source.
                    if ( $var == 'Content' && $hit->Content == '' && $hit->body != '' ) {
                        $obj->Content = $hit->body;
                    } else {
    			        $obj->$var = $hit->$var;
    			    }
                } catch (Exception $e) { }
			}
			$obj->score = $hit->score;
			$obj->Number = $k + 1; // which number result it is...
			$results->push($obj);
		}
		
	    $data['Results'] = $results;
	    $data['Query']   = DBField::create('Text', $request->requestVar('Search'));
	    $data['TotalResults'] = DBField::create('Text', count($hits));
        $data['TotalPages'] = DBField::create('Text', $totalPages);
        $data['ThisPage'] = DBField::create('Text', $currentPage);
        $data['StartResult'] = $start + 1;
        $data['EndResult'] = $start + count($results);

{'fraction_non_alphanumeric': 0.14483906770255273, 'fraction_numerical': 0.0015538290788013318, 'mean_word_length': 2.948729184925504, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 83, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7237726', 'n_tokens_mistral': 2884, 'n_tokens_neox': 2611, 'n_words': 690}