<commit_before><?php
// (c) Copyright 2002-2012 by authors of the Tiki Wiki CMS Groupware Project
// 
// All Rights Reserved. See copyright.txt for details and a complete list of authors.
// Licensed under the GNU LESSER GENERAL PUBLIC LICENSE. See license.txt for details.
// $Id$

class Search_Index_Lucene implements Search_Index_Interface
{
	private $lucene;
	private $highlight = true;
	private $cache;
	private $lastModif;
	private $directory;
	private $maxResults = 0;
	private $resultSetLimit = 0;

	function __construct($directory, $lang = 'en', $highlight = true)
	{
		switch($lang) {
		case 'en':
		default:
			Zend_Search_Lucene_Analysis_Analyzer::setDefault(new StandardAnalyzer_Analyzer_Standard_English());
		}

		$this->directory = $directory;
		$this->lastModif = file_exists($directory) ? filemtime($directory) : 0;

		$this->highlight = (bool) $highlight;
	}

	private function getLucene()
	{
		if ($this->lucene) {
			return $this->lucene;
		}

		try {
			$this->lucene = Zend_Search_Lucene::open($this->directory);
		} catch (Zend_Search_Lucene_Exception $e) {
			$this->lucene = Zend_Search_Lucene::create($this->directory);
		}
		global $prefs;
		if (!empty($prefs['unified_lucene_max_buffered_docs'])) {							// these break indexing if set empty
			$this->lucene->setMaxBufferedDocs($prefs['unified_lucene_max_buffered_docs']);	// default is 10
		}
		if (!empty($prefs['unified_lucene_max_merge_docs'])) {
			$this->lucene->setMaxMergeDocs($prefs['unified_lucene_max_merge_docs']);		// default is PHP_INT_MAX (effectively "infinite")
		}
		if (!empty($prefs['unified_lucene_merge_factor'])) {
			$this->lucene->setMergeFactor($prefs['unified_lucene_merge_factor']);			// default is 10
		}
		$this->lucene->setResultSetLimit($this->resultSetLimit);

		return $this->lucene;
	}

	function addDocument(array $data)
	{
		$document = $this->generateDocument($data);

		$this->getLucene()->addDocument($document);
	}

	function optimize()
	{
		$this->getLucene()->optimize();
	}

	function invalidateMultiple(Search_Expr_Interface $expr)
	{
		$documents = array();

		$lucene = $this->getLucene();
		$query = $this->buildQuery($expr);
		foreach ($lucene->find($query) as $hit) {
			$document = $hit->getDocument();
			$documents[] = array(
				'object_type' => $document->object_type,
				'object_id' => $document->object_id,
			);
			$lucene->delete($hit->id);
		}

		return $documents;
	}

	function find(Search_Expr_Interface $query, Search_Query_Order $sortOrder, $resultStart, $resultCount)
	{
		$data = $this->internalFind($query, $sortOrder);

		$result = array_slice($data['result'], $resultStart, $resultCount);

		$resultSet = new Search_ResultSet($result, count($data['result']), $resultStart, $resultCount);
		$resultSet->setEstimate($data['count']);

		if ($this->highlight) {
			$resultSet->setHighlightHelper(new Search_Index_Lucene_HighlightHelper($query));
		} else {
			$resultSet->setHighlightHelper(new Search_ResultSet_SnippetHelper);
		}

		return $resultSet;
	}

	function setCache($cache)
	{
		$this->cache = $cache;
	}

	function setMaxResults($max)
	{
		$this->maxResults = (int) $max;
	}

	public function setResultSetLimit($resultSetLimit) 
	{
		$this->resultSetLimit = $resultSetLimit;
	}

	public function getResultSetLimit() 
	{
		return $this->resultSetLimit;
	}
	
	private function internalFind(& $query, $sortOrder)
	{
		if ($this->cache) {
			$args = func_get_args();
			$cacheKey = serialize($args);

			$entry = $this->cache->getSerialized($cacheKey, 'searchresult', $this->lastModif);

			if ($entry) {
				$query = $entry['query'];
				return $entry['hits'];
			}
		}

		$query = $this->buildQuery($query);
		try {
			$hits = $this->getLucene()->find($query, $this->getSortField($sortOrder), $this->getSortType($sortOrder), $this->getSortOrder($sortOrder));
		} catch (Exception $e) {
			TikiLib::lib('errorreport')->report($e->getMessage());
		}

		$result = array();
		foreach ($hits as $key => $hit) {
			$result[] = array_merge($this->extractValues($hit->getDocument()), array('relevance' => round($hit->score, 2)));

			if ($this->maxResults && count($result) >= $this->maxResults) {
				break;
			}
		}

		$return = array(
			'result' => $result,
			'count' => count($hits),
		);

		if ($this->cache) {
			$this->cache->cacheItem(
							$cacheKey, 
							serialize(
											array(
												'query' => $query,
												'hits' => $return,<commit_msg>[FIX] uni-search: Filter out duplicate object hits that can creep into the index somehow<commit_after><?php
// (c) Copyright 2002-2012 by authors of the Tiki Wiki CMS Groupware Project
// 
// All Rights Reserved. See copyright.txt for details and a complete list of authors.
// Licensed under the GNU LESSER GENERAL PUBLIC LICENSE. See license.txt for details.
// $Id$

class Search_Index_Lucene implements Search_Index_Interface
{
	private $lucene;
	private $highlight = true;
	private $cache;
	private $lastModif;
	private $directory;
	private $maxResults = 0;
	private $resultSetLimit = 0;

	function __construct($directory, $lang = 'en', $highlight = true)
	{
		switch($lang) {
		case 'en':
		default:
			Zend_Search_Lucene_Analysis_Analyzer::setDefault(new StandardAnalyzer_Analyzer_Standard_English());
		}

		$this->directory = $directory;
		$this->lastModif = file_exists($directory) ? filemtime($directory) : 0;

		$this->highlight = (bool) $highlight;
	}

	private function getLucene()
	{
		if ($this->lucene) {
			return $this->lucene;
		}

		try {
			$this->lucene = Zend_Search_Lucene::open($this->directory);
		} catch (Zend_Search_Lucene_Exception $e) {
			$this->lucene = Zend_Search_Lucene::create($this->directory);
		}
		global $prefs;
		if (!empty($prefs['unified_lucene_max_buffered_docs'])) {							// these break indexing if set empty
			$this->lucene->setMaxBufferedDocs($prefs['unified_lucene_max_buffered_docs']);	// default is 10
		}
		if (!empty($prefs['unified_lucene_max_merge_docs'])) {
			$this->lucene->setMaxMergeDocs($prefs['unified_lucene_max_merge_docs']);		// default is PHP_INT_MAX (effectively "infinite")
		}
		if (!empty($prefs['unified_lucene_merge_factor'])) {
			$this->lucene->setMergeFactor($prefs['unified_lucene_merge_factor']);			// default is 10
		}
		$this->lucene->setResultSetLimit($this->resultSetLimit);

		return $this->lucene;
	}

	function addDocument(array $data)
	{
		$document = $this->generateDocument($data);

		$this->getLucene()->addDocument($document);
	}

	function optimize()
	{
		$this->getLucene()->optimize();
	}

	function invalidateMultiple(Search_Expr_Interface $expr)
	{
		$documents = array();

		$lucene = $this->getLucene();
		$query = $this->buildQuery($expr);
		foreach ($lucene->find($query) as $hit) {
			$document = $hit->getDocument();
			$documents[] = array(
				'object_type' => $document->object_type,
				'object_id' => $document->object_id,
			);
			$lucene->delete($hit->id);
		}

		return $documents;
	}

	function find(Search_Expr_Interface $query, Search_Query_Order $sortOrder, $resultStart, $resultCount)
	{
		$data = $this->internalFind($query, $sortOrder);

		$result = array_slice($data['result'], $resultStart, $resultCount);

		$resultSet = new Search_ResultSet($result, count($data['result']), $resultStart, $resultCount);
		$resultSet->setEstimate($data['count']);

		if ($this->highlight) {
			$resultSet->setHighlightHelper(new Search_Index_Lucene_HighlightHelper($query));
		} else {
			$resultSet->setHighlightHelper(new Search_ResultSet_SnippetHelper);
		}

		return $resultSet;
	}

	function setCache($cache)
	{
		$this->cache = $cache;
	}

	function setMaxResults($max)
	{
		$this->maxResults = (int) $max;
	}

	public function setResultSetLimit($resultSetLimit) 
	{
		$this->resultSetLimit = $resultSetLimit;
	}

	public function getResultSetLimit() 
	{
		return $this->resultSetLimit;
	}
	
	private function internalFind(& $query, $sortOrder)
	{
		if ($this->cache) {
			$args = func_get_args();
			$cacheKey = serialize($args);

			$entry = $this->cache->getSerialized($cacheKey, 'searchresult', $this->lastModif);

			if ($entry) {
				$query = $entry['query'];
				return $entry['hits'];
			}
		}

		$query = $this->buildQuery($query);
		try {
			$hits = $this->getLucene()->find($query, $this->getSortField($sortOrder), $this->getSortType($sortOrder), $this->getSortOrder($sortOrder));
		} catch (Exception $e) {
			TikiLib::lib('errorreport')->report($e->getMessage());
		}

		$result = array();
		foreach ($hits as $key => $hit) {
			$res = array_merge($this->extractValues($hit->getDocument()), array('relevance' => round($hit->score, 2)));

			$found = false;
			if (!empty($res['object_id']) && !empty($res['object_type'])) {	// filter out duplicates here
				foreach($result as $r) {
					if ($r['object_id'] === $res['object_id'] && $r['object_type'] === $res['object_type']) {
						$found = true;
						break;
					}
				}
			}
			if (!$found) {
				$result[] = $res;
			}

			if ($this->maxResults && count($result) >= $this->maxResults) {
				break;
			}
		}

		$return = array(
			'result' => $result,
			'count' => count($hits),
		);

		if ($this->cache) {
			$this->cache->cacheItem(
							$cacheKey, 
							serialize(
											array(
												'query' => $query,
												'hits' => $return,
{'fraction_non_alphanumeric': 0.16889993516317267, 'fraction_numerical': 0.0034579641236222175, 'mean_word_length': 4.3776873910517144, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 143, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3929940', 'n_tokens_mistral': 3655, 'n_tokens_neox': 3105, 'n_words': 586}