<commit_before><?php

class Search_Indexer
{
	private $searchIndex;
	private $contentSources = array();
	private $globalSources = array();

	function __construct(Search_Index_Interface $searchIndex)
	{
		$this->searchIndex = $searchIndex;
	}

	function addContentSource($objectType, Search_ContentSource_Interface $contentSource)
	{
		$this->contentSources[$objectType] = $contentSource;
	}

	function addGlobalSource(Search_GlobalSource_Interface $globalSource)
	{
		$this->globalSources[] = $globalSource;
	}

	/**
	 * Rebuild the entire index.
	 */
	function rebuild()
	{
		foreach ($this->contentSources as $objectType => $contentSource) {
			foreach ($contentSource->getDocuments() as $objectId) {
				$this->addDocument($objectType, $objectId);
			}
		}
	}

	function update(array $objectList)
	{
		$this->searchIndex->invalidateMultiple($objectList);

		foreach ($objectList as $object) {
			$this->addDocument($object['object_type'], $object['object_id']);
		}
	}

	private function addDocument($objectType, $objectId)
	{
		$typeFactory = $this->searchIndex->getTypeFactory();
		if (isset($this->contentSources[$objectType])) {
			$contentSource = $this->contentSources[$objectType];

			if (false !== $data = $contentSource->getDocument($objectId, $typeFactory)) {
				$initialData = $data;

				foreach ($this->globalSources as $globalSource) {<commit_msg>[FIX] missing commit to 31220<commit_after><?php

class Search_Indexer
{
	private $searchIndex;
	private $contentSources = array();
	private $globalSources = array();

	function __construct(Search_Index_Interface $searchIndex)
	{
		$this->searchIndex = $searchIndex;
	}

	function addContentSource($objectType, Search_ContentSource_Interface $contentSource)
	{
		$this->contentSources[$objectType] = $contentSource;
	}

	function addGlobalSource(Search_GlobalSource_Interface $globalSource)
	{
		$this->globalSources[] = $globalSource;
	}

	/**
	 * Rebuild the entire index.
	 */
	function rebuild()
	{
		$stat = array();
		foreach ($this->contentSources as $objectType => $contentSource) {
			foreach ($contentSource->getDocuments() as $objectId) {
				$this->addDocument($objectType, $objectId);
				$stat[$objectType] = empty($stat[$objectType])? 1: $stat[$objectType] + 1;
			}
		}
		return $stat;
	}

	function update(array $objectList)
	{
		$this->searchIndex->invalidateMultiple($objectList);

		foreach ($objectList as $object) {
			$this->addDocument($object['object_type'], $object['object_id']);
		}
	}

	private function addDocument($objectType, $objectId)
	{
		$typeFactory = $this->searchIndex->getTypeFactory();
		if (isset($this->contentSources[$objectType])) {
			$contentSource = $this->contentSources[$objectType];

			if (false !== $data = $contentSource->getDocument($objectId, $typeFactory)) {
				$initialData = $data;

				foreach ($this->globalSources as $globalSource) {
{'fraction_non_alphanumeric': 0.1488469601677149, 'fraction_numerical': 0.002445842068483578, 'mean_word_length': 5.462753950338601, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7029354', 'n_tokens_mistral': 960, 'n_tokens_neox': 867, 'n_words': 163}