<commit_before><?php

class SolrSearch_IndexAll extends ProcessAbstract
{
	public function run($args)
	{
		$solr = new Apache_Solr_Service(SOLR_SERVER, SOLR_PORT, SOLR_CORE);

		$db = get_db();
		$items = $db->getTable('Item')->findAll();



		foreach ($items as $item){
			//only index items if they are public
			if ($item['public'] == '1'){
				//define docs
				$docs = array();
				$elementTexts = $db->getTable('ElementText')->findBySql('record_id = ?', array($item['id']));
				$doc = new Apache_Solr_Document();
				$doc->id = $item['id'];
				foreach ($elementTexts as $elementText){
					$fieldName = $elementText['element_id'] . '_s';
					$doc->setMultiValue($fieldName, $elementText['text']);
					//store Dublin Core titles as separate fields
					if ($elementText['element_id'] == 50){
						$doc->setMultiValue('title', $elementText['text']);
					}
				}

				//add tags
				foreach($item->Tags as $key => $tag){
					$doc->setMultiValue('tag', $tag);
				}

				//add collection
				if ($item['collection_id'] > 0){
					$collectionName = $db->getTable('Collection')->find($item['collection_id'])->name;
					$doc->collection = $collectionName;
				}

				//add item type
				if ($item['item_type_id'] > 0){
					$itemType = $db->getTable('ItemType')->find($item['item_type_id'])->name;
					$doc->itemtype = $itemType;
				}

				//add images or index XML files
				$files = $item->Files;
				foreach ($files as $file){
					$mimeType = $file->mime_browser;
					if($file['has_derivative_image'] == 1){
						$doc->setMultiValue('image', $file['id']);
					}
					//if the file is XML, index the full text
					if ($mimeType == 'application/xml' || $mimeType == 'text/xml'){
						$teiFile = $file->getPath('archive');
						$xml_doc = new DomDocument;
						$xml_doc->load($teiFile);
						$xpath = new DOMXPath($xml_doc);
						$nodes = $xpath->query('//text()');
						foreach ($nodes as $node){
							$value = preg_replace('/\s\s+/', ' ', trim($node->nodeValue));
							if ($value != ' ' && $value != ''){
								$doc->setMultiValue('fulltext', $value);
							}
						}
					}
				}
				//if FedoraConnector is installed, index fulltext of XML
				if (function_exists('fedora_connector_installed')){
					$datastreams = $db->getTable('FedoraConnector_Datastream')->findBySql('mime_type = ? AND item_id = ?', array('text/xml', $item->id));
					foreach($datastreams as $datastream){
						$teiFile = fedora_connector_content_url($datastream);
						$fedora_doc = new DomDocument;
						$fedora_doc->load($teiFile);
						$xpath = new DOMXPath($fedora_doc);
						$nodes = $xpath->query('//text()');
						foreach ($nodes as $node){
							$value = preg_replace('/\s\s+/', ' ', trim($node->nodeValue));
							if ($value != ' ' && $value != ''){
								$doc->setMultiValue('fulltext', $value);
							}
						}
					}
				}

			//add docs to array to be posted to Solr
			$docs[] = $doc;
			$solr->addDocuments($docs);
			}
		}
		try {
	    	$solr->commit();
			$solr->optimize();
		}
		catch ( Exception $e ) {
			echo $e->getMessage();
		}
	}
}<commit_msg>Use the database pager when calling SolrSearch_IndexAll.<commit_after><?php

include 'SolrSearch/DbPager.php';

class SolrSearch_IndexAll extends ProcessAbstract
{
	public function run($args)
	{
		$solr  = new Apache_Solr_Service(SOLR_SERVER, SOLR_PORT, SOLR_CORE);
		$db    = get_db();
        $sql   = $db
            ->getTable('Item')
            ->where('public=1')
            ->order('id');
        $pager = new SolrSearch_DbPager($db, $sql);

        while ($items = $pager->next()) {
            foreach ($items as $item) {
                //define docs
                $docs = array();

                $doc = new Apache_Solr_Document();
                $doc->id = $item['id'];

                $elementTexts = $db
                    ->getTable('ElementText')
                    ->findBySql('record_id = ?', array($item['id']));
                foreach ($elementTexts as $elementText) {
                    $fieldName = $elementText['element_id'] . '_s';
                    $doc->setMultiValue($fieldName, $elementText['text']);
                    if ($elementText['element_id'] == 50) {
                        $doc->setMultiValue('title', $elementText['text']);
                    }
                }

                foreach($item->Tags as $key => $tag) {
                    $doc->setMultiValue('tag', $tag);
                }

                if ($item['collection_id'] > 0) {
                    $collectionName = $db
                        ->getTable('Collection')
                        ->find($item['collection_id'])
                        ->name;
                    $doc->collection = $collectionName;
                }

                //add item type
                if ($item['item_type_id'] > 0) {
                    $itemType = $db
                        ->getTable('ItemType')
                        ->find($item['item_type_id'])
                        ->name;
                    $doc->itemtype = $itemType;
                }

                //add images or index XML files
                $files = $item->Files;
                foreach ($files as $file) {

                    $mimeType = $file->mime_browser;
                    if($file['has_derivative_image'] == 1) {
                        $doc->setMultiValue('image', $file['id']);
                    }

                    //if the file is XML, index the full text
                    if ($mimeType == 'application/xml' || $mimeType == 'text/xml') {
                        $teiFile = $file->getPath('archive');
                        $xml_doc = new DomDocument;
                        $xml_doc->load($teiFile);
                        $xpath = new DOMXPath($xml_doc);
                        $nodes = $xpath->query('//text()');
                        foreach ($nodes as $node) {
                            $value = preg_replace('/\s\s+/', ' ', trim($node->nodeValue));
                            if ($value != ' ' && $value != '') {
                                $doc->setMultiValue('fulltext', $value);
                            }
                        }
                    }
                }
                //if FedoraConnector is installed, index fulltext of XML
                if (function_exists('fedora_connector_installed')) {
                    $datastreams = $db->getTable('FedoraConnector_Datastream')->findBySql('mime_type = ? AND item_id = ?', array('text/xml', $item->id));
                    foreach($datastreams as $datastream) {
                        $teiFile = fedora_connector_content_url($datastream);
                        $fedora_doc = new DomDocument;
                        $fedora_doc->load($teiFile);
                        $xpath = new DOMXPath($fedora_doc);
                        $nodes = $xpath->query('//text()');
                        foreach ($nodes as $node) {
                            $value = preg_replace('/\s\s+/', ' ', trim($node->nodeValue));
                            if ($value != ' ' && $value != '') {
                                $doc->setMultiValue('fulltext', $value);
                            }
                        }
                    }
                }

                //add docs to array to be posted to Solr
                $docs[] = $doc;
                $solr->addDocuments($docs);
            }
        }
		try {
	    	$solr->commit();
			$solr->optimize();
		}
		catch ( Exception $e ) {
			echo $e->getMessage();
		}
	}
}
{'fraction_non_alphanumeric': 0.16160405009325873, 'fraction_numerical': 0.0015987210231814548, 'mean_word_length': 1.6535878402262283, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 83, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 12}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '752677', 'n_tokens_mistral': 2581, 'n_tokens_neox': 2138, 'n_words': 421}