<commit_before><?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
	xmlns:batch="http://www.springframework.org/schema/batch"
	xmlns:context="http://www.springframework.org/schema/context"
	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
	xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd
	                    http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context.xsd
	                    http://www.springframework.org/schema/batch http://www.springframework.org/schema/batch/spring-batch.xsd">
	                    
	<context:property-placeholder ignore-resource-not-found="true" location="classpath*:META-INF/spring/application.properties,classpath*:META-INF/spring/database.properties,classpath*:META-INF/spring/world-flora-harvest.properties,classpath*:META-INF/spring/world-flora-harvest-db.properties"/>
	<context:annotation-config />

	<batch:job id="ReIndex">	   
		<batch:step id="reIndex">
			<batch:description>Re-index some or all of the eMonocot database</batch:description>
			<batch:tasklet>
				<batch:chunk reader="itemReader" writer="itemIndexer" commit-interval="10" />
			</batch:tasklet>
			<batch:next on="COMPLETED" to="removeMissingDocuments"/>		
		</batch:step>
		<batch:step id="removeMissingDocuments">
			<batch:description>Remove missing solr documents</batch:description>
			<batch:tasklet>
				<batch:chunk reader="solrDocumentItemReader" processor="solrMissingObjectProcessor" writer="solrDeletingWriter" commit-interval="10" />
			</batch:tasklet>
			<batch:end on="COMPLETED"/>		
		</batch:step>
	</batch:job>
		
	<bean id="itemReader" class="org.springframework.batch.item.database.HibernateCursorItemReader"	scope="step">
		<property name="sessionFactory" ref="sessionFactory" />
		<property name="queryString" value="#{jobParameters['query.string']}" />		
	</bean>
    
    <bean id="itemIndexer" class="org.emonocot.harvest.common.SolrIndexingWriter" scope="step">
      <property name="type" value="#{jobParameters['query.type']}" />
      <property name="sessionFactory" ref="sessionFactory" />
      <property name="solrIndexingListener" ref="solrIndexingListener" />
    </bean>
    
    <bean id="solrDocumentItemReader" class="org.emonocot.harvest.common.SolrDocumentItemReader" scope="step">
		<property name="searchableObjectService" ref="searchableObjectServiceImpl" />		
		<property name="queryString" value="#{jobParameters['solr.query.string']}" />
		<property name="sort" value="#{jobParameters['solr.sort']}" />	
		<property name="selectedFacets" value="#{jobParameters['solr.selectedFacets']}" />
	</bean>
	
	<bean id="solrMissingObjectProcessor" class="org.emonocot.harvest.common.SolrMissingObjectProcessor">
		<property name="searchableObjectService" ref="searchableObjectServiceImpl" />
	</bean>
    
    <bean id="solrDeletingWriter"  class="org.emonocot.harvest.common.SolrDeletingWriter">
      <property name="solrServer" ref="solrServer" />
    </bean>
	
</beans>
<commit_msg>Modified commit intervals on reindex job to improve speed<commit_after><?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
	xmlns:batch="http://www.springframework.org/schema/batch"
	xmlns:context="http://www.springframework.org/schema/context"
	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
	xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd
	                    http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context.xsd
	                    http://www.springframework.org/schema/batch http://www.springframework.org/schema/batch/spring-batch.xsd">
	                    
	<context:property-placeholder ignore-resource-not-found="true" location="classpath*:META-INF/spring/application.properties,classpath*:META-INF/spring/database.properties,classpath*:META-INF/spring/world-flora-harvest.properties,classpath*:META-INF/spring/world-flora-harvest-db.properties"/>
	<context:annotation-config />

	<batch:job id="ReIndex">	   
		<batch:step id="reIndex">
			<batch:description>Re-index some or all of the eMonocot database</batch:description>
			<batch:tasklet>
				<batch:chunk reader="itemReader" writer="itemIndexer" commit-interval="1000" />
			</batch:tasklet>
			<batch:next on="COMPLETED" to="removeMissingDocuments"/>		
		</batch:step>
		<batch:step id="removeMissingDocuments">
			<batch:description>Remove missing solr documents</batch:description>
			<batch:tasklet>
				<batch:chunk reader="solrDocumentItemReader" processor="solrMissingObjectProcessor" writer="solrDeletingWriter" commit-interval="10000" />
			</batch:tasklet>
			<batch:end on="COMPLETED"/>		
		</batch:step>
	</batch:job>
		
	<bean id="itemReader" class="org.springframework.batch.item.database.HibernateCursorItemReader"	scope="step">
		<property name="sessionFactory" ref="sessionFactory" />
		<property name="queryString" value="#{jobParameters['query.string']}" />		
	</bean>
    
    <bean id="itemIndexer" class="org.emonocot.harvest.common.SolrIndexingWriter" scope="step">
      <property name="type" value="#{jobParameters['query.type']}" />
      <property name="sessionFactory" ref="sessionFactory" />
      <property name="solrIndexingListener" ref="solrIndexingListener" />
    </bean>
    
    <bean id="solrDocumentItemReader" class="org.emonocot.harvest.common.SolrDocumentItemReader" scope="step">
		<property name="searchableObjectService" ref="searchableObjectServiceImpl" />		
		<property name="queryString" value="#{jobParameters['solr.query.string']}" />
		<property name="sort" value="#{jobParameters['solr.sort']}" />	
		<property name="selectedFacets" value="#{jobParameters['solr.selectedFacets']}" />
	</bean>
	
	<bean id="solrMissingObjectProcessor" class="org.emonocot.harvest.common.SolrMissingObjectProcessor">
		<property name="searchableObjectService" ref="searchableObjectServiceImpl" />
	</bean>
    
    <bean id="solrDeletingWriter"  class="org.emonocot.harvest.common.SolrDeletingWriter">
      <property name="solrServer" ref="solrServer" />
    </bean>
	
</beans>

{'fraction_non_alphanumeric': 0.15969460792110704, 'fraction_numerical': 0.0046126928582789885, 'mean_word_length': 5.2567164179104475, 'pattern_counts': {'":': 0, '<': 91, '<?xml version=': 2, '>': 91, 'https://': 0, 'lorem ipsum': 0, 'www.': 20, 'xml': 10}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '377768', 'n_tokens_mistral': 2051, 'n_tokens_neox': 1911, 'n_words': 236}