<commit_before><?xml version="1.0"?>
<!--
 Copyright 2005-2007 The Kuali Foundation.

 Licensed under the Educational Community License, Version 1.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at

 http://www.opensource.org/licenses/ecl1.php

 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
-->
<project name="kuali" default="dist-local" basedir=".">
	<!-- run the help target to view usage instructions for public targets -->
	
	<target name="dist-local" if="is.local.build" depends="clean,filter-project,filter-local" description="Prepare local configuration files and tokenized resources and deploy tomcat context file on development workstation">
		<copy file="${metainf.directory}/${context.file}" tofile="${appserver.localhost.dir}/${ant.project.name}-${build.environment}.xml" overwrite="true" />
	</target>

	<target name="test-local" depends="dist-local,make-tests,test" description="Run all unit tests and format results"/>

	<target name="echo-properties" depends="init-properties" description="Print all build properties that have been set">
		<echoproperties />
	</target>

	<target name="clean" depends="clean-project,clean-local" description="Remove all build output" />

	<target name="init-property-files">
		<property file="${user.home}/${ant.project.name}-build.properties" />
		<property file="${shared.external.build.properties}" />
		<property file="build.properties" />
	</target>
	
	<target name="init-properties" depends="init-property-files">
		<tstamp><format property="build.version" pattern="MM/dd/yyyy hh:mm aa" /></tstamp>
		<condition property="context.docbase" value=' docBase="${basedir}${file.separator}work${file.separator}web-root"'><equals arg1="${is.local.build}" arg2="true" /></condition>
		<property name="context.docbase" value="" />
		<condition property="datasource.url" value="${mysql.datasource.url}"><equals arg1="${datasource.ojb.platform}" arg2="${mysql.ojb.platform}" /></condition>
		<property name="datasource.url" value="${oracle.datasource.url}" />
		<condition property="use.p6spy.local">
			<and>
				<equals arg1="${is.local.build}" arg2="true" />
				<equals arg1="${use.p6spy}" arg2="true" />
			</and>
		</condition>
		<condition property="datasource.driver.class" value="${p6spy.driver.class}"><equals arg1="${use.p6spy.local}" arg2="true"/></condition>
		<condition property="datasource.driver.class" value="${mysql.driver.class}"><equals arg1="${datasource.ojb.platform}" arg2="${mysql.ojb.platform}" /></condition>
		<property name="datasource.driver.class" value="${oracle.driver.class}" />
		<condition property="p6spy.real.driver.class" value="${mysql.driver.class}"><equals arg1="${datasource.ojb.platform}" arg2="${mysql.ojb.platform}" /></condition>
		<property name="p6spy.real.driver.class" value="${oracle.driver.class}" />
		<condition property="datasource.ojb.sequence.manager" value="${mysql.ojb.sequence.manager}"><equals arg1="${datasource.ojb.platform}" arg2="${mysql.ojb.platform}" /></condition>
		<property name="datasource.ojb.sequence.manager" value="${oracle.ojb.sequence.manager}" />
	    <condition property="rice.workflow.datasource.platform" value="${mysql.workflow.datasource.platform}"><equals arg1="${datasource.ojb.platform}" arg2="${mysql.ojb.platform}" /></condition>
		<property name="rice.workflow.datasource.platform" value="${oracle.workflow.datasource.platform}" />
		<property name="rice.workflow.datasource.url" value="${datasource.url}" />
		<property name="rice.workflow.datasource.driver.class" value="${datasource.driver.class}" />
		<condition property="appenders" value="INFO, LogFile, StdOut"><equals arg1="${is.local.build}" arg2="true" /></condition>
		<property name="appenders" value="INFO, LogFile" />
		<condition property="use.quartz.jdbc.jobstore" value="false"><equals arg1="${is.local.build}" arg2="true" /></condition>
		<property name="use.quartz.jdbc.jobstore" value="true" />
		<condition property="rice.dev.mode" value="true"><equals arg1="${is.local.build}" arg2="true" /></condition>
		<property name="rice.dev.mode" value="false" />
		<condition property="rice.message.persistence" value="false"><equals arg1="${is.local.build}" arg2="true" /></condition>
		<property name="rice.message.persistence" value="true" />
		<condition property="rice.use.quartz.database" value="true">
			<and>
				<not><equals arg1="${use.quartz.scheduling}" arg2="true" /></not>
				<not><equals arg1="${is.local.build}" arg2="true" /></not>
			</and>
		</condition>
		<property name="rice.use.quartz.database" value="false" />
		<condition property="rice.exception.routing.immediate" value="true"><equals arg1="${use.quartz.scheduling}" arg2="false" /></condition>
		<property name="rice.exception.routing.immediate" value="false" />
		<loadfile property="configuration" srcfile="${build.project.directory}/${configuration.file}"><filterchain><expandproperties/></filterchain></loadfile>
		<loadfile property="p6spy" srcfile="${build.project.directory}/${p6spy.file}"><filterchain><expandproperties/></filterchain></loadfile>
		<loadfile property="web" srcfile="${build.project.directory}/${web.file}"><filterchain><expandproperties/></filterchain></loadfile>
		<loadfile property="context" srcfile="${build.appserver.directory}/${context.file}"><filterchain><expandproperties/></filterchain></loadfile>
		<loadfile property="log4j" srcfile="${user.home}/${log4j.file}" failonerror="false"><filterchain><expandproperties/></filterchain></loadfile>
		<loadfile property="log4j" srcfile="${build.external.directory}/${log4j.file}"><filterchain><expandproperties/></filterchain></loadfile>
		<loadfile property="security" srcfile="${build.external.directory}/${security.file}"><filterchain><expandproperties/></filterchain></loadfile>
		<loadfile property="standalone-workflow" srcfile="${build.appserver.directory}/standalone-workflow.xml"><filterchain><expandproperties/></filterchain></loadfile>
	</target>

	<target name="init-make-references" depends="init-properties">
		<path id="compile.source.classpath">
			<fileset dir="${appserver.lib.dir}" includes="*.jar" />
			<fileset dir="${lib.directory}" includes="*.jar" />
		</path>
		<path id="compile.tests.classpath">
			<path refid="compile.source.classpath" />
			<fileset dir="${test.lib.directory}" includes="*.jar" />
		</path>
		<!-- the test classes must come first so they can override classes as necessary in
		     the loaded libraries -->
		<path id="run.tests.classpath">
			<pathelement location="${test.classes.directory}" />
			<path refid="compile.tests.classpath" />
		</path>
	</target>
	
	<target name="make-javadoc" depends="init-make-references">
		<mkdir dir="${build.doc.directory}"/>
		<javadoc destdir="${build.doc.directory}" packagenames="org.*" locale="en" useexternalfile="yes" maxmemory="${javadoc.max.memory}">
			<sourcepath><pathelement location="${source.directory}"/></sourcepath>
			<classpath refid="compile.source.classpath"/>
		</javadoc>
	</target>

	<target name="clean-help" depends="init-property-files" unless="no.filter.project.help">
		<delete failonerror="false">
			<fileset dir="${basedir}">
				<include name="${help.directory}/**" />
			</fileset>
		</delete>
	</target>

	<target name="clean-rice" depends="init-property-files" unless="no.filter.project.rice">
		<delete failonerror="false">
			<fileset dir="${basedir}">
				<include name="${licenses.directory}/${rice.name}/**" />
				<include name="${webroot.directory}/${rice.application}/**" />
				<include name="${webroot.directory}/${tags.directory}/${rice.application}/**" />
				<include name="${webroot.directory}/${workflow.application}/**" />
			</fileset>
		</delete>
	</target>
	
	<target name="clean-spring-ide" depends="init-property-files" unless="no.filter.project.spring.ide">
		<delete dir="${rice.datadictionary.directory}" failonerror="false" />
		<delete file="${spring.ide.support.file}" failonerror="false" />
	</target>

	<target name="clean-project" depends="init-property-files,clean-help,clean-rice,clean-spring-ide">
		<delete failonerror="false">
			<fileset dir="${basedir}">
				<include name="${source.directory}/${configuration.file}" />
				<include name="${source.directory}/${p6spy.file}" />
				<include name="${webinf.directory}/${web.file}" />
				<include name="${lib.directory}/${log4j.library.file}" />
				<include name="${war.directory}/**/**" />
				<include name="${test.directory}/**/**" />
				<include name="*.war" />
				<include name="*.zip" />
				<include name="workflow.xml"/>
			</fileset>
		</delete>
	</target>

	<target name="clean-local" if="is.local.build" depends="init-property-files">
		<delete failonerror="false" includeemptydirs="true">
			<fileset dir="${base.security.directory}" includes="**/${ant.project.name}/"/>
			<fileset dir="${base.settings.directory}" includes="**/${ant.project.name}/"/>
			<fileset dir="${base.logs.directory}" includes="**/${ant.project.name}/"/>
			<fileset dir="${base.work.directory}" includes="**/${ant.project.name}/"/>
			<fileset dir="${base.deploy.directory}" includes="**/${ant.project.name}/"/>
		</delete>
		<delete failonerror="false">
			<fileset dir="${appserver.lib.dir}">
				<or>
					<present targetdir="${build.directory}" />
					<present targetdir="${build.appserver.directory}" />
					<present targetdir="${drivers.directory}" />
				</or>
			</fileset>
		</delete>
		<delete failonerror="false">
			<fileset dir="${appserver.classes.dir}">
				<present targetdir="${build.appserver.directory}" />
			</fileset>
		</delete>
		<delete failonerror="false">
			<fileset dir="${appserver.localhost.dir}" includes="${ant.project.name}-*.xml" />
		</delete>
		<delete failonerror="false">
			<fileset dir="${appserver.deploy.dir}" includes="${ant.project.name}-*.war,${ant.project.name}-*/**" />
		</delete>
	</target>

	<target name="filter-project-log4j" unless="use.p6spy.local" depends="init-properties">
		<copy file="${build.directory}/${log4j.library.file}" todir="${lib.directory}"/>
	</target>

	<target name="filter-project-p6spy" if="use.p6spy.local" depends="filter-project-log4j">
		<copy file="${build.directory}/${log4j.library.file}" todir="${appserver.lib.dir}"/>
	</target>
	
	<target name="filter-project-help" unless="no.filter.project.help" depends="init-properties">
		<unzip dest="${help.directory}" src="${build.project.directory}/help.zip" />
	</target>

	<target name="filter-project-rice" unless="no.filter.project.rice" depends="init-properties">
		<unzip dest="${webroot.directory}" src="${build.project.directory}/${rice.name}-${rice.version}${rice.output.web.suffix}" />
		<unzip dest="${licenses.directory}/${rice.name}" src="${build.project.directory}/${rice.name}-${rice.version}${rice.output.licenses.suffix}" />
		<!-- delete Rice JSP and tag compiled files to force re-compilation -->
		<delete dir="${appserver.work.dir}/${ant.project.name}-${build.environment}/org/apache/jsp/kr" />
		<delete dir="${appserver.work.dir}/${ant.project.name}-${build.environment}/org/apache/jsp/en" />
		<delete dir="${appserver.work.dir}/${ant.project.name}-${build.environment}/org/apache/jsp/tag/web/kr" />
		<delete dir="${appserver.work.dir}/${ant.project.name}-${build.environment}/org/apache/jsp/tag/web/en" />
	</target>
	
  	<target name="filter-project-spring-ide" unless="no.filter.project.spring.ide" depends="init-properties">
		<delete dir="${rice.datadictionary.directory}" failonerror="false" quiet="true"/>
		<mkdir dir="${rice.datadictionary.directory}"/>
		<unjar src="${lib.directory}/${rice.name}-${rice.impl.name}-${rice.version}.jar" dest="${rice.datadictionary.directory}">
			<patternset>
				<include name="${spring.ide.support.datadictionary.files}"/>
				<exclude name="**/conversion/**"/>
			</patternset>
		</unjar>
		<path id="rice.datadictionary.files"><fileset dir="${rice.datadictionary.directory}" includes="${spring.ide.support.datadictionary.files}"><contains casesensitive="true" text="&lt;beans"/></fileset></path>
		<path id="kfs.datadictionary.files"><fileset dir="${source.directory}" includes="${spring.ide.support.datadictionary.files}"><contains casesensitive="true" text="&lt;beans"/></fileset></path><commit_msg>put back full purge of work directory<commit_after><?xml version="1.0"?>
<!--
 Copyright 2005-2007 The Kuali Foundation.

 Licensed under the Educational Community License, Version 1.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at

 http://www.opensource.org/licenses/ecl1.php

 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
-->
<project name="kuali" default="dist-local" basedir=".">
	<!-- run the help target to view usage instructions for public targets -->
	
	<target name="dist-local" if="is.local.build" depends="clean,filter-project,filter-local" description="Prepare local configuration files and tokenized resources and deploy tomcat context file on development workstation">
		<copy file="${metainf.directory}/${context.file}" tofile="${appserver.localhost.dir}/${ant.project.name}-${build.environment}.xml" overwrite="true" />
	</target>

	<target name="test-local" depends="dist-local,make-tests,test" description="Run all unit tests and format results"/>

	<target name="echo-properties" depends="init-properties" description="Print all build properties that have been set">
		<echoproperties />
	</target>

	<target name="clean" depends="clean-project,clean-local" description="Remove all build output" />

	<target name="init-property-files">
		<property file="${user.home}/${ant.project.name}-build.properties" />
		<property file="${shared.external.build.properties}" />
		<property file="build.properties" />
	</target>
	
	<target name="init-properties" depends="init-property-files">
		<tstamp><format property="build.version" pattern="MM/dd/yyyy hh:mm aa" /></tstamp>
		<condition property="context.docbase" value=' docBase="${basedir}${file.separator}work${file.separator}web-root"'><equals arg1="${is.local.build}" arg2="true" /></condition>
		<property name="context.docbase" value="" />
		<condition property="datasource.url" value="${mysql.datasource.url}"><equals arg1="${datasource.ojb.platform}" arg2="${mysql.ojb.platform}" /></condition>
		<property name="datasource.url" value="${oracle.datasource.url}" />
		<condition property="use.p6spy.local">
			<and>
				<equals arg1="${is.local.build}" arg2="true" />
				<equals arg1="${use.p6spy}" arg2="true" />
			</and>
		</condition>
		<condition property="datasource.driver.class" value="${p6spy.driver.class}"><equals arg1="${use.p6spy.local}" arg2="true"/></condition>
		<condition property="datasource.driver.class" value="${mysql.driver.class}"><equals arg1="${datasource.ojb.platform}" arg2="${mysql.ojb.platform}" /></condition>
		<property name="datasource.driver.class" value="${oracle.driver.class}" />
		<condition property="p6spy.real.driver.class" value="${mysql.driver.class}"><equals arg1="${datasource.ojb.platform}" arg2="${mysql.ojb.platform}" /></condition>
		<property name="p6spy.real.driver.class" value="${oracle.driver.class}" />
		<condition property="datasource.ojb.sequence.manager" value="${mysql.ojb.sequence.manager}"><equals arg1="${datasource.ojb.platform}" arg2="${mysql.ojb.platform}" /></condition>
		<property name="datasource.ojb.sequence.manager" value="${oracle.ojb.sequence.manager}" />
	    <condition property="rice.workflow.datasource.platform" value="${mysql.workflow.datasource.platform}"><equals arg1="${datasource.ojb.platform}" arg2="${mysql.ojb.platform}" /></condition>
		<property name="rice.workflow.datasource.platform" value="${oracle.workflow.datasource.platform}" />
		<property name="rice.workflow.datasource.url" value="${datasource.url}" />
		<property name="rice.workflow.datasource.driver.class" value="${datasource.driver.class}" />
		<condition property="appenders" value="INFO, LogFile, StdOut"><equals arg1="${is.local.build}" arg2="true" /></condition>
		<property name="appenders" value="INFO, LogFile" />
		<condition property="use.quartz.jdbc.jobstore" value="false"><equals arg1="${is.local.build}" arg2="true" /></condition>
		<property name="use.quartz.jdbc.jobstore" value="true" />
		<condition property="rice.dev.mode" value="true"><equals arg1="${is.local.build}" arg2="true" /></condition>
		<property name="rice.dev.mode" value="false" />
		<condition property="rice.message.persistence" value="false"><equals arg1="${is.local.build}" arg2="true" /></condition>
		<property name="rice.message.persistence" value="true" />
		<condition property="rice.use.quartz.database" value="true">
			<and>
				<not><equals arg1="${use.quartz.scheduling}" arg2="true" /></not>
				<not><equals arg1="${is.local.build}" arg2="true" /></not>
			</and>
		</condition>
		<property name="rice.use.quartz.database" value="false" />
		<condition property="rice.exception.routing.immediate" value="true"><equals arg1="${use.quartz.scheduling}" arg2="false" /></condition>
		<property name="rice.exception.routing.immediate" value="false" />
		<loadfile property="configuration" srcfile="${build.project.directory}/${configuration.file}"><filterchain><expandproperties/></filterchain></loadfile>
		<loadfile property="p6spy" srcfile="${build.project.directory}/${p6spy.file}"><filterchain><expandproperties/></filterchain></loadfile>
		<loadfile property="web" srcfile="${build.project.directory}/${web.file}"><filterchain><expandproperties/></filterchain></loadfile>
		<loadfile property="context" srcfile="${build.appserver.directory}/${context.file}"><filterchain><expandproperties/></filterchain></loadfile>
		<loadfile property="log4j" srcfile="${user.home}/${log4j.file}" failonerror="false"><filterchain><expandproperties/></filterchain></loadfile>
		<loadfile property="log4j" srcfile="${build.external.directory}/${log4j.file}"><filterchain><expandproperties/></filterchain></loadfile>
		<loadfile property="security" srcfile="${build.external.directory}/${security.file}"><filterchain><expandproperties/></filterchain></loadfile>
		<loadfile property="standalone-workflow" srcfile="${build.appserver.directory}/standalone-workflow.xml"><filterchain><expandproperties/></filterchain></loadfile>
	</target>

	<target name="init-make-references" depends="init-properties">
		<path id="compile.source.classpath">
			<fileset dir="${appserver.lib.dir}" includes="*.jar" />
			<fileset dir="${lib.directory}" includes="*.jar" />
		</path>
		<path id="compile.tests.classpath">
			<path refid="compile.source.classpath" />
			<fileset dir="${test.lib.directory}" includes="*.jar" />
		</path>
		<!-- the test classes must come first so they can override classes as necessary in
		     the loaded libraries -->
		<path id="run.tests.classpath">
			<pathelement location="${test.classes.directory}" />
			<path refid="compile.tests.classpath" />
		</path>
	</target>
	
	<target name="make-javadoc" depends="init-make-references">
		<mkdir dir="${build.doc.directory}"/>
		<javadoc destdir="${build.doc.directory}" packagenames="org.*" locale="en" useexternalfile="yes" maxmemory="${javadoc.max.memory}">
			<sourcepath><pathelement location="${source.directory}"/></sourcepath>
			<classpath refid="compile.source.classpath"/>
		</javadoc>
	</target>

	<target name="clean-help" depends="init-property-files" unless="no.filter.project.help">
		<delete failonerror="false">
			<fileset dir="${basedir}">
				<include name="${help.directory}/**" />
			</fileset>
		</delete>
	</target>

	<target name="clean-rice" depends="init-property-files" unless="no.filter.project.rice">
		<delete failonerror="false">
			<fileset dir="${basedir}">
				<include name="${licenses.directory}/${rice.name}/**" />
				<include name="${webroot.directory}/${rice.application}/**" />
				<include name="${webroot.directory}/${tags.directory}/${rice.application}/**" />
				<include name="${webroot.directory}/${workflow.application}/**" />
			</fileset>
		</delete>
	</target>
	
	<target name="clean-spring-ide" depends="init-property-files" unless="no.filter.project.spring.ide">
		<delete dir="${rice.datadictionary.directory}" failonerror="false" />
		<delete file="${spring.ide.support.file}" failonerror="false" />
	</target>

	<target name="clean-project" depends="init-property-files,clean-help,clean-rice,clean-spring-ide">
		<delete failonerror="false">
			<fileset dir="${basedir}">
				<include name="${source.directory}/${configuration.file}" />
				<include name="${source.directory}/${p6spy.file}" />
				<include name="${webinf.directory}/${web.file}" />
				<include name="${lib.directory}/${log4j.library.file}" />
				<include name="${war.directory}/**/**" />
				<include name="${test.directory}/**/**" />
				<include name="*.war" />
				<include name="*.zip" />
				<include name="workflow.xml"/>
			</fileset>
		</delete>
	</target>

	<target name="clean-local-work" if="is.local.build" unless="preserve.work.directory" depends="init-property-files">
		<delete failonerror="false" includeemptydirs="true">
			<fileset dir="${appserver.work.dir}" includes="${ant.project.name}-*/**" />
		</delete>
	</target>

	<target name="clean-local" if="is.local.build" depends="clean-local-work">
		<delete failonerror="false" includeemptydirs="true">
			<fileset dir="${base.security.directory}" includes="**/${ant.project.name}/"/>
			<fileset dir="${base.settings.directory}" includes="**/${ant.project.name}/"/>
			<fileset dir="${base.logs.directory}" includes="**/${ant.project.name}/"/>
			<fileset dir="${base.work.directory}" includes="**/${ant.project.name}/"/>
			<fileset dir="${base.deploy.directory}" includes="**/${ant.project.name}/"/>
		</delete>
		<delete failonerror="false">
			<fileset dir="${appserver.lib.dir}">
				<or>
					<present targetdir="${build.directory}" />
					<present targetdir="${build.appserver.directory}" />
					<present targetdir="${drivers.directory}" />
				</or>
			</fileset>
		</delete>
		<delete failonerror="false">
			<fileset dir="${appserver.classes.dir}">
				<present targetdir="${build.appserver.directory}" />
			</fileset>
		</delete>
		<delete failonerror="false">
			<fileset dir="${appserver.localhost.dir}" includes="${ant.project.name}-*.xml" />
		</delete>
		<delete failonerror="false">
			<fileset dir="${appserver.deploy.dir}" includes="${ant.project.name}-*.war,${ant.project.name}-*/**" />
		</delete>
	</target>

	<target name="filter-project-log4j" unless="use.p6spy.local" depends="init-properties">
		<copy file="${build.directory}/${log4j.library.file}" todir="${lib.directory}"/>
	</target>

	<target name="filter-project-p6spy" if="use.p6spy.local" depends="filter-project-log4j">
		<copy file="${build.directory}/${log4j.library.file}" todir="${appserver.lib.dir}"/>
	</target>
	
	<target name="filter-project-help" unless="no.filter.project.help" depends="init-properties">
		<unzip dest="${help.directory}" src="${build.project.directory}/help.zip" />
	</target>

	<target name="filter-project-rice" unless="no.filter.project.rice" depends="init-properties">
		<unzip dest="${webroot.directory}" src="${build.project.directory}/${rice.name}-${rice.version}${rice.output.web.suffix}" />
		<unzip dest="${licenses.directory}/${rice.name}" src="${build.project.directory}/${rice.name}-${rice.version}${rice.output.licenses.suffix}" />
	</target>
	
  	<target name="filter-project-spring-ide" unless="no.filter.project.spring.ide" depends="init-properties">
		<delete dir="${rice.datadictionary.directory}" failonerror="false" quiet="true"/>
		<mkdir dir="${rice.datadictionary.directory}"/>
		<unjar src="${lib.directory}/${rice.name}-${rice.impl.name}-${rice.version}.jar" dest="${rice.datadictionary.directory}">
			<patternset>
				<include name="${spring.ide.support.datadictionary.files}"/>
				<exclude name="**/conversion/**"/>
			</patternset>
		</unjar>
		<path id="rice.datadictionary.files"><fileset dir="${rice.datadictionary.directory}" includes="${spring.ide.support.datadictionary.files}"><contains casesensitive="true" text="&lt;beans"/></fileset></path>
		<path id="kfs.datadictionary.files"><fileset dir="${source.directory}" includes="${spring.ide.support.datadictionary.files}"><contains casesensitive="true" text="&lt;beans"/></fileset></path>
{'fraction_non_alphanumeric': 0.20757758968158002, 'fraction_numerical': 0.00532043530834341, 'mean_word_length': 5.682197683813627, 'pattern_counts': {'":': 0, '<': 511, '<?xml version=': 2, '>': 511, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 10}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2385726', 'n_tokens_mistral': 8189, 'n_tokens_neox': 7526, 'n_words': 1234}