<commit_before><project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
	<modelVersion>4.0.0</modelVersion>

	<groupId>newsleak</groupId>
	<artifactId>preprocessing</artifactId>
	<version>0.9</version>
	<build>
		<plugins>
			<plugin>
				<groupId>org.apache.maven.plugins</groupId>
				<artifactId>maven-compiler-plugin</artifactId>
				<configuration>
					<source>1.8</source>
					<target>1.8</target>
				</configuration>
			</plugin>
			<plugin>
				<artifactId>maven-assembly-plugin</artifactId>
				<configuration>
					<archive>
						<manifest>
							<mainClass>uhh_lt.newsleak.preprocessing.CreateCollection</mainClass>
						</manifest>
					</archive>
					<descriptorRefs>
						<descriptorRef>jar-with-dependencies</descriptorRef>
					</descriptorRefs>
				</configuration>
			</plugin>
		</plugins>
	</build>
	<packaging>jar</packaging>

	<name>preprocessing</name>
	<url>http://maven.apache.org</url>

	<properties>
		<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
		<uimaDependencyVersion>2.10.2</uimaDependencyVersion>
		<uimafitDependencyVersion>2.4.0</uimafitDependencyVersion>
	</properties>



	<repositories>
		<repository>
			<id>jitpack.io</id>
			<url>https://jitpack.io</url>
		</repository>
	</repositories>

	<dependencies>
		<dependency>
			<groupId>org.apache.uima</groupId>
			<artifactId>uimaj-core</artifactId>
			<version>${uimaDependencyVersion}</version>
		</dependency>
		<dependency>
			<groupId>org.apache.uima</groupId>
			<artifactId>uimafit-core</artifactId>
			<version>${uimafitDependencyVersion}</version>
		</dependency>
		<dependency>
			<groupId>org.apache.uima</groupId>
			<artifactId>uimaj-document-annotation</artifactId>
			<version>${uimafitDependencyVersion}</version>
		</dependency>
		<dependency>
			<groupId>org.apache.uima</groupId>
			<artifactId>uimafit-cpe</artifactId>
			<version>${uimafitDependencyVersion}</version>
		</dependency>
		<dependency>
			<groupId>org.apache.uima</groupId>
			<artifactId>uimafit-examples</artifactId>
			<version>${uimafitDependencyVersion}</version>
		</dependency>
		<dependency>
			<groupId>org.apache.commons</groupId>
			<artifactId>commons-csv</artifactId>
			<version>1.4</version>
		</dependency>
		<dependency>
			<groupId>commons-io</groupId>
			<artifactId>commons-io</artifactId>
			<version>2.4</version>
		</dependency>
		<dependency>
			<groupId>commons-cli</groupId>
			<artifactId>commons-cli</artifactId>
			<version>1.4</version>
		</dependency>
		<dependency>
			<groupId>com.google.code.gson</groupId>
			<artifactId>gson</artifactId>
			<version>2.8.2</version>
		</dependency>
		<dependency>
			<groupId>org.apache.commons</groupId>
			<artifactId>commons-lang3</artifactId>
			<version>3.0</version>
		</dependency>
		<dependency>
			<groupId>org.apache.opennlp</groupId>
			<artifactId>opennlp-tools</artifactId>
			<version>1.8.3</version>
		</dependency>
		<dependency>
			<groupId>org.apache.opennlp</groupId>
			<artifactId>opennlp-uima</artifactId>
			<version>1.8.3</version>
		</dependency>
		<dependency>
			<groupId>org.apache.httpcomponents</groupId>
			<artifactId>httpclient</artifactId>
			<version>4.5.3</version>
		</dependency>
		<dependency>
			<groupId>org.apache.httpcomponents</groupId>
			<artifactId>httpcore</artifactId>
			<version>4.4.6</version>
		</dependency>
		<dependency>
			<groupId>com.github.heideltime</groupId>
			<artifactId>heideltime</artifactId>
			<version>2.2.1</version>
		</dependency>

		<dependency>
			<groupId>org.elasticsearch</groupId>
			<artifactId>elasticsearch</artifactId>
			<version>2.4.6</version>
		</dependency>

		<dependency>
			<groupId>io.searchbox</groupId>
			<artifactId>jest</artifactId>
			<version>6.3.1</version>
		</dependency>
		<dependency>
			<groupId>io.searchbox</groupId>
			<artifactId>jest-common</artifactId>
			<version>6.3.1</version>
		</dependency>
		
		<dependency>
			<groupId>postgresql</groupId>
			<artifactId>postgresql</artifactId>
			<version>9.1-901.jdbc4</version>
		</dependency>

		<dependency>
			<groupId>com.github.rholder</groupId>
			<artifactId>snowball-stemmer</artifactId>
			<version>1.3.0.581.1</version>
		</dependency>

		<dependency>
			<groupId>com.ibm.icu</groupId>
			<artifactId>icu4j</artifactId><commit_msg>fix stemmer class incompatibility induced by lucene<commit_after><project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
	<modelVersion>4.0.0</modelVersion>

	<groupId>newsleak</groupId>
	<artifactId>preprocessing</artifactId>
	<version>0.9</version>
	<build>
		<plugins>
			<plugin>
				<groupId>org.apache.maven.plugins</groupId>
				<artifactId>maven-compiler-plugin</artifactId>
				<configuration>
					<source>1.8</source>
					<target>1.8</target>
				</configuration>
			</plugin>
			<plugin>
				<artifactId>maven-assembly-plugin</artifactId>
				<configuration>
					<archive>
						<manifest>
							<mainClass>uhh_lt.newsleak.preprocessing.CreateCollection</mainClass>
						</manifest>
					</archive>
					<descriptorRefs>
						<descriptorRef>jar-with-dependencies</descriptorRef>
					</descriptorRefs>
				</configuration>
			</plugin>
		</plugins>
	</build>
	<packaging>jar</packaging>

	<name>preprocessing</name>
	<url>http://maven.apache.org</url>

	<properties>
		<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
		<uimaDependencyVersion>2.10.2</uimaDependencyVersion>
		<uimafitDependencyVersion>2.4.0</uimafitDependencyVersion>
	</properties>



	<repositories>
		<repository>
			<id>jitpack.io</id>
			<url>https://jitpack.io</url>
		</repository>
	</repositories>

	<dependencies>
		<dependency>
			<groupId>org.apache.uima</groupId>
			<artifactId>uimaj-core</artifactId>
			<version>${uimaDependencyVersion}</version>
		</dependency>
		<dependency>
			<groupId>org.apache.uima</groupId>
			<artifactId>uimafit-core</artifactId>
			<version>${uimafitDependencyVersion}</version>
		</dependency>
		<dependency>
			<groupId>org.apache.uima</groupId>
			<artifactId>uimaj-document-annotation</artifactId>
			<version>${uimafitDependencyVersion}</version>
		</dependency>
		<dependency>
			<groupId>org.apache.uima</groupId>
			<artifactId>uimafit-cpe</artifactId>
			<version>${uimafitDependencyVersion}</version>
		</dependency>
		<dependency>
			<groupId>org.apache.uima</groupId>
			<artifactId>uimafit-examples</artifactId>
			<version>${uimafitDependencyVersion}</version>
		</dependency>
		<dependency>
			<groupId>org.apache.commons</groupId>
			<artifactId>commons-csv</artifactId>
			<version>1.4</version>
		</dependency>
		<dependency>
			<groupId>commons-io</groupId>
			<artifactId>commons-io</artifactId>
			<version>2.4</version>
		</dependency>
		<dependency>
			<groupId>commons-cli</groupId>
			<artifactId>commons-cli</artifactId>
			<version>1.4</version>
		</dependency>
		<dependency>
			<groupId>com.google.code.gson</groupId>
			<artifactId>gson</artifactId>
			<version>2.8.2</version>
		</dependency>
		<dependency>
			<groupId>org.apache.commons</groupId>
			<artifactId>commons-lang3</artifactId>
			<version>3.0</version>
		</dependency>
		<dependency>
			<groupId>org.apache.opennlp</groupId>
			<artifactId>opennlp-tools</artifactId>
			<version>1.8.3</version>
		</dependency>
		<dependency>
			<groupId>org.apache.opennlp</groupId>
			<artifactId>opennlp-uima</artifactId>
			<version>1.8.3</version>
		</dependency>
		<dependency>
			<groupId>org.apache.httpcomponents</groupId>
			<artifactId>httpclient</artifactId>
			<version>4.5.3</version>
		</dependency>
		<dependency>
			<groupId>org.apache.httpcomponents</groupId>
			<artifactId>httpcore</artifactId>
			<version>4.4.6</version>
		</dependency>
		<dependency>
			<groupId>com.github.heideltime</groupId>
			<artifactId>heideltime</artifactId>
			<version>2.2.1</version>
		</dependency>

		<dependency>
			<groupId>org.elasticsearch</groupId>
			<artifactId>elasticsearch</artifactId>
			<version>2.4.6</version>
			<exclusions>
                <exclusion>
                    <artifactId>lucene-analyzers-common</artifactId>
                    <groupId>org.apache.lucene</groupId>
                </exclusion>
            </exclusions>
		</dependency>

		<dependency>
			<groupId>io.searchbox</groupId>
			<artifactId>jest</artifactId>
			<version>6.3.1</version>
		</dependency>
		<dependency>
			<groupId>io.searchbox</groupId>
			<artifactId>jest-common</artifactId>
			<version>6.3.1</version>
		</dependency>
		
		<dependency>
			<groupId>postgresql</groupId>
			<artifactId>postgresql</artifactId>
			<version>9.1-901.jdbc4</version>
		</dependency>

		<dependency>
			<groupId>com.github.rholder</groupId>
			<artifactId>snowball-stemmer</artifactId>
			<version>1.3.0.581.1</version>
		</dependency>

		<dependency>
			<groupId>com.ibm.icu</groupId>
			<artifactId>icu4j</artifactId>
{'fraction_non_alphanumeric': 0.1679674618005936, 'fraction_numerical': 0.017808068594041992, 'mean_word_length': 4.192922374429224, 'pattern_counts': {'":': 0, '<': 469, '<?xml version=': 0, '>': 469, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 4}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1510876', 'n_tokens_mistral': 4044, 'n_tokens_neox': 3077, 'n_words': 315}