<commit_before><?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
	<modelVersion>4.0.0</modelVersion>

	<parent>
		<groupId>is.landsbokasafn.deduplicator</groupId>
		<artifactId>deduplicator</artifactId>
		<version>3.1.0-SNAPSHOT</version>
	</parent>
	<artifactId>deduplicator-commons</artifactId>
	<name>DeDuplicator Commons</name>
	<packaging>jar</packaging>

    <properties>
        <maven-site-plugin.skip>true</maven-site-plugin.skip>
    </properties>

	<dependencies>
		<dependency>
			<groupId>org.apache.lucene</groupId>
			<artifactId>lucene-core</artifactId>
			<version>4.7.1</version>
		</dependency>
		<dependency>
			<groupId>org.apache.lucene</groupId>
			<artifactId>lucene-analyzers-common</artifactId>
			<version>4.7.1</version>
		</dependency>
		<dependency>
			<groupId>org.apache.lucene</groupId>
			<artifactId>lucene-queryparser</artifactId>
			<version>4.7.1</version>
		</dependency>
		<!-- Currently OpenWayback-Core is only imported for the AggressiveUrlCanonicalizer -->
		<dependency>
			<groupId>org.netpreserve.openwayback</groupId>
			<artifactId>openwayback-core</artifactId>
			<version>2.0.0</version>
			<exclusions>
				<exclusion>
					<artifactId>webarchive-commons</artifactId>
					<groupId>org.netpreserve.commons</groupId>
				</exclusion>
				<exclusion>
					<artifactId>openwayback-cdx-server</artifactId>
					<groupId>org.netpreserve.openwayback</groupId>
				</exclusion>
				<exclusion>
					<artifactId>spring-core</artifactId>
					<groupId>org.springframework</groupId>
				</exclusion>
				<exclusion>
					<artifactId>jwat-warc</artifactId>
					<groupId>org.jwat</groupId>
				</exclusion>
				<exclusion>
					<artifactId>je</artifactId>
					<groupId>com.sleepycat</groupId>
				</exclusion>
				<exclusion>
					<artifactId>transform</artifactId>
					<groupId>com.flagstone</groupId>
				</exclusion>
				<exclusion>
					<artifactId>juniversalchardet</artifactId>
					<groupId>com.googlecode.juniversalchardet</groupId>
				</exclusion>
				<exclusion>
					<artifactId>jedis</artifactId>
					<groupId>redis.clients</groupId>
				</exclusion>
				<exclusion>
					<artifactId>bsh</artifactId>
					<groupId>org.beanshell</groupId>
				</exclusion>
				<exclusion>
					<artifactId>spring-beans</artifactId>
					<groupId>org.springframework</groupId>
				</exclusion>
				<exclusion>
					<artifactId>jwat-arc</artifactId>
					<groupId>org.jwat</groupId>
				</exclusion>
				<exclusion>
					<artifactId>
						openwayback-access-control-core
					</artifactId>
					<groupId>org.netpreserve.openwayback</groupId>
				</exclusion>
				<exclusion>
					<artifactId>htmlparser</artifactId>
					<groupId>org.htmlparser</groupId>
				</exclusion>
			</exclusions>
		</dependency>
	</dependencies>
</project>
<commit_msg>Have property for Lucene version<commit_after><?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
	<modelVersion>4.0.0</modelVersion>

	<parent>
		<groupId>is.landsbokasafn.deduplicator</groupId>
		<artifactId>deduplicator</artifactId>
		<version>3.1.0-SNAPSHOT</version>
	</parent>
	<artifactId>deduplicator-commons</artifactId>
	<name>DeDuplicator Commons</name>
	<packaging>jar</packaging>

    <properties>
        <maven-site-plugin.skip>true</maven-site-plugin.skip>
        <org.apache.lucene.version>4.7.1</org.apache.lucene.version>
    </properties>

	<dependencies>
		<dependency>
			<groupId>org.apache.lucene</groupId>
			<artifactId>lucene-core</artifactId>
			<version>${org.apache.lucene.version}</version>
		</dependency>
		<dependency>
			<groupId>org.apache.lucene</groupId>
			<artifactId>lucene-analyzers-common</artifactId>
			<version>${org.apache.lucene.version}</version>
		</dependency>
		<dependency>
			<groupId>org.apache.lucene</groupId>
			<artifactId>lucene-queryparser</artifactId>
			<version>${org.apache.lucene.version}</version>
		</dependency>
		<!-- Currently OpenWayback-Core is only imported for the AggressiveUrlCanonicalizer -->
		<dependency>
			<groupId>org.netpreserve.openwayback</groupId>
			<artifactId>openwayback-core</artifactId>
			<version>2.0.0</version>
			<exclusions>
				<exclusion>
					<artifactId>webarchive-commons</artifactId>
					<groupId>org.netpreserve.commons</groupId>
				</exclusion>
				<exclusion>
					<artifactId>openwayback-cdx-server</artifactId>
					<groupId>org.netpreserve.openwayback</groupId>
				</exclusion>
				<exclusion>
					<artifactId>spring-core</artifactId>
					<groupId>org.springframework</groupId>
				</exclusion>
				<exclusion>
					<artifactId>jwat-warc</artifactId>
					<groupId>org.jwat</groupId>
				</exclusion>
				<exclusion>
					<artifactId>je</artifactId>
					<groupId>com.sleepycat</groupId>
				</exclusion>
				<exclusion>
					<artifactId>transform</artifactId>
					<groupId>com.flagstone</groupId>
				</exclusion>
				<exclusion>
					<artifactId>juniversalchardet</artifactId>
					<groupId>com.googlecode.juniversalchardet</groupId>
				</exclusion>
				<exclusion>
					<artifactId>jedis</artifactId>
					<groupId>redis.clients</groupId>
				</exclusion>
				<exclusion>
					<artifactId>bsh</artifactId>
					<groupId>org.beanshell</groupId>
				</exclusion>
				<exclusion>
					<artifactId>spring-beans</artifactId>
					<groupId>org.springframework</groupId>
				</exclusion>
				<exclusion>
					<artifactId>jwat-arc</artifactId>
					<groupId>org.jwat</groupId>
				</exclusion>
				<exclusion>
					<artifactId>
						openwayback-access-control-core
					</artifactId>
					<groupId>org.netpreserve.openwayback</groupId>
				</exclusion>
				<exclusion>
					<artifactId>htmlparser</artifactId>
					<groupId>org.htmlparser</groupId>
				</exclusion>
			</exclusions>
		</dependency>
	</dependencies>
</project>

{'fraction_non_alphanumeric': 0.1538339670468948, 'fraction_numerical': 0.010139416983523447, 'mean_word_length': 3.3210130047912387, 'pattern_counts': {'":': 0, '<': 281, '<?xml version=': 2, '>': 281, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '94677', 'n_tokens_mistral': 2822, 'n_tokens_neox': 2031, 'n_words': 221}