<commit_before><?xml version="1.0" encoding="UTF-8"?>

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
    xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">

  <modelVersion>4.0.0</modelVersion>
  <groupId>cc.twittertools</groupId>
  <artifactId>twitter-tools</artifactId>
  <packaging>jar</packaging>
  <name>Twitter Tools</name>
  <version>1.1.1</version>
  <description>Twitter tools for researchers</description>
  <url>http://twittertools.cc</url>
  <licenses>
    <license>
      <name>The Apache Software License, Version 2.0</name>
      <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url>
      <distribution>repo</distribution>
    </license>
  </licenses>
  
  <scm>
    <connection>scm:git:git@github.com:budgefeeney/twitter-tools.git</connection>
    <developerConnection>scm:git:git@github.com:budgefeeney/twitter-tools.git</developerConnection>
    <url>git@github.com:budgefeeney/twitter-tools.git</url>
  </scm>
  
  <properties>
    <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
  </properties>
  
  <developers>
    <developer>
      <id>lintool</id>
      <name><NAME></name>
      <email><EMAIL></email>
    </developer>
    <developer>
      <id>budgefeeney</id>
      <name><NAME></name>
      <email><EMAIL></email>
    </developer>
  </developers>
  
  <build>
  <plugins>
    <plugin>
      <artifactId>maven-assembly-plugin</artifactId>
      <configuration>
        <archive>
          <manifest>
            <mainClass>cc.twittertools.scripts</mainClass>
          </manifest>
        </archive>
        <descriptorRefs>
          <descriptorRef>jar-with-dependencies</descriptorRef>
        </descriptorRefs>
      </configuration>
    </plugin>
  </plugins>
  </build>
  
  <repositories>
  	<repository>
  	  <id>maven</id>
  	  <name>Official Maven Repository (UK Mirror)</name>
  	  <url>http://uk.maven.org/maven2</url>
    </repository>
    <repository>
      <id>twitter</id>
      <name>Repository for Twitter Java Tools</name>
      <url>http://maven.twttr.com</url>
    </repository>
    <repository>
      <id>twitter</id>
      <name>Repository for Twitter Java Tools</name>
      <url>http://maven.twttr.com</url>
    </repository>
    <repository>
      <id>javanet</id>
      <name>Repository for Maven2</name>
      <url>http://download.java.net/maven/2</url>
    </repository>
    <repository>
      <id>google</id>
      <name>Google repository for Maven2</name>
      <url>https://oss.sonatype.org/content/repositories/google-releases/</url>
    </repository>
    <repository>
      <id>jboss</id>
      <name>JBoss repository for Maven2</name>
      <url>http://repository.jboss.org/maven2/</url>
    </repository>
  </repositories>
  
  <!-- Some of these dependencies could probably be culled ~~~ Bryan -->
  <dependencies>
    <dependency>
      <groupId>org.twitter4j</groupId>
      <artifactId>twitter4j-core</artifactId>
      <version>3.0.3</version>
      <optional>true</optional>
    </dependency>
    <dependency>
      <groupId>org.twitter4j</groupId>
      <artifactId>twitter4j-stream</artifactId>
      <version>3.0.3</version>
      <optional>true</optional>
    </dependency>
	<dependency>
		<groupId>org.jsoup</groupId>
		<artifactId>jsoup</artifactId>
		<version>1.7.1</version>
	</dependency>
    <dependency>
      <groupId>com.google.guava</groupId>
      <artifactId>guava</artifactId>
      <version>14.0.1</version>
      <optional>true</optional>
    </dependency>
    <dependency>
      <groupId>commons-cli</groupId>
      <artifactId>commons-cli</artifactId>
      <version>1.2</version>
      <optional>true</optional>
    </dependency>
	<dependency> <!-- Should phase this out in favour of 3.1 below -->
	  <groupId>commons-lang</groupId>
	  <artifactId>commons-lang</artifactId>
	  <version>2.6</version>
	</dependency>
	<dependency>
	  <groupId>org.apache.commons</groupId>
	  <artifactId>commons-lang3</artifactId>
	  <version>3.1</version>
    </dependency>
    <dependency>
      <groupId>com.ning</groupId>
      <artifactId>async-http-client</artifactId>
      <version>1.7.5</version>
    </dependency>
    <dependency>
      <groupId>com.google.code.gson</groupId>
      <artifactId>gson</artifactId>
      <version>1.6</version>
    </dependency>
    <dependency>
      <groupId>org.apache.hadoop</groupId>
      <artifactId>hadoop-core</artifactId>
      <version>0.20.2</version>
    </dependency>
    <dependency>
      <groupId>org.apache.lucene</groupId>
      <artifactId>lucene-core</artifactId>
      <version>3.6.2</version>
    </dependency>
    <dependency>
      <groupId>org.apache.lucene</groupId>
      <artifactId>lucene-analyzers</artifactId>
      <version>3.6.2</version>
    </dependency>
    <dependency>
      <groupId>org.slf4j</groupId>
      <artifactId>slf4j-log4j12</artifactId>
      <version>1.6.1</version>
    </dependency>
    <dependency>
      <groupId>junit</groupId>
      <artifactId>junit-dep</artifactId>
      <version>4.8.2</version>
      <scope>test</scope>
    </dependency>
    <dependency>
      <groupId>log4j</groupId>
      <artifactId>log4j</artifactId>
      <version>1.2.17</version>
    </dependency>
    <dependency>
      <groupId>log4j</groupId>
      <artifactId>apache-log4j-extras</artifactId>
      <version>1.1</version>
    </dependency>
    <dependency>
      <groupId>joda-time</groupId>
      <artifactId>joda-time</artifactId>
      <version>2.2</version>
    </dependency>
    <dependency>
      <groupId>it.unimi.dsi</groupId>
      <artifactId>fastutil</artifactId>
      <version>6.5.7</version>
    </dependency>
    <dependency>
      <groupId>it.unimi.di</groupId>
      <artifactId>mg4j</artifactId>
      <version>5.2</version>
    </dependency>
    <dependency> <!-- TODO: remove getopt in favour of args4j -->
      <groupId>gnu.getopt</groupId>
      <artifactId>java-getopt</artifactId>
      <version>1.0.13</version>
    </dependency>
    <dependency>
	  <groupId>args4j</groupId>
	  <artifactId>args4j</artifactId>
	  <version>2.0.25</version>
    </dependency>
    <dependency>
      <groupId>colt</groupId>
      <artifactId>colt</artifactId>
      <version>1.2.0</version>
    </dependency>
    <dependency>
      <groupId>org.hamcrest</groupId>
      <artifactId>hamcrest-all</artifactId>
      <version>1.3</version>
      <scope>test</scope>
    </dependency>
    <dependency>
      <groupId>com.j256.simplejmx</groupId>
      <artifactId>simplejmx</artifactId>
      <version>1.1</version>
    </dependency>
    <dependency>
      <groupId>org.apache.httpcomponents</groupId>
      <artifactId>httpclient</artifactId>
      <version>4.2.5</version>
    </dependency>
    <dependency>
      <groupId>com.twitter.common</groupId>
      <artifactId>text</artifactId>
      <version>0.1.9</version>
    </dependency>
  </dependencies>
</project>
<commit_msg>Remove duplicate twitter repository reference.<commit_after><?xml version="1.0" encoding="UTF-8"?>

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
    xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">

  <modelVersion>4.0.0</modelVersion>
  <groupId>cc.twittertools</groupId>
  <artifactId>twitter-tools</artifactId>
  <packaging>jar</packaging>
  <name>Twitter Tools</name>
  <version>1.1.1</version>
  <description>Twitter tools for researchers</description>
  <url>http://twittertools.cc</url>
  <licenses>
    <license>
      <name>The Apache Software License, Version 2.0</name>
      <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url>
      <distribution>repo</distribution>
    </license>
  </licenses>
  
  <scm>
    <connection>scm:git:git@github.<EMAIL>:budgefeeney/twitter-tools.git</connection>
    <developerConnection>scm:git:git<EMAIL>:budgefeeney/twitter-tools.git</developerConnection>
    <url><EMAIL>:budgefeeney/twitter-tools.git</url>
  </scm>
  
  <properties>
    <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
  </properties>
  
  <developers>
    <developer>
      <id>lintool</id>
      <name><NAME></name>
      <email><EMAIL></email>
    </developer>
    <developer>
      <id>budgefeeney</id>
      <name><NAME></name>
      <email><EMAIL></email>
    </developer>
  </developers>
  
  <build>
  <plugins>
    <plugin>
      <artifactId>maven-assembly-plugin</artifactId>
      <configuration>
        <archive>
          <manifest>
            <mainClass>cc.twittertools.scripts</mainClass>
          </manifest>
        </archive>
        <descriptorRefs>
          <descriptorRef>jar-with-dependencies</descriptorRef>
        </descriptorRefs>
      </configuration>
    </plugin>
  </plugins>
  </build>
  
  <repositories>
  	<repository>
  	  <id>maven</id>
  	  <name>Official Maven Repository (UK Mirror)</name>
  	  <url>http://uk.maven.org/maven2</url>
    </repository>
    <repository>
      <id>twitter</id>
      <name>Repository for Twitter Java Tools</name>
      <url>http://maven.twttr.com</url>
    </repository>
    <repository>
      <id>javanet</id>
      <name>Repository for Maven2</name>
      <url>http://download.java.net/maven/2</url>
    </repository>
    <repository>
      <id>google</id>
      <name>Google repository for Maven2</name>
      <url>https://oss.sonatype.org/content/repositories/google-releases/</url>
    </repository>
    <repository>
      <id>jboss</id>
      <name>JBoss repository for Maven2</name>
      <url>http://repository.jboss.org/maven2/</url>
    </repository>
  </repositories>
  
  <!-- Some of these dependencies could probably be culled ~~~ Bryan -->
  <dependencies>
    <dependency>
      <groupId>org.twitter4j</groupId>
      <artifactId>twitter4j-core</artifactId>
      <version>3.0.3</version>
      <optional>true</optional>
    </dependency>
    <dependency>
      <groupId>org.twitter4j</groupId>
      <artifactId>twitter4j-stream</artifactId>
      <version>3.0.3</version>
      <optional>true</optional>
    </dependency>
	<dependency>
		<groupId>org.jsoup</groupId>
		<artifactId>jsoup</artifactId>
		<version>1.7.1</version>
	</dependency>
    <dependency>
      <groupId>com.google.guava</groupId>
      <artifactId>guava</artifactId>
      <version>14.0.1</version>
      <optional>true</optional>
    </dependency>
    <dependency>
      <groupId>commons-cli</groupId>
      <artifactId>commons-cli</artifactId>
      <version>1.2</version>
      <optional>true</optional>
    </dependency>
	<dependency> <!-- Should phase this out in favour of 3.1 below -->
	  <groupId>commons-lang</groupId>
	  <artifactId>commons-lang</artifactId>
	  <version>2.6</version>
	</dependency>
	<dependency>
	  <groupId>org.apache.commons</groupId>
	  <artifactId>commons-lang3</artifactId>
	  <version>3.1</version>
    </dependency>
    <dependency>
      <groupId>com.ning</groupId>
      <artifactId>async-http-client</artifactId>
      <version>1.7.5</version>
    </dependency>
    <dependency>
      <groupId>com.google.code.gson</groupId>
      <artifactId>gson</artifactId>
      <version>1.6</version>
    </dependency>
    <dependency>
      <groupId>org.apache.hadoop</groupId>
      <artifactId>hadoop-core</artifactId>
      <version>0.20.2</version>
    </dependency>
    <dependency>
      <groupId>org.apache.lucene</groupId>
      <artifactId>lucene-core</artifactId>
      <version>3.6.2</version>
    </dependency>
    <dependency>
      <groupId>org.apache.lucene</groupId>
      <artifactId>lucene-analyzers</artifactId>
      <version>3.6.2</version>
    </dependency>
    <dependency>
      <groupId>org.slf4j</groupId>
      <artifactId>slf4j-log4j12</artifactId>
      <version>1.6.1</version>
    </dependency>
    <dependency>
      <groupId>junit</groupId>
      <artifactId>junit-dep</artifactId>
      <version>4.8.2</version>
      <scope>test</scope>
    </dependency>
    <dependency>
      <groupId>log4j</groupId>
      <artifactId>log4j</artifactId>
      <version>1.2.17</version>
    </dependency>
    <dependency>
      <groupId>log4j</groupId>
      <artifactId>apache-log4j-extras</artifactId>
      <version>1.1</version>
    </dependency>
    <dependency>
      <groupId>joda-time</groupId>
      <artifactId>joda-time</artifactId>
      <version>2.2</version>
    </dependency>
    <dependency>
      <groupId>it.unimi.dsi</groupId>
      <artifactId>fastutil</artifactId>
      <version>6.5.7</version>
    </dependency>
    <dependency>
      <groupId>it.unimi.di</groupId>
      <artifactId>mg4j</artifactId>
      <version>5.2</version>
    </dependency>
    <dependency> <!-- TODO: remove getopt in favour of args4j -->
      <groupId>gnu.getopt</groupId>
      <artifactId>java-getopt</artifactId>
      <version>1.0.13</version>
    </dependency>
    <dependency>
	  <groupId>args4j</groupId>
	  <artifactId>args4j</artifactId>
	  <version>2.0.25</version>
    </dependency>
    <dependency>
      <groupId>colt</groupId>
      <artifactId>colt</artifactId>
      <version>1.2.0</version>
    </dependency>
    <dependency>
      <groupId>org.hamcrest</groupId>
      <artifactId>hamcrest-all</artifactId>
      <version>1.3</version>
      <scope>test</scope>
    </dependency>
    <dependency>
      <groupId>com.j256.simplejmx</groupId>
      <artifactId>simplejmx</artifactId>
      <version>1.1</version>
    </dependency>
    <dependency>
      <groupId>org.apache.httpcomponents</groupId>
      <artifactId>httpclient</artifactId>
      <version>4.2.5</version>
    </dependency>
    <dependency>
      <groupId>com.twitter.common</groupId>
      <artifactId>text</artifactId>
      <version>0.1.9</version>
    </dependency>
  </dependencies>
</project>

{'fraction_non_alphanumeric': 0.1662924603462012, 'fraction_numerical': 0.018975881799087418, 'mean_word_length': 2.968956596723196, 'pattern_counts': {'":': 0, '<': 714, '<?xml version=': 2, '>': 714, 'https://': 2, 'lorem ipsum': 0, 'www.': 4, 'xml': 6}, 'pii_count': 3, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7494739', 'n_tokens_mistral': 5115, 'n_tokens_neox': 4538, 'n_words': 565}