<commit_before><!--
   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
-->

<project name="lz4" default="dist" basedir="."
    xmlns:cpptasks="antlib:net.sf.antcontrib.cpptasks"
    xmlns:ivy="antlib:org.apache.ivy.ant"
    xmlns:junit4="antlib:com.carrotsearch.junit4"
    xmlns:mvn="antlib:org.apache.maven.artifact.ant">

  <property name="src" location="src"/>
  <property name="build" location="build"/>
  <property name="dist"  location="dist"/>
  <property name="lib" location="lib" />

  <property name="javac.source" value="1.6" />
  <property name="javac.target" value="1.6" />

  <property name="ivy.jar.version" value="2.2.0"/>
  <property name="ivy.jar.name" value="ivy-${ivy.jar.version}.jar"/>

  <condition property="platform" value="linux">
   <os name="Linux"/>
  </condition>
  <condition property="platform" value="win32">
   <os family="windows"/>
  </condition>
  <condition property="platform" value="solaris">
   <os name="SunOS"/>
  </condition>
  <condition property="platform" value="darwin">
   <os family="mac"/>
  </condition>

  <target name="clean" description="clean working copy">
    <delete dir="${build}" />
    <delete dir="${dist}" />
    <delete dir="${lib}" />
  </target>

  <target name="ivy-bootstrap" description="install ivy" unless="ivy.available">
    <condition property="ivy.available">
      <typefound uri="antlib:org.apache.ivy.ant" name="configure" />
    </condition>
    <antcall target="-ivy-install" />
  </target>

  <target name="-ivy-install" unless="ivy.available">
    <mkdir dir="${user.home}/.ant/lib" />
    <get src="http://repo1.maven.org/maven2/org/apache/ivy/ivy/${ivy.jar.version}/${ivy.jar.name}" dest="${user.home}/.ant/lib/${ivy.jar.name}"/>
  </target>

  <target name="install-cpptasks" unless="cpptasks.available">
    <ivy:cachepath organisation="ant-contrib" module="cpptasks" revision="1.0b5"
      inline="true" conf="default" transitive="true" pathid="cpptasks.classpath"/>
    <taskdef uri="antlib:net.sf.antcontrib.cpptasks" resource="net/sf/antcontrib/cpptasks/antlib.xml" classpathref="cpptasks.classpath"/>
    <property name="cpptasks.available" value="true"/>
  </target>

  <target name="install-maven-ant-tasks" unless="maven-ant-tasks.available">
    <ivy:cachepath organisation="org.apache.maven" module="maven-ant-tasks" revision="2.1.3"
      inline="true" conf="default" transitive="true" pathid="maven-ant-tasks.classpath"/>
    <taskdef uri="antlib:org.apache.maven.artifact.ant" resource="org/apache/maven/artifact/ant/antlib.xml" classpathref="maven-ant-tasks.classpath"/>
    <property name="maven-ant-tasks.available" value="true"/>
  </target>

  <target name="install-forbidden-apis" unless="forbidden-apis.available">
    <ivy:cachepath organisation="de.thetaphi" module="forbiddenapis" revision="1.0"
      inline="true" conf="default" transitive="true" pathid="forbidden-apis.classpath"/>
    <taskdef name="forbidden-apis" classname="de.thetaphi.forbiddenapis.AntTask" classpathref="forbidden-apis.classpath"/>
    <property name="forbidden-apis.available" value="true"/>
  </target>

  <target name="init">
    <tstamp />
    <ivy:resolve conf="test" />
    <ivy:retrieve />
  </target>

  <target name="install-junit4" depends="init" unless="junit4.available">
    <taskdef uri="antlib:com.carrotsearch.junit4">
      <classpath>
        <fileset dir="${lib}" includes="*.jar" />
      </classpath>
    </taskdef>
    <property name="junit4.available" value="true" />
  </target>

  <target name="compile-java" depends="init">
    <mkdir dir="${build}/classes" />
    <javac
      includeAntRuntime="false"
      srcdir="${src}/java"
      source="${javac.source}"
      target="${javac.target}"
      encoding="UTF-8"
      debug="true"
      destdir="${build}/classes"/>
    <mkdir dir="${build}/unsafe-classes" />
    <javac
      includeAntRuntime="false"
      srcdir="${src}/java-unsafe"
      source="${javac.source}"
      target="${javac.target}"
      encoding="UTF-8"
      debug="true"
      classpath="${build}/classes"
      destdir="${build}/unsafe-classes"/>
  </target>

  <target name="generate-headers" depends="compile-java" unless="${skip.jni}">
    <mkdir dir="${build}/jni-headers" />
    <javah
      destDir="${build}/jni-headers">
      <classpath>
        <pathelement location="${build}/classes/" />
      </classpath>
      <class name="net.jpountz.lz4.LZ4JNI" />
      <class name="net.jpountz.xxhash.XXHashJNI" />
    </javah>
  </target>

  <target name="compile-jni" depends="install-cpptasks,generate-headers" unless="${skip.jni}">
    <mkdir dir="${build}/objects" />
    <mkdir dir="${build}/jni/${platform}/${os.arch}" />
    <cpptasks:cc
      debug="false"
      optimize="extreme"
      objdir="${build}/objects"
      outfile="${build}/jni/${platform}/${os.arch}/lz4-java"
      outtype="shared">
      <includepath path="${src}/lz4" />
      <includepath path="${src}/xxhash" />
      <includepath path="${java.home}/../include" />
      <includepath path="${java.home}/../include/${platform}/" />
      <includepath path="${build}/jni-headers" />
      <fileset dir="${src}/lz4" includes="lz4.c, lz4hc.c" />
      <fileset dir="${src}/xxhash" includes="xxhash.c" />
      <fileset dir="${src}/jni" includes="*.c" />
      <linker />
    </cpptasks:cc>
  </target>

  <target name="compile" depends="compile-java, compile-jni" />

  <target name="compile-tests" depends="compile">
    <mkdir dir="${build}/test-classes" />
    <javac
      includeAntRuntime="false"
      srcdir="${src}/test"
      encoding="UTF-8"
      debug="true"
      destdir="${build}/test-classes">
      <classpath>
        <path location="${build}/classes" />
        <path location="${build}/unsafe-classes" />
        <fileset dir="lib" includes="*.jar" />
      </classpath>
    </javac>
  </target>

  <target name="test" description="run tests" depends="compile-tests, install-junit4">
    <mkdir dir="${build}/tests" />
    <junit4:junit4
      dir="${java.io.tmpdir}"
      maxmemory="300m"
      parallelism="auto">
      <assertions>
        <enable package="net.jpountz.lz4"/>
      </assertions>
      <classpath>
        <path location="${src}/resources" />
        <path location="${src}/test-resources" />
        <path location="${build}/classes" />
        <path location="${build}/unsafe-classes" />
        <path location="${build}/test-classes" />
        <path location="${build}/jni" />
        <fileset dir="${lib}" includes="*.jar" />
      </classpath>
      <fileset dir="${build}/test-classes/">
        <include name="**/*Test.class" />
        <exclude name="**/*$*" />
        <exclude name="**/Abstract*" />
      </fileset>
      <listeners>
        <junit4:report-text
        showThrowable="true"
        showStackTraces="true"
        showOutput="never"

        showStatusOk="true"
        showStatusError="true"
        showStatusFailure="true"
        showStatusIgnored="true"

        showSuiteSummary="false" />
        <!-- For enkins -->
        <junit4:report-ant-xml dir="${build}/tests" />
      </listeners>
    </junit4:junit4>
  </target>

  <target name="sources" description="package sources">
    <mkdir dir="${dist}" />
    <jar
      destfile="${dist}/${ivy.module}-${ivy.revision}-sources.jar">
      <fileset dir="${src}/java" />
      <fileset dir="${src}/java-unsafe" />
    </jar>
  </target>

  <target name="docs" description="generate javadocs" depends="init">
    <mkdir dir="${build}/docs" />
    <javadoc
        overview="${src}/java/overview.html"
        packagenames="net.jpountz.lz4,net.jpountz.xxhash"
        windowtitle="lz4 / xxhash - Java API ${ivy.revision}"
        destDir="${build}/docs">
      <link href="http://download.oracle.com/javase/6/docs/api/" />
      <sourcepath>
        <pathelement location="${src}/java"/>
      </sourcepath>
    </javadoc>
    <mkdir dir="${dist}" />
    <jar
      destfile="${dist}/${ivy.module}-${ivy.revision}-javadoc.jar">
      <fileset dir="${build}/docs" />
    </jar>
  </target>

  <target name="jar" description="generate JAR" depends="compile">
    <mkdir dir="${dist}" />
    <jar
      destfile="${dist}/${ivy.module}-${ivy.revision}.jar">
      <fileset dir="${src}/resources" />
      <fileset dir="${build}/classes" />
      <fileset dir="${build}/unsafe-classes" />
      <fileset dir="${build}/jni" erroronmissingdir="false" />
    </jar>
  </target>

  <target name="makepom" description="generate a pom file">
    <ivy:makepom
        ivyfile="${basedir}/ivy.xml"
        pomfile="${dist}/${ivy.module}-${ivy.revision}.pom"
        templatefile="${src}/build/pom.template">
      <mapping conf="default" scope="compile" />
      <mapping conf="test" scope="test" />
    </ivy:makepom>
  </target>

  <target name="dist" description="package" depends="jar,docs,sources,makepom" />

  <target name="forbidden-apis" description="check API usage" depends="install-forbidden-apis,compile">
    <forbidden-apis internalRuntimeForbidden="true" classpathref="forbidden-apis.classpath">
      <bundledSignatures name="jdk-unsafe-${javac.target}"/>
      <bundledSignatures name="jdk-deprecated-${javac.target}"/>
      <fileset dir="${build}/classes" includes="**/*.class" />
    </forbidden-apis>
    <forbidden-apis classpathref="forbidden-apis.classpath">
      <bundledSignatures name="jdk-unsafe-${javac.target}"/>
      <bundledSignatures name="jdk-deprecated-${javac.target}"/>
      <fileset dir="${build}/classes" includes="**/*.class" />
      <fileset dir="${build}/unsafe-classes" includes="**/*.class" />
    </forbidden-apis>
  </target>

  <macrodef name="deploy">
    <attribute name="repositoryid" />
    <attribute name="repositoryurl" />
    <sequential>
      <mvn:mvn>
        <arg value="org.apache.maven.plugins:maven-gpg-plugin:1.4:sign-and-deploy-file" />
        <arg value="-Durl=@{repositoryurl}" />
        <arg value="-DrepositoryId=@{repositoryid}" />
        <arg value="-DpomFile=${dist}/${ivy.module}-${ivy.revision}.pom" />
        <arg value="-Dfile=${dist}/${ivy.module}-${ivy.revision}.jar" />
        <arg value="-Dfiles=${dist}/${ivy.module}-${ivy.revision}-sources.jar,${dist}/${ivy.module}-${ivy.revision}-javadoc.jar" />
        <arg value="-Dclassifiers=sources,javadoc" />
        <arg value="-Dtypes=jar,jar" />
        <arg value="-Pgpg" />
      </mvn:mvn>
    </sequential>
  </macrodef>

  <target name="deploy" description="deploy snapshot to snapshot repo" depends="install-maven-ant-tasks">
    <property name="skip.jni" value="true" />
    <antcall target="-deploy" />
  </target>

  <target name="-deploy" depends="clean,dist">
    <deploy repositoryid="sonatype-nexus-snapshots" repositoryurl="https://oss.sonatype.org/content/repositories/snapshots" />
  </target>

  <!-- before this, update project version from SNAPSHOT to RELEASE -->
  <target name="stage" description="deploy snapshot to snapshot repo" depends="install-maven-ant-tasks,test,forbidden-apis">
    <property name="skip.jni" value="true" />
    <antcall target="-stage" />
  </target>

  <target name="-stage" depends="clean,dist">
    <deploy repositoryid="sonatype-nexus-staging" repositoryurl="https://oss.sonatype.org/service/local/staging/deploy/maven2" />
  </target>

</project>
<commit_msg>Update forbiddenapis to version 1.3.<commit_after><!--
   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
-->

<project name="lz4" default="dist" basedir="."
    xmlns:cpptasks="antlib:net.sf.antcontrib.cpptasks"
    xmlns:ivy="antlib:org.apache.ivy.ant"
    xmlns:junit4="antlib:com.carrotsearch.junit4"
    xmlns:mvn="antlib:org.apache.maven.artifact.ant">

  <property name="src" location="src"/>
  <property name="build" location="build"/>
  <property name="dist"  location="dist"/>
  <property name="lib" location="lib" />

  <property name="javac.source" value="1.6" />
  <property name="javac.target" value="1.6" />

  <property name="ivy.jar.version" value="2.2.0"/>
  <property name="ivy.jar.name" value="ivy-${ivy.jar.version}.jar"/>

  <condition property="platform" value="linux">
   <os name="Linux"/>
  </condition>
  <condition property="platform" value="win32">
   <os family="windows"/>
  </condition>
  <condition property="platform" value="solaris">
   <os name="SunOS"/>
  </condition>
  <condition property="platform" value="darwin">
   <os family="mac"/>
  </condition>

  <target name="clean" description="clean working copy">
    <delete dir="${build}" />
    <delete dir="${dist}" />
    <delete dir="${lib}" />
  </target>

  <target name="ivy-bootstrap" description="install ivy" unless="ivy.available">
    <condition property="ivy.available">
      <typefound uri="antlib:org.apache.ivy.ant" name="configure" />
    </condition>
    <antcall target="-ivy-install" />
  </target>

  <target name="-ivy-install" unless="ivy.available">
    <mkdir dir="${user.home}/.ant/lib" />
    <get src="http://repo1.maven.org/maven2/org/apache/ivy/ivy/${ivy.jar.version}/${ivy.jar.name}" dest="${user.home}/.ant/lib/${ivy.jar.name}"/>
  </target>

  <target name="install-cpptasks" unless="cpptasks.available">
    <ivy:cachepath organisation="ant-contrib" module="cpptasks" revision="1.0b5"
      inline="true" conf="default" transitive="true" pathid="cpptasks.classpath"/>
    <taskdef uri="antlib:net.sf.antcontrib.cpptasks" resource="net/sf/antcontrib/cpptasks/antlib.xml" classpathref="cpptasks.classpath"/>
    <property name="cpptasks.available" value="true"/>
  </target>

  <target name="install-maven-ant-tasks" unless="maven-ant-tasks.available">
    <ivy:cachepath organisation="org.apache.maven" module="maven-ant-tasks" revision="2.1.3"
      inline="true" conf="default" transitive="true" pathid="maven-ant-tasks.classpath"/>
    <taskdef uri="antlib:org.apache.maven.artifact.ant" resource="org/apache/maven/artifact/ant/antlib.xml" classpathref="maven-ant-tasks.classpath"/>
    <property name="maven-ant-tasks.available" value="true"/>
  </target>

  <target name="install-forbidden-apis" unless="forbidden-apis.available">
    <ivy:cachepath organisation="de.thetaphi" module="forbiddenapis" revision="1.3"
      inline="true" conf="default" transitive="true" pathid="forbidden-apis.classpath"/>
    <taskdef name="forbidden-apis" classname="de.thetaphi.forbiddenapis.AntTask" classpathref="forbidden-apis.classpath"/>
    <property name="forbidden-apis.available" value="true"/>
  </target>

  <target name="init">
    <tstamp />
    <ivy:resolve conf="test" />
    <ivy:retrieve />
  </target>

  <target name="install-junit4" depends="init" unless="junit4.available">
    <taskdef uri="antlib:com.carrotsearch.junit4">
      <classpath>
        <fileset dir="${lib}" includes="*.jar" />
      </classpath>
    </taskdef>
    <property name="junit4.available" value="true" />
  </target>

  <target name="compile-java" depends="init">
    <mkdir dir="${build}/classes" />
    <javac
      includeAntRuntime="false"
      srcdir="${src}/java"
      source="${javac.source}"
      target="${javac.target}"
      encoding="UTF-8"
      debug="true"
      destdir="${build}/classes"/>
    <mkdir dir="${build}/unsafe-classes" />
    <javac
      includeAntRuntime="false"
      srcdir="${src}/java-unsafe"
      source="${javac.source}"
      target="${javac.target}"
      encoding="UTF-8"
      debug="true"
      classpath="${build}/classes"
      destdir="${build}/unsafe-classes"/>
  </target>

  <target name="generate-headers" depends="compile-java" unless="${skip.jni}">
    <mkdir dir="${build}/jni-headers" />
    <javah
      destDir="${build}/jni-headers">
      <classpath>
        <pathelement location="${build}/classes/" />
      </classpath>
      <class name="net.jpountz.lz4.LZ4JNI" />
      <class name="net.jpountz.xxhash.XXHashJNI" />
    </javah>
  </target>

  <target name="compile-jni" depends="install-cpptasks,generate-headers" unless="${skip.jni}">
    <mkdir dir="${build}/objects" />
    <mkdir dir="${build}/jni/${platform}/${os.arch}" />
    <cpptasks:cc
      debug="false"
      optimize="extreme"
      objdir="${build}/objects"
      outfile="${build}/jni/${platform}/${os.arch}/lz4-java"
      outtype="shared">
      <includepath path="${src}/lz4" />
      <includepath path="${src}/xxhash" />
      <includepath path="${java.home}/../include" />
      <includepath path="${java.home}/../include/${platform}/" />
      <includepath path="${build}/jni-headers" />
      <fileset dir="${src}/lz4" includes="lz4.c, lz4hc.c" />
      <fileset dir="${src}/xxhash" includes="xxhash.c" />
      <fileset dir="${src}/jni" includes="*.c" />
      <linker />
    </cpptasks:cc>
  </target>

  <target name="compile" depends="compile-java, compile-jni" />

  <target name="compile-tests" depends="compile">
    <mkdir dir="${build}/test-classes" />
    <javac
      includeAntRuntime="false"
      srcdir="${src}/test"
      encoding="UTF-8"
      debug="true"
      destdir="${build}/test-classes">
      <classpath>
        <path location="${build}/classes" />
        <path location="${build}/unsafe-classes" />
        <fileset dir="lib" includes="*.jar" />
      </classpath>
    </javac>
  </target>

  <target name="test" description="run tests" depends="compile-tests, install-junit4">
    <mkdir dir="${build}/tests" />
    <junit4:junit4
      dir="${java.io.tmpdir}"
      maxmemory="300m"
      parallelism="auto">
      <assertions>
        <enable package="net.jpountz.lz4"/>
      </assertions>
      <classpath>
        <path location="${src}/resources" />
        <path location="${src}/test-resources" />
        <path location="${build}/classes" />
        <path location="${build}/unsafe-classes" />
        <path location="${build}/test-classes" />
        <path location="${build}/jni" />
        <fileset dir="${lib}" includes="*.jar" />
      </classpath>
      <fileset dir="${build}/test-classes/">
        <include name="**/*Test.class" />
        <exclude name="**/*$*" />
        <exclude name="**/Abstract*" />
      </fileset>
      <listeners>
        <junit4:report-text
        showThrowable="true"
        showStackTraces="true"
        showOutput="never"

        showStatusOk="true"
        showStatusError="true"
        showStatusFailure="true"
        showStatusIgnored="true"

        showSuiteSummary="false" />
        <!-- For enkins -->
        <junit4:report-ant-xml dir="${build}/tests" />
      </listeners>
    </junit4:junit4>
  </target>

  <target name="sources" description="package sources">
    <mkdir dir="${dist}" />
    <jar
      destfile="${dist}/${ivy.module}-${ivy.revision}-sources.jar">
      <fileset dir="${src}/java" />
      <fileset dir="${src}/java-unsafe" />
    </jar>
  </target>

  <target name="docs" description="generate javadocs" depends="init">
    <mkdir dir="${build}/docs" />
    <javadoc
        overview="${src}/java/overview.html"
        packagenames="net.jpountz.lz4,net.jpountz.xxhash"
        windowtitle="lz4 / xxhash - Java API ${ivy.revision}"
        destDir="${build}/docs">
      <link href="http://download.oracle.com/javase/6/docs/api/" />
      <sourcepath>
        <pathelement location="${src}/java"/>
      </sourcepath>
    </javadoc>
    <mkdir dir="${dist}" />
    <jar
      destfile="${dist}/${ivy.module}-${ivy.revision}-javadoc.jar">
      <fileset dir="${build}/docs" />
    </jar>
  </target>

  <target name="jar" description="generate JAR" depends="compile">
    <mkdir dir="${dist}" />
    <jar
      destfile="${dist}/${ivy.module}-${ivy.revision}.jar">
      <fileset dir="${src}/resources" />
      <fileset dir="${build}/classes" />
      <fileset dir="${build}/unsafe-classes" />
      <fileset dir="${build}/jni" erroronmissingdir="false" />
    </jar>
  </target>

  <target name="makepom" description="generate a pom file">
    <ivy:makepom
        ivyfile="${basedir}/ivy.xml"
        pomfile="${dist}/${ivy.module}-${ivy.revision}.pom"
        templatefile="${src}/build/pom.template">
      <mapping conf="default" scope="compile" />
      <mapping conf="test" scope="test" />
    </ivy:makepom>
  </target>

  <target name="dist" description="package" depends="jar,docs,sources,makepom" />

  <target name="forbidden-apis" description="check API usage" depends="install-forbidden-apis,compile">
    <forbidden-apis internalRuntimeForbidden="true" classpathref="forbidden-apis.classpath">
      <bundledSignatures name="jdk-unsafe-${javac.target}"/>
      <bundledSignatures name="jdk-deprecated-${javac.target}"/>
      <fileset dir="${build}/classes" includes="**/*.class" />
    </forbidden-apis>
    <forbidden-apis classpathref="forbidden-apis.classpath">
      <bundledSignatures name="jdk-unsafe-${javac.target}"/>
      <bundledSignatures name="jdk-deprecated-${javac.target}"/>
      <fileset dir="${build}/classes" includes="**/*.class" />
      <fileset dir="${build}/unsafe-classes" includes="**/*.class" />
    </forbidden-apis>
  </target>

  <macrodef name="deploy">
    <attribute name="repositoryid" />
    <attribute name="repositoryurl" />
    <sequential>
      <mvn:mvn>
        <arg value="org.apache.maven.plugins:maven-gpg-plugin:1.4:sign-and-deploy-file" />
        <arg value="-Durl=@{repositoryurl}" />
        <arg value="-DrepositoryId=@{repositoryid}" />
        <arg value="-DpomFile=${dist}/${ivy.module}-${ivy.revision}.pom" />
        <arg value="-Dfile=${dist}/${ivy.module}-${ivy.revision}.jar" />
        <arg value="-Dfiles=${dist}/${ivy.module}-${ivy.revision}-sources.jar,${dist}/${ivy.module}-${ivy.revision}-javadoc.jar" />
        <arg value="-Dclassifiers=sources,javadoc" />
        <arg value="-Dtypes=jar,jar" />
        <arg value="-Pgpg" />
      </mvn:mvn>
    </sequential>
  </macrodef>

  <target name="deploy" description="deploy snapshot to snapshot repo" depends="install-maven-ant-tasks">
    <property name="skip.jni" value="true" />
    <antcall target="-deploy" />
  </target>

  <target name="-deploy" depends="clean,dist">
    <deploy repositoryid="sonatype-nexus-snapshots" repositoryurl="https://oss.sonatype.org/content/repositories/snapshots" />
  </target>

  <!-- before this, update project version from SNAPSHOT to RELEASE -->
  <target name="stage" description="deploy snapshot to snapshot repo" depends="install-maven-ant-tasks,test,forbidden-apis">
    <property name="skip.jni" value="true" />
    <antcall target="-stage" />
  </target>

  <target name="-stage" depends="clean,dist">
    <deploy repositoryid="sonatype-nexus-staging" repositoryurl="https://oss.sonatype.org/service/local/staging/deploy/maven2" />
  </target>

</project>

{'fraction_non_alphanumeric': 0.19556272006108683, 'fraction_numerical': 0.004920884062274636, 'mean_word_length': 3.4605487228003784, 'pattern_counts': {'":': 0, '<': 435, '<?xml version=': 0, '>': 435, 'https://': 4, 'lorem ipsum': 0, 'www.': 2, 'xml': 16}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3155134', 'n_tokens_mistral': 7768, 'n_tokens_neox': 7573, 'n_words': 1232}