<commit_before><!--
  Ant build script for FindBugs
  Contributed by <NAME> <<EMAIL>>
  With changes by <NAME> <<EMAIL>>

  If you are building FindBugs for the first time, please see
  the manual for important information.  You can find it in
  "doc/manual/index.html" in the FindBugs directory, or online at

	http://findbugs.sourceforge.net/manual/index.html

  The general idea behind building FindBugs is that the top-level
  working directory (i.e., the directory containing "build.xml")
  looks like a findbugs binary installation (once the default "build"
  target has executed).

  The "release" target builds an actual binary installation
  by copying files and directories from the top-level working
  directory into a subdirectory of build/release.
-->

<project name="find-bugs" default="build">
    <!--
      *****************************************************************
      All properties that need to be customized for your system
      (e.g., location of third-party libraries) are
      contained in a separate properties file, build.properties.

      You should not need to change anything in build.xml.
      *****************************************************************
    -->
    <property file="build.properties"/>

    <property name="javac.target" value="jsr14" />
    <property name="findbugs.dir" value="." />
    <property name="findbugs.src.dir" value="${findbugs.dir}/src/java" />
    <property name="anttask.src.dir" value="${findbugs.dir}/src/antTask" />
    <property name="validatortask.src.dir" value="${findbugs.dir}/src/validator" />
    <property name="tools.src.dir" value="${findbugs.dir}/src/tools"/>
    <property name="script.src.dir" value="${findbugs.dir}/src/scripts"/>
    <property name="findbugs.jar.name" value="findbugs.jar" />
    <property name="findbugs.annotations.jar.name" value="findbugsAnnotations.jar"/>
    <property name="findbugsgui.jar.name" value="findbugsGUI.jar" />
    <property name="coreplugin.jar.name" value="coreplugin.jar" />
    <property name="styleplugin.jar.name" value="styleplugin.jar" />
    <property name="anttask.jar.name" value="findbugs-ant.jar" />
    <property name="validatortask.jar.name" value="validatortask.jar" />
    <property name="doc.src.dir" value="${findbugs.dir}/src/doc" />
    <property name="doc.dir" value="${findbugs.dir}/doc" />
    <property name="javadoc.dir" value="${findbugs.dir}/doc/api" />
    <property name="build.dir" value="${findbugs.dir}/build" />
    <property name="jar.dir" value="lib" />
    <property name="script.dir" value="bin" />
    <property name="plugin.dir" value="plugin" />
    <property name="class.dir" value="${build.dir}/classes" />
    <property name="tools.dir" value="${build.dir}/tools"/>
    <property name="test.dir" value="${findbugs.dir}/test" />
    <property name="junit.src.dir" value="${findbugs.dir}/src/junit"/>
    <property name="junit.build.dir" value="${build.dir}/junit"/>
    <property name="release.prop.file" value="${build.dir}/release.properties" />
    <property name="html.xsl.stylesheet" value="${xsl.stylesheet.home}/html/chunk.xsl" />
    <property name="fo.xsl.stylesheet" value="${xsl.stylesheet.home}/fo/docbook.xsl" />
    <property name="eclipse.xsl.stylesheet" value="${xsl.stylesheet.home}/eclipse/eclipse.xsl" />
    <property name="doc.props.file" value="${findbugs.dir}/etc/doc.properties"/>
    <property name="html.uptodate.stamp" value="${build.dir}/html-uptodate.stamp"/>
    <property name="script.props.file" value="${findbugs.dir}/etc/script.properties"/>
    <property name="scripts.uptodate.stamp" value="${findbugs.dir}/build/scripts-uptodate.stamp"/>

    <!--
      *****************************************************************
      Class paths
      *****************************************************************
    -->

    <!-- Classpath for compiling FindBugs. -->
    <path id="classpath">
      <pathelement location="${jar.dir}/clover.jar" />
      <pathelement location="${jar.dir}/bcel.jar" />
      <pathelement location="${jar.dir}/dom4j-full.jar" />
      <pathelement location="${jar.dir}/AppleJavaExtensions.jar" />
    </path>

    <!-- Classpath for running Saxon -->
    <path id="saxon.classpath">
      <pathelement location="${saxon.home}/saxon.jar"/>
      <pathelement path="${java.class.path}"/>
    </path>

    <!-- Classpath for running FOP -->
    <path id="fop.classpath">
      <pathelement path="${fop.home}/build/fop.jar" />
      <fileset dir="${fop.home}/lib">
        <include name="**/*.jar"/>
      </fileset>
      <pathelement path="${java.class.path}"/>
    </path>

    <!-- Classpath for building tools. -->
    <path id="tools.buildclasspath">
      <path refid="classpath"/>
      <pathelement path="${class.dir}"/>
      <pathelement location="${jar.dir}/junit.jar"/>
    </path>

    <!-- Classpath for running tools. -->
    <path id="tools.classpath">
      <path refid="tools.buildclasspath"/>
      <pathelement location="${tools.dir}"/>
    </path>

    <!-- Classpath for compiling and running junit tests. -->
    <path id="junit.classpath">
      <pathelement location="${jar.dir}/findbugs.jar"/>
      <pathelement location="${jar.dir}/bcel.jar"/>
      <pathelement location="${jar.dir}/dom4j-full.jar" />
      <pathelement location="${jar.dir}/junit.jar"/>
    </path>
    
    <path id="test.classpath">
      <pathelement location="${jar.dir}/junit.jar"/>
      <pathelement location="${jar.dir}/findbugs.jar"/>
    </path>

    <!--
      *****************************************************************
      FileSets
      *****************************************************************
    -->

    <!-- All Java source files used to generated HTML. -->
    <fileset id="tools.html.src" dir="${tools.src.dir}"
             includes="edu/umd/cs/findbugs/tools/html/*.java"/>

    <!--
      *****************************************************************
      Targets
      *****************************************************************
    -->

    <target name="clean">
      <delete dir="${build.dir}" />
      <delete dir="${plugin.dir}" />
      <delete dir="${doc.dir}" />
      <delete file="${doc.src.dir}/bugDescriptions.html"/>
      <delete file="${release.prop.file}" />
      <delete file="${jar.dir}/${findbugs.jar.name}" />
      <delete file="${jar.dir}/${findbugs.annotations.jar.name}" />
      <delete file="${jar.dir}/${findbugsgui.jar.name}" />
      <delete file="${jar.dir}/${anttask.jar.name}" />
      <delete file="${jar.dir}/${validatortask.jar.name}" />
      <delete>
        <fileset dir="${test.dir}" includes="**/*.class" />
        <fileset dir="${test.dir}" includes="**/*.jar" />
      </delete>
    </target>

    <target name="init" >
      <mkdir dir="${jar.dir}" />
      <mkdir dir="${plugin.dir}" />
      <mkdir dir="${class.dir}" />
      <mkdir dir="${doc.dir}" />
      <mkdir dir="${build.dir}" />
      <mkdir dir="${tools.dir}"/>
    </target>


<target name="pmd-rt">
    <taskdef name="pmd" classname="net.sourceforge.pmd.ant.PMDTask"/>
    <pmd rulesetfiles="recommendedruleset.xml">
        <formatter type="html" toFile="pmd-rt_report.html"/>
        <fileset dir="/user/pugh/jdk1.5/src">
            <include name="**/*.java"/>
        </fileset>
    </pmd>
</target>

<target name="pmd">
    <taskdef name="pmd" classname="net.sourceforge.pmd.ant.PMDTask"/>
    <pmd rulesetfiles="customruleset.xml">
        <formatter type="html" toFile="pmd_report.html"/>
        <fileset dir="${findbugs.src.dir}">
            <include name="**/*.java"/>
        </fileset>
    </pmd>
</target>

<target name="cpd">
    <taskdef name="cpd" classname="net.sourceforge.pmd.cpd.CPDTask" />
    <cpd minimumTokenCount="100" outputFile="cpd.txt">
        <fileset dir="${findbugs.src.dir}">
            <include name="**/*.java"/>
        </fileset>
    </cpd>
</target>



    <!-- Target to compile classes. -->
    <target name="classes" depends="init">
      <javac srcdir="${findbugs.src.dir}" destdir="${class.dir}"
             source="1.5"
             target="${javac.target}"
             fork="true"
             debug="true">
        <classpath refid="classpath" />
        <compilerarg compiler="javac1.5" value="-Xlint:unchecked"/>
        <compilerarg compiler="javac1.5" value="-Xlint:deprecation"/>
      </javac>
    </target>

    <!-- Target to build jar file for the main engine. -->
    <target name="findbugs" depends="classes">
      <copy todir="${class.dir}">
        <fileset dir="${findbugs.src.dir}" includes="**/*.properties"/>
      </copy>
      <copy file="${findbugs.dir}/etc/summary.xsl" todir="${class.dir}" />
      <copy todir="${class.dir}">
        <fileset dir="${findbugs.dir}/src/xsl" includes="*.xsl"/>
      </copy>

      <!--
        FindBugs engine - all classes except those in the GUI,
        the Detector implementations (which are part of the core plugin),
        and the Ant task (which is put in its own Jar file).
      -->
      <jar destfile="${jar.dir}/${findbugs.jar.name}"
           basedir="${class.dir}" manifest="${findbugs.dir}/etc/MANIFEST-findbugs.MF">
        <exclude name="edu/umd/cs/findbugs/gui/**" />
        <exclude name="edu/umd/cs/findbugs/detect/*" />
        <exclude name="*.xml" />
        <exclude name="edu/umd/cs/findbugs/anttask/**" />
      </jar>

      <!--
        JDK 1.5 annotations supported by FindBugs.
      -->
      <jar destfile="${jar.dir}/${findbugs.annotations.jar.name}"
           basedir="${class.dir}">
        <include name="edu/umd/cs/findbugs/annotations/**"/>
      </jar>
    </target>

    <!-- Check XML files (plugin descriptor and messages files)
         for the plugin being built to ensure
         that the messages files actually define all of the messages
         required by the plugin (as defined in the plugin descriptor). -->
    <target name="pluginXML" depends="classes">
      <java dir="${findbugs.dir}"
            fork="true"
            classname="edu.umd.cs.findbugs.tools.xml.CheckMessages"
            failonerror="true">
         <classpath refid="tools.classpath"/>
         <arg value="etc/findbugs.xml"/>
         <arg value="etc/messages.xml"/>
      </java>
    </target>

    <!-- Generic target to build a plugin.
         Use antcall to invoke.
         Parameters:
            plugin.xml.dir - where plugin descriptor and messages file(s) are
            plugin.subpkg.name - subpkg of edu.umd.cs.findbugs containing classes
            plugin.jar.name - filename of jar file to create
    -->
    <target name="plugin" depends="classes,tools,pluginXML">
      <jar destfile="${plugin.dir}/${plugin.jar.name}">
         <fileset dir="${plugin.xml.dir}" includes="*.xml"/>
         <fileset dir="${plugin.xml.dir}" includes="*.xsd"/>
         <fileset dir="${class.dir}" includes="edu/umd/cs/findbugs/${plugin.subpkg.name}/*.class" />
      </jar>
    </target>

  <!-- Target to build all standard plugins. -->
  <target name="allplugins" depends="coreplugin"/>

   <!-- Target to build the core plugin. -->
   <target name="coreplugin">
     <taskdef name="SchemaValidator" classname="edu.umd.cs.findbugs.validatortask.SchemaValidatorTask" classpath="${jar.dir}/${validatortask.jar.name}"/>

	 <SchemaValidator xml="${findbugs.dir}/etc/findbugs.xml" failonerror="true"/>
	 <SchemaValidator xml="${findbugs.dir}/etc/messages.xml" failonerror="true"/>
	 <SchemaValidator xml="${findbugs.dir}/etc/messages_fr.xml" failonerror="true"/>
	 <SchemaValidator xml="${findbugs.dir}/etc/messages_ja.xml" failonerror="true"/>
     <antcall target="plugin">
       <param name="plugin.xml.dir" value="${findbugs.dir}/etc"/>
       <param name="plugin.subpkg.name" value="detect"/>
       <param name="plugin.jar.name" value="${coreplugin.jar.name}"/>
     </antcall>
   </target>

  <!-- Target to build the style plugin. -->
<!--
  <target name="styleplugin">
    <antcall target="plugin">
       <param name="plugin.xml.dir" value="${findbugs.dir}/etc/style"/>
       <param name="plugin.subpkg.name" value="style"/>
       <param name="plugin.jar.name" value="${styleplugin.jar.name}"/>
    </antcall><commit_msg>hmm, charset problems, me thinks.<commit_after><!--
  Ant build script for FindBugs
  Contributed by <NAME> <<EMAIL>>
  With changes by <NAME> <<EMAIL>>

  If you are building FindBugs for the first time, please see
  the manual for important information.  You can find it in
  "doc/manual/index.html" in the FindBugs directory, or online at

	http://findbugs.sourceforge.net/manual/index.html

  The general idea behind building FindBugs is that the top-level
  working directory (i.e., the directory containing "build.xml")
  looks like a findbugs binary installation (once the default "build"
  target has executed).

  The "release" target builds an actual binary installation
  by copying files and directories from the top-level working
  directory into a subdirectory of build/release.
-->

<project name="find-bugs" default="build">
    <!--
      *****************************************************************
      All properties that need to be customized for your system
      (e.g., location of third-party libraries) are
      contained in a separate properties file, build.properties.

      You should not need to change anything in build.xml.
      *****************************************************************
    -->
    <property file="build.properties"/>

    <property name="javac.target" value="jsr14" />
    <property name="findbugs.dir" value="." />
    <property name="findbugs.src.dir" value="${findbugs.dir}/src/java" />
    <property name="anttask.src.dir" value="${findbugs.dir}/src/antTask" />
    <property name="validatortask.src.dir" value="${findbugs.dir}/src/validator" />
    <property name="tools.src.dir" value="${findbugs.dir}/src/tools"/>
    <property name="script.src.dir" value="${findbugs.dir}/src/scripts"/>
    <property name="findbugs.jar.name" value="findbugs.jar" />
    <property name="findbugs.annotations.jar.name" value="findbugsAnnotations.jar"/>
    <property name="findbugsgui.jar.name" value="findbugsGUI.jar" />
    <property name="coreplugin.jar.name" value="coreplugin.jar" />
    <property name="styleplugin.jar.name" value="styleplugin.jar" />
    <property name="anttask.jar.name" value="findbugs-ant.jar" />
    <property name="validatortask.jar.name" value="validatortask.jar" />
    <property name="doc.src.dir" value="${findbugs.dir}/src/doc" />
    <property name="doc.dir" value="${findbugs.dir}/doc" />
    <property name="javadoc.dir" value="${findbugs.dir}/doc/api" />
    <property name="build.dir" value="${findbugs.dir}/build" />
    <property name="jar.dir" value="lib" />
    <property name="script.dir" value="bin" />
    <property name="plugin.dir" value="plugin" />
    <property name="class.dir" value="${build.dir}/classes" />
    <property name="tools.dir" value="${build.dir}/tools"/>
    <property name="test.dir" value="${findbugs.dir}/test" />
    <property name="junit.src.dir" value="${findbugs.dir}/src/junit"/>
    <property name="junit.build.dir" value="${build.dir}/junit"/>
    <property name="release.prop.file" value="${build.dir}/release.properties" />
    <property name="html.xsl.stylesheet" value="${xsl.stylesheet.home}/html/chunk.xsl" />
    <property name="fo.xsl.stylesheet" value="${xsl.stylesheet.home}/fo/docbook.xsl" />
    <property name="eclipse.xsl.stylesheet" value="${xsl.stylesheet.home}/eclipse/eclipse.xsl" />
    <property name="doc.props.file" value="${findbugs.dir}/etc/doc.properties"/>
    <property name="html.uptodate.stamp" value="${build.dir}/html-uptodate.stamp"/>
    <property name="script.props.file" value="${findbugs.dir}/etc/script.properties"/>
    <property name="scripts.uptodate.stamp" value="${findbugs.dir}/build/scripts-uptodate.stamp"/>

    <!--
      *****************************************************************
      Class paths
      *****************************************************************
    -->

    <!-- Classpath for compiling FindBugs. -->
    <path id="classpath">
      <pathelement location="${jar.dir}/clover.jar" />
      <pathelement location="${jar.dir}/bcel.jar" />
      <pathelement location="${jar.dir}/dom4j-full.jar" />
      <pathelement location="${jar.dir}/AppleJavaExtensions.jar" />
    </path>

    <!-- Classpath for running Saxon -->
    <path id="saxon.classpath">
      <pathelement location="${saxon.home}/saxon.jar"/>
      <pathelement path="${java.class.path}"/>
    </path>

    <!-- Classpath for running FOP -->
    <path id="fop.classpath">
      <pathelement path="${fop.home}/build/fop.jar" />
      <fileset dir="${fop.home}/lib">
        <include name="**/*.jar"/>
      </fileset>
      <pathelement path="${java.class.path}"/>
    </path>

    <!-- Classpath for building tools. -->
    <path id="tools.buildclasspath">
      <path refid="classpath"/>
      <pathelement path="${class.dir}"/>
      <pathelement location="${jar.dir}/junit.jar"/>
    </path>

    <!-- Classpath for running tools. -->
    <path id="tools.classpath">
      <path refid="tools.buildclasspath"/>
      <pathelement location="${tools.dir}"/>
    </path>

    <!-- Classpath for compiling and running junit tests. -->
    <path id="junit.classpath">
      <pathelement location="${jar.dir}/findbugs.jar"/>
      <pathelement location="${jar.dir}/bcel.jar"/>
      <pathelement location="${jar.dir}/dom4j-full.jar" />
      <pathelement location="${jar.dir}/junit.jar"/>
    </path>
    
    <path id="test.classpath">
      <pathelement location="${jar.dir}/junit.jar"/>
      <pathelement location="${jar.dir}/findbugs.jar"/>
    </path>

    <!--
      *****************************************************************
      FileSets
      *****************************************************************
    -->

    <!-- All Java source files used to generated HTML. -->
    <fileset id="tools.html.src" dir="${tools.src.dir}"
             includes="edu/umd/cs/findbugs/tools/html/*.java"/>

    <!--
      *****************************************************************
      Targets
      *****************************************************************
    -->

    <target name="clean">
      <delete dir="${build.dir}" />
      <delete dir="${plugin.dir}" />
      <delete dir="${doc.dir}" />
      <delete file="${doc.src.dir}/bugDescriptions.html"/>
      <delete file="${release.prop.file}" />
      <delete file="${jar.dir}/${findbugs.jar.name}" />
      <delete file="${jar.dir}/${findbugs.annotations.jar.name}" />
      <delete file="${jar.dir}/${findbugsgui.jar.name}" />
      <delete file="${jar.dir}/${anttask.jar.name}" />
      <delete file="${jar.dir}/${validatortask.jar.name}" />
      <delete>
        <fileset dir="${test.dir}" includes="**/*.class" />
        <fileset dir="${test.dir}" includes="**/*.jar" />
      </delete>
    </target>

    <target name="init" >
      <mkdir dir="${jar.dir}" />
      <mkdir dir="${plugin.dir}" />
      <mkdir dir="${class.dir}" />
      <mkdir dir="${doc.dir}" />
      <mkdir dir="${build.dir}" />
      <mkdir dir="${tools.dir}"/>
    </target>


<target name="pmd-rt">
    <taskdef name="pmd" classname="net.sourceforge.pmd.ant.PMDTask"/>
    <pmd rulesetfiles="recommendedruleset.xml">
        <formatter type="html" toFile="pmd-rt_report.html"/>
        <fileset dir="/user/pugh/jdk1.5/src">
            <include name="**/*.java"/>
        </fileset>
    </pmd>
</target>

<target name="pmd">
    <taskdef name="pmd" classname="net.sourceforge.pmd.ant.PMDTask"/>
    <pmd rulesetfiles="customruleset.xml">
        <formatter type="html" toFile="pmd_report.html"/>
        <fileset dir="${findbugs.src.dir}">
            <include name="**/*.java"/>
        </fileset>
    </pmd>
</target>

<target name="cpd">
    <taskdef name="cpd" classname="net.sourceforge.pmd.cpd.CPDTask" />
    <cpd minimumTokenCount="100" outputFile="cpd.txt">
        <fileset dir="${findbugs.src.dir}">
            <include name="**/*.java"/>
        </fileset>
    </cpd>
</target>



    <!-- Target to compile classes. -->
    <target name="classes" depends="init">
      <javac srcdir="${findbugs.src.dir}" destdir="${class.dir}"
             source="1.5"
             target="${javac.target}"
             fork="true"
             debug="true">
        <classpath refid="classpath" />
        <compilerarg compiler="javac1.5" value="-Xlint:unchecked"/>
        <compilerarg compiler="javac1.5" value="-Xlint:deprecation"/>
      </javac>
    </target>

    <!-- Target to build jar file for the main engine. -->
    <target name="findbugs" depends="classes">
      <copy todir="${class.dir}">
        <fileset dir="${findbugs.src.dir}" includes="**/*.properties"/>
      </copy>
      <copy file="${findbugs.dir}/etc/summary.xsl" todir="${class.dir}" />
      <copy todir="${class.dir}">
        <fileset dir="${findbugs.dir}/src/xsl" includes="*.xsl"/>
      </copy>

      <!--
        FindBugs engine - all classes except those in the GUI,
        the Detector implementations (which are part of the core plugin),
        and the Ant task (which is put in its own Jar file).
      -->
      <jar destfile="${jar.dir}/${findbugs.jar.name}"
           basedir="${class.dir}" manifest="${findbugs.dir}/etc/MANIFEST-findbugs.MF">
        <exclude name="edu/umd/cs/findbugs/gui/**" />
        <exclude name="edu/umd/cs/findbugs/detect/*" />
        <exclude name="*.xml" />
        <exclude name="edu/umd/cs/findbugs/anttask/**" />
      </jar>

      <!--
        JDK 1.5 annotations supported by FindBugs.
      -->
      <jar destfile="${jar.dir}/${findbugs.annotations.jar.name}"
           basedir="${class.dir}">
        <include name="edu/umd/cs/findbugs/annotations/**"/>
      </jar>
    </target>

    <!-- Check XML files (plugin descriptor and messages files)
         for the plugin being built to ensure
         that the messages files actually define all of the messages
         required by the plugin (as defined in the plugin descriptor). -->
    <target name="pluginXML" depends="classes">
      <java dir="${findbugs.dir}"
            fork="true"
            classname="edu.umd.cs.findbugs.tools.xml.CheckMessages"
            failonerror="true">
         <classpath refid="tools.classpath"/>
         <arg value="etc/findbugs.xml"/>
         <arg value="etc/messages.xml"/>
      </java>
    </target>

    <!-- Generic target to build a plugin.
         Use antcall to invoke.
         Parameters:
            plugin.xml.dir - where plugin descriptor and messages file(s) are
            plugin.subpkg.name - subpkg of edu.umd.cs.findbugs containing classes
            plugin.jar.name - filename of jar file to create
    -->
    <target name="plugin" depends="classes,tools,pluginXML">
      <jar destfile="${plugin.dir}/${plugin.jar.name}">
         <fileset dir="${plugin.xml.dir}" includes="*.xml"/>
         <fileset dir="${plugin.xml.dir}" includes="*.xsd"/>
         <fileset dir="${class.dir}" includes="edu/umd/cs/findbugs/${plugin.subpkg.name}/*.class" />
      </jar>
    </target>

  <!-- Target to build all standard plugins. -->
  <target name="allplugins" depends="coreplugin"/>

   <!-- Target to build the core plugin. -->
   <target name="coreplugin">
     <taskdef name="SchemaValidator" classname="edu.umd.cs.findbugs.validatortask.SchemaValidatorTask" classpath="${jar.dir}/${validatortask.jar.name}"/>

	 <SchemaValidator xml="${findbugs.dir}/etc/findbugs.xml" failonerror="true"/>
	 <SchemaValidator xml="${findbugs.dir}/etc/messages.xml" failonerror="true"/>
<!--	 <SchemaValidator xml="${findbugs.dir}/etc/messages_fr.xml" failonerror="true"/> -->
<!--	 <SchemaValidator xml="${findbugs.dir}/etc/messages_ja.xml" failonerror="true"/> -->
     <antcall target="plugin">
       <param name="plugin.xml.dir" value="${findbugs.dir}/etc"/>
       <param name="plugin.subpkg.name" value="detect"/>
       <param name="plugin.jar.name" value="${coreplugin.jar.name}"/>
     </antcall>
   </target>

  <!-- Target to build the style plugin. -->
<!--
  <target name="styleplugin">
    <antcall target="plugin">
       <param name="plugin.xml.dir" value="${findbugs.dir}/etc/style"/>
       <param name="plugin.subpkg.name" value="style"/>
       <param name="plugin.jar.name" value="${styleplugin.jar.name}"/>
    </antcall>
{'fraction_non_alphanumeric': 0.22391304347826088, 'fraction_numerical': 0.001394585726004922, 'mean_word_length': 3.074364973262032, 'pattern_counts': {'":': 0, '<': 423, '<?xml version=': 0, '>': 421, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 44}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7085628', 'n_tokens_mistral': 7506, 'n_tokens_neox': 7179, 'n_words': 1427}