<commit_before><project name="AppEngineMapReduce" default="dist" basedir=".">
  <description>
    A user space MapReduce library for AppEngine.
  </description>
  <property name="src" location="src" />
  <property name="build" location="build" />
  <property name="lib" location="lib" />
  <property name="dist" location="dist" />
  <property name="static" location="static" />

  <property name="testsrc" location="test" />
  <property name="testbuild" location="test_build" />
  <property name="testlib" location="test_lib" />

  <property name="sdk" location="sdk" />
  <property name="version" value="1.7.6" />
  <property name="sdkdirname" value="appengine-java-sdk-${version}" />
  <property name="sdkfilename" value="${sdkdirname}.zip" />
  <property name="sdklibdir" location="${sdk}/${sdkdirname}/lib" />

  <path id="sdkjars">
    <fileset dir="${sdklibdir}/user">
      <include name="appengine-api-1.0-sdk-${version}.jar" />
    </fileset>
    <fileset dir="${sdklibdir}/shared">
      <include name="servlet-api.jar" />
    </fileset>
  </path>

  <!-- We plan to move to downloading all of these on the fly, but for
       now json is jar'ed up, and the other two are released
       relatively infrequently. -->
  <path id="thirdpartyjars">
    <fileset dir="${lib}">
      <include name="*.jar" />
    </fileset>
  </path>

  <path id="testjars">
    <pathelement path="${testlib}/easymock-3.0.jar" />
    <pathelement path="${testlib}/junit-4.8.2.jar" />
    <pathelement path="${testlib}/cglib-nodep-2.2.jar" />
    <pathelement path="${testlib}/objenesis-1.2.jar" />
    <pathelement path="${sdklibdir}/testing/appengine-testing.jar" />
    <pathelement path="${sdklibdir}/impl/appengine-api-stubs.jar" />
  </path>

  <target name="init">
    <mkdir dir="${build}" />
  </target>

  <target name="compile" depends="init,download_sdk">
    <javac srcdir="${src}" destdir="${build}" debug="on" debuglevel="lines,source">
      <compilerarg value="-Xlint:all"/>
      <compilerarg value="-Xlint:-sunapi"/>
      <classpath>
        <path refid="sdkjars" />
        <path refid="thirdpartyjars" />
      </classpath>
    </javac>
  </target>

  <target name="test_init">
    <mkdir dir="${testbuild}" />
  </target>

  <target name="test_compile" depends="test_init,dist,download_sdk">
    <javac srcdir="${testsrc}" destdir="${testbuild}" debug="on" debuglevel="lines,source">
      <classpath>
        <path refid="sdkjars" />
        <path refid="thirdpartyjars" />
        <path refid="testjars" />
        <pathelement location="${dist}/lib/appengine-mapper.jar" />
      </classpath>
    </javac>
  </target>

  <target name="test" depends="test_compile">
    <junit filtertrace="off">
      <batchtest>
        <fileset dir="${testsrc}">
          <include name="**/*Test*.java" />
          <!-- We must exclude class that has "Test" in the name AND does not have tests (i.e. a 
               test helper or abstract class) to prevent spurious JUnit errors -->
          <exclude name="com/google/appengine/tools/mapreduce/EndToEndTestCase.java" />
          <exclude name="com/google/appengine/tools/mapreduce/impl/TestWorkerTask.java" />
          <exclude name="com/google/appengine/tools/mapreduce/impl/shardedjob/EndToEndTestCase.java" />
          <exclude name="com/google/appengine/tools/mapreduce/impl/shardedjob/TestController.java" />
          <exclude name="com/google/appengine/tools/mapreduce/impl/shardedjob/TestTask.java" />
          <exclude name="com/google/appengine/tools/mapreduce/inputs/BlobstoreInputTestCase.java" />
        </fileset>
      </batchtest>
      <classpath>
        <path refid="sdkjars" />
        <path refid="thirdpartyjars" />
        <path refid="testjars" />
        <pathelement location="${dist}/lib/appengine-mapper.jar" />
        <pathelement location="${testbuild}" />
      </classpath>
      <formatter type="brief" usefile="false" />
    </junit>
  </target>

  <target name="dist" depends="compile">
    <mkdir dir="${dist}/lib" />
    <copy todir="${build}/com/google/appengine/tools/mapreduce">
      <fileset dir="${static}" />
    </copy>
    <copy todir="${build}/com/google/appengine/tools/pipeline/impl/servlets/ui">
      <fileset dir="pipeline-ui" />
    </copy>
    <copy todir="${build}">
      <fileset dir="${src}" />
    </copy>
    <jar jarfile="${dist}/lib/appengine-mapper.jar" basedir="${build}" />
    <copy todir="${dist}/lib">
      <path refid="thirdpartyjars" />
    </copy>
  </target>

  <target name="clean">
    <delete dir="${build}" />
    <delete dir="${dist}" />
    <delete dir="${testbuild}" />
  </target>

  <target name="check_sdk_extracted">
    <available file="${sdk}/${sdkdirname}" property="sdkextracted" />
  </target>

  <target name="download_sdk" description="Gets the AppEngine SDK" depends="check_sdk_extracted" unless="sdkextracted">
    <mkdir dir="${sdk}" />
    <get src="http://googleappengine.googlecode.com/files/${sdkfilename}" dest="${sdk}/${sdkfilename}" />
    <unzip src="${sdk}/${sdkfilename}" dest="${sdk}" />
  </target>

  <!-- Separate target because downloading the SDK can be expensive and
       usually only needs to be done once. -->
  <target name="clean_sdk">
    <delete dir="${sdk}" />
  </target>

  <target name="copy_libs_to_example" depends="dist">
    <copy todir="example/war/WEB-INF/lib">
      <fileset dir="${dist}/lib" />
      <path refid="sdkjars" />
    </copy>
  </target>

  <target name="compile_example" depends="copy_libs_to_example">
    <mkdir dir="example/war/WEB-INF/classes" /><commit_msg>Have ant download GCS libary into the MapReduce build. Minor build fixes.<commit_after><project name="AppEngineMapReduce" default="dist" basedir=".">
  <description>
    A user space MapReduce library for AppEngine.
  </description>
  <property name="src" location="src" />
  <property name="build" location="build" />
  <property name="lib" location="lib" />
  <property name="dist" location="dist" />
  <property name="static" location="static" />

  <property name="testsrc" location="test" />
  <property name="testbuild" location="test_build" />
  <property name="testlib" location="test_lib" />

  <property name="sdk" location="sdk" />
  <property name="version" value="1.7.7" />
  <property name="sdkdirname" value="appengine-java-sdk-${version}" />
  <property name="sdkfilename" value="${sdkdirname}.zip" />
  <property name="sdklibdir" location="${sdk}/${sdkdirname}/lib" />

  <property name="gcsversion" value="r34" />
  <property name="gcsclientzip" value="appengine-gcs-client-java.zip" />
  <property name="gcsclientjar" value="appengine-gcs-client.jar" />

  <path id="sdkjars">
    <fileset dir="${sdklibdir}/user">
      <include name="appengine-api-1.0-sdk-${version}.jar" />
    </fileset>
    <fileset dir="${sdklibdir}/shared">
      <include name="servlet-api.jar" />
    </fileset>
  </path>

  <!-- We plan to move to downloading all of these on the fly, but for
       now json is jar'ed up, and the other two are released
       relatively infrequently. -->
  <path id="thirdpartyjars">
    <fileset dir="${lib}">
      <include name="*.jar" />
    </fileset>
  </path>

  <path id="testjars">
    <pathelement path="${testlib}/easymock-3.0.jar" />
    <pathelement path="${testlib}/junit-4.8.2.jar" />
    <pathelement path="${testlib}/cglib-nodep-2.2.jar" />
    <pathelement path="${testlib}/objenesis-1.2.jar" />
    <pathelement path="${sdklibdir}/testing/appengine-testing.jar" />
    <pathelement path="${sdklibdir}/impl/appengine-api-stubs.jar" />
  </path>

  <target name="init">
    <mkdir dir="${build}" />
  </target>

  <target name="compile" depends="init,download_sdk,download_gcs">
    <javac srcdir="${src}" destdir="${build}" debug="on" debuglevel="lines,source">
      <compilerarg value="-Xlint:all"/>
      <compilerarg value="-Xlint:-sunapi"/>
      <classpath>
        <path refid="sdkjars" />
        <path refid="thirdpartyjars" />
      </classpath>
    </javac>
  </target>

  <target name="test_init">
    <mkdir dir="${testbuild}" />
  </target>

  <target name="test_compile" depends="test_init,dist,download_sdk,download_gcs">
    <javac srcdir="${testsrc}" destdir="${testbuild}" debug="on" debuglevel="lines,source">
      <classpath>
        <path refid="sdkjars" />
        <path refid="thirdpartyjars" />
        <path refid="testjars" />
        <pathelement location="${dist}/lib/appengine-mapper.jar" />
      </classpath>
    </javac>
  </target>

  <target name="test" depends="test_compile">
    <junit filtertrace="off" maxmemory="256m" fork="true">
      <batchtest>
        <fileset dir="${testsrc}">
          <include name="**/*Test*.java" />
          <!-- We must exclude class that has "Test" in the name AND does not have tests (i.e. a 
               test helper or abstract class) to prevent spurious JUnit errors -->
          <exclude name="com/google/appengine/tools/mapreduce/EndToEndTestCase.java" />
          <exclude name="com/google/appengine/tools/mapreduce/impl/TestWorkerTask.java" />
          <exclude name="com/google/appengine/tools/mapreduce/impl/shardedjob/EndToEndTestCase.java" />
          <exclude name="com/google/appengine/tools/mapreduce/impl/shardedjob/TestController.java" />
          <exclude name="com/google/appengine/tools/mapreduce/impl/shardedjob/TestTask.java" />
          <exclude name="com/google/appengine/tools/mapreduce/inputs/BlobstoreInputTestCase.java" />
          <exclude name="com/google/appengine/tools/mapreduce/inputs/CloudStorageLineInputTestCase.java" />
        </fileset>
      </batchtest>
      <classpath>
        <path refid="sdkjars" />
        <path refid="thirdpartyjars" />
        <path refid="testjars" />
        <pathelement location="${dist}/lib/appengine-mapper.jar" />
        <pathelement location="${testbuild}" />
      </classpath>
      <formatter type="brief" usefile="false" />
    </junit>
  </target>

  <target name="dist" depends="compile">
    <mkdir dir="${dist}/lib" />
    <copy todir="${build}/com/google/appengine/tools/mapreduce">
      <fileset dir="${static}" />
    </copy>
    <copy todir="${build}/com/google/appengine/tools/pipeline/impl/servlets/ui">
      <fileset dir="pipeline-ui" />
    </copy>
    <copy todir="${build}">
      <fileset dir="${src}" />
    </copy>
    <jar jarfile="${dist}/lib/appengine-mapper.jar" basedir="${build}" />
    <copy todir="${dist}/lib">
      <path refid="thirdpartyjars" />
    </copy>
  </target>

  <target name="clean">
    <delete dir="${build}" />
    <delete dir="${dist}" />
    <delete dir="${testbuild}" />
  </target>

  <target name="check_sdk_extracted">
    <available file="${sdk}/${sdkdirname}" property="sdkextracted" />
  </target>

  <target name="download_sdk" description="Gets the AppEngine SDK" depends="check_sdk_extracted" unless="sdkextracted">
    <mkdir dir="${sdk}" />
    <get src="http://googleappengine.googlecode.com/files/${sdkfilename}" dest="${sdk}/${sdkfilename}" />
    <unzip src="${sdk}/${sdkfilename}" dest="${sdk}" />
  </target>

  <!-- Helper macro to delete a directory only if it is empty -->
  <macrodef name="deleteEmptyFolder">
    <attribute name="dir" />
    <sequential>
      <delete includeemptydirs="true" verbose="true">
        <fileset dir="@{dir}">
          <and>
            <size value="0" />
            <type type="dir" />
          </and>
        </fileset>
      </delete>
    </sequential>
  </macrodef>

  <target name="check_gcs_extracted">
    <available file="${lib}/${gcsclientjar}" property="gcsextracted" />
  </target>

  <target name="download_gcs" description="Gets the GCS Library" depends="check_gcs_extracted" unless="gcsextracted">
    <mkdir dir="${lib}/tmp" />
    <get src="https://appengine-gcs-client.googlecode.com/files/appengine-gcs-client-java-${gcsversion}.zip"
      dest="${lib}/tmp/${gcsclientzip}" />
    <unzip src="${lib}/tmp/${gcsclientzip}" dest="${lib}">
      <patternset>
        <include name="${gcsclientjar}"/>
      </patternset>
    </unzip>
    <delete file="${lib}/tmp/${gcsclientzip}" />
    <deleteEmptyFolder dir="${lib}/tmp" />
  </target>

  <!-- Separate target because downloading the SDK can be expensive and
       usually only needs to be done once. -->
  <target name="clean_sdk">
    <delete dir="${sdk}" />
  </target>

  <target name="copy_libs_to_example" depends="dist">
    <copy todir="example/war/WEB-INF/lib">
      <fileset dir="${dist}/lib" />
      <path refid="sdkjars" />
    </copy>
  </target>

  <target name="compile_example" depends="copy_libs_to_example">
    <mkdir dir="example/war/WEB-INF/classes" />
{'fraction_non_alphanumeric': 0.1878019801980198, 'fraction_numerical': 0.002693069306930693, 'mean_word_length': 3.3962395543175488, 'pattern_counts': {'":': 0, '<': 293, '<?xml version=': 0, '>': 293, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6763552', 'n_tokens_mistral': 4128, 'n_tokens_neox': 4042, 'n_words': 751}